Don't panic in forkchoiceUpdated handler (#3165)

## Issue Addressed

Fix a panic due to misuse of the Tokio executor when processing a forkchoiceUpdated response. We were previously calling `process_invalid_execution_payload` from the async function `update_execution_engine_forkchoice_async`, which resulted in a panic because `process_invalid_execution_payload` contains a call to fork choice, which ultimately calls `block_on`.

An example backtrace can be found here: https://gist.github.com/michaelsproul/ac5da03e203d6ffac672423eaf52fb20

## Proposed Changes

Wrap the call to `process_invalid_execution_payload` in a `spawn_blocking` so that `block_on` is no longer called from an async context.

## Additional Info

- I've been thinking about how to catch bugs like this with static analysis (a new Clippy lint).
- The payload validation tests have been re-worked to support distinct responses from the mock EE for newPayload and forkchoiceUpdated. Three new tests have been added covering the `Invalid`, `InvalidBlockHash` and `InvalidTerminalBlock` cases.
- I think we need a bunch more tests of different legal and illegal variations

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -2208,7 +2208,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// This method is generally much more efficient than importing each block using
     /// `Self::process_block`.
     pub fn process_chain_segment(
-        &self,
+        self: &Arc<Self>,
         chain_segment: Vec<SignedBeaconBlock<T::EthSpec>>,
     ) -> ChainSegmentResult<T::EthSpec> {
         let mut filtered_chain_segment = Vec::with_capacity(chain_segment.len());
@@ -2402,7 +2402,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Returns an `Err` if the given block was invalid, or an error was encountered during
     /// verification.
     pub fn process_block<B: IntoFullyVerifiedBlock<T>>(
-        &self,
+        self: &Arc<Self>,
         unverified_block: B,
     ) -> Result<Hash256, BlockError<T::EthSpec>> {
         // Start the Prometheus timer.
@@ -3234,7 +3234,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     ///
     /// See the documentation of `InvalidationOperation` for information about defining `op`.
     pub fn process_invalid_execution_payload(
-        &self,
+        self: &Arc<Self>,
         op: &InvalidationOperation,
     ) -> Result<(), Error> {
         debug!(
@@ -3302,7 +3302,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     }
 
     /// Execute the fork choice algorithm and enthrone the result as the canonical head.
-    pub fn fork_choice(&self) -> Result<(), Error> {
+    pub fn fork_choice(self: &Arc<Self>) -> Result<(), Error> {
         metrics::inc_counter(&metrics::FORK_CHOICE_REQUESTS);
         let _timer = metrics::start_timer(&metrics::FORK_CHOICE_TIMES);
 
@@ -3315,7 +3315,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         result
     }
 
-    fn fork_choice_internal(&self) -> Result<(), Error> {
+    fn fork_choice_internal(self: &Arc<Self>) -> Result<(), Error> {
         // Atomically obtain the head block root and the finalized block.
         let (beacon_block_root, finalized_block) = {
             let mut fork_choice = self.fork_choice.write();
@@ -3718,7 +3718,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         Ok(())
     }
 
-    pub fn prepare_beacon_proposer_blocking(&self) -> Result<(), Error> {
+    pub fn prepare_beacon_proposer_blocking(self: &Arc<Self>) -> Result<(), Error> {
         let current_slot = self.slot()?;
 
         // Avoids raising an error before Bellatrix.
@@ -3750,7 +3750,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// 1. We're in the tail-end of the slot (as defined by PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR)
     /// 2. The head block is one slot (or less) behind the prepare slot (e.g., we're preparing for
     ///    the next slot and the block at the current slot is already known).
-    pub async fn prepare_beacon_proposer_async(&self, current_slot: Slot) -> Result<(), Error> {
+    pub async fn prepare_beacon_proposer_async(
+        self: &Arc<Self>,
+        current_slot: Slot,
+    ) -> Result<(), Error> {
         let prepare_slot = current_slot + 1;
         let prepare_epoch = prepare_slot.epoch(T::EthSpec::slots_per_epoch());
 
@@ -3952,7 +3955,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     }
 
     pub fn update_execution_engine_forkchoice_blocking(
-        &self,
+        self: &Arc<Self>,
         current_slot: Slot,
     ) -> Result<(), Error> {
         // Avoids raising an error before Bellatrix.
@@ -3973,7 +3976,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     }
 
     pub async fn update_execution_engine_forkchoice_async(
-        &self,
+        self: &Arc<Self>,
         current_slot: Slot,
     ) -> Result<(), Error> {
         let next_slot = current_slot + 1;
@@ -4091,7 +4094,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         drop(forkchoice_lock);
 
         match forkchoice_updated_response {
-            Ok(status) => match &status {
+            Ok(status) => match status {
                 PayloadStatus::Valid => {
                     // Ensure that fork choice knows that the block is no longer optimistic.
                     if let Err(e) = self
@@ -4134,13 +4137,24 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     );
                     // The execution engine has stated that all blocks between the
                     // `head_execution_block_hash` and `latest_valid_hash` are invalid.
-                    self.process_invalid_execution_payload(
-                        &InvalidationOperation::InvalidateMany {
-                            head_block_root,
-                            always_invalidate_head: true,
-                            latest_valid_ancestor: *latest_valid_hash,
-                        },
-                    )?;
+                    let chain = self.clone();
+                    execution_layer
+                        .executor()
+                        .spawn_blocking_handle(
+                            move || {
+                                chain.process_invalid_execution_payload(
+                                    &InvalidationOperation::InvalidateMany {
+                                        head_block_root,
+                                        always_invalidate_head: true,
+                                        latest_valid_ancestor: latest_valid_hash,
+                                    },
+                                )
+                            },
+                            "process_invalid_execution_payload_many",
+                        )
+                        .ok_or(BeaconChainError::RuntimeShutdown)?
+                        .await
+                        .map_err(BeaconChainError::ProcessInvalidExecutionPayload)??;
 
                     Err(BeaconChainError::ExecutionForkChoiceUpdateInvalid { status })
                 }
@@ -4156,11 +4170,22 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     //
                     // Using a `None` latest valid ancestor will result in only the head block
                     // being invalidated (no ancestors).
-                    self.process_invalid_execution_payload(
-                        &InvalidationOperation::InvalidateOne {
-                            block_root: head_block_root,
-                        },
-                    )?;
+                    let chain = self.clone();
+                    execution_layer
+                        .executor()
+                        .spawn_blocking_handle(
+                            move || {
+                                chain.process_invalid_execution_payload(
+                                    &InvalidationOperation::InvalidateOne {
+                                        block_root: head_block_root,
+                                    },
+                                )
+                            },
+                            "process_invalid_execution_payload_single",
+                        )
+                        .ok_or(BeaconChainError::RuntimeShutdown)?
+                        .await
+                        .map_err(BeaconChainError::ProcessInvalidExecutionPayload)??;
 
                     Err(BeaconChainError::ExecutionForkChoiceUpdateInvalid { status })
                 }

beacon_node/beacon_chain/src/block_verification.rs

@@ -72,6 +72,7 @@ use state_processing::{
 use std::borrow::Cow;
 use std::fs;
 use std::io::Write;
+use std::sync::Arc;
 use std::time::Duration;
 use store::{Error as DBError, HotColdDB, HotStateSummary, KeyValueStore, StoreOp};
 use tree_hash::TreeHash;
@@ -577,7 +578,7 @@ pub struct FullyVerifiedBlock<'a, T: BeaconChainTypes> {
 pub trait IntoFullyVerifiedBlock<T: BeaconChainTypes>: Sized {
     fn into_fully_verified_block(
         self,
-        chain: &BeaconChain<T>,
+        chain: &Arc<BeaconChain<T>>,
     ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
         self.into_fully_verified_block_slashable(chain)
             .map(|fully_verified| {
@@ -593,7 +594,7 @@ pub trait IntoFullyVerifiedBlock<T: BeaconChainTypes>: Sized {
     /// Convert the block to fully-verified form while producing data to aid checking slashability.
     fn into_fully_verified_block_slashable(
         self,
-        chain: &BeaconChain<T>,
+        chain: &Arc<BeaconChain<T>>,
     ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>>;
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec>;
@@ -828,7 +829,7 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for GossipVerifiedBlock<T> {
     /// Completes verification of the wrapped `block`.
     fn into_fully_verified_block_slashable(
         self,
-        chain: &BeaconChain<T>,
+        chain: &Arc<BeaconChain<T>>,
     ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>> {
         let fully_verified =
             SignatureVerifiedBlock::from_gossip_verified_block_check_slashable(self, chain)?;
@@ -948,7 +949,7 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignatureVerifiedBlock<T
     /// Completes verification of the wrapped `block`.
     fn into_fully_verified_block_slashable(
         self,
-        chain: &BeaconChain<T>,
+        chain: &Arc<BeaconChain<T>>,
     ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>> {
         let header = self.block.signed_block_header();
         let (parent, block) = if let Some(parent) = self.parent {
@@ -977,7 +978,7 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignedBeaconBlock<T::Eth
     /// and then using that implementation of `IntoFullyVerifiedBlock` to complete verification.
     fn into_fully_verified_block_slashable(
         self,
-        chain: &BeaconChain<T>,
+        chain: &Arc<BeaconChain<T>>,
     ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>> {
         // Perform an early check to prevent wasting time on irrelevant blocks.
         let block_root = check_block_relevancy(&self, None, chain)
@@ -1004,7 +1005,7 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
         block: SignedBeaconBlock<T::EthSpec>,
         block_root: Hash256,
         parent: PreProcessingSnapshot<T::EthSpec>,
-        chain: &BeaconChain<T>,
+        chain: &Arc<BeaconChain<T>>,
     ) -> Result<Self, BlockError<T::EthSpec>> {
         if let Some(parent) = chain.fork_choice.read().get_block(&block.parent_root()) {
             // Reject any block where the parent has an invalid payload. It's impossible for a valid

beacon_node/beacon_chain/src/errors.rs

@@ -26,6 +26,7 @@ use state_processing::{
 };
 use std::time::Duration;
 use task_executor::ShutdownReason;
+use tokio::task::JoinError;
 use types::*;
 
 macro_rules! easy_from_to {
@@ -170,6 +171,8 @@ pub enum BeaconChainError {
     CannotAttestToFinalizedBlock {
         beacon_block_root: Hash256,
     },
+    RuntimeShutdown,
+    ProcessInvalidExecutionPayload(JoinError),
 }
 
 easy_from_to!(SlotProcessingError, BeaconChainError);

beacon_node/beacon_chain/src/execution_payload.rs

@@ -20,6 +20,7 @@ use state_processing::per_block_processing::{
     compute_timestamp_at_slot, is_execution_enabled, is_merge_transition_complete,
     partially_verify_execution_payload,
 };
+use std::sync::Arc;
 use types::*;
 
 /// Verify that `execution_payload` contained by `block` is considered valid by an execution
@@ -32,7 +33,7 @@ use types::*;
 ///
 /// https://github.com/ethereum/consensus-specs/blob/v1.1.9/specs/bellatrix/beacon-chain.md#notify_new_payload
 pub fn notify_new_payload<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
+    chain: &Arc<BeaconChain<T>>,
     state: &BeaconState<T::EthSpec>,
     block: BeaconBlockRef<T::EthSpec>,
 ) -> Result<PayloadVerificationStatus, BlockError<T::EthSpec>> {

beacon_node/beacon_chain/src/state_advance_timer.rs

@@ -172,7 +172,7 @@ async fn state_advance_timer<T: BeaconChainTypes>(
 ///
 /// See the module-level documentation for rationale.
 fn advance_head<T: BeaconChainTypes>(
-    beacon_chain: &BeaconChain<T>,
+    beacon_chain: &Arc<BeaconChain<T>>,
     log: &Logger,
 ) -> Result<(), Error> {
     let current_slot = beacon_chain.slot()?;