Don't return errors when fork choice fails (#3370)

## Issue Addressed

NA

## Proposed Changes

There are scenarios where the only viable head will have an invalid execution payload, in this scenario the `get_head` function on `proto_array` will return an error. We must recover from this scenario by importing blocks from the network.

This PR stops `BeaconChain::recompute_head` from returning an error so that we can't accidentally start down-scoring peers or aborting block import just because the current head has an invalid payload.

## Reviewer Notes

The following changes are included:

1. Allow `fork_choice.get_head` to fail gracefully in `BeaconChain::process_block` when trying to update the `early_attester_cache`; simply don't add the block to the cache rather than aborting the entire process.
1. Don't return an error from `BeaconChain::recompute_head_at_current_slot` and `BeaconChain::recompute_head` to defensively prevent calling functions from aborting any process just because the fork choice function failed to run.
    - This should have practically no effect, since most callers were still continuing if recomputing the head failed.
    - The outlier is that the API will return 200 rather than a 500 when fork choice fails.
1. Add the `ProtoArrayForkChoice::set_all_blocks_to_optimistic` function to recover from the scenario where we've rebooted and the persisted fork choice has an invalid head.

beacon_node/network/src/beacon_processor/tests.rs

@@ -221,7 +221,7 @@ impl TestRig {
     }
 
     pub async fn recompute_head(&self) {
-        self.chain.recompute_head_at_current_slot().await.unwrap()
+        self.chain.recompute_head_at_current_slot().await
     }
 
     pub fn head_root(&self) -> Hash256 {

beacon_node/network/src/beacon_processor/worker/gossip_methods.rs

@@ -927,21 +927,7 @@ impl<T: BeaconChainTypes> Worker<T> {
                     "peer_id" => %peer_id
                 );
 
-                if let Err(e) = self.chain.recompute_head_at_current_slot().await {
-                    error!(
-                        self.log,
-                        "Fork choice failed";
-                        "error" => ?e,
-                        "location" => "block_gossip"
-                    )
-                } else {
-                    debug!(
-                        self.log,
-                        "Fork choice success";
-                        "block" => ?block_root,
-                        "location" => "block_gossip"
-                    )
-                }
+                self.chain.recompute_head_at_current_slot().await;
             }
             Err(BlockError::ParentUnknown { .. }) => {
                 // Inform the sync manager to find parents for this block

beacon_node/network/src/beacon_processor/worker/sync_methods.rs

@@ -111,7 +111,7 @@ impl<T: BeaconChainTypes> Worker<T> {
                     None,
                 );
 
-                self.recompute_head("process_rpc_block").await;
+                self.chain.recompute_head_at_current_slot().await;
             }
         }
         // Sync handles these results
@@ -248,7 +248,7 @@ impl<T: BeaconChainTypes> Worker<T> {
             ChainSegmentResult::Successful { imported_blocks } => {
                 metrics::inc_counter(&metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_SUCCESS_TOTAL);
                 if imported_blocks > 0 {
-                    self.recompute_head("process_blocks_ok").await;
+                    self.chain.recompute_head_at_current_slot().await;
                 }
                 (imported_blocks, Ok(()))
             }
@@ -259,7 +259,7 @@ impl<T: BeaconChainTypes> Worker<T> {
                 metrics::inc_counter(&metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_FAILED_TOTAL);
                 let r = self.handle_failed_chain_segment(error);
                 if imported_blocks > 0 {
-                    self.recompute_head("process_blocks_err").await;
+                    self.chain.recompute_head_at_current_slot().await;
                 }
                 (imported_blocks, r)
             }
@@ -392,24 +392,6 @@ impl<T: BeaconChainTypes> Worker<T> {
         }
     }
 
-    /// Runs fork-choice on a given chain. This is used during block processing after one successful
-    /// block import.
-    async fn recompute_head(&self, location: &str) {
-        match self.chain.recompute_head_at_current_slot().await {
-            Ok(()) => debug!(
-                self.log,
-                "Fork choice success";
-                "location" => location
-            ),
-            Err(e) => error!(
-                self.log,
-                "Fork choice failed";
-                "error" => ?e,
-                "location" => location
-            ),
-        }
-    }
-
     /// Helper function to handle a `BlockError` from `process_chain_segment`
     fn handle_failed_chain_segment(
         &self,