Initial Commit of Retrospective OTB Verification (#3372)

## Issue Addressed

* #2983 

## Proposed Changes

Basically followed the [instructions laid out here](https://github.com/sigp/lighthouse/issues/2983#issuecomment-1062494947)


Co-authored-by: Paul Hauner <paul@paulhauner.com>
Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com>

beacon_node/execution_layer/src/test_utils/execution_block_generator.rs

@@ -105,6 +105,7 @@ pub struct PoWBlock {
     pub timestamp: u64,
 }
 
+#[derive(Clone)]
 pub struct ExecutionBlockGenerator<T: EthSpec> {
     /*
      * Common database

beacon_node/execution_layer/src/test_utils/handle_rpc.rs

@@ -48,6 +48,12 @@ pub async fn handle_rpc<T: EthSpec>(
                     s.parse()
                         .map_err(|e| format!("unable to parse hash: {:?}", e))
                 })?;
+
+            // If we have a static response set, just return that.
+            if let Some(response) = *ctx.static_get_block_by_hash_response.lock() {
+                return Ok(serde_json::to_value(response).unwrap());
+            }
+
             let full_tx = params
                 .get(1)
                 .and_then(JsonValue::as_bool)

beacon_node/execution_layer/src/test_utils/mod.rs

@@ -2,7 +2,7 @@
 
 use crate::engine_api::auth::JwtKey;
 use crate::engine_api::{
-    auth::Auth, http::JSONRPC_VERSION, PayloadStatusV1, PayloadStatusV1Status,
+    auth::Auth, http::JSONRPC_VERSION, ExecutionBlock, PayloadStatusV1, PayloadStatusV1Status,
 };
 use bytes::Bytes;
 use environment::null_logger;
@@ -96,6 +96,7 @@ impl<T: EthSpec> MockServer<T> {
             preloaded_responses,
             static_new_payload_response: <_>::default(),
             static_forkchoice_updated_response: <_>::default(),
+            static_get_block_by_hash_response: <_>::default(),
             _phantom: PhantomData,
         });
 
@@ -317,6 +318,16 @@ impl<T: EthSpec> MockServer<T> {
         self.set_forkchoice_updated_response(Self::invalid_terminal_block_status());
     }
 
+    /// This will make the node appear like it is syncing.
+    pub fn all_get_block_by_hash_requests_return_none(&self) {
+        *self.ctx.static_get_block_by_hash_response.lock() = Some(None);
+    }
+
+    /// The node will respond "naturally"; it will return blocks if they're known to it.
+    pub fn all_get_block_by_hash_requests_return_natural_value(&self) {
+        *self.ctx.static_get_block_by_hash_response.lock() = None;
+    }
+
     /// Disables any static payload responses so the execution block generator will do its own
     /// verification.
     pub fn full_payload_verification(&self) {
@@ -406,6 +417,7 @@ pub struct Context<T: EthSpec> {
     pub previous_request: Arc<Mutex<Option<serde_json::Value>>>,
     pub static_new_payload_response: Arc<Mutex<Option<StaticNewPayloadResponse>>>,
     pub static_forkchoice_updated_response: Arc<Mutex<Option<PayloadStatusV1>>>,
+    pub static_get_block_by_hash_response: Arc<Mutex<Option<Option<ExecutionBlock>>>>,
     pub _phantom: PhantomData<T>,
 }