ContextDeserialize and Beacon API Improvements (#7372)

* #7286
* BeaconAPI is not returning a versioned response when it should for some V1 endpoints
* these [strange functions with vX in the name that still accept `endpoint_version` arguments](https://github.com/sigp/lighthouse/blob/stable/beacon_node/http_api/src/produce_block.rs#L192)

This refactor is a prerequisite to get the fulu EF tests running.

common/eth2/src/lib.rs

@@ -16,7 +16,7 @@ pub mod types;
 
 use self::mixin::{RequestAccept, ResponseOptional};
 use self::types::{Error as ResponseError, *};
-use ::types::fork_versioned_response::ExecutionOptimisticFinalizedForkVersionedResponse;
+use ::types::beacon_response::ExecutionOptimisticFinalizedBeaconResponse;
 use derivative::Derivative;
 use either::Either;
 use futures::Stream;
@@ -283,6 +283,52 @@ impl BeaconNodeHttpClient {
         }
     }
 
+    pub async fn get_fork_contextual<T, U, Ctx, Meta>(
+        &self,
+        url: U,
+        ctx_constructor: impl Fn(ForkName) -> Ctx,
+    ) -> Result<Option<ForkVersionedResponse<T, Meta>>, Error>
+    where
+        U: IntoUrl,
+        T: ContextDeserialize<'static, Ctx>,
+        Meta: DeserializeOwned,
+        Ctx: Clone,
+    {
+        let response = self
+            .get_response(url, |b| b.accept(Accept::Json))
+            .await
+            .optional()?;
+
+        let Some(resp) = response else {
+            return Ok(None);
+        };
+
+        let bytes = resp.bytes().await?;
+
+        #[derive(serde::Deserialize)]
+        struct Helper {
+            version: ForkName,
+            #[serde(flatten)]
+            metadata: serde_json::Value,
+            data: serde_json::Value,
+        }
+
+        let helper: Helper = serde_json::from_slice(&bytes).map_err(Error::InvalidJson)?;
+
+        let metadata: Meta = serde_json::from_value(helper.metadata).map_err(Error::InvalidJson)?;
+
+        let ctx = ctx_constructor(helper.version);
+
+        let data: T = ContextDeserialize::context_deserialize(helper.data, ctx)
+            .map_err(Error::InvalidJson)?;
+
+        Ok(Some(ForkVersionedResponse {
+            version: helper.version,
+            metadata,
+            data,
+        }))
+    }
+
     /// Perform a HTTP GET request using an 'accept' header, returning `None` on a 404 error.
     pub async fn get_bytes_opt_accept_header<U: IntoUrl>(
         &self,
@@ -850,7 +896,7 @@ impl BeaconNodeHttpClient {
         &self,
         start_period: u64,
         count: u64,
-    ) -> Result<Option<Vec<ForkVersionedResponse<LightClientUpdate<E>>>>, Error> {
+    ) -> Result<Option<Vec<BeaconResponse<LightClientUpdate<E>>>>, Error> {
         let mut path = self.eth_path(V1)?;
 
         path.path_segments_mut()
@@ -865,7 +911,14 @@ impl BeaconNodeHttpClient {
         path.query_pairs_mut()
             .append_pair("count", &count.to_string());
 
-        self.get_opt(path).await
+        self.get_opt(path).await.map(|opt| {
+            opt.map(|updates: Vec<_>| {
+                updates
+                    .into_iter()
+                    .map(BeaconResponse::ForkVersioned)
+                    .collect()
+            })
+        })
     }
 
     /// `GET beacon/light_client/bootstrap`
@@ -874,7 +927,7 @@ impl BeaconNodeHttpClient {
     pub async fn get_light_client_bootstrap<E: EthSpec>(
         &self,
         block_root: Hash256,
-    ) -> Result<Option<ForkVersionedResponse<LightClientBootstrap<E>>>, Error> {
+    ) -> Result<Option<BeaconResponse<LightClientBootstrap<E>>>, Error> {
         let mut path = self.eth_path(V1)?;
 
         path.path_segments_mut()
@@ -884,7 +937,9 @@ impl BeaconNodeHttpClient {
             .push("bootstrap")
             .push(&format!("{:?}", block_root));
 
-        self.get_opt(path).await
+        self.get_opt(path)
+            .await
+            .map(|opt| opt.map(BeaconResponse::ForkVersioned))
     }
 
     /// `GET beacon/light_client/optimistic_update`
@@ -892,7 +947,7 @@ impl BeaconNodeHttpClient {
     /// Returns `Ok(None)` on a 404 error.
     pub async fn get_beacon_light_client_optimistic_update<E: EthSpec>(
         &self,
-    ) -> Result<Option<ForkVersionedResponse<LightClientOptimisticUpdate<E>>>, Error> {
+    ) -> Result<Option<BeaconResponse<LightClientOptimisticUpdate<E>>>, Error> {
         let mut path = self.eth_path(V1)?;
 
         path.path_segments_mut()
@@ -901,7 +956,9 @@ impl BeaconNodeHttpClient {
             .push("light_client")
             .push("optimistic_update");
 
-        self.get_opt(path).await
+        self.get_opt(path)
+            .await
+            .map(|opt| opt.map(BeaconResponse::ForkVersioned))
     }
 
     /// `GET beacon/light_client/finality_update`
@@ -909,7 +966,7 @@ impl BeaconNodeHttpClient {
     /// Returns `Ok(None)` on a 404 error.
     pub async fn get_beacon_light_client_finality_update<E: EthSpec>(
         &self,
-    ) -> Result<Option<ForkVersionedResponse<LightClientFinalityUpdate<E>>>, Error> {
+    ) -> Result<Option<BeaconResponse<LightClientFinalityUpdate<E>>>, Error> {
         let mut path = self.eth_path(V1)?;
 
         path.path_segments_mut()
@@ -918,7 +975,9 @@ impl BeaconNodeHttpClient {
             .push("light_client")
             .push("finality_update");
 
-        self.get_opt(path).await
+        self.get_opt(path)
+            .await
+            .map(|opt| opt.map(BeaconResponse::ForkVersioned))
     }
 
     /// `GET beacon/headers?slot,parent_root`
@@ -981,8 +1040,14 @@ impl BeaconNodeHttpClient {
             .push("beacon")
             .push("blocks");
 
-        self.post_with_timeout(path, block_contents, self.timeouts.proposal)
-            .await?;
+        let fork_name = block_contents.signed_block().fork_name_unchecked();
+        self.post_generic_with_consensus_version(
+            path,
+            block_contents,
+            Some(self.timeouts.proposal),
+            fork_name,
+        )
+        .await?;
 
         Ok(())
     }
@@ -1200,16 +1265,12 @@ impl BeaconNodeHttpClient {
     pub async fn get_beacon_blocks<E: EthSpec>(
         &self,
         block_id: BlockId,
-    ) -> Result<
-        Option<ExecutionOptimisticFinalizedForkVersionedResponse<SignedBeaconBlock<E>>>,
-        Error,
-    > {
+    ) -> Result<Option<ExecutionOptimisticFinalizedBeaconResponse<SignedBeaconBlock<E>>>, Error>
+    {
         let path = self.get_beacon_blocks_path(block_id)?;
-        let Some(response) = self.get_response(path, |b| b).await.optional()? else {
-            return Ok(None);
-        };
-
-        Ok(Some(response.json().await?))
+        self.get_opt(path)
+            .await
+            .map(|opt| opt.map(BeaconResponse::ForkVersioned))
     }
 
     /// `GET v1/beacon/blob_sidecars/{block_id}`
@@ -1219,8 +1280,8 @@ impl BeaconNodeHttpClient {
         &self,
         block_id: BlockId,
         indices: Option<&[u64]>,
-    ) -> Result<Option<ExecutionOptimisticFinalizedForkVersionedResponse<BlobSidecarList<E>>>, Error>
-    {
+        spec: &ChainSpec,
+    ) -> Result<Option<ExecutionOptimisticFinalizedBeaconResponse<BlobSidecarList<E>>>, Error> {
         let mut path = self.get_blobs_path(block_id)?;
         if let Some(indices) = indices {
             let indices_string = indices
@@ -1232,11 +1293,11 @@ impl BeaconNodeHttpClient {
                 .append_pair("indices", &indices_string);
         }
 
-        let Some(response) = self.get_response(path, |b| b).await.optional()? else {
-            return Ok(None);
-        };
-
-        Ok(Some(response.json().await?))
+        self.get_fork_contextual(path, |fork| {
+            (fork, spec.max_blobs_per_block_by_fork(fork) as usize)
+        })
+        .await
+        .map(|opt| opt.map(BeaconResponse::ForkVersioned))
     }
 
     /// `GET v1/beacon/blinded_blocks/{block_id}`
@@ -1246,15 +1307,13 @@ impl BeaconNodeHttpClient {
         &self,
         block_id: BlockId,
     ) -> Result<
-        Option<ExecutionOptimisticFinalizedForkVersionedResponse<SignedBlindedBeaconBlock<E>>>,
+        Option<ExecutionOptimisticFinalizedBeaconResponse<SignedBlindedBeaconBlock<E>>>,
         Error,
     > {
         let path = self.get_beacon_blinded_blocks_path(block_id)?;
-        let Some(response) = self.get_response(path, |b| b).await.optional()? else {
-            return Ok(None);
-        };
-
-        Ok(Some(response.json().await?))
+        self.get_opt(path)
+            .await
+            .map(|opt| opt.map(BeaconResponse::ForkVersioned))
     }
 
     /// `GET v1/beacon/blocks` (LEGACY)
@@ -1263,7 +1322,7 @@ impl BeaconNodeHttpClient {
     pub async fn get_beacon_blocks_v1<E: EthSpec>(
         &self,
         block_id: BlockId,
-    ) -> Result<Option<ForkVersionedResponse<SignedBeaconBlock<E>>>, Error> {
+    ) -> Result<Option<BeaconResponse<SignedBeaconBlock<E>>>, Error> {
         let mut path = self.eth_path(V1)?;
 
         path.path_segments_mut()
@@ -1272,7 +1331,9 @@ impl BeaconNodeHttpClient {
             .push("blocks")
             .push(&block_id.to_string());
 
-        self.get_opt(path).await
+        self.get_opt(path)
+            .await
+            .map(|opt| opt.map(BeaconResponse::Unversioned))
     }
 
     /// `GET beacon/blocks` as SSZ
@@ -1353,7 +1414,7 @@ impl BeaconNodeHttpClient {
     pub async fn get_beacon_blocks_attestations_v2<E: EthSpec>(
         &self,
         block_id: BlockId,
-    ) -> Result<Option<ExecutionOptimisticFinalizedForkVersionedResponse<Vec<Attestation<E>>>>, Error>
+    ) -> Result<Option<ExecutionOptimisticFinalizedBeaconResponse<Vec<Attestation<E>>>>, Error>
     {
         let mut path = self.eth_path(V2)?;
 
@@ -1364,7 +1425,9 @@ impl BeaconNodeHttpClient {
             .push(&block_id.to_string())
             .push("attestations");
 
-        self.get_opt(path).await
+        self.get_opt(path)
+            .await
+            .map(|opt| opt.map(BeaconResponse::ForkVersioned))
     }
 
     /// `POST v1/beacon/pool/attestations`
@@ -1456,7 +1519,7 @@ impl BeaconNodeHttpClient {
         &self,
         slot: Option<Slot>,
         committee_index: Option<u64>,
-    ) -> Result<ForkVersionedResponse<Vec<Attestation<E>>>, Error> {
+    ) -> Result<BeaconResponse<Vec<Attestation<E>>>, Error> {
         let mut path = self.eth_path(V2)?;
 
         path.path_segments_mut()
@@ -1475,7 +1538,7 @@ impl BeaconNodeHttpClient {
                 .append_pair("committee_index", &index.to_string());
         }
 
-        self.get(path).await
+        self.get(path).await.map(BeaconResponse::ForkVersioned)
     }
 
     /// `POST v1/beacon/pool/attester_slashings`
@@ -1534,7 +1597,7 @@ impl BeaconNodeHttpClient {
     /// `GET v2/beacon/pool/attester_slashings`
     pub async fn get_beacon_pool_attester_slashings_v2<E: EthSpec>(
         &self,
-    ) -> Result<ForkVersionedResponse<Vec<AttesterSlashing<E>>>, Error> {
+    ) -> Result<BeaconResponse<Vec<AttesterSlashing<E>>>, Error> {
         let mut path = self.eth_path(V2)?;
 
         path.path_segments_mut()
@@ -1543,7 +1606,7 @@ impl BeaconNodeHttpClient {
             .push("pool")
             .push("attester_slashings");
 
-        self.get(path).await
+        self.get(path).await.map(BeaconResponse::ForkVersioned)
     }
 
     /// `POST beacon/pool/proposer_slashings`
@@ -1964,10 +2027,11 @@ impl BeaconNodeHttpClient {
     pub async fn get_debug_beacon_states<E: EthSpec>(
         &self,
         state_id: StateId,
-    ) -> Result<Option<ExecutionOptimisticFinalizedForkVersionedResponse<BeaconState<E>>>, Error>
-    {
+    ) -> Result<Option<ExecutionOptimisticFinalizedBeaconResponse<BeaconState<E>>>, Error> {
         let path = self.get_debug_beacon_states_path(state_id)?;
-        self.get_opt(path).await
+        self.get_opt(path)
+            .await
+            .map(|opt| opt.map(BeaconResponse::ForkVersioned))
     }
 
     /// `GET debug/beacon/states/{state_id}`
@@ -2051,7 +2115,7 @@ impl BeaconNodeHttpClient {
         slot: Slot,
         randao_reveal: &SignatureBytes,
         graffiti: Option<&Graffiti>,
-    ) -> Result<ForkVersionedResponse<FullBlockContents<E>>, Error> {
+    ) -> Result<BeaconResponse<FullBlockContents<E>>, Error> {
         self.get_validator_blocks_modular(slot, randao_reveal, graffiti, SkipRandaoVerification::No)
             .await
     }
@@ -2063,12 +2127,12 @@ impl BeaconNodeHttpClient {
         randao_reveal: &SignatureBytes,
         graffiti: Option<&Graffiti>,
         skip_randao_verification: SkipRandaoVerification,
-    ) -> Result<ForkVersionedResponse<FullBlockContents<E>>, Error> {
+    ) -> Result<BeaconResponse<FullBlockContents<E>>, Error> {
         let path = self
             .get_validator_blocks_path::<E>(slot, randao_reveal, graffiti, skip_randao_verification)
             .await?;
 
-        self.get(path).await
+        self.get(path).await.map(BeaconResponse::ForkVersioned)
     }
 
     /// returns `GET v2/validator/blocks/{slot}` URL path
@@ -2324,7 +2388,7 @@ impl BeaconNodeHttpClient {
         slot: Slot,
         randao_reveal: &SignatureBytes,
         graffiti: Option<&Graffiti>,
-    ) -> Result<ForkVersionedResponse<BlindedBeaconBlock<E>>, Error> {
+    ) -> Result<BeaconResponse<BlindedBeaconBlock<E>>, Error> {
         self.get_validator_blinded_blocks_modular(
             slot,
             randao_reveal,
@@ -2373,7 +2437,7 @@ impl BeaconNodeHttpClient {
         randao_reveal: &SignatureBytes,
         graffiti: Option<&Graffiti>,
         skip_randao_verification: SkipRandaoVerification,
-    ) -> Result<ForkVersionedResponse<BlindedBeaconBlock<E>>, Error> {
+    ) -> Result<BeaconResponse<BlindedBeaconBlock<E>>, Error> {
         let path = self
             .get_validator_blinded_blocks_path::<E>(
                 slot,
@@ -2383,7 +2447,7 @@ impl BeaconNodeHttpClient {
             )
             .await?;
 
-        self.get(path).await
+        self.get(path).await.map(BeaconResponse::ForkVersioned)
     }
 
     /// `GET v2/validator/blinded_blocks/{slot}` in ssz format
@@ -2472,7 +2536,7 @@ impl BeaconNodeHttpClient {
         slot: Slot,
         attestation_data_root: Hash256,
         committee_index: CommitteeIndex,
-    ) -> Result<Option<ForkVersionedResponse<Attestation<E>>>, Error> {
+    ) -> Result<Option<BeaconResponse<Attestation<E>>>, Error> {
         let mut path = self.eth_path(V2)?;
 
         path.path_segments_mut()
@@ -2490,6 +2554,7 @@ impl BeaconNodeHttpClient {
 
         self.get_opt_with_timeout(path, self.timeouts.attestation)
             .await
+            .map(|opt| opt.map(BeaconResponse::ForkVersioned))
     }
 
     /// `GET validator/sync_committee_contribution`