ContextDeserialize and Beacon API Improvements (#7372)

* #7286
* BeaconAPI is not returning a versioned response when it should for some V1 endpoints
* these [strange functions with vX in the name that still accept `endpoint_version` arguments](https://github.com/sigp/lighthouse/blob/stable/beacon_node/http_api/src/produce_block.rs#L192)

This refactor is a prerequisite to get the fulu EF tests running.

consensus/context_deserialize/src/impls.rs

@@ -0,0 +1,103 @@
+use crate::ContextDeserialize;
+use serde::de::{Deserialize, DeserializeSeed, Deserializer, SeqAccess, Visitor};
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+impl<'de, C, T> ContextDeserialize<'de, T> for Arc<C>
+where
+    C: ContextDeserialize<'de, T>,
+{
+    fn context_deserialize<D>(deserializer: D, context: T) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Ok(Arc::new(C::context_deserialize(deserializer, context)?))
+    }
+}
+
+impl<'de, T, C> ContextDeserialize<'de, C> for Vec<T>
+where
+    T: ContextDeserialize<'de, C>,
+    C: Clone,
+{
+    fn context_deserialize<D>(deserializer: D, context: C) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        // Our Visitor, which owns one copy of the context T
+        struct ContextVisitor<C, T> {
+            context: T,
+            _marker: PhantomData<C>,
+        }
+
+        impl<'de, C, T> Visitor<'de> for ContextVisitor<C, T>
+        where
+            C: ContextDeserialize<'de, T>,
+            T: Clone,
+        {
+            type Value = Vec<C>;
+
+            fn expecting(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+                fmt.write_str("a sequence of context‐deserialized elements")
+            }
+
+            fn visit_seq<A>(self, mut seq: A) -> Result<Vec<C>, A::Error>
+            where
+                A: SeqAccess<'de>,
+            {
+                let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0));
+                // for each element, we clone the context and hand it to the seed
+                while let Some(elem) = seq.next_element_seed(ContextSeed {
+                    context: self.context.clone(),
+                    _marker: PhantomData,
+                })? {
+                    out.push(elem);
+                }
+                Ok(out)
+            }
+        }
+
+        // A little seed that hands the deserializer + context into C::context_deserialize
+        struct ContextSeed<T, C> {
+            context: C,
+            _marker: PhantomData<T>,
+        }
+
+        impl<'de, T, C> DeserializeSeed<'de> for ContextSeed<T, C>
+        where
+            T: ContextDeserialize<'de, C>,
+            C: Clone,
+        {
+            type Value = T;
+
+            fn deserialize<D>(self, deserializer: D) -> Result<T, D::Error>
+            where
+                D: Deserializer<'de>,
+            {
+                T::context_deserialize(deserializer, self.context)
+            }
+        }
+
+        deserializer.deserialize_seq(ContextVisitor {
+            context,
+            _marker: PhantomData,
+        })
+    }
+}
+
+macro_rules! trivial_deserialize {
+    ($($t:ty),* $(,)?) => {
+        $(
+            impl<'de, T> ContextDeserialize<'de, T> for $t {
+                fn context_deserialize<D>(deserializer: D, _context: T) -> Result<Self, D::Error>
+                where
+                    D: Deserializer<'de>,
+                {
+                    <$t>::deserialize(deserializer)
+                }
+            }
+        )*
+    };
+}
+
+trivial_deserialize!(bool, u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, f32, f64);

consensus/context_deserialize/src/lib.rs

@@ -0,0 +1,13 @@
+pub mod impls;
+pub mod milhouse;
+pub mod ssz_impls;
+
+extern crate serde;
+use serde::de::Deserializer;
+
+/// General-purpose deserialization trait that accepts extra context `C`.
+pub trait ContextDeserialize<'de, C>: Sized {
+    fn context_deserialize<D>(deserializer: D, context: C) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>;
+}

consensus/context_deserialize/src/milhouse.rs

@@ -0,0 +1,45 @@
+use crate::ContextDeserialize;
+use milhouse::{List, Value, Vector};
+use serde::de::Deserializer;
+use ssz_types::typenum::Unsigned;
+
+impl<'de, C, T, N> ContextDeserialize<'de, C> for List<T, N>
+where
+    T: ContextDeserialize<'de, C> + Value,
+    N: Unsigned,
+    C: Clone,
+{
+    fn context_deserialize<D>(deserializer: D, context: C) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        // First deserialize as a Vec.
+        // This is not the most efficient implementation as it allocates a temporary Vec. In future
+        // we could write a more performant implementation using `List::builder()`.
+        let vec = Vec::<T>::context_deserialize(deserializer, context)?;
+
+        // Then convert to List, which will check the length.
+        List::new(vec)
+            .map_err(|e| serde::de::Error::custom(format!("Failed to create List: {:?}", e)))
+    }
+}
+
+impl<'de, C, T, N> ContextDeserialize<'de, C> for Vector<T, N>
+where
+    T: ContextDeserialize<'de, C> + Value,
+    N: Unsigned,
+    C: Clone,
+{
+    fn context_deserialize<D>(deserializer: D, context: C) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        // First deserialize as a List
+        let list = List::<T, N>::context_deserialize(deserializer, context)?;
+
+        // Then convert to Vector, which will check the length
+        Vector::try_from(list).map_err(|e| {
+            serde::de::Error::custom(format!("Failed to convert List to Vector: {:?}", e))
+        })
+    }
+}

consensus/context_deserialize/src/ssz_impls.rs

@@ -0,0 +1,48 @@
+use crate::serde::de::Error;
+use crate::ContextDeserialize;
+use serde::de::Deserializer;
+use serde::Deserialize;
+use ssz_types::length::{Fixed, Variable};
+use ssz_types::typenum::Unsigned;
+use ssz_types::{Bitfield, FixedVector};
+
+impl<'de, C, T, N> ContextDeserialize<'de, C> for FixedVector<T, N>
+where
+    T: ContextDeserialize<'de, C>,
+    N: Unsigned,
+    C: Clone,
+{
+    fn context_deserialize<D>(deserializer: D, context: C) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let vec = Vec::<T>::context_deserialize(deserializer, context)?;
+        FixedVector::new(vec).map_err(|e| D::Error::custom(format!("{:?}", e)))
+    }
+}
+
+impl<'de, C, N> ContextDeserialize<'de, C> for Bitfield<Variable<N>>
+where
+    N: Unsigned + Clone,
+{
+    fn context_deserialize<D>(deserializer: D, _context: C) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Bitfield::<Variable<N>>::deserialize(deserializer)
+            .map_err(|e| D::Error::custom(format!("{:?}", e)))
+    }
+}
+
+impl<'de, C, N> ContextDeserialize<'de, C> for Bitfield<Fixed<N>>
+where
+    N: Unsigned + Clone,
+{
+    fn context_deserialize<D>(deserializer: D, _context: C) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Bitfield::<Fixed<N>>::deserialize(deserializer)
+            .map_err(|e| D::Error::custom(format!("{:?}", e)))
+    }
+}