Implement standard eth2.0 API (#1569)

- Resolves #1550
- Resolves #824
- Resolves #825
- Resolves #1131
- Resolves #1411
- Resolves #1256
- Resolve #1177

- Includes the `ShufflingId` struct initially defined in #1492. That PR is now closed and the changes are included here, with significant bug fixes.
- Implement the https://github.com/ethereum/eth2.0-APIs in a new `http_api` crate using `warp`. This replaces the `rest_api` crate.
- Add a new `common/eth2` crate which provides a wrapper around `reqwest`, providing the HTTP client that is used by the validator client and for testing. This replaces the `common/remote_beacon_node` crate.
- Create a `http_metrics` crate which is a dedicated server for Prometheus metrics (they are no longer served on the same port as the REST API). We now have flags for `--metrics`, `--metrics-address`, etc.
- Allow the `subnet_id` to be an optional parameter for `VerifiedUnaggregatedAttestation::verify`. This means it does not need to be provided unnecessarily by the validator client.
- Move `fn map_attestation_committee` in `mod beacon_chain::attestation_verification` to a new `fn with_committee_cache` on the `BeaconChain` so the same cache can be used for obtaining validator duties.
- Add some other helpers to `BeaconChain` to assist with common API duties (e.g., `block_root_at_slot`, `head_beacon_block_root`).
- Change the `NaiveAggregationPool` so it can index attestations by `hash_tree_root(attestation.data)`. This is a requirement of the API.
- Add functions to `BeaconChainHarness` to allow it to create slashings and exits.
- Allow for `eth1::Eth1NetworkId` to go to/from a `String`.
- Add functions to the `OperationPool` to allow getting all objects in the pool.
- Add function to `BeaconState` to check if a committee cache is initialized.
- Fix bug where `seconds_per_eth1_block` was not transferring over from `YamlConfig` to `ChainSpec`.
- Add the `deposit_contract_address` to `YamlConfig` and `ChainSpec`. We needed to be able to return it in an API response.
- Change some uses of serde `serialize_with` and `deserialize_with` to a single use of `with` (code quality).
- Impl `Display` and `FromStr` for several BLS fields.
- Check for clock discrepancy when VC polls BN for sync state (with +/- 1 slot tolerance). This is not intended to be comprehensive, it was just easy to do.

- See #1434 for a per-endpoint overview.
- Seeking clarity here: https://github.com/ethereum/eth2.0-APIs/issues/75

- [x] Add docs for prom port to close #1256
- [x] Follow up on this #1177
- [x] ~~Follow up with #1424~~ Will fix in future PR.
- [x] Follow up with #1411
- [x] ~~Follow up with  #1260~~ Will fix in future PR.
- [x] Add quotes to all integers.
- [x] Remove `rest_types`
- [x] Address missing beacon block error. (#1629)
- [x] ~~Add tests for lighthouse/peers endpoints~~ Wontfix
- [x] ~~Follow up with validator status proposal~~ Tracked in #1434
- [x] Unify graffiti structs
- [x] ~~Start server when waiting for genesis?~~ Will fix in future PR.
- [x] TODO in http_api tests
- [x] Move lighthouse endpoints off /eth/v1
- [x] Update docs to link to standard

- ~~Blocked on #1586~~

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

consensus/serde_utils/src/bytes_4_hex.rs

@@ -0,0 +1,38 @@
+//! Formats `[u8; 4]` as a 0x-prefixed hex string.
+//!
+//! E.g., `[0, 1, 2, 3]` serializes as `"0x00010203"`.
+
+use crate::hex::PrefixedHexVisitor;
+use serde::de::Error;
+use serde::{Deserializer, Serializer};
+
+const BYTES_LEN: usize = 4;
+
+pub fn serialize<S>(bytes: &[u8; BYTES_LEN], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&bytes));
+
+    serializer.serialize_str(&hex_string)
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<[u8; BYTES_LEN], D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let decoded = deserializer.deserialize_str(PrefixedHexVisitor)?;
+
+    if decoded.len() != BYTES_LEN {
+        return Err(D::Error::custom(format!(
+            "expected {} bytes for array, got {}",
+            BYTES_LEN,
+            decoded.len()
+        )));
+    }
+
+    let mut array = [0; BYTES_LEN];
+    array.copy_from_slice(&decoded);
+    Ok(array)
+}

consensus/serde_utils/src/hex.rs

@@ -0,0 +1,81 @@
+//! Provides utilities for parsing 0x-prefixed hex strings.
+
+use serde::de::{self, Visitor};
+use std::fmt;
+
+/// Encode `data` as a 0x-prefixed hex string.
+pub fn encode<T: AsRef<[u8]>>(data: T) -> String {
+    let hex = hex::encode(data);
+    let mut s = "0x".to_string();
+    s.push_str(hex.as_str());
+    s
+}
+
+/// Decode `data` from a 0x-prefixed hex string.
+pub fn decode(s: &str) -> Result<Vec<u8>, String> {
+    if s.starts_with("0x") {
+        hex::decode(&s[2..]).map_err(|e| format!("invalid hex: {:?}", e))
+    } else {
+        Err("hex must have 0x prefix".to_string())
+    }
+}
+
+pub struct PrefixedHexVisitor;
+
+impl<'de> Visitor<'de> for PrefixedHexVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a hex string with 0x prefix")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if value.starts_with("0x") {
+            Ok(hex::decode(&value[2..])
+                .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
+        } else {
+            Err(de::Error::custom("missing 0x prefix"))
+        }
+    }
+}
+
+pub struct HexVisitor;
+
+impl<'de> Visitor<'de> for HexVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a hex string (irrelevant of prefix)")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(hex::decode(value.trim_start_matches("0x"))
+            .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn encoding() {
+        let bytes = vec![0, 255];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x00ff");
+
+        let bytes = vec![];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x");
+
+        let bytes = vec![1, 2, 3];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x010203");
+    }
+}

consensus/serde_utils/src/lib.rs

@@ -1,2 +1,9 @@
-pub mod quoted_u64;
+mod quoted_int;
+
+pub mod bytes_4_hex;
+pub mod hex;
 pub mod quoted_u64_vec;
+pub mod u32_hex;
+pub mod u8_hex;
+
+pub use quoted_int::{quoted_u32, quoted_u64, quoted_u8};

consensus/serde_utils/src/quoted_int.rs

@@ -0,0 +1,144 @@
+//! Formats some integer types using quotes.
+//!
+//! E.g., `1` serializes as `"1"`.
+//!
+//! Quotes can be optional during decoding.
+
+use serde::{Deserializer, Serializer};
+use serde_derive::{Deserialize, Serialize};
+use std::convert::TryFrom;
+use std::marker::PhantomData;
+
+macro_rules! define_mod {
+    ($int: ty, $visit_fn: ident) => {
+        /// Serde support for deserializing quoted integers.
+        ///
+        /// Configurable so that quotes are either required or optional.
+        pub struct QuotedIntVisitor<T> {
+            require_quotes: bool,
+            _phantom: PhantomData<T>,
+        }
+
+        impl<'a, T> serde::de::Visitor<'a> for QuotedIntVisitor<T>
+        where
+            T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
+        {
+            type Value = T;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                if self.require_quotes {
+                    write!(formatter, "a quoted integer")
+                } else {
+                    write!(formatter, "a quoted or unquoted integer")
+                }
+            }
+
+            fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
+            where
+                E: serde::de::Error,
+            {
+                s.parse::<$int>()
+                    .map(T::from)
+                    .map_err(serde::de::Error::custom)
+            }
+
+            fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
+            where
+                E: serde::de::Error,
+            {
+                if self.require_quotes {
+                    Err(serde::de::Error::custom(
+                        "received unquoted integer when quotes are required",
+                    ))
+                } else {
+                    T::try_from(v).map_err(|_| serde::de::Error::custom("invalid integer"))
+                }
+            }
+        }
+
+        /// Wrapper type for requiring quotes on a `$int`-like type.
+        ///
+        /// Unlike using `serde(with = "quoted_$int::require_quotes")` this is composable, and can be nested
+        /// inside types like `Option`, `Result` and `Vec`.
+        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
+        #[serde(transparent)]
+        pub struct Quoted<T>
+        where
+            T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
+        {
+            #[serde(with = "require_quotes")]
+            pub value: T,
+        }
+
+        /// Serialize with quotes.
+        pub fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
+        where
+            S: Serializer,
+            T: From<$int> + Into<$int> + Copy,
+        {
+            let v: $int = (*value).into();
+            serializer.serialize_str(&format!("{}", v))
+        }
+
+        /// Deserialize with or without quotes.
+        pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+        where
+            D: Deserializer<'de>,
+            T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
+        {
+            deserializer.deserialize_any(QuotedIntVisitor {
+                require_quotes: false,
+                _phantom: PhantomData,
+            })
+        }
+
+        /// Requires quotes when deserializing.
+        ///
+        /// Usage: `#[serde(with = "quoted_u64::require_quotes")]`.
+        pub mod require_quotes {
+            pub use super::serialize;
+            use super::*;
+
+            pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+            where
+                D: Deserializer<'de>,
+                T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
+            {
+                deserializer.deserialize_any(QuotedIntVisitor {
+                    require_quotes: true,
+                    _phantom: PhantomData,
+                })
+            }
+        }
+
+        #[cfg(test)]
+        mod test {
+            use super::*;
+
+            #[test]
+            fn require_quotes() {
+                let x = serde_json::from_str::<Quoted<$int>>("\"8\"").unwrap();
+                assert_eq!(x.value, 8);
+                serde_json::from_str::<Quoted<$int>>("8").unwrap_err();
+            }
+        }
+    };
+}
+
+pub mod quoted_u8 {
+    use super::*;
+
+    define_mod!(u8, visit_u8);
+}
+
+pub mod quoted_u32 {
+    use super::*;
+
+    define_mod!(u32, visit_u32);
+}
+
+pub mod quoted_u64 {
+    use super::*;
+
+    define_mod!(u64, visit_u64);
+}

consensus/serde_utils/src/quoted_u64.rs

@@ -1,115 +0,0 @@
-use serde::{Deserializer, Serializer};
-use serde_derive::{Deserialize, Serialize};
-use std::marker::PhantomData;
-
-/// Serde support for deserializing quoted integers.
-///
-/// Configurable so that quotes are either required or optional.
-pub struct QuotedIntVisitor<T> {
-    require_quotes: bool,
-    _phantom: PhantomData<T>,
-}
-
-impl<'a, T> serde::de::Visitor<'a> for QuotedIntVisitor<T>
-where
-    T: From<u64> + Into<u64> + Copy,
-{
-    type Value = T;
-
-    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-        if self.require_quotes {
-            write!(formatter, "a quoted integer")
-        } else {
-            write!(formatter, "a quoted or unquoted integer")
-        }
-    }
-
-    fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        s.parse::<u64>()
-            .map(T::from)
-            .map_err(serde::de::Error::custom)
-    }
-
-    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        if self.require_quotes {
-            Err(serde::de::Error::custom(
-                "received unquoted integer when quotes are required",
-            ))
-        } else {
-            Ok(T::from(v))
-        }
-    }
-}
-
-/// Wrapper type for requiring quotes on a `u64`-like type.
-///
-/// Unlike using `serde(with = "quoted_u64::require_quotes")` this is composable, and can be nested
-/// inside types like `Option`, `Result` and `Vec`.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
-#[serde(transparent)]
-pub struct Quoted<T>
-where
-    T: From<u64> + Into<u64> + Copy,
-{
-    #[serde(with = "require_quotes")]
-    pub value: T,
-}
-
-/// Serialize with quotes.
-pub fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: Serializer,
-    T: From<u64> + Into<u64> + Copy,
-{
-    let v: u64 = (*value).into();
-    serializer.serialize_str(&format!("{}", v))
-}
-
-/// Deserialize with or without quotes.
-pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
-where
-    D: Deserializer<'de>,
-    T: From<u64> + Into<u64> + Copy,
-{
-    deserializer.deserialize_any(QuotedIntVisitor {
-        require_quotes: false,
-        _phantom: PhantomData,
-    })
-}
-
-/// Requires quotes when deserializing.
-///
-/// Usage: `#[serde(with = "quoted_u64::require_quotes")]`.
-pub mod require_quotes {
-    pub use super::serialize;
-    use super::*;
-
-    pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
-    where
-        D: Deserializer<'de>,
-        T: From<u64> + Into<u64> + Copy,
-    {
-        deserializer.deserialize_any(QuotedIntVisitor {
-            require_quotes: true,
-            _phantom: PhantomData,
-        })
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn require_quotes() {
-        let x = serde_json::from_str::<Quoted<u64>>("\"8\"").unwrap();
-        assert_eq!(x.value, 8);
-        serde_json::from_str::<Quoted<u64>>("8").unwrap_err();
-    }
-}

consensus/serde_utils/src/quoted_u64_vec.rs

@@ -1,3 +1,9 @@
+//! Formats `Vec<u64>` using quotes.
+//!
+//! E.g., `vec![0, 1, 2]` serializes as `["0", "1", "2"]`.
+//!
+//! Quotes can be optional during decoding.
+
 use serde::ser::SerializeSeq;
 use serde::{Deserializer, Serializer};
 use serde_derive::{Deserialize, Serialize};
@@ -6,7 +12,7 @@ use serde_derive::{Deserialize, Serialize};
 #[serde(transparent)]
 pub struct QuotedIntWrapper {
     #[serde(with = "crate::quoted_u64")]
-    int: u64,
+    pub int: u64,
 }
 
 pub struct QuotedIntVecVisitor;

consensus/serde_utils/src/u32_hex.rs

@@ -0,0 +1,21 @@
+//! Formats `u32` as a 0x-prefixed, little-endian hex string.
+//!
+//! E.g., `0` serializes as `"0x00000000"`.
+
+use crate::bytes_4_hex;
+use serde::{Deserializer, Serializer};
+
+pub fn serialize<S>(num: &u32, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let hex = format!("0x{}", hex::encode(num.to_le_bytes()));
+    serializer.serialize_str(&hex)
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<u32, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    bytes_4_hex::deserialize(deserializer).map(u32::from_le_bytes)
+}

consensus/serde_utils/src/u8_hex.rs

@@ -0,0 +1,29 @@
+//! Formats `u8` as a 0x-prefixed hex string.
+//!
+//! E.g., `0` serializes as `"0x00"`.
+
+use crate::hex::PrefixedHexVisitor;
+use serde::de::Error;
+use serde::{Deserializer, Serializer};
+
+pub fn serialize<S>(byte: &u8, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let hex = format!("0x{}", hex::encode([*byte]));
+    serializer.serialize_str(&hex)
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<u8, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+    if bytes.len() != 1 {
+        return Err(D::Error::custom(format!(
+            "expected 1 byte for u8, got {}",
+            bytes.len()
+        )));
+    }
+    Ok(bytes[0])
+}