Implement standard eth2.0 API (#1569)

- Resolves #1550
- Resolves #824
- Resolves #825
- Resolves #1131
- Resolves #1411
- Resolves #1256
- Resolve #1177

- Includes the `ShufflingId` struct initially defined in #1492. That PR is now closed and the changes are included here, with significant bug fixes.
- Implement the https://github.com/ethereum/eth2.0-APIs in a new `http_api` crate using `warp`. This replaces the `rest_api` crate.
- Add a new `common/eth2` crate which provides a wrapper around `reqwest`, providing the HTTP client that is used by the validator client and for testing. This replaces the `common/remote_beacon_node` crate.
- Create a `http_metrics` crate which is a dedicated server for Prometheus metrics (they are no longer served on the same port as the REST API). We now have flags for `--metrics`, `--metrics-address`, etc.
- Allow the `subnet_id` to be an optional parameter for `VerifiedUnaggregatedAttestation::verify`. This means it does not need to be provided unnecessarily by the validator client.
- Move `fn map_attestation_committee` in `mod beacon_chain::attestation_verification` to a new `fn with_committee_cache` on the `BeaconChain` so the same cache can be used for obtaining validator duties.
- Add some other helpers to `BeaconChain` to assist with common API duties (e.g., `block_root_at_slot`, `head_beacon_block_root`).
- Change the `NaiveAggregationPool` so it can index attestations by `hash_tree_root(attestation.data)`. This is a requirement of the API.
- Add functions to `BeaconChainHarness` to allow it to create slashings and exits.
- Allow for `eth1::Eth1NetworkId` to go to/from a `String`.
- Add functions to the `OperationPool` to allow getting all objects in the pool.
- Add function to `BeaconState` to check if a committee cache is initialized.
- Fix bug where `seconds_per_eth1_block` was not transferring over from `YamlConfig` to `ChainSpec`.
- Add the `deposit_contract_address` to `YamlConfig` and `ChainSpec`. We needed to be able to return it in an API response.
- Change some uses of serde `serialize_with` and `deserialize_with` to a single use of `with` (code quality).
- Impl `Display` and `FromStr` for several BLS fields.
- Check for clock discrepancy when VC polls BN for sync state (with +/- 1 slot tolerance). This is not intended to be comprehensive, it was just easy to do.

- See #1434 for a per-endpoint overview.
- Seeking clarity here: https://github.com/ethereum/eth2.0-APIs/issues/75

- [x] Add docs for prom port to close #1256
- [x] Follow up on this #1177
- [x] ~~Follow up with #1424~~ Will fix in future PR.
- [x] Follow up with #1411
- [x] ~~Follow up with  #1260~~ Will fix in future PR.
- [x] Add quotes to all integers.
- [x] Remove `rest_types`
- [x] Address missing beacon block error. (#1629)
- [x] ~~Add tests for lighthouse/peers endpoints~~ Wontfix
- [x] ~~Follow up with validator status proposal~~ Tracked in #1434
- [x] Unify graffiti structs
- [x] ~~Start server when waiting for genesis?~~ Will fix in future PR.
- [x] TODO in http_api tests
- [x] Move lighthouse endpoints off /eth/v1
- [x] Update docs to link to standard

- ~~Blocked on #1586~~

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

crypto/bls/Cargo.toml

@@ -11,7 +11,7 @@ milagro_bls = { git = "https://github.com/sigp/milagro_bls", branch = "paulh" }
 rand = "0.7.2"
 serde = "1.0.102"
 serde_derive = "1.0.102"
-serde_hex = { path = "../../consensus/serde_hex" }
+serde_utils = { path = "../../consensus/serde_utils" }
 hex = "0.3"
 eth2_hashing = "0.1.0"
 ethereum-types = "0.9.1"

crypto/bls/src/generic_aggregate_signature.rs

@@ -6,7 +6,7 @@ use crate::{
 };
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
-use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
+use serde_utils::hex::encode as hex_encode;
 use ssz::{Decode, Encode};
 use std::fmt;
 use std::marker::PhantomData;
@@ -245,6 +245,23 @@ where
     impl_tree_hash!(SIGNATURE_BYTES_LEN);
 }
 
+impl<Pub, AggPub, Sig, AggSig> fmt::Display for GenericAggregateSignature<Pub, AggPub, Sig, AggSig>
+where
+    Sig: TSignature<Pub>,
+    AggSig: TAggregateSignature<Pub, AggPub, Sig>,
+{
+    impl_display!();
+}
+
+impl<Pub, AggPub, Sig, AggSig> std::str::FromStr
+    for GenericAggregateSignature<Pub, AggPub, Sig, AggSig>
+where
+    Sig: TSignature<Pub>,
+    AggSig: TAggregateSignature<Pub, AggPub, Sig>,
+{
+    impl_from_str!();
+}
+
 impl<Pub, AggPub, Sig, AggSig> Serialize for GenericAggregateSignature<Pub, AggPub, Sig, AggSig>
 where
     Sig: TSignature<Pub>,

crypto/bls/src/generic_public_key.rs

@@ -1,7 +1,7 @@
 use crate::Error;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
-use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
+use serde_utils::hex::encode as hex_encode;
 use ssz::{Decode, Encode};
 use std::fmt;
 use std::hash::{Hash, Hasher};
@@ -97,6 +97,14 @@ impl<Pub: TPublicKey> TreeHash for GenericPublicKey<Pub> {
     impl_tree_hash!(PUBLIC_KEY_BYTES_LEN);
 }
 
+impl<Pub: TPublicKey> fmt::Display for GenericPublicKey<Pub> {
+    impl_display!();
+}
+
+impl<Pub: TPublicKey> std::str::FromStr for GenericPublicKey<Pub> {
+    impl_from_str!();
+}
+
 impl<Pub: TPublicKey> Serialize for GenericPublicKey<Pub> {
     impl_serde_serialize!();
 }

crypto/bls/src/generic_public_key_bytes.rs

@@ -4,7 +4,7 @@ use crate::{
 };
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
-use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
+use serde_utils::hex::encode as hex_encode;
 use ssz::{Decode, Encode};
 use std::convert::TryInto;
 use std::fmt;
@@ -101,6 +101,16 @@ where
     Pub: TPublicKey,
 {
     fn from(pk: GenericPublicKey<Pub>) -> Self {
+        Self::from(&pk)
+    }
+}
+
+/// Serializes the `PublicKey` in compressed form, storing the bytes in the newly created `Self`.
+impl<Pub> From<&GenericPublicKey<Pub>> for GenericPublicKeyBytes<Pub>
+where
+    Pub: TPublicKey,
+{
+    fn from(pk: &GenericPublicKey<Pub>) -> Self {
         Self {
             bytes: pk.serialize(),
             _phantom: PhantomData,
@@ -132,6 +142,14 @@ impl<Pub> TreeHash for GenericPublicKeyBytes<Pub> {
     impl_tree_hash!(PUBLIC_KEY_BYTES_LEN);
 }
 
+impl<Pub> fmt::Display for GenericPublicKeyBytes<Pub> {
+    impl_display!();
+}
+
+impl<Pub> std::str::FromStr for GenericPublicKeyBytes<Pub> {
+    impl_from_str!();
+}
+
 impl<Pub> Serialize for GenericPublicKeyBytes<Pub> {
     impl_serde_serialize!();
 }

crypto/bls/src/generic_signature.rs

@@ -4,7 +4,7 @@ use crate::{
 };
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
-use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
+use serde_utils::hex::encode as hex_encode;
 use ssz::{Decode, Encode};
 use std::fmt;
 use std::marker::PhantomData;
@@ -149,6 +149,14 @@ impl<PublicKey, T: TSignature<PublicKey>> TreeHash for GenericSignature<PublicKe
     impl_tree_hash!(SIGNATURE_BYTES_LEN);
 }
 
+impl<PublicKey, T: TSignature<PublicKey>> fmt::Display for GenericSignature<PublicKey, T> {
+    impl_display!();
+}
+
+impl<PublicKey, T: TSignature<PublicKey>> std::str::FromStr for GenericSignature<PublicKey, T> {
+    impl_from_str!();
+}
+
 impl<PublicKey, T: TSignature<PublicKey>> Serialize for GenericSignature<PublicKey, T> {
     impl_serde_serialize!();
 }

crypto/bls/src/generic_signature_bytes.rs

@@ -5,7 +5,7 @@ use crate::{
 };
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
-use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
+use serde_utils::hex::encode as hex_encode;
 use ssz::{Decode, Encode};
 use std::convert::TryInto;
 use std::fmt;
@@ -124,6 +124,14 @@ impl<Pub, Sig> TreeHash for GenericSignatureBytes<Pub, Sig> {
     impl_tree_hash!(SIGNATURE_BYTES_LEN);
 }
 
+impl<Pub, Sig> fmt::Display for GenericSignatureBytes<Pub, Sig> {
+    impl_display!();
+}
+
+impl<Pub, Sig> std::str::FromStr for GenericSignatureBytes<Pub, Sig> {
+    impl_from_str!();
+}
+
 impl<Pub, Sig> Serialize for GenericSignatureBytes<Pub, Sig> {
     impl_serde_serialize!();
 }

crypto/bls/src/macros.rs

@@ -76,6 +76,35 @@ macro_rules! impl_ssz_decode {
     };
 }
 
+/// Contains the functions required for a `fmt::Display` implementation.
+///
+/// Does not include the `Impl` section since it gets very complicated when it comes to generics.
+macro_rules! impl_display {
+    () => {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, "{}", hex_encode(self.serialize().to_vec()))
+        }
+    };
+}
+
+/// Contains the functions required for a `fmt::Display` implementation.
+///
+/// Does not include the `Impl` section since it gets very complicated when it comes to generics.
+macro_rules! impl_from_str {
+    () => {
+        type Err = String;
+
+        fn from_str(s: &str) -> Result<Self, Self::Err> {
+            if s.starts_with("0x") {
+                let bytes = hex::decode(&s[2..]).map_err(|e| e.to_string())?;
+                Self::deserialize(&bytes[..]).map_err(|e| format!("{:?}", e))
+            } else {
+                Err("must start with 0x".to_string())
+            }
+        }
+    };
+}
+
 /// Contains the functions required for a `serde::Serialize` implementation.
 ///
 /// Does not include the `Impl` section since it gets very complicated when it comes to generics.
@@ -85,7 +114,7 @@ macro_rules! impl_serde_serialize {
         where
             S: Serializer,
         {
-            serializer.serialize_str(&hex_encode(self.serialize().to_vec()))
+            serializer.serialize_str(&self.to_string())
         }
     };
 }
@@ -99,9 +128,25 @@ macro_rules! impl_serde_deserialize {
         where
             D: Deserializer<'de>,
         {
-            let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
-            Self::deserialize(&bytes[..])
-                .map_err(|e| serde::de::Error::custom(format!("invalid pubkey ({:?})", e)))
+            pub struct StringVisitor;
+
+            impl<'de> serde::de::Visitor<'de> for StringVisitor {
+                type Value = String;
+
+                fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+                    formatter.write_str("a hex string with 0x prefix")
+                }
+
+                fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+                where
+                    E: serde::de::Error,
+                {
+                    Ok(value.to_string())
+                }
+            }
+
+            let string = deserializer.deserialize_str(StringVisitor)?;
+            <Self as std::str::FromStr>::from_str(&string).map_err(serde::de::Error::custom)
         }
     };
 }