Implement standard eth2.0 API (#1569)

- Resolves #1550
- Resolves #824
- Resolves #825
- Resolves #1131
- Resolves #1411
- Resolves #1256
- Resolve #1177

- Includes the `ShufflingId` struct initially defined in #1492. That PR is now closed and the changes are included here, with significant bug fixes.
- Implement the https://github.com/ethereum/eth2.0-APIs in a new `http_api` crate using `warp`. This replaces the `rest_api` crate.
- Add a new `common/eth2` crate which provides a wrapper around `reqwest`, providing the HTTP client that is used by the validator client and for testing. This replaces the `common/remote_beacon_node` crate.
- Create a `http_metrics` crate which is a dedicated server for Prometheus metrics (they are no longer served on the same port as the REST API). We now have flags for `--metrics`, `--metrics-address`, etc.
- Allow the `subnet_id` to be an optional parameter for `VerifiedUnaggregatedAttestation::verify`. This means it does not need to be provided unnecessarily by the validator client.
- Move `fn map_attestation_committee` in `mod beacon_chain::attestation_verification` to a new `fn with_committee_cache` on the `BeaconChain` so the same cache can be used for obtaining validator duties.
- Add some other helpers to `BeaconChain` to assist with common API duties (e.g., `block_root_at_slot`, `head_beacon_block_root`).
- Change the `NaiveAggregationPool` so it can index attestations by `hash_tree_root(attestation.data)`. This is a requirement of the API.
- Add functions to `BeaconChainHarness` to allow it to create slashings and exits.
- Allow for `eth1::Eth1NetworkId` to go to/from a `String`.
- Add functions to the `OperationPool` to allow getting all objects in the pool.
- Add function to `BeaconState` to check if a committee cache is initialized.
- Fix bug where `seconds_per_eth1_block` was not transferring over from `YamlConfig` to `ChainSpec`.
- Add the `deposit_contract_address` to `YamlConfig` and `ChainSpec`. We needed to be able to return it in an API response.
- Change some uses of serde `serialize_with` and `deserialize_with` to a single use of `with` (code quality).
- Impl `Display` and `FromStr` for several BLS fields.
- Check for clock discrepancy when VC polls BN for sync state (with +/- 1 slot tolerance). This is not intended to be comprehensive, it was just easy to do.

- See #1434 for a per-endpoint overview.
- Seeking clarity here: https://github.com/ethereum/eth2.0-APIs/issues/75

- [x] Add docs for prom port to close #1256
- [x] Follow up on this #1177
- [x] ~~Follow up with #1424~~ Will fix in future PR.
- [x] Follow up with #1411
- [x] ~~Follow up with  #1260~~ Will fix in future PR.
- [x] Add quotes to all integers.
- [x] Remove `rest_types`
- [x] Address missing beacon block error. (#1629)
- [x] ~~Add tests for lighthouse/peers endpoints~~ Wontfix
- [x] ~~Follow up with validator status proposal~~ Tracked in #1434
- [x] Unify graffiti structs
- [x] ~~Start server when waiting for genesis?~~ Will fix in future PR.
- [x] TODO in http_api tests
- [x] Move lighthouse endpoints off /eth/v1
- [x] Update docs to link to standard

- ~~Blocked on #1586~~

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

consensus/proto_array/src/fork_choice_test_definition.rs

@@ -4,7 +4,7 @@ mod votes;
 
 use crate::proto_array_fork_choice::{Block, ProtoArrayForkChoice};
 use serde_derive::{Deserialize, Serialize};
-use types::{Epoch, Hash256, Slot};
+use types::{Epoch, Hash256, ShufflingId, Slot};
 
 pub use ffg_updates::*;
 pub use no_votes::*;
@@ -55,12 +55,15 @@ pub struct ForkChoiceTestDefinition {
 
 impl ForkChoiceTestDefinition {
     pub fn run(self) {
+        let junk_shuffling_id = ShufflingId::from_components(Epoch::new(0), Hash256::zero());
         let mut fork_choice = ProtoArrayForkChoice::new(
             self.finalized_block_slot,
             Hash256::zero(),
             self.justified_epoch,
             self.finalized_epoch,
             self.finalized_root,
+            junk_shuffling_id.clone(),
+            junk_shuffling_id,
         )
         .expect("should create fork choice struct");
 
@@ -125,6 +128,14 @@ impl ForkChoiceTestDefinition {
                         parent_root: Some(parent_root),
                         state_root: Hash256::zero(),
                         target_root: Hash256::zero(),
+                        current_epoch_shuffling_id: ShufflingId::from_components(
+                            Epoch::new(0),
+                            Hash256::zero(),
+                        ),
+                        next_epoch_shuffling_id: ShufflingId::from_components(
+                            Epoch::new(0),
+                            Hash256::zero(),
+                        ),
                         justified_epoch,
                         finalized_epoch,
                     };

consensus/proto_array/src/proto_array.rs

@@ -2,7 +2,7 @@ use crate::{error::Error, Block};
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use std::collections::HashMap;
-use types::{Epoch, Hash256, Slot};
+use types::{Epoch, Hash256, ShufflingId, Slot};
 
 #[derive(Clone, PartialEq, Debug, Encode, Decode, Serialize, Deserialize)]
 pub struct ProtoNode {
@@ -18,6 +18,8 @@ pub struct ProtoNode {
     /// The `target_root` is not necessary for `ProtoArray` either, it also just exists for upstream
     /// components (namely fork choice attestation verification).
     pub target_root: Hash256,
+    pub current_epoch_shuffling_id: ShufflingId,
+    pub next_epoch_shuffling_id: ShufflingId,
     pub root: Hash256,
     pub parent: Option<usize>,
     pub justified_epoch: Epoch,
@@ -142,6 +144,8 @@ impl ProtoArray {
             slot: block.slot,
             root: block.root,
             target_root: block.target_root,
+            current_epoch_shuffling_id: block.current_epoch_shuffling_id,
+            next_epoch_shuffling_id: block.next_epoch_shuffling_id,
             state_root: block.state_root,
             parent: block
                 .parent_root

consensus/proto_array/src/proto_array_fork_choice.rs

@@ -4,7 +4,7 @@ use crate::ssz_container::SszContainer;
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
 use std::collections::HashMap;
-use types::{Epoch, Hash256, Slot};
+use types::{Epoch, Hash256, ShufflingId, Slot};
 
 pub const DEFAULT_PRUNE_THRESHOLD: usize = 256;
 
@@ -25,6 +25,8 @@ pub struct Block {
     pub parent_root: Option<Hash256>,
     pub state_root: Hash256,
     pub target_root: Hash256,
+    pub current_epoch_shuffling_id: ShufflingId,
+    pub next_epoch_shuffling_id: ShufflingId,
     pub justified_epoch: Epoch,
     pub finalized_epoch: Epoch,
 }
@@ -70,6 +72,8 @@ impl ProtoArrayForkChoice {
         justified_epoch: Epoch,
         finalized_epoch: Epoch,
         finalized_root: Hash256,
+        current_epoch_shuffling_id: ShufflingId,
+        next_epoch_shuffling_id: ShufflingId,
     ) -> Result<Self, String> {
         let mut proto_array = ProtoArray {
             prune_threshold: DEFAULT_PRUNE_THRESHOLD,
@@ -87,6 +91,8 @@ impl ProtoArrayForkChoice {
             // We are using the finalized_root as the target_root, since it always lies on an
             // epoch boundary.
             target_root: finalized_root,
+            current_epoch_shuffling_id,
+            next_epoch_shuffling_id,
             justified_epoch,
             finalized_epoch,
         };
@@ -194,6 +200,8 @@ impl ProtoArrayForkChoice {
             parent_root,
             state_root: block.state_root,
             target_root: block.target_root,
+            current_epoch_shuffling_id: block.current_epoch_shuffling_id.clone(),
+            next_epoch_shuffling_id: block.next_epoch_shuffling_id.clone(),
             justified_epoch: block.justified_epoch,
             finalized_epoch: block.finalized_epoch,
         })
@@ -341,6 +349,7 @@ mod test_compute_deltas {
         let finalized_desc = Hash256::from_low_u64_be(2);
         let not_finalized_desc = Hash256::from_low_u64_be(3);
         let unknown = Hash256::from_low_u64_be(4);
+        let junk_shuffling_id = ShufflingId::from_components(Epoch::new(0), Hash256::zero());
 
         let mut fc = ProtoArrayForkChoice::new(
             genesis_slot,
@@ -348,6 +357,8 @@ mod test_compute_deltas {
             genesis_epoch,
             genesis_epoch,
             finalized_root,
+            junk_shuffling_id.clone(),
+            junk_shuffling_id.clone(),
         )
         .unwrap();
 
@@ -359,6 +370,8 @@ mod test_compute_deltas {
                 parent_root: Some(finalized_root),
                 state_root,
                 target_root: finalized_root,
+                current_epoch_shuffling_id: junk_shuffling_id.clone(),
+                next_epoch_shuffling_id: junk_shuffling_id.clone(),
                 justified_epoch: genesis_epoch,
                 finalized_epoch: genesis_epoch,
             })
@@ -372,6 +385,8 @@ mod test_compute_deltas {
                 parent_root: None,
                 state_root,
                 target_root: finalized_root,
+                current_epoch_shuffling_id: junk_shuffling_id.clone(),
+                next_epoch_shuffling_id: junk_shuffling_id.clone(),
                 justified_epoch: genesis_epoch,
                 finalized_epoch: genesis_epoch,
             })