Implement standard eth2.0 API (#1569)

- Resolves #1550
- Resolves #824
- Resolves #825
- Resolves #1131
- Resolves #1411
- Resolves #1256
- Resolve #1177

- Includes the `ShufflingId` struct initially defined in #1492. That PR is now closed and the changes are included here, with significant bug fixes.
- Implement the https://github.com/ethereum/eth2.0-APIs in a new `http_api` crate using `warp`. This replaces the `rest_api` crate.
- Add a new `common/eth2` crate which provides a wrapper around `reqwest`, providing the HTTP client that is used by the validator client and for testing. This replaces the `common/remote_beacon_node` crate.
- Create a `http_metrics` crate which is a dedicated server for Prometheus metrics (they are no longer served on the same port as the REST API). We now have flags for `--metrics`, `--metrics-address`, etc.
- Allow the `subnet_id` to be an optional parameter for `VerifiedUnaggregatedAttestation::verify`. This means it does not need to be provided unnecessarily by the validator client.
- Move `fn map_attestation_committee` in `mod beacon_chain::attestation_verification` to a new `fn with_committee_cache` on the `BeaconChain` so the same cache can be used for obtaining validator duties.
- Add some other helpers to `BeaconChain` to assist with common API duties (e.g., `block_root_at_slot`, `head_beacon_block_root`).
- Change the `NaiveAggregationPool` so it can index attestations by `hash_tree_root(attestation.data)`. This is a requirement of the API.
- Add functions to `BeaconChainHarness` to allow it to create slashings and exits.
- Allow for `eth1::Eth1NetworkId` to go to/from a `String`.
- Add functions to the `OperationPool` to allow getting all objects in the pool.
- Add function to `BeaconState` to check if a committee cache is initialized.
- Fix bug where `seconds_per_eth1_block` was not transferring over from `YamlConfig` to `ChainSpec`.
- Add the `deposit_contract_address` to `YamlConfig` and `ChainSpec`. We needed to be able to return it in an API response.
- Change some uses of serde `serialize_with` and `deserialize_with` to a single use of `with` (code quality).
- Impl `Display` and `FromStr` for several BLS fields.
- Check for clock discrepancy when VC polls BN for sync state (with +/- 1 slot tolerance). This is not intended to be comprehensive, it was just easy to do.

- See #1434 for a per-endpoint overview.
- Seeking clarity here: https://github.com/ethereum/eth2.0-APIs/issues/75

- [x] Add docs for prom port to close #1256
- [x] Follow up on this #1177
- [x] ~~Follow up with #1424~~ Will fix in future PR.
- [x] Follow up with #1411
- [x] ~~Follow up with  #1260~~ Will fix in future PR.
- [x] Add quotes to all integers.
- [x] Remove `rest_types`
- [x] Address missing beacon block error. (#1629)
- [x] ~~Add tests for lighthouse/peers endpoints~~ Wontfix
- [x] ~~Follow up with validator status proposal~~ Tracked in #1434
- [x] Unify graffiti structs
- [x] ~~Start server when waiting for genesis?~~ Will fix in future PR.
- [x] TODO in http_api tests
- [x] Move lighthouse endpoints off /eth/v1
- [x] Update docs to link to standard

- ~~Blocked on #1586~~

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

testing/node_test_rig/Cargo.toml

@@ -15,6 +15,6 @@ url = "2.1.1"
 serde = "1.0.110"
 futures = "0.3.5"
 genesis = { path = "../../beacon_node/genesis" }
-remote_beacon_node = { path = "../../common/remote_beacon_node" }
+eth2 = { path = "../../common/eth2" }
 validator_client = { path = "../../validator_client" }
 validator_dir = { path = "../../common/validator_dir", features = ["insecure_keys"] }

testing/node_test_rig/src/lib.rs

@@ -4,7 +4,12 @@
 
 use beacon_node::ProductionBeaconNode;
 use environment::RuntimeContext;
+use eth2::{
+    reqwest::{ClientBuilder, Url},
+    BeaconNodeHttpClient,
+};
 use std::path::PathBuf;
+use std::time::Duration;
 use std::time::{SystemTime, UNIX_EPOCH};
 use tempdir::TempDir;
 use types::EthSpec;
@@ -13,9 +18,12 @@ use validator_dir::insecure_keys::build_deterministic_validator_dirs;
 
 pub use beacon_node::{ClientConfig, ClientGenesis, ProductionClient};
 pub use environment;
-pub use remote_beacon_node::RemoteBeaconNode;
+pub use eth2;
 pub use validator_client::Config as ValidatorConfig;
 
+/// The global timeout for HTTP requests to the beacon node.
+const HTTP_TIMEOUT: Duration = Duration::from_secs(4);
+
 /// Provides a beacon node that is running in the current process on a given tokio executor (it
 /// is _local_ to this process).
 ///
@@ -52,16 +60,23 @@ impl<E: EthSpec> LocalBeaconNode<E> {
 impl<E: EthSpec> LocalBeaconNode<E> {
     /// Returns a `RemoteBeaconNode` that can connect to `self`. Useful for testing the node as if
     /// it were external this process.
-    pub fn remote_node(&self) -> Result<RemoteBeaconNode<E>, String> {
-        let socket_addr = self
+    pub fn remote_node(&self) -> Result<BeaconNodeHttpClient, String> {
+        let listen_addr = self
             .client
-            .http_listen_addr()
+            .http_api_listen_addr()
             .ok_or_else(|| "A remote beacon node must have a http server".to_string())?;
-        Ok(RemoteBeaconNode::new(format!(
-            "http://{}:{}",
-            socket_addr.ip(),
-            socket_addr.port()
-        ))?)
+
+        let beacon_node_url: Url = format!("http://{}:{}", listen_addr.ip(), listen_addr.port())
+            .parse()
+            .map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?;
+        let beacon_node_http_client = ClientBuilder::new()
+            .timeout(HTTP_TIMEOUT)
+            .build()
+            .map_err(|e| format!("Unable to build HTTP client: {:?}", e))?;
+        Ok(BeaconNodeHttpClient::from_components(
+            beacon_node_url,
+            beacon_node_http_client,
+        ))
     }
 }
 
@@ -71,8 +86,8 @@ pub fn testing_client_config() -> ClientConfig {
     // Setting ports to `0` means that the OS will choose some available port.
     client_config.network.libp2p_port = 0;
     client_config.network.discovery_port = 0;
-    client_config.rest_api.enabled = true;
-    client_config.rest_api.port = 0;
+    client_config.http_api.enabled = true;
+    client_config.http_api.listen_port = 0;
     client_config.websocket_server.enabled = true;
     client_config.websocket_server.port = 0;