Implement standard eth2.0 API (#1569)

- Resolves #1550
- Resolves #824
- Resolves #825
- Resolves #1131
- Resolves #1411
- Resolves #1256
- Resolve #1177

- Includes the `ShufflingId` struct initially defined in #1492. That PR is now closed and the changes are included here, with significant bug fixes.
- Implement the https://github.com/ethereum/eth2.0-APIs in a new `http_api` crate using `warp`. This replaces the `rest_api` crate.
- Add a new `common/eth2` crate which provides a wrapper around `reqwest`, providing the HTTP client that is used by the validator client and for testing. This replaces the `common/remote_beacon_node` crate.
- Create a `http_metrics` crate which is a dedicated server for Prometheus metrics (they are no longer served on the same port as the REST API). We now have flags for `--metrics`, `--metrics-address`, etc.
- Allow the `subnet_id` to be an optional parameter for `VerifiedUnaggregatedAttestation::verify`. This means it does not need to be provided unnecessarily by the validator client.
- Move `fn map_attestation_committee` in `mod beacon_chain::attestation_verification` to a new `fn with_committee_cache` on the `BeaconChain` so the same cache can be used for obtaining validator duties.
- Add some other helpers to `BeaconChain` to assist with common API duties (e.g., `block_root_at_slot`, `head_beacon_block_root`).
- Change the `NaiveAggregationPool` so it can index attestations by `hash_tree_root(attestation.data)`. This is a requirement of the API.
- Add functions to `BeaconChainHarness` to allow it to create slashings and exits.
- Allow for `eth1::Eth1NetworkId` to go to/from a `String`.
- Add functions to the `OperationPool` to allow getting all objects in the pool.
- Add function to `BeaconState` to check if a committee cache is initialized.
- Fix bug where `seconds_per_eth1_block` was not transferring over from `YamlConfig` to `ChainSpec`.
- Add the `deposit_contract_address` to `YamlConfig` and `ChainSpec`. We needed to be able to return it in an API response.
- Change some uses of serde `serialize_with` and `deserialize_with` to a single use of `with` (code quality).
- Impl `Display` and `FromStr` for several BLS fields.
- Check for clock discrepancy when VC polls BN for sync state (with +/- 1 slot tolerance). This is not intended to be comprehensive, it was just easy to do.

- See #1434 for a per-endpoint overview.
- Seeking clarity here: https://github.com/ethereum/eth2.0-APIs/issues/75

- [x] Add docs for prom port to close #1256
- [x] Follow up on this #1177
- [x] ~~Follow up with #1424~~ Will fix in future PR.
- [x] Follow up with #1411
- [x] ~~Follow up with  #1260~~ Will fix in future PR.
- [x] Add quotes to all integers.
- [x] Remove `rest_types`
- [x] Address missing beacon block error. (#1629)
- [x] ~~Add tests for lighthouse/peers endpoints~~ Wontfix
- [x] ~~Follow up with validator status proposal~~ Tracked in #1434
- [x] Unify graffiti structs
- [x] ~~Start server when waiting for genesis?~~ Will fix in future PR.
- [x] TODO in http_api tests
- [x] Move lighthouse endpoints off /eth/v1
- [x] Update docs to link to standard

- ~~Blocked on #1586~~

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

beacon_node/src/cli.rs

@@ -142,7 +142,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         .arg(
             Arg::with_name("http")
                 .long("http")
-                .help("Enable RESTful HTTP API server. Disabled by default.")
+                .help("Enable the RESTful HTTP API server. Disabled by default.")
                 .takes_value(false),
         )
         .arg(
@@ -169,6 +169,38 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .default_value("")
                 .takes_value(true),
         )
+        /* Prometheus metrics HTTP server related arguments */
+        .arg(
+            Arg::with_name("metrics")
+                .long("metrics")
+                .help("Enable the Prometheus metrics HTTP server. Disabled by default.")
+                .takes_value(false),
+        )
+        .arg(
+            Arg::with_name("metrics-address")
+                .long("metrics-address")
+                .value_name("ADDRESS")
+                .help("Set the listen address for the Prometheus metrics HTTP server.")
+                .default_value("127.0.0.1")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("metrics-port")
+                .long("metrics-port")
+                .value_name("PORT")
+                .help("Set the listen TCP port for the Prometheus metrics HTTP server.")
+                .default_value("5054")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("metrics-allow-origin")
+                .long("metrics-allow-origin")
+                .value_name("ORIGIN")
+                .help("Set the value of the Access-Control-Allow-Origin response HTTP header for the Prometheus metrics HTTP server. \
+                    Use * to allow any origin (not recommended in production)")
+                .default_value("")
+                .takes_value(true),
+        )
         /* Websocket related arguments */
         .arg(
             Arg::with_name("ws")

beacon_node/src/config.rs

@@ -87,26 +87,26 @@ pub fn get_config<E: EthSpec>(
      */
 
     if cli_args.is_present("staking") {
-        client_config.rest_api.enabled = true;
+        client_config.http_api.enabled = true;
         client_config.sync_eth1_chain = true;
     }
 
     /*
-     * Http server
+     * Http API server
      */
 
     if cli_args.is_present("http") {
-        client_config.rest_api.enabled = true;
+        client_config.http_api.enabled = true;
     }
 
     if let Some(address) = cli_args.value_of("http-address") {
-        client_config.rest_api.listen_address = address
+        client_config.http_api.listen_addr = address
             .parse::<Ipv4Addr>()
             .map_err(|_| "http-address is not a valid IPv4 address.")?;
     }
 
     if let Some(port) = cli_args.value_of("http-port") {
-        client_config.rest_api.port = port
+        client_config.http_api.listen_port = port
             .parse::<u16>()
             .map_err(|_| "http-port is not a valid u16.")?;
     }
@@ -117,7 +117,36 @@ pub fn get_config<E: EthSpec>(
         hyper::header::HeaderValue::from_str(allow_origin)
             .map_err(|_| "Invalid allow-origin value")?;
 
-        client_config.rest_api.allow_origin = allow_origin.to_string();
+        client_config.http_api.allow_origin = Some(allow_origin.to_string());
+    }
+
+    /*
+     * Prometheus metrics HTTP server
+     */
+
+    if cli_args.is_present("metrics") {
+        client_config.http_metrics.enabled = true;
+    }
+
+    if let Some(address) = cli_args.value_of("metrics-address") {
+        client_config.http_metrics.listen_addr = address
+            .parse::<Ipv4Addr>()
+            .map_err(|_| "metrics-address is not a valid IPv4 address.")?;
+    }
+
+    if let Some(port) = cli_args.value_of("metrics-port") {
+        client_config.http_metrics.listen_port = port
+            .parse::<u16>()
+            .map_err(|_| "metrics-port is not a valid u16.")?;
+    }
+
+    if let Some(allow_origin) = cli_args.value_of("metrics-allow-origin") {
+        // Pre-validate the config value to give feedback to the user on node startup, instead of
+        // as late as when the first API response is produced.
+        hyper::header::HeaderValue::from_str(allow_origin)
+            .map_err(|_| "Invalid allow-origin value")?;
+
+        client_config.http_metrics.allow_origin = Some(allow_origin.to_string());
     }
 
     // Log a warning indicating an open HTTP server if it wasn't specified explicitly
@@ -125,7 +154,7 @@ pub fn get_config<E: EthSpec>(
     if cli_args.is_present("staking") {
         warn!(
             log,
-            "Running HTTP server on port {}", client_config.rest_api.port
+            "Running HTTP server on port {}", client_config.http_api.listen_port
         );
     }
 
@@ -219,7 +248,8 @@ pub fn get_config<E: EthSpec>(
             unused_port("tcp").map_err(|e| format!("Failed to get port for libp2p: {}", e))?;
         client_config.network.discovery_port =
             unused_port("udp").map_err(|e| format!("Failed to get port for discovery: {}", e))?;
-        client_config.rest_api.port = 0;
+        client_config.http_api.listen_port = 0;
+        client_config.http_metrics.listen_port = 0;
         client_config.websocket_server.port = 0;
     }
 
@@ -230,6 +260,11 @@ pub fn get_config<E: EthSpec>(
 
     client_config.eth1.deposit_contract_address =
         format!("{:?}", eth2_testnet_config.deposit_contract_address()?);
+    let spec_contract_address = format!("{:?}", spec.deposit_contract_address);
+    if client_config.eth1.deposit_contract_address != spec_contract_address {
+        return Err("Testnet contract address does not match spec".into());
+    }
+
     client_config.eth1.deposit_contract_deploy_block =
         eth2_testnet_config.deposit_contract_deploy_block;
     client_config.eth1.lowest_cached_block_number =
@@ -265,7 +300,7 @@ pub fn get_config<E: EthSpec>(
     };
 
     let trimmed_graffiti_len = cmp::min(raw_graffiti.len(), GRAFFITI_BYTES_LEN);
-    client_config.graffiti[..trimmed_graffiti_len]
+    client_config.graffiti.0[..trimmed_graffiti_len]
         .copy_from_slice(&raw_graffiti[..trimmed_graffiti_len]);
 
     if let Some(max_skip_slots) = cli_args.value_of("max-skip-slots") {

beacon_node/src/lib.rs

@@ -71,7 +71,6 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
         context: RuntimeContext<E>,
         mut client_config: ClientConfig,
     ) -> Result<Self, String> {
-        let http_eth2_config = context.eth2_config().clone();
         let spec = context.eth2_config().spec.clone();
         let client_config_1 = client_config.clone();
         let client_genesis = client_config.genesis.clone();
@@ -118,26 +117,22 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
             builder.no_eth1_backend()?
         };
 
-        let (builder, events) = builder
+        let (builder, _events) = builder
             .system_time_slot_clock()?
             .tee_event_handler(client_config.websocket_server.clone())?;
 
         // Inject the executor into the discv5 network config.
         client_config.network.discv5_config.executor = Some(Box::new(executor));
 
-        let builder = builder
+        builder
             .build_beacon_chain()?
             .network(&client_config.network)
             .await?
-            .notifier()?;
-
-        let builder = if client_config.rest_api.enabled {
-            builder.http_server(&client_config, &http_eth2_config, events)?
-        } else {
-            builder
-        };
-
-        Ok(Self(builder.build()))
+            .notifier()?
+            .http_api_config(client_config.http_api.clone())
+            .http_metrics_config(client_config.http_metrics.clone())
+            .build()
+            .map(Self)
     }
 
     pub fn into_inner(self) -> ProductionClient<E> {