Implement standard eth2.0 API (#1569)

- Resolves #1550
- Resolves #824
- Resolves #825
- Resolves #1131
- Resolves #1411
- Resolves #1256
- Resolve #1177

- Includes the `ShufflingId` struct initially defined in #1492. That PR is now closed and the changes are included here, with significant bug fixes.
- Implement the https://github.com/ethereum/eth2.0-APIs in a new `http_api` crate using `warp`. This replaces the `rest_api` crate.
- Add a new `common/eth2` crate which provides a wrapper around `reqwest`, providing the HTTP client that is used by the validator client and for testing. This replaces the `common/remote_beacon_node` crate.
- Create a `http_metrics` crate which is a dedicated server for Prometheus metrics (they are no longer served on the same port as the REST API). We now have flags for `--metrics`, `--metrics-address`, etc.
- Allow the `subnet_id` to be an optional parameter for `VerifiedUnaggregatedAttestation::verify`. This means it does not need to be provided unnecessarily by the validator client.
- Move `fn map_attestation_committee` in `mod beacon_chain::attestation_verification` to a new `fn with_committee_cache` on the `BeaconChain` so the same cache can be used for obtaining validator duties.
- Add some other helpers to `BeaconChain` to assist with common API duties (e.g., `block_root_at_slot`, `head_beacon_block_root`).
- Change the `NaiveAggregationPool` so it can index attestations by `hash_tree_root(attestation.data)`. This is a requirement of the API.
- Add functions to `BeaconChainHarness` to allow it to create slashings and exits.
- Allow for `eth1::Eth1NetworkId` to go to/from a `String`.
- Add functions to the `OperationPool` to allow getting all objects in the pool.
- Add function to `BeaconState` to check if a committee cache is initialized.
- Fix bug where `seconds_per_eth1_block` was not transferring over from `YamlConfig` to `ChainSpec`.
- Add the `deposit_contract_address` to `YamlConfig` and `ChainSpec`. We needed to be able to return it in an API response.
- Change some uses of serde `serialize_with` and `deserialize_with` to a single use of `with` (code quality).
- Impl `Display` and `FromStr` for several BLS fields.
- Check for clock discrepancy when VC polls BN for sync state (with +/- 1 slot tolerance). This is not intended to be comprehensive, it was just easy to do.

- See #1434 for a per-endpoint overview.
- Seeking clarity here: https://github.com/ethereum/eth2.0-APIs/issues/75

- [x] Add docs for prom port to close #1256
- [x] Follow up on this #1177
- [x] ~~Follow up with #1424~~ Will fix in future PR.
- [x] Follow up with #1411
- [x] ~~Follow up with  #1260~~ Will fix in future PR.
- [x] Add quotes to all integers.
- [x] Remove `rest_types`
- [x] Address missing beacon block error. (#1629)
- [x] ~~Add tests for lighthouse/peers endpoints~~ Wontfix
- [x] ~~Follow up with validator status proposal~~ Tracked in #1434
- [x] Unify graffiti structs
- [x] ~~Start server when waiting for genesis?~~ Will fix in future PR.
- [x] TODO in http_api tests
- [x] Move lighthouse endpoints off /eth/v1
- [x] Update docs to link to standard

- ~~Blocked on #1586~~

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

beacon_node/http_metrics/src/metrics.rs

@@ -0,0 +1,105 @@
+use crate::Context;
+use beacon_chain::BeaconChainTypes;
+use eth2::lighthouse::Health;
+use lighthouse_metrics::{Encoder, TextEncoder};
+
+pub use lighthouse_metrics::*;
+
+lazy_static! {
+    pub static ref PROCESS_NUM_THREADS: Result<IntGauge> = try_create_int_gauge(
+        "process_num_threads",
+        "Number of threads used by the current process"
+    );
+    pub static ref PROCESS_RES_MEM: Result<IntGauge> = try_create_int_gauge(
+        "process_resident_memory_bytes",
+        "Resident memory used by the current process"
+    );
+    pub static ref PROCESS_VIRT_MEM: Result<IntGauge> = try_create_int_gauge(
+        "process_virtual_memory_bytes",
+        "Virtual memory used by the current process"
+    );
+    pub static ref SYSTEM_VIRT_MEM_TOTAL: Result<IntGauge> =
+        try_create_int_gauge("system_virt_mem_total_bytes", "Total system virtual memory");
+    pub static ref SYSTEM_VIRT_MEM_AVAILABLE: Result<IntGauge> = try_create_int_gauge(
+        "system_virt_mem_available_bytes",
+        "Available system virtual memory"
+    );
+    pub static ref SYSTEM_VIRT_MEM_USED: Result<IntGauge> =
+        try_create_int_gauge("system_virt_mem_used_bytes", "Used system virtual memory");
+    pub static ref SYSTEM_VIRT_MEM_FREE: Result<IntGauge> =
+        try_create_int_gauge("system_virt_mem_free_bytes", "Free system virtual memory");
+    pub static ref SYSTEM_VIRT_MEM_PERCENTAGE: Result<Gauge> = try_create_float_gauge(
+        "system_virt_mem_percentage",
+        "Percentage of used virtual memory"
+    );
+    pub static ref SYSTEM_LOADAVG_1: Result<Gauge> =
+        try_create_float_gauge("system_loadavg_1", "Loadavg over 1 minute");
+    pub static ref SYSTEM_LOADAVG_5: Result<Gauge> =
+        try_create_float_gauge("system_loadavg_5", "Loadavg over 5 minutes");
+    pub static ref SYSTEM_LOADAVG_15: Result<Gauge> =
+        try_create_float_gauge("system_loadavg_15", "Loadavg over 15 minutes");
+}
+
+pub fn gather_prometheus_metrics<T: BeaconChainTypes>(
+    ctx: &Context<T>,
+) -> std::result::Result<String, String> {
+    let mut buffer = vec![];
+    let encoder = TextEncoder::new();
+
+    // There are two categories of metrics:
+    //
+    // - Dynamically updated: things like histograms and event counters that are updated on the
+    // fly.
+    // - Statically updated: things which are only updated at the time of the scrape (used where we
+    // can avoid cluttering up code with metrics calls).
+    //
+    // The `lighthouse_metrics` crate has a `DEFAULT_REGISTRY` global singleton (via `lazy_static`)
+    // which keeps the state of all the metrics. Dynamically updated things will already be
+    // up-to-date in the registry (because they update themselves) however statically updated
+    // things need to be "scraped".
+    //
+    // We proceed by, first updating all the static metrics using `scrape_for_metrics(..)`. Then,
+    // using `lighthouse_metrics::gather(..)` to collect the global `DEFAULT_REGISTRY` metrics into
+    // a string that can be returned via HTTP.
+
+    if let Some(beacon_chain) = ctx.chain.as_ref() {
+        slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&beacon_chain.slot_clock);
+        beacon_chain::scrape_for_metrics(beacon_chain);
+    }
+
+    if let (Some(db_path), Some(freezer_db_path)) =
+        (ctx.db_path.as_ref(), ctx.freezer_db_path.as_ref())
+    {
+        store::scrape_for_metrics(db_path, freezer_db_path);
+    }
+
+    eth2_libp2p::scrape_discovery_metrics();
+
+    // This will silently fail if we are unable to observe the health. This is desired behaviour
+    // since we don't support `Health` for all platforms.
+    if let Ok(health) = Health::observe() {
+        set_gauge(&PROCESS_NUM_THREADS, health.pid_num_threads as i64);
+        set_gauge(&PROCESS_RES_MEM, health.pid_mem_resident_set_size as i64);
+        set_gauge(&PROCESS_VIRT_MEM, health.pid_mem_virtual_memory_size as i64);
+        set_gauge(&SYSTEM_VIRT_MEM_TOTAL, health.sys_virt_mem_total as i64);
+        set_gauge(
+            &SYSTEM_VIRT_MEM_AVAILABLE,
+            health.sys_virt_mem_available as i64,
+        );
+        set_gauge(&SYSTEM_VIRT_MEM_USED, health.sys_virt_mem_used as i64);
+        set_gauge(&SYSTEM_VIRT_MEM_FREE, health.sys_virt_mem_free as i64);
+        set_float_gauge(
+            &SYSTEM_VIRT_MEM_PERCENTAGE,
+            health.sys_virt_mem_percent as f64,
+        );
+        set_float_gauge(&SYSTEM_LOADAVG_1, health.sys_loadavg_1);
+        set_float_gauge(&SYSTEM_LOADAVG_5, health.sys_loadavg_5);
+        set_float_gauge(&SYSTEM_LOADAVG_15, health.sys_loadavg_15);
+    }
+
+    encoder
+        .encode(&lighthouse_metrics::gather(), &mut buffer)
+        .unwrap();
+
+    String::from_utf8(buffer).map_err(|e| format!("Failed to encode prometheus info: {:?}", e))
+}