Implement standard eth2.0 API (#1569)

- Resolves #1550
- Resolves #824
- Resolves #825
- Resolves #1131
- Resolves #1411
- Resolves #1256
- Resolve #1177

- Includes the `ShufflingId` struct initially defined in #1492. That PR is now closed and the changes are included here, with significant bug fixes.
- Implement the https://github.com/ethereum/eth2.0-APIs in a new `http_api` crate using `warp`. This replaces the `rest_api` crate.
- Add a new `common/eth2` crate which provides a wrapper around `reqwest`, providing the HTTP client that is used by the validator client and for testing. This replaces the `common/remote_beacon_node` crate.
- Create a `http_metrics` crate which is a dedicated server for Prometheus metrics (they are no longer served on the same port as the REST API). We now have flags for `--metrics`, `--metrics-address`, etc.
- Allow the `subnet_id` to be an optional parameter for `VerifiedUnaggregatedAttestation::verify`. This means it does not need to be provided unnecessarily by the validator client.
- Move `fn map_attestation_committee` in `mod beacon_chain::attestation_verification` to a new `fn with_committee_cache` on the `BeaconChain` so the same cache can be used for obtaining validator duties.
- Add some other helpers to `BeaconChain` to assist with common API duties (e.g., `block_root_at_slot`, `head_beacon_block_root`).
- Change the `NaiveAggregationPool` so it can index attestations by `hash_tree_root(attestation.data)`. This is a requirement of the API.
- Add functions to `BeaconChainHarness` to allow it to create slashings and exits.
- Allow for `eth1::Eth1NetworkId` to go to/from a `String`.
- Add functions to the `OperationPool` to allow getting all objects in the pool.
- Add function to `BeaconState` to check if a committee cache is initialized.
- Fix bug where `seconds_per_eth1_block` was not transferring over from `YamlConfig` to `ChainSpec`.
- Add the `deposit_contract_address` to `YamlConfig` and `ChainSpec`. We needed to be able to return it in an API response.
- Change some uses of serde `serialize_with` and `deserialize_with` to a single use of `with` (code quality).
- Impl `Display` and `FromStr` for several BLS fields.
- Check for clock discrepancy when VC polls BN for sync state (with +/- 1 slot tolerance). This is not intended to be comprehensive, it was just easy to do.

- See #1434 for a per-endpoint overview.
- Seeking clarity here: https://github.com/ethereum/eth2.0-APIs/issues/75

- [x] Add docs for prom port to close #1256
- [x] Follow up on this #1177
- [x] ~~Follow up with #1424~~ Will fix in future PR.
- [x] Follow up with #1411
- [x] ~~Follow up with  #1260~~ Will fix in future PR.
- [x] Add quotes to all integers.
- [x] Remove `rest_types`
- [x] Address missing beacon block error. (#1629)
- [x] ~~Add tests for lighthouse/peers endpoints~~ Wontfix
- [x] ~~Follow up with validator status proposal~~ Tracked in #1434
- [x] Unify graffiti structs
- [x] ~~Start server when waiting for genesis?~~ Will fix in future PR.
- [x] TODO in http_api tests
- [x] Move lighthouse endpoints off /eth/v1
- [x] Update docs to link to standard

- ~~Blocked on #1586~~

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

beacon_node/http_api/src/beacon_proposer_cache.rs

@@ -0,0 +1,185 @@
+use crate::metrics;
+use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use eth2::types::ProposerData;
+use fork_choice::ProtoBlock;
+use slot_clock::SlotClock;
+use state_processing::per_slot_processing;
+use types::{BeaconState, Epoch, EthSpec, Hash256, PublicKeyBytes};
+
+/// This sets a maximum bound on the number of epochs to skip whilst instantiating the cache for
+/// the first time.
+const EPOCHS_TO_SKIP: u64 = 2;
+
+/// Caches the beacon block proposers for a given `epoch` and `epoch_boundary_root`.
+///
+/// This cache is only able to contain a single set of proposers and is only
+/// intended to cache the proposers for the current epoch according to the head
+/// of the chain. A change in epoch or re-org to a different chain may cause a
+/// cache miss and rebuild.
+pub struct BeaconProposerCache {
+    epoch: Epoch,
+    decision_block_root: Hash256,
+    proposers: Vec<ProposerData>,
+}
+
+impl BeaconProposerCache {
+    /// Create a new cache for the current epoch of the `chain`.
+    pub fn new<T: BeaconChainTypes>(chain: &BeaconChain<T>) -> Result<Self, BeaconChainError> {
+        let head_root = chain.head_beacon_block_root()?;
+        let head_block = chain
+            .fork_choice
+            .read()
+            .get_block(&head_root)
+            .ok_or_else(|| BeaconChainError::MissingBeaconBlock(head_root))?;
+
+        // If the head epoch is more than `EPOCHS_TO_SKIP` in the future, just build the cache at
+        // the epoch of the head. This prevents doing a massive amount of skip slots when starting
+        // a new database from genesis.
+        let epoch = {
+            let epoch_now = chain
+                .epoch()
+                .unwrap_or_else(|_| chain.spec.genesis_slot.epoch(T::EthSpec::slots_per_epoch()));
+            let head_epoch = head_block.slot.epoch(T::EthSpec::slots_per_epoch());
+            if epoch_now > head_epoch + EPOCHS_TO_SKIP {
+                head_epoch
+            } else {
+                epoch_now
+            }
+        };
+
+        Self::for_head_block(chain, epoch, head_root, head_block)
+    }
+
+    /// Create a new cache that contains the shuffling for `current_epoch`,
+    /// assuming that `head_root` and `head_block` represents the most recent
+    /// canonical block.
+    fn for_head_block<T: BeaconChainTypes>(
+        chain: &BeaconChain<T>,
+        current_epoch: Epoch,
+        head_root: Hash256,
+        head_block: ProtoBlock,
+    ) -> Result<Self, BeaconChainError> {
+        let _timer = metrics::start_timer(&metrics::HTTP_API_BEACON_PROPOSER_CACHE_TIMES);
+
+        let mut head_state = chain
+            .get_state(&head_block.state_root, Some(head_block.slot))?
+            .ok_or_else(|| BeaconChainError::MissingBeaconState(head_block.state_root))?;
+
+        let decision_block_root = Self::decision_block_root(current_epoch, head_root, &head_state)?;
+
+        // We *must* skip forward to the current epoch to obtain valid proposer
+        // duties. We cannot skip to the previous epoch, like we do with
+        // attester duties.
+        while head_state.current_epoch() < current_epoch {
+            // Skip slots until the current epoch, providing `Hash256::zero()` as the state root
+            // since we don't require it to be valid to identify producers.
+            per_slot_processing(&mut head_state, Some(Hash256::zero()), &chain.spec)?;
+        }
+
+        let proposers = current_epoch
+            .slot_iter(T::EthSpec::slots_per_epoch())
+            .map(|slot| {
+                head_state
+                    .get_beacon_proposer_index(slot, &chain.spec)
+                    .map_err(BeaconChainError::from)
+                    .and_then(|i| {
+                        let pubkey = chain
+                            .validator_pubkey(i)?
+                            .ok_or_else(|| BeaconChainError::ValidatorPubkeyCacheIncomplete(i))?;
+
+                        Ok(ProposerData {
+                            pubkey: PublicKeyBytes::from(pubkey),
+                            slot,
+                        })
+                    })
+            })
+            .collect::<Result<_, _>>()?;
+
+        Ok(Self {
+            epoch: current_epoch,
+            decision_block_root,
+            proposers,
+        })
+    }
+
+    /// Returns a block root which can be used to key the shuffling obtained from the following
+    /// parameters:
+    ///
+    /// - `shuffling_epoch`: the epoch for which the shuffling pertains.
+    /// - `head_block_root`: the block root at the head of the chain.
+    /// - `head_block_state`: the state of `head_block_root`.
+    pub fn decision_block_root<E: EthSpec>(
+        shuffling_epoch: Epoch,
+        head_block_root: Hash256,
+        head_block_state: &BeaconState<E>,
+    ) -> Result<Hash256, BeaconChainError> {
+        let decision_slot = shuffling_epoch
+            .start_slot(E::slots_per_epoch())
+            .saturating_sub(1_u64);
+
+        // If decision slot is equal to or ahead of the head, the block root is the head block root
+        if decision_slot >= head_block_state.slot {
+            Ok(head_block_root)
+        } else {
+            head_block_state
+                .get_block_root(decision_slot)
+                .map(|root| *root)
+                .map_err(Into::into)
+        }
+    }
+
+    /// Return the proposers for the given `Epoch`.
+    ///
+    /// The cache may be rebuilt if:
+    ///
+    /// - The epoch has changed since the last cache build.
+    /// - There has been a re-org that crosses an epoch boundary.
+    pub fn get_proposers<T: BeaconChainTypes>(
+        &mut self,
+        chain: &BeaconChain<T>,
+        epoch: Epoch,
+    ) -> Result<Vec<ProposerData>, warp::Rejection> {
+        let current_epoch = chain
+            .slot_clock
+            .now_or_genesis()
+            .ok_or_else(|| {
+                warp_utils::reject::custom_server_error("unable to read slot clock".to_string())
+            })?
+            .epoch(T::EthSpec::slots_per_epoch());
+
+        // Disallow requests that are outside the current epoch. This ensures the cache doesn't get
+        // washed-out with old values.
+        if current_epoch != epoch {
+            return Err(warp_utils::reject::custom_bad_request(format!(
+                "requested epoch is {} but only current epoch {} is allowed",
+                epoch, current_epoch
+            )));
+        }
+
+        let (head_block_root, head_decision_block_root) = chain
+            .with_head(|head| {
+                Self::decision_block_root(current_epoch, head.beacon_block_root, &head.beacon_state)
+                    .map(|decision_root| (head.beacon_block_root, decision_root))
+            })
+            .map_err(warp_utils::reject::beacon_chain_error)?;
+
+        let head_block = chain
+            .fork_choice
+            .read()
+            .get_block(&head_block_root)
+            .ok_or_else(|| BeaconChainError::MissingBeaconBlock(head_block_root))
+            .map_err(warp_utils::reject::beacon_chain_error)?;
+
+        // Rebuild the cache if this call causes a cache-miss.
+        if self.epoch != current_epoch || self.decision_block_root != head_decision_block_root {
+            metrics::inc_counter(&metrics::HTTP_API_BEACON_PROPOSER_CACHE_MISSES_TOTAL);
+
+            *self = Self::for_head_block(chain, current_epoch, head_block_root, head_block)
+                .map_err(warp_utils::reject::beacon_chain_error)?;
+        } else {
+            metrics::inc_counter(&metrics::HTTP_API_BEACON_PROPOSER_CACHE_HITS_TOTAL);
+        }
+
+        Ok(self.proposers.clone())
+    }
+}

beacon_node/http_api/src/block_id.rs

@@ -0,0 +1,87 @@
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use eth2::types::BlockId as CoreBlockId;
+use std::str::FromStr;
+use types::{Hash256, SignedBeaconBlock, Slot};
+
+/// Wraps `eth2::types::BlockId` and provides a simple way to obtain a block or root for a given
+/// `BlockId`.
+#[derive(Debug)]
+pub struct BlockId(pub CoreBlockId);
+
+impl BlockId {
+    pub fn from_slot(slot: Slot) -> Self {
+        Self(CoreBlockId::Slot(slot))
+    }
+
+    pub fn from_root(root: Hash256) -> Self {
+        Self(CoreBlockId::Root(root))
+    }
+
+    /// Return the block root identified by `self`.
+    pub fn root<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<Hash256, warp::Rejection> {
+        match &self.0 {
+            CoreBlockId::Head => chain
+                .head_info()
+                .map(|head| head.block_root)
+                .map_err(warp_utils::reject::beacon_chain_error),
+            CoreBlockId::Genesis => Ok(chain.genesis_block_root),
+            CoreBlockId::Finalized => chain
+                .head_info()
+                .map(|head| head.finalized_checkpoint.root)
+                .map_err(warp_utils::reject::beacon_chain_error),
+            CoreBlockId::Justified => chain
+                .head_info()
+                .map(|head| head.current_justified_checkpoint.root)
+                .map_err(warp_utils::reject::beacon_chain_error),
+            CoreBlockId::Slot(slot) => chain
+                .block_root_at_slot(*slot)
+                .map_err(warp_utils::reject::beacon_chain_error)
+                .and_then(|root_opt| {
+                    root_opt.ok_or_else(|| {
+                        warp_utils::reject::custom_not_found(format!(
+                            "beacon block at slot {}",
+                            slot
+                        ))
+                    })
+                }),
+            CoreBlockId::Root(root) => Ok(*root),
+        }
+    }
+
+    /// Return the `SignedBeaconBlock` identified by `self`.
+    pub fn block<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<SignedBeaconBlock<T::EthSpec>, warp::Rejection> {
+        match &self.0 {
+            CoreBlockId::Head => chain
+                .head_beacon_block()
+                .map_err(warp_utils::reject::beacon_chain_error),
+            _ => {
+                let root = self.root(chain)?;
+                chain
+                    .get_block(&root)
+                    .map_err(warp_utils::reject::beacon_chain_error)
+                    .and_then(|root_opt| {
+                        root_opt.ok_or_else(|| {
+                            warp_utils::reject::custom_not_found(format!(
+                                "beacon block with root {}",
+                                root
+                            ))
+                        })
+                    })
+            }
+        }
+    }
+}
+
+impl FromStr for BlockId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        CoreBlockId::from_str(s).map(Self)
+    }
+}

beacon_node/http_api/src/metrics.rs

@@ -0,0 +1,32 @@
+pub use lighthouse_metrics::*;
+
+lazy_static::lazy_static! {
+    pub static ref HTTP_API_PATHS_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "http_api_paths_total",
+        "Count of HTTP requests received",
+        &["path"]
+    );
+    pub static ref HTTP_API_STATUS_CODES_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "http_api_status_codes_total",
+        "Count of HTTP status codes returned",
+        &["status"]
+    );
+    pub static ref HTTP_API_PATHS_TIMES: Result<HistogramVec> = try_create_histogram_vec(
+        "http_api_paths_times",
+        "Duration to process HTTP requests per path",
+        &["path"]
+    );
+
+    pub static ref HTTP_API_BEACON_PROPOSER_CACHE_TIMES: Result<Histogram> = try_create_histogram(
+        "http_api_beacon_proposer_cache_build_times",
+        "Duration to process HTTP requests per path",
+    );
+    pub static ref HTTP_API_BEACON_PROPOSER_CACHE_HITS_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "http_api_beacon_proposer_cache_hits_total",
+        "Count of times the proposer cache has been hit",
+    );
+    pub static ref HTTP_API_BEACON_PROPOSER_CACHE_MISSES_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "http_api_beacon_proposer_cache_misses_total",
+        "Count of times the proposer cache has been missed",
+    );
+}

beacon_node/http_api/src/state_id.rs

@@ -0,0 +1,118 @@
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use eth2::types::StateId as CoreStateId;
+use std::str::FromStr;
+use types::{BeaconState, EthSpec, Fork, Hash256, Slot};
+
+/// Wraps `eth2::types::StateId` and provides common state-access functionality. E.g., reading
+/// states or parts of states from the database.
+pub struct StateId(CoreStateId);
+
+impl StateId {
+    pub fn head() -> Self {
+        Self(CoreStateId::Head)
+    }
+
+    pub fn slot(slot: Slot) -> Self {
+        Self(CoreStateId::Slot(slot))
+    }
+
+    /// Return the state root identified by `self`.
+    pub fn root<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<Hash256, warp::Rejection> {
+        let slot = match &self.0 {
+            CoreStateId::Head => {
+                return chain
+                    .head_info()
+                    .map(|head| head.state_root)
+                    .map_err(warp_utils::reject::beacon_chain_error)
+            }
+            CoreStateId::Genesis => return Ok(chain.genesis_state_root),
+            CoreStateId::Finalized => chain.head_info().map(|head| {
+                head.finalized_checkpoint
+                    .epoch
+                    .start_slot(T::EthSpec::slots_per_epoch())
+            }),
+            CoreStateId::Justified => chain.head_info().map(|head| {
+                head.current_justified_checkpoint
+                    .epoch
+                    .start_slot(T::EthSpec::slots_per_epoch())
+            }),
+            CoreStateId::Slot(slot) => Ok(*slot),
+            CoreStateId::Root(root) => return Ok(*root),
+        }
+        .map_err(warp_utils::reject::beacon_chain_error)?;
+
+        chain
+            .state_root_at_slot(slot)
+            .map_err(warp_utils::reject::beacon_chain_error)?
+            .ok_or_else(|| {
+                warp_utils::reject::custom_not_found(format!("beacon state at slot {}", slot))
+            })
+    }
+
+    /// Return the `fork` field of the state identified by `self`.
+    pub fn fork<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<Fork, warp::Rejection> {
+        self.map_state(chain, |state| Ok(state.fork))
+    }
+
+    /// Return the `BeaconState` identified by `self`.
+    pub fn state<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<BeaconState<T::EthSpec>, warp::Rejection> {
+        let (state_root, slot_opt) = match &self.0 {
+            CoreStateId::Head => {
+                return chain
+                    .head_beacon_state()
+                    .map_err(warp_utils::reject::beacon_chain_error)
+            }
+            CoreStateId::Slot(slot) => (self.root(chain)?, Some(*slot)),
+            _ => (self.root(chain)?, None),
+        };
+
+        chain
+            .get_state(&state_root, slot_opt)
+            .map_err(warp_utils::reject::beacon_chain_error)
+            .and_then(|opt| {
+                opt.ok_or_else(|| {
+                    warp_utils::reject::custom_not_found(format!(
+                        "beacon state at root {}",
+                        state_root
+                    ))
+                })
+            })
+    }
+
+    /// Map a function across the `BeaconState` identified by `self`.
+    ///
+    /// This function will avoid instantiating/copying a new state when `self` points to the head
+    /// of the chain.
+    pub fn map_state<T: BeaconChainTypes, F, U>(
+        &self,
+        chain: &BeaconChain<T>,
+        func: F,
+    ) -> Result<U, warp::Rejection>
+    where
+        F: Fn(&BeaconState<T::EthSpec>) -> Result<U, warp::Rejection>,
+    {
+        match &self.0 {
+            CoreStateId::Head => chain
+                .with_head(|snapshot| Ok(func(&snapshot.beacon_state)))
+                .map_err(warp_utils::reject::beacon_chain_error)?,
+            _ => func(&self.state(chain)?),
+        }
+    }
+}
+
+impl FromStr for StateId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        CoreStateId::from_str(s).map(Self)
+    }
+}

beacon_node/http_api/src/validator_inclusion.rs

@@ -0,0 +1,88 @@
+use crate::state_id::StateId;
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use eth2::{
+    lighthouse::{GlobalValidatorInclusionData, ValidatorInclusionData},
+    types::ValidatorId,
+};
+use state_processing::per_epoch_processing::ValidatorStatuses;
+use types::{Epoch, EthSpec};
+
+/// Returns information about *all validators* (i.e., global) and how they performed during a given
+/// epoch.
+pub fn global_validator_inclusion_data<T: BeaconChainTypes>(
+    epoch: Epoch,
+    chain: &BeaconChain<T>,
+) -> Result<GlobalValidatorInclusionData, warp::Rejection> {
+    let target_slot = epoch.end_slot(T::EthSpec::slots_per_epoch());
+
+    let state = StateId::slot(target_slot).state(chain)?;
+
+    let mut validator_statuses = ValidatorStatuses::new(&state, &chain.spec)
+        .map_err(warp_utils::reject::beacon_state_error)?;
+    validator_statuses
+        .process_attestations(&state, &chain.spec)
+        .map_err(warp_utils::reject::beacon_state_error)?;
+
+    let totals = validator_statuses.total_balances;
+
+    Ok(GlobalValidatorInclusionData {
+        current_epoch_active_gwei: totals.current_epoch(),
+        previous_epoch_active_gwei: totals.previous_epoch(),
+        current_epoch_attesting_gwei: totals.current_epoch_attesters(),
+        current_epoch_target_attesting_gwei: totals.current_epoch_target_attesters(),
+        previous_epoch_attesting_gwei: totals.previous_epoch_attesters(),
+        previous_epoch_target_attesting_gwei: totals.previous_epoch_target_attesters(),
+        previous_epoch_head_attesting_gwei: totals.previous_epoch_head_attesters(),
+    })
+}
+
+/// Returns information about a single validator and how it performed during a given epoch.
+pub fn validator_inclusion_data<T: BeaconChainTypes>(
+    epoch: Epoch,
+    validator_id: &ValidatorId,
+    chain: &BeaconChain<T>,
+) -> Result<Option<ValidatorInclusionData>, warp::Rejection> {
+    let target_slot = epoch.end_slot(T::EthSpec::slots_per_epoch());
+
+    let mut state = StateId::slot(target_slot).state(chain)?;
+
+    let mut validator_statuses = ValidatorStatuses::new(&state, &chain.spec)
+        .map_err(warp_utils::reject::beacon_state_error)?;
+    validator_statuses
+        .process_attestations(&state, &chain.spec)
+        .map_err(warp_utils::reject::beacon_state_error)?;
+
+    state
+        .update_pubkey_cache()
+        .map_err(warp_utils::reject::beacon_state_error)?;
+
+    let validator_index = match validator_id {
+        ValidatorId::Index(index) => *index as usize,
+        ValidatorId::PublicKey(pubkey) => {
+            if let Some(index) = state
+                .get_validator_index(pubkey)
+                .map_err(warp_utils::reject::beacon_state_error)?
+            {
+                index
+            } else {
+                return Ok(None);
+            }
+        }
+    };
+
+    Ok(validator_statuses
+        .statuses
+        .get(validator_index)
+        .map(|vote| ValidatorInclusionData {
+            is_slashed: vote.is_slashed,
+            is_withdrawable_in_current_epoch: vote.is_withdrawable_in_current_epoch,
+            is_active_in_current_epoch: vote.is_active_in_current_epoch,
+            is_active_in_previous_epoch: vote.is_active_in_previous_epoch,
+            current_epoch_effective_balance_gwei: vote.current_epoch_effective_balance,
+            is_current_epoch_attester: vote.is_current_epoch_attester,
+            is_current_epoch_target_attester: vote.is_current_epoch_target_attester,
+            is_previous_epoch_attester: vote.is_previous_epoch_attester,
+            is_previous_epoch_target_attester: vote.is_previous_epoch_target_attester,
+            is_previous_epoch_head_attester: vote.is_previous_epoch_head_attester,
+        }))
+}