Implement standard eth2.0 API (#1569)

- Resolves #1550
- Resolves #824
- Resolves #825
- Resolves #1131
- Resolves #1411
- Resolves #1256
- Resolve #1177

- Includes the `ShufflingId` struct initially defined in #1492. That PR is now closed and the changes are included here, with significant bug fixes.
- Implement the https://github.com/ethereum/eth2.0-APIs in a new `http_api` crate using `warp`. This replaces the `rest_api` crate.
- Add a new `common/eth2` crate which provides a wrapper around `reqwest`, providing the HTTP client that is used by the validator client and for testing. This replaces the `common/remote_beacon_node` crate.
- Create a `http_metrics` crate which is a dedicated server for Prometheus metrics (they are no longer served on the same port as the REST API). We now have flags for `--metrics`, `--metrics-address`, etc.
- Allow the `subnet_id` to be an optional parameter for `VerifiedUnaggregatedAttestation::verify`. This means it does not need to be provided unnecessarily by the validator client.
- Move `fn map_attestation_committee` in `mod beacon_chain::attestation_verification` to a new `fn with_committee_cache` on the `BeaconChain` so the same cache can be used for obtaining validator duties.
- Add some other helpers to `BeaconChain` to assist with common API duties (e.g., `block_root_at_slot`, `head_beacon_block_root`).
- Change the `NaiveAggregationPool` so it can index attestations by `hash_tree_root(attestation.data)`. This is a requirement of the API.
- Add functions to `BeaconChainHarness` to allow it to create slashings and exits.
- Allow for `eth1::Eth1NetworkId` to go to/from a `String`.
- Add functions to the `OperationPool` to allow getting all objects in the pool.
- Add function to `BeaconState` to check if a committee cache is initialized.
- Fix bug where `seconds_per_eth1_block` was not transferring over from `YamlConfig` to `ChainSpec`.
- Add the `deposit_contract_address` to `YamlConfig` and `ChainSpec`. We needed to be able to return it in an API response.
- Change some uses of serde `serialize_with` and `deserialize_with` to a single use of `with` (code quality).
- Impl `Display` and `FromStr` for several BLS fields.
- Check for clock discrepancy when VC polls BN for sync state (with +/- 1 slot tolerance). This is not intended to be comprehensive, it was just easy to do.

- See #1434 for a per-endpoint overview.
- Seeking clarity here: https://github.com/ethereum/eth2.0-APIs/issues/75

- [x] Add docs for prom port to close #1256
- [x] Follow up on this #1177
- [x] ~~Follow up with #1424~~ Will fix in future PR.
- [x] Follow up with #1411
- [x] ~~Follow up with  #1260~~ Will fix in future PR.
- [x] Add quotes to all integers.
- [x] Remove `rest_types`
- [x] Address missing beacon block error. (#1629)
- [x] ~~Add tests for lighthouse/peers endpoints~~ Wontfix
- [x] ~~Follow up with validator status proposal~~ Tracked in #1434
- [x] Unify graffiti structs
- [x] ~~Start server when waiting for genesis?~~ Will fix in future PR.
- [x] TODO in http_api tests
- [x] Move lighthouse endpoints off /eth/v1
- [x] Update docs to link to standard

- ~~Blocked on #1586~~

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

common/eth2/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "eth2"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+serde = { version = "1.0.110", features = ["derive"] }
+serde_json = "1.0.52"
+types = { path = "../../consensus/types" }
+hex = "0.4.2"
+reqwest = { version = "0.10.8", features = ["json"] }
+eth2_libp2p = { path = "../../beacon_node/eth2_libp2p" }
+proto_array = { path = "../../consensus/proto_array", optional = true }
+serde_utils = { path = "../../consensus/serde_utils" }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+psutil = { version = "3.1.0", optional = true }
+procinfo = { version = "0.4.2", optional = true }
+
+[features]
+default = ["lighthouse"]
+lighthouse = ["proto_array", "psutil", "procinfo"]

common/eth2/src/lib.rs

@@ -0,0 +1,784 @@
+//! This crate provides two major things:
+//!
+//! 1. The types served by the `http_api` crate.
+//! 2. A wrapper around `reqwest` that forms a HTTP client, able of consuming the endpoints served
+//!    by the `http_api` crate.
+//!
+//! Eventually it would be ideal to publish this crate on crates.io, however we have some local
+//! dependencies preventing this presently.
+
+#[cfg(feature = "lighthouse")]
+pub mod lighthouse;
+pub mod types;
+
+use self::types::*;
+use reqwest::{IntoUrl, Response};
+use serde::{de::DeserializeOwned, Serialize};
+use std::convert::TryFrom;
+use std::fmt;
+
+pub use reqwest;
+pub use reqwest::{StatusCode, Url};
+
+#[derive(Debug)]
+pub enum Error {
+    /// The `reqwest` client raised an error.
+    Reqwest(reqwest::Error),
+    /// The server returned an error message where the body was able to be parsed.
+    ServerMessage(ErrorMessage),
+    /// The server returned an error message where the body was unable to be parsed.
+    StatusCode(StatusCode),
+    /// The supplied URL is badly formatted. It should look something like `http://127.0.0.1:5052`.
+    InvalidUrl(Url),
+}
+
+impl Error {
+    /// If the error has a HTTP status code, return it.
+    pub fn status(&self) -> Option<StatusCode> {
+        match self {
+            Error::Reqwest(error) => error.status(),
+            Error::ServerMessage(msg) => StatusCode::try_from(msg.code).ok(),
+            Error::StatusCode(status) => Some(*status),
+            Error::InvalidUrl(_) => None,
+        }
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+/// A wrapper around `reqwest::Client` which provides convenience methods for interfacing with a
+/// Lighthouse Beacon Node HTTP server (`http_api`).
+#[derive(Clone)]
+pub struct BeaconNodeHttpClient {
+    client: reqwest::Client,
+    server: Url,
+}
+
+impl BeaconNodeHttpClient {
+    pub fn new(server: Url) -> Self {
+        Self {
+            client: reqwest::Client::new(),
+            server,
+        }
+    }
+
+    pub fn from_components(server: Url, client: reqwest::Client) -> Self {
+        Self { client, server }
+    }
+
+    /// Return the path with the standard `/eth1/v1` prefix applied.
+    fn eth_path(&self) -> Result<Url, Error> {
+        let mut path = self.server.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("eth")
+            .push("v1");
+
+        Ok(path)
+    }
+
+    /// Perform a HTTP GET request.
+    async fn get<T: DeserializeOwned, U: IntoUrl>(&self, url: U) -> Result<T, Error> {
+        let response = self.client.get(url).send().await.map_err(Error::Reqwest)?;
+        ok_or_error(response)
+            .await?
+            .json()
+            .await
+            .map_err(Error::Reqwest)
+    }
+
+    /// Perform a HTTP GET request, returning `None` on a 404 error.
+    async fn get_opt<T: DeserializeOwned, U: IntoUrl>(&self, url: U) -> Result<Option<T>, Error> {
+        let response = self.client.get(url).send().await.map_err(Error::Reqwest)?;
+        match ok_or_error(response).await {
+            Ok(resp) => resp.json().await.map(Option::Some).map_err(Error::Reqwest),
+            Err(err) => {
+                if err.status() == Some(StatusCode::NOT_FOUND) {
+                    Ok(None)
+                } else {
+                    Err(err)
+                }
+            }
+        }
+    }
+
+    /// Perform a HTTP POST request.
+    async fn post<T: Serialize, U: IntoUrl>(&self, url: U, body: &T) -> Result<(), Error> {
+        let response = self
+            .client
+            .post(url)
+            .json(body)
+            .send()
+            .await
+            .map_err(Error::Reqwest)?;
+        ok_or_error(response).await?;
+        Ok(())
+    }
+
+    /// `GET beacon/genesis`
+    ///
+    /// ## Errors
+    ///
+    /// May return a `404` if beacon chain genesis has not yet occurred.
+    pub async fn get_beacon_genesis(&self) -> Result<GenericResponse<GenesisData>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("genesis");
+
+        self.get(path).await
+    }
+
+    /// `GET beacon/states/{state_id}/root`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_states_root(
+        &self,
+        state_id: StateId,
+    ) -> Result<Option<GenericResponse<RootData>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("states")
+            .push(&state_id.to_string())
+            .push("root");
+
+        self.get_opt(path).await
+    }
+
+    /// `GET beacon/states/{state_id}/fork`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_states_fork(
+        &self,
+        state_id: StateId,
+    ) -> Result<Option<GenericResponse<Fork>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("states")
+            .push(&state_id.to_string())
+            .push("fork");
+
+        self.get_opt(path).await
+    }
+
+    /// `GET beacon/states/{state_id}/finality_checkpoints`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_states_finality_checkpoints(
+        &self,
+        state_id: StateId,
+    ) -> Result<Option<GenericResponse<FinalityCheckpointsData>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("states")
+            .push(&state_id.to_string())
+            .push("finality_checkpoints");
+
+        self.get_opt(path).await
+    }
+
+    /// `GET beacon/states/{state_id}/validators`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_states_validators(
+        &self,
+        state_id: StateId,
+    ) -> Result<Option<GenericResponse<Vec<ValidatorData>>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("states")
+            .push(&state_id.to_string())
+            .push("validators");
+
+        self.get_opt(path).await
+    }
+
+    /// `GET beacon/states/{state_id}/committees?slot,index`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_states_committees(
+        &self,
+        state_id: StateId,
+        epoch: Epoch,
+        slot: Option<Slot>,
+        index: Option<u64>,
+    ) -> Result<Option<GenericResponse<Vec<CommitteeData>>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("states")
+            .push(&state_id.to_string())
+            .push("committees")
+            .push(&epoch.to_string());
+
+        if let Some(slot) = slot {
+            path.query_pairs_mut()
+                .append_pair("slot", &slot.to_string());
+        }
+
+        if let Some(index) = index {
+            path.query_pairs_mut()
+                .append_pair("index", &index.to_string());
+        }
+
+        self.get_opt(path).await
+    }
+
+    /// `GET beacon/states/{state_id}/validators/{validator_id}`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_states_validator_id(
+        &self,
+        state_id: StateId,
+        validator_id: &ValidatorId,
+    ) -> Result<Option<GenericResponse<ValidatorData>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("states")
+            .push(&state_id.to_string())
+            .push("validators")
+            .push(&validator_id.to_string());
+
+        self.get_opt(path).await
+    }
+
+    /// `GET beacon/headers?slot,parent_root`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_headers(
+        &self,
+        slot: Option<Slot>,
+        parent_root: Option<Hash256>,
+    ) -> Result<Option<GenericResponse<Vec<BlockHeaderData>>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("headers");
+
+        if let Some(slot) = slot {
+            path.query_pairs_mut()
+                .append_pair("slot", &slot.to_string());
+        }
+
+        if let Some(root) = parent_root {
+            path.query_pairs_mut()
+                .append_pair("parent_root", &format!("{:?}", root));
+        }
+
+        self.get_opt(path).await
+    }
+
+    /// `GET beacon/headers/{block_id}`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_headers_block_id(
+        &self,
+        block_id: BlockId,
+    ) -> Result<Option<GenericResponse<BlockHeaderData>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("headers")
+            .push(&block_id.to_string());
+
+        self.get_opt(path).await
+    }
+
+    /// `POST beacon/blocks`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn post_beacon_blocks<T: EthSpec>(
+        &self,
+        block: &SignedBeaconBlock<T>,
+    ) -> Result<(), Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("blocks");
+
+        self.post(path, block).await?;
+
+        Ok(())
+    }
+
+    /// `GET beacon/blocks`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_blocks<T: EthSpec>(
+        &self,
+        block_id: BlockId,
+    ) -> Result<Option<GenericResponse<SignedBeaconBlock<T>>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("blocks")
+            .push(&block_id.to_string());
+
+        self.get_opt(path).await
+    }
+
+    /// `GET beacon/blocks/{block_id}/root`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_blocks_root(
+        &self,
+        block_id: BlockId,
+    ) -> Result<Option<GenericResponse<RootData>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("blocks")
+            .push(&block_id.to_string())
+            .push("root");
+
+        self.get_opt(path).await
+    }
+
+    /// `GET beacon/blocks/{block_id}/attestations`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_blocks_attestations<T: EthSpec>(
+        &self,
+        block_id: BlockId,
+    ) -> Result<Option<GenericResponse<Vec<Attestation<T>>>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("blocks")
+            .push(&block_id.to_string())
+            .push("attestations");
+
+        self.get_opt(path).await
+    }
+
+    /// `POST beacon/pool/attestations`
+    pub async fn post_beacon_pool_attestations<T: EthSpec>(
+        &self,
+        attestation: &Attestation<T>,
+    ) -> Result<(), Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("pool")
+            .push("attestations");
+
+        self.post(path, attestation).await?;
+
+        Ok(())
+    }
+
+    /// `GET beacon/pool/attestations`
+    pub async fn get_beacon_pool_attestations<T: EthSpec>(
+        &self,
+    ) -> Result<GenericResponse<Vec<Attestation<T>>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("pool")
+            .push("attestations");
+
+        self.get(path).await
+    }
+
+    /// `POST beacon/pool/attester_slashings`
+    pub async fn post_beacon_pool_attester_slashings<T: EthSpec>(
+        &self,
+        slashing: &AttesterSlashing<T>,
+    ) -> Result<(), Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("pool")
+            .push("attester_slashings");
+
+        self.post(path, slashing).await?;
+
+        Ok(())
+    }
+
+    /// `GET beacon/pool/attester_slashings`
+    pub async fn get_beacon_pool_attester_slashings<T: EthSpec>(
+        &self,
+    ) -> Result<GenericResponse<Vec<AttesterSlashing<T>>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("pool")
+            .push("attester_slashings");
+
+        self.get(path).await
+    }
+
+    /// `POST beacon/pool/proposer_slashings`
+    pub async fn post_beacon_pool_proposer_slashings(
+        &self,
+        slashing: &ProposerSlashing,
+    ) -> Result<(), Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("pool")
+            .push("proposer_slashings");
+
+        self.post(path, slashing).await?;
+
+        Ok(())
+    }
+
+    /// `GET beacon/pool/proposer_slashings`
+    pub async fn get_beacon_pool_proposer_slashings(
+        &self,
+    ) -> Result<GenericResponse<Vec<ProposerSlashing>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("pool")
+            .push("proposer_slashings");
+
+        self.get(path).await
+    }
+
+    /// `POST beacon/pool/voluntary_exits`
+    pub async fn post_beacon_pool_voluntary_exits(
+        &self,
+        exit: &SignedVoluntaryExit,
+    ) -> Result<(), Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("pool")
+            .push("voluntary_exits");
+
+        self.post(path, exit).await?;
+
+        Ok(())
+    }
+
+    /// `GET beacon/pool/voluntary_exits`
+    pub async fn get_beacon_pool_voluntary_exits(
+        &self,
+    ) -> Result<GenericResponse<Vec<SignedVoluntaryExit>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("pool")
+            .push("voluntary_exits");
+
+        self.get(path).await
+    }
+
+    /// `GET config/fork_schedule`
+    pub async fn get_config_fork_schedule(&self) -> Result<GenericResponse<Vec<Fork>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("config")
+            .push("fork_schedule");
+
+        self.get(path).await
+    }
+
+    /// `GET config/fork_schedule`
+    pub async fn get_config_spec(&self) -> Result<GenericResponse<YamlConfig>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("config")
+            .push("spec");
+
+        self.get(path).await
+    }
+
+    /// `GET config/deposit_contract`
+    pub async fn get_config_deposit_contract(
+        &self,
+    ) -> Result<GenericResponse<DepositContractData>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("config")
+            .push("deposit_contract");
+
+        self.get(path).await
+    }
+
+    /// `GET node/version`
+    pub async fn get_node_version(&self) -> Result<GenericResponse<VersionData>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("node")
+            .push("version");
+
+        self.get(path).await
+    }
+
+    /// `GET node/syncing`
+    pub async fn get_node_syncing(&self) -> Result<GenericResponse<SyncingData>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("node")
+            .push("syncing");
+
+        self.get(path).await
+    }
+
+    /// `GET debug/beacon/states/{state_id}`
+    pub async fn get_debug_beacon_states<T: EthSpec>(
+        &self,
+        state_id: StateId,
+    ) -> Result<Option<GenericResponse<BeaconState<T>>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("debug")
+            .push("beacon")
+            .push("states")
+            .push(&state_id.to_string());
+
+        self.get_opt(path).await
+    }
+
+    /// `GET debug/beacon/heads`
+    pub async fn get_debug_beacon_heads(
+        &self,
+    ) -> Result<GenericResponse<Vec<ChainHeadData>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("debug")
+            .push("beacon")
+            .push("heads");
+
+        self.get(path).await
+    }
+
+    /// `GET validator/duties/attester/{epoch}?index`
+    ///
+    /// ## Note
+    ///
+    /// The `index` query parameter accepts a list of validator indices.
+    pub async fn get_validator_duties_attester(
+        &self,
+        epoch: Epoch,
+        index: Option<&[u64]>,
+    ) -> Result<GenericResponse<Vec<AttesterData>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("validator")
+            .push("duties")
+            .push("attester")
+            .push(&epoch.to_string());
+
+        if let Some(index) = index {
+            let string = index
+                .iter()
+                .map(|i| i.to_string())
+                .collect::<Vec<_>>()
+                .join(",");
+            path.query_pairs_mut().append_pair("index", &string);
+        }
+
+        self.get(path).await
+    }
+
+    /// `GET validator/duties/proposer/{epoch}`
+    pub async fn get_validator_duties_proposer(
+        &self,
+        epoch: Epoch,
+    ) -> Result<GenericResponse<Vec<ProposerData>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("validator")
+            .push("duties")
+            .push("proposer")
+            .push(&epoch.to_string());
+
+        self.get(path).await
+    }
+
+    /// `GET validator/duties/attester/{epoch}?index`
+    ///
+    /// ## Note
+    ///
+    /// The `index` query parameter accepts a list of validator indices.
+    pub async fn get_validator_blocks<T: EthSpec>(
+        &self,
+        slot: Slot,
+        randao_reveal: SignatureBytes,
+        graffiti: Option<&Graffiti>,
+    ) -> Result<GenericResponse<BeaconBlock<T>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("validator")
+            .push("blocks")
+            .push(&slot.to_string());
+
+        path.query_pairs_mut()
+            .append_pair("randao_reveal", &randao_reveal.to_string());
+
+        if let Some(graffiti) = graffiti {
+            path.query_pairs_mut()
+                .append_pair("graffiti", &graffiti.to_string());
+        }
+
+        self.get(path).await
+    }
+
+    /// `GET validator/attestation_data?slot,committee_index`
+    pub async fn get_validator_attestation_data(
+        &self,
+        slot: Slot,
+        committee_index: CommitteeIndex,
+    ) -> Result<GenericResponse<AttestationData>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("validator")
+            .push("attestation_data");
+
+        path.query_pairs_mut()
+            .append_pair("slot", &slot.to_string())
+            .append_pair("committee_index", &committee_index.to_string());
+
+        self.get(path).await
+    }
+
+    /// `GET validator/attestation_attestation?slot,attestation_data_root`
+    pub async fn get_validator_aggregate_attestation<T: EthSpec>(
+        &self,
+        slot: Slot,
+        attestation_data_root: Hash256,
+    ) -> Result<Option<GenericResponse<Attestation<T>>>, Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("validator")
+            .push("aggregate_attestation");
+
+        path.query_pairs_mut()
+            .append_pair("slot", &slot.to_string())
+            .append_pair(
+                "attestation_data_root",
+                &format!("{:?}", attestation_data_root),
+            );
+
+        self.get_opt(path).await
+    }
+
+    /// `POST validator/aggregate_and_proofs`
+    pub async fn post_validator_aggregate_and_proof<T: EthSpec>(
+        &self,
+        aggregate: &SignedAggregateAndProof<T>,
+    ) -> Result<(), Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("validator")
+            .push("aggregate_and_proofs");
+
+        self.post(path, aggregate).await?;
+
+        Ok(())
+    }
+
+    /// `POST validator/beacon_committee_subscriptions`
+    pub async fn post_validator_beacon_committee_subscriptions(
+        &self,
+        subscriptions: &[BeaconCommitteeSubscription],
+    ) -> Result<(), Error> {
+        let mut path = self.eth_path()?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("validator")
+            .push("beacon_committee_subscriptions");
+
+        self.post(path, &subscriptions).await?;
+
+        Ok(())
+    }
+}
+
+/// Returns `Ok(response)` if the response is a `200 OK` response. Otherwise, creates an
+/// appropriate error message.
+async fn ok_or_error(response: Response) -> Result<Response, Error> {
+    let status = response.status();
+
+    if status == StatusCode::OK {
+        Ok(response)
+    } else if let Ok(message) = response.json().await {
+        Err(Error::ServerMessage(message))
+    } else {
+        Err(Error::StatusCode(status))
+    }
+}

common/eth2/src/lighthouse.rs

@@ -0,0 +1,224 @@
+//! This module contains endpoints that are non-standard and only available on Lighthouse servers.
+
+use crate::{
+    types::{Epoch, EthSpec, GenericResponse, ValidatorId},
+    BeaconNodeHttpClient, Error,
+};
+use proto_array::core::ProtoArray;
+use serde::{Deserialize, Serialize};
+
+pub use eth2_libp2p::{types::SyncState, PeerInfo};
+
+/// Information returned by `peers` and `connected_peers`.
+// TODO: this should be deserializable..
+#[derive(Debug, Clone, Serialize)]
+#[serde(bound = "T: EthSpec")]
+pub struct Peer<T: EthSpec> {
+    /// The Peer's ID
+    pub peer_id: String,
+    /// The PeerInfo associated with the peer.
+    pub peer_info: PeerInfo<T>,
+}
+
+/// The results of validators voting during an epoch.
+///
+/// Provides information about the current and previous epochs.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct GlobalValidatorInclusionData {
+    /// The total effective balance of all active validators during the _current_ epoch.
+    pub current_epoch_active_gwei: u64,
+    /// The total effective balance of all active validators during the _previous_ epoch.
+    pub previous_epoch_active_gwei: u64,
+    /// The total effective balance of all validators who attested during the _current_ epoch.
+    pub current_epoch_attesting_gwei: u64,
+    /// The total effective balance of all validators who attested during the _current_ epoch and
+    /// agreed with the state about the beacon block at the first slot of the _current_ epoch.
+    pub current_epoch_target_attesting_gwei: u64,
+    /// The total effective balance of all validators who attested during the _previous_ epoch.
+    pub previous_epoch_attesting_gwei: u64,
+    /// The total effective balance of all validators who attested during the _previous_ epoch and
+    /// agreed with the state about the beacon block at the first slot of the _previous_ epoch.
+    pub previous_epoch_target_attesting_gwei: u64,
+    /// The total effective balance of all validators who attested during the _previous_ epoch and
+    /// agreed with the state about the beacon block at the time of attestation.
+    pub previous_epoch_head_attesting_gwei: u64,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ValidatorInclusionData {
+    /// True if the validator has been slashed, ever.
+    pub is_slashed: bool,
+    /// True if the validator can withdraw in the current epoch.
+    pub is_withdrawable_in_current_epoch: bool,
+    /// True if the validator was active in the state's _current_ epoch.
+    pub is_active_in_current_epoch: bool,
+    /// True if the validator was active in the state's _previous_ epoch.
+    pub is_active_in_previous_epoch: bool,
+    /// The validator's effective balance in the _current_ epoch.
+    pub current_epoch_effective_balance_gwei: u64,
+    /// True if the validator had an attestation included in the _current_ epoch.
+    pub is_current_epoch_attester: bool,
+    /// True if the validator's beacon block root attestation for the first slot of the _current_
+    /// epoch matches the block root known to the state.
+    pub is_current_epoch_target_attester: bool,
+    /// True if the validator had an attestation included in the _previous_ epoch.
+    pub is_previous_epoch_attester: bool,
+    /// True if the validator's beacon block root attestation for the first slot of the _previous_
+    /// epoch matches the block root known to the state.
+    pub is_previous_epoch_target_attester: bool,
+    /// True if the validator's beacon block root attestation in the _previous_ epoch at the
+    /// attestation's slot (`attestation_data.slot`) matches the block root known to the state.
+    pub is_previous_epoch_head_attester: bool,
+}
+
+#[cfg(target_os = "linux")]
+use {procinfo::pid, psutil::process::Process};
+
+/// Reports on the health of the Lighthouse instance.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct Health {
+    /// The pid of this process.
+    pub pid: u32,
+    /// The number of threads used by this pid.
+    pub pid_num_threads: i32,
+    /// The total resident memory used by this pid.
+    pub pid_mem_resident_set_size: u64,
+    /// The total virtual memory used by this pid.
+    pub pid_mem_virtual_memory_size: u64,
+    /// Total virtual memory on the system
+    pub sys_virt_mem_total: u64,
+    /// Total virtual memory available for new processes.
+    pub sys_virt_mem_available: u64,
+    /// Total virtual memory used on the system
+    pub sys_virt_mem_used: u64,
+    /// Total virtual memory not used on the system
+    pub sys_virt_mem_free: u64,
+    /// Percentage of virtual memory used on the system
+    pub sys_virt_mem_percent: f32,
+    /// System load average over 1 minute.
+    pub sys_loadavg_1: f64,
+    /// System load average over 5 minutes.
+    pub sys_loadavg_5: f64,
+    /// System load average over 15 minutes.
+    pub sys_loadavg_15: f64,
+}
+
+impl Health {
+    #[cfg(not(target_os = "linux"))]
+    pub fn observe() -> Result<Self, String> {
+        Err("Health is only available on Linux".into())
+    }
+
+    #[cfg(target_os = "linux")]
+    pub fn observe() -> Result<Self, String> {
+        let process =
+            Process::current().map_err(|e| format!("Unable to get current process: {:?}", e))?;
+
+        let process_mem = process
+            .memory_info()
+            .map_err(|e| format!("Unable to get process memory info: {:?}", e))?;
+
+        let stat = pid::stat_self().map_err(|e| format!("Unable to get stat: {:?}", e))?;
+
+        let vm = psutil::memory::virtual_memory()
+            .map_err(|e| format!("Unable to get virtual memory: {:?}", e))?;
+        let loadavg =
+            psutil::host::loadavg().map_err(|e| format!("Unable to get loadavg: {:?}", e))?;
+
+        Ok(Self {
+            pid: process.pid(),
+            pid_num_threads: stat.num_threads,
+            pid_mem_resident_set_size: process_mem.rss(),
+            pid_mem_virtual_memory_size: process_mem.vms(),
+            sys_virt_mem_total: vm.total(),
+            sys_virt_mem_available: vm.available(),
+            sys_virt_mem_used: vm.used(),
+            sys_virt_mem_free: vm.free(),
+            sys_virt_mem_percent: vm.percent(),
+            sys_loadavg_1: loadavg.one,
+            sys_loadavg_5: loadavg.five,
+            sys_loadavg_15: loadavg.fifteen,
+        })
+    }
+}
+
+impl BeaconNodeHttpClient {
+    /// `GET lighthouse/health`
+    pub async fn get_lighthouse_health(&self) -> Result<GenericResponse<Health>, Error> {
+        let mut path = self.server.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("health");
+
+        self.get(path).await
+    }
+
+    /// `GET lighthouse/syncing`
+    pub async fn get_lighthouse_syncing(&self) -> Result<GenericResponse<SyncState>, Error> {
+        let mut path = self.server.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("syncing");
+
+        self.get(path).await
+    }
+
+    /*
+     * Note:
+     *
+     * The `lighthouse/peers` endpoints do not have functions here. We are yet to implement
+     * `Deserialize` on the `PeerInfo` struct since it contains use of `Instant`. This could be
+     * fairly simply achieved, if desired.
+     */
+
+    /// `GET lighthouse/proto_array`
+    pub async fn get_lighthouse_proto_array(&self) -> Result<GenericResponse<ProtoArray>, Error> {
+        let mut path = self.server.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("proto_array");
+
+        self.get(path).await
+    }
+
+    /// `GET lighthouse/validator_inclusion/{epoch}/global`
+    pub async fn get_lighthouse_validator_inclusion_global(
+        &self,
+        epoch: Epoch,
+    ) -> Result<GenericResponse<GlobalValidatorInclusionData>, Error> {
+        let mut path = self.server.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("validator_inclusion")
+            .push(&epoch.to_string())
+            .push("global");
+
+        self.get(path).await
+    }
+
+    /// `GET lighthouse/validator_inclusion/{epoch}/{validator_id}`
+    pub async fn get_lighthouse_validator_inclusion(
+        &self,
+        epoch: Epoch,
+        validator_id: ValidatorId,
+    ) -> Result<GenericResponse<Option<ValidatorInclusionData>>, Error> {
+        let mut path = self.server.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("validator_inclusion")
+            .push(&epoch.to_string())
+            .push(&validator_id.to_string());
+
+        self.get(path).await
+    }
+}

common/eth2/src/types.rs

@@ -0,0 +1,432 @@
+//! This module exposes a superset of the `types` crate. It adds additional types that are only
+//! required for the HTTP API.
+
+use eth2_libp2p::{Enr, Multiaddr};
+use serde::{Deserialize, Serialize};
+use std::convert::TryFrom;
+use std::fmt;
+use std::str::FromStr;
+
+pub use types::*;
+
+/// An API error serializable to JSON.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ErrorMessage {
+    pub code: u16,
+    pub message: String,
+    #[serde(default)]
+    pub stacktraces: Vec<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct GenesisData {
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub genesis_time: u64,
+    pub genesis_validators_root: Hash256,
+    #[serde(with = "serde_utils::bytes_4_hex")]
+    pub genesis_fork_version: [u8; 4],
+}
+
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum BlockId {
+    Head,
+    Genesis,
+    Finalized,
+    Justified,
+    Slot(Slot),
+    Root(Hash256),
+}
+
+impl FromStr for BlockId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "head" => Ok(BlockId::Head),
+            "genesis" => Ok(BlockId::Genesis),
+            "finalized" => Ok(BlockId::Finalized),
+            "justified" => Ok(BlockId::Justified),
+            other => {
+                if other.starts_with("0x") {
+                    Hash256::from_str(&s[2..])
+                        .map(BlockId::Root)
+                        .map_err(|e| format!("{} cannot be parsed as a root", e))
+                } else {
+                    u64::from_str(s)
+                        .map(Slot::new)
+                        .map(BlockId::Slot)
+                        .map_err(|_| format!("{} cannot be parsed as a parameter", s))
+                }
+            }
+        }
+    }
+}
+
+impl fmt::Display for BlockId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BlockId::Head => write!(f, "head"),
+            BlockId::Genesis => write!(f, "genesis"),
+            BlockId::Finalized => write!(f, "finalized"),
+            BlockId::Justified => write!(f, "justified"),
+            BlockId::Slot(slot) => write!(f, "{}", slot),
+            BlockId::Root(root) => write!(f, "{:?}", root),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum StateId {
+    Head,
+    Genesis,
+    Finalized,
+    Justified,
+    Slot(Slot),
+    Root(Hash256),
+}
+
+impl FromStr for StateId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "head" => Ok(StateId::Head),
+            "genesis" => Ok(StateId::Genesis),
+            "finalized" => Ok(StateId::Finalized),
+            "justified" => Ok(StateId::Justified),
+            other => {
+                if other.starts_with("0x") {
+                    Hash256::from_str(&s[2..])
+                        .map(StateId::Root)
+                        .map_err(|e| format!("{} cannot be parsed as a root", e))
+                } else {
+                    u64::from_str(s)
+                        .map(Slot::new)
+                        .map(StateId::Slot)
+                        .map_err(|_| format!("{} cannot be parsed as a slot", s))
+                }
+            }
+        }
+    }
+}
+
+impl fmt::Display for StateId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            StateId::Head => write!(f, "head"),
+            StateId::Genesis => write!(f, "genesis"),
+            StateId::Finalized => write!(f, "finalized"),
+            StateId::Justified => write!(f, "justified"),
+            StateId::Slot(slot) => write!(f, "{}", slot),
+            StateId::Root(root) => write!(f, "{:?}", root),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+#[serde(bound = "T: Serialize + serde::de::DeserializeOwned")]
+pub struct GenericResponse<T: Serialize + serde::de::DeserializeOwned> {
+    pub data: T,
+}
+
+impl<T: Serialize + serde::de::DeserializeOwned> From<T> for GenericResponse<T> {
+    fn from(data: T) -> Self {
+        Self { data }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone, Serialize)]
+#[serde(bound = "T: Serialize")]
+pub struct GenericResponseRef<'a, T: Serialize> {
+    pub data: &'a T,
+}
+
+impl<'a, T: Serialize> From<&'a T> for GenericResponseRef<'a, T> {
+    fn from(data: &'a T) -> Self {
+        Self { data }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
+pub struct RootData {
+    pub root: Hash256,
+}
+
+impl From<Hash256> for RootData {
+    fn from(root: Hash256) -> Self {
+        Self { root }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct FinalityCheckpointsData {
+    pub previous_justified: Checkpoint,
+    pub current_justified: Checkpoint,
+    pub finalized: Checkpoint,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum ValidatorId {
+    PublicKey(PublicKeyBytes),
+    Index(u64),
+}
+
+impl FromStr for ValidatorId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if s.starts_with("0x") {
+            PublicKeyBytes::from_str(s)
+                .map(ValidatorId::PublicKey)
+                .map_err(|e| format!("{} cannot be parsed as a public key: {}", s, e))
+        } else {
+            u64::from_str(s)
+                .map(ValidatorId::Index)
+                .map_err(|e| format!("{} cannot be parsed as a slot: {}", s, e))
+        }
+    }
+}
+
+impl fmt::Display for ValidatorId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ValidatorId::PublicKey(pubkey) => write!(f, "{:?}", pubkey),
+            ValidatorId::Index(index) => write!(f, "{}", index),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ValidatorData {
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub index: u64,
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub balance: u64,
+    pub status: ValidatorStatus,
+    pub validator: Validator,
+}
+
+// TODO: This does not currently match the spec, but I'm going to try and change the spec using
+// this proposal:
+//
+// https://hackmd.io/bQxMDRt1RbS1TLno8K4NPg?view
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
+pub enum ValidatorStatus {
+    Unknown,
+    WaitingForEligibility,
+    WaitingForFinality,
+    WaitingInQueue,
+    StandbyForActive(Epoch),
+    Active,
+    ActiveAwaitingVoluntaryExit(Epoch),
+    ActiveAwaitingSlashedExit(Epoch),
+    ExitedVoluntarily(Epoch),
+    ExitedSlashed(Epoch),
+    Withdrawable,
+    Withdrawn,
+}
+
+impl ValidatorStatus {
+    pub fn from_validator(
+        validator_opt: Option<&Validator>,
+        epoch: Epoch,
+        finalized_epoch: Epoch,
+        far_future_epoch: Epoch,
+    ) -> Self {
+        if let Some(validator) = validator_opt {
+            if validator.is_withdrawable_at(epoch) {
+                ValidatorStatus::Withdrawable
+            } else if validator.is_exited_at(epoch) {
+                if validator.slashed {
+                    ValidatorStatus::ExitedSlashed(validator.withdrawable_epoch)
+                } else {
+                    ValidatorStatus::ExitedVoluntarily(validator.withdrawable_epoch)
+                }
+            } else if validator.is_active_at(epoch) {
+                if validator.exit_epoch < far_future_epoch {
+                    if validator.slashed {
+                        ValidatorStatus::ActiveAwaitingSlashedExit(validator.exit_epoch)
+                    } else {
+                        ValidatorStatus::ActiveAwaitingVoluntaryExit(validator.exit_epoch)
+                    }
+                } else {
+                    ValidatorStatus::Active
+                }
+            } else if validator.activation_epoch < far_future_epoch {
+                ValidatorStatus::StandbyForActive(validator.activation_epoch)
+            } else if validator.activation_eligibility_epoch < far_future_epoch {
+                if finalized_epoch < validator.activation_eligibility_epoch {
+                    ValidatorStatus::WaitingForFinality
+                } else {
+                    ValidatorStatus::WaitingInQueue
+                }
+            } else {
+                ValidatorStatus::WaitingForEligibility
+            }
+        } else {
+            ValidatorStatus::Unknown
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CommitteesQuery {
+    pub slot: Option<Slot>,
+    pub index: Option<u64>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct CommitteeData {
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub index: u64,
+    pub slot: Slot,
+    #[serde(with = "serde_utils::quoted_u64_vec")]
+    pub validators: Vec<u64>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct HeadersQuery {
+    pub slot: Option<Slot>,
+    pub parent_root: Option<Hash256>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct BlockHeaderAndSignature {
+    pub message: BeaconBlockHeader,
+    pub signature: SignatureBytes,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct BlockHeaderData {
+    pub root: Hash256,
+    pub canonical: bool,
+    pub header: BlockHeaderAndSignature,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct DepositContractData {
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub chain_id: u64,
+    pub address: Address,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ChainHeadData {
+    pub slot: Slot,
+    pub root: Hash256,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct IdentityData {
+    pub peer_id: String,
+    pub enr: Enr,
+    pub p2p_addresses: Vec<Multiaddr>,
+    // TODO: missing the following fields:
+    //
+    // - discovery_addresses
+    // - metadata
+    //
+    // Tracked here: https://github.com/sigp/lighthouse/issues/1434
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct VersionData {
+    pub version: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct SyncingData {
+    pub is_syncing: bool,
+    pub head_slot: Slot,
+    pub sync_distance: Slot,
+}
+
+#[derive(Clone, PartialEq, Debug, Deserialize)]
+#[serde(try_from = "String", bound = "T: FromStr")]
+pub struct QueryVec<T: FromStr>(pub Vec<T>);
+
+impl<T: FromStr> TryFrom<String> for QueryVec<T> {
+    type Error = String;
+
+    fn try_from(string: String) -> Result<Self, Self::Error> {
+        if string == "" {
+            return Ok(Self(vec![]));
+        }
+
+        string
+            .split(',')
+            .map(|s| s.parse().map_err(|_| "unable to parse".to_string()))
+            .collect::<Result<Vec<T>, String>>()
+            .map(Self)
+    }
+}
+
+#[derive(Clone, Deserialize)]
+pub struct ValidatorDutiesQuery {
+    pub index: Option<QueryVec<u64>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct AttesterData {
+    pub pubkey: PublicKeyBytes,
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub validator_index: u64,
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub committees_at_slot: u64,
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub committee_index: CommitteeIndex,
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub committee_length: u64,
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub validator_committee_index: u64,
+    pub slot: Slot,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ProposerData {
+    pub pubkey: PublicKeyBytes,
+    pub slot: Slot,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct ValidatorBlocksQuery {
+    pub randao_reveal: SignatureBytes,
+    pub graffiti: Option<Graffiti>,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct ValidatorAttestationDataQuery {
+    pub slot: Slot,
+    pub committee_index: CommitteeIndex,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct ValidatorAggregateAttestationQuery {
+    pub attestation_data_root: Hash256,
+    pub slot: Slot,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct BeaconCommitteeSubscription {
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub validator_index: u64,
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub committee_index: u64,
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub committees_at_slot: u64,
+    pub slot: Slot,
+    pub is_aggregator: bool,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn query_vec() {
+        assert_eq!(
+            QueryVec::try_from("0,1,2".to_string()).unwrap(),
+            QueryVec(vec![0_u64, 1, 2])
+        );
+    }
+}

common/lighthouse_metrics/src/lib.rs

@@ -55,6 +55,7 @@
 //! ```
 
 use prometheus::{HistogramOpts, HistogramTimer, Opts};
+use std::time::Duration;
 
 pub use prometheus::{
     Encoder, Gauge, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge,
@@ -221,6 +222,19 @@ pub fn start_timer(histogram: &Result<Histogram>) -> Option<HistogramTimer> {
     }
 }
 
+/// Starts a timer on `vec` with the given `name`.
+pub fn observe_timer_vec(vec: &Result<HistogramVec>, name: &[&str], duration: Duration) {
+    // This conversion was taken from here:
+    //
+    // https://docs.rs/prometheus/0.5.0/src/prometheus/histogram.rs.html#550-555
+    let nanos = f64::from(duration.subsec_nanos()) / 1e9;
+    let secs = duration.as_secs() as f64 + nanos;
+
+    if let Some(h) = get_histogram(vec, name) {
+        h.observe(secs)
+    }
+}
+
 /// Stops a timer created with `start_timer(..)`.
 pub fn stop_timer(timer: Option<HistogramTimer>) {
     if let Some(t) = timer {

common/remote_beacon_node/Cargo.toml

@@ -1,21 +0,0 @@
-[package]
-name = "remote_beacon_node"
-version = "0.2.0"
-authors = ["Paul Hauner <paul@paulhauner.com>"]
-edition = "2018"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-reqwest = { version = "0.10.4", features = ["json", "native-tls-vendored"] }
-url = "2.1.1"
-serde = "1.0.110"
-futures = "0.3.5"
-types = { path = "../../consensus/types" }
-rest_types = { path = "../rest_types" }
-hex = "0.4.2"
-eth2_ssz = "0.1.2"
-serde_json = "1.0.52"
-eth2_config = { path = "../eth2_config" }
-proto_array = { path = "../../consensus/proto_array" }
-operation_pool = { path = "../../beacon_node/operation_pool" }

common/remote_beacon_node/src/lib.rs

@@ -1,732 +0,0 @@
-//! Provides a `RemoteBeaconNode` which interacts with a HTTP API on another Lighthouse (or
-//! compatible) instance.
-//!
-//! Presently, this is only used for testing but it _could_ become a user-facing library.
-
-use eth2_config::Eth2Config;
-use reqwest::{Client, ClientBuilder, Response, StatusCode};
-use serde::{de::DeserializeOwned, Deserialize, Serialize};
-use ssz::Encode;
-use std::marker::PhantomData;
-use std::time::Duration;
-use types::{
-    Attestation, AttestationData, AttesterSlashing, BeaconBlock, BeaconState, CommitteeIndex,
-    Epoch, EthSpec, Fork, Graffiti, Hash256, ProposerSlashing, PublicKey, PublicKeyBytes,
-    Signature, SignedAggregateAndProof, SignedBeaconBlock, Slot, SubnetId,
-};
-use url::Url;
-
-pub use operation_pool::PersistedOperationPool;
-pub use proto_array::core::ProtoArray;
-pub use rest_types::{
-    CanonicalHeadResponse, Committee, HeadBeaconBlock, Health, IndividualVotesRequest,
-    IndividualVotesResponse, SyncingResponse, ValidatorDutiesRequest, ValidatorDutyBytes,
-    ValidatorRequest, ValidatorResponse, ValidatorSubscription,
-};
-
-// Setting a long timeout for debug ensures that crypto-heavy operations can still succeed.
-#[cfg(debug_assertions)]
-pub const REQUEST_TIMEOUT_SECONDS: u64 = 15;
-
-#[cfg(not(debug_assertions))]
-pub const REQUEST_TIMEOUT_SECONDS: u64 = 5;
-
-#[derive(Clone)]
-/// Connects to a remote Lighthouse (or compatible) node via HTTP.
-pub struct RemoteBeaconNode<E: EthSpec> {
-    pub http: HttpClient<E>,
-}
-
-impl<E: EthSpec> RemoteBeaconNode<E> {
-    /// Uses the default HTTP timeout.
-    pub fn new(http_endpoint: String) -> Result<Self, String> {
-        Self::new_with_timeout(http_endpoint, Duration::from_secs(REQUEST_TIMEOUT_SECONDS))
-    }
-
-    pub fn new_with_timeout(http_endpoint: String, timeout: Duration) -> Result<Self, String> {
-        Ok(Self {
-            http: HttpClient::new(http_endpoint, timeout)
-                .map_err(|e| format!("Unable to create http client: {:?}", e))?,
-        })
-    }
-}
-
-#[derive(Debug)]
-pub enum Error {
-    /// Unable to parse a URL. Check the server URL.
-    UrlParseError(url::ParseError),
-    /// The `reqwest` library returned an error.
-    ReqwestError(reqwest::Error),
-    /// There was an error when encoding/decoding an object using serde.
-    SerdeJsonError(serde_json::Error),
-    /// The server responded to the request, however it did not return a 200-type success code.
-    DidNotSucceed { status: StatusCode, body: String },
-    /// The request input was invalid.
-    InvalidInput,
-}
-
-#[derive(Clone)]
-pub struct HttpClient<E> {
-    client: Client,
-    url: Url,
-    timeout: Duration,
-    _phantom: PhantomData<E>,
-}
-
-impl<E: EthSpec> HttpClient<E> {
-    /// Creates a new instance (without connecting to the node).
-    pub fn new(server_url: String, timeout: Duration) -> Result<Self, Error> {
-        Ok(Self {
-            client: ClientBuilder::new()
-                .timeout(timeout)
-                .build()
-                .expect("should build from static configuration"),
-            url: Url::parse(&server_url)?,
-            timeout: Duration::from_secs(15),
-            _phantom: PhantomData,
-        })
-    }
-
-    pub fn beacon(&self) -> Beacon<E> {
-        Beacon(self.clone())
-    }
-
-    pub fn validator(&self) -> Validator<E> {
-        Validator(self.clone())
-    }
-
-    pub fn spec(&self) -> Spec<E> {
-        Spec(self.clone())
-    }
-
-    pub fn node(&self) -> Node<E> {
-        Node(self.clone())
-    }
-
-    pub fn advanced(&self) -> Advanced<E> {
-        Advanced(self.clone())
-    }
-
-    pub fn consensus(&self) -> Consensus<E> {
-        Consensus(self.clone())
-    }
-
-    fn url(&self, path: &str) -> Result<Url, Error> {
-        self.url.join(path).map_err(|e| e.into())
-    }
-
-    pub async fn json_post<T: Serialize>(&self, url: Url, body: T) -> Result<Response, Error> {
-        self.client
-            .post(&url.to_string())
-            .json(&body)
-            .send()
-            .await
-            .map_err(Error::from)
-    }
-
-    pub async fn json_get<T: DeserializeOwned>(
-        &self,
-        mut url: Url,
-        query_pairs: Vec<(String, String)>,
-    ) -> Result<T, Error> {
-        query_pairs.into_iter().for_each(|(key, param)| {
-            url.query_pairs_mut().append_pair(&key, &param);
-        });
-
-        let response = self
-            .client
-            .get(&url.to_string())
-            .send()
-            .await
-            .map_err(Error::from)?;
-
-        let success = error_for_status(response).await.map_err(Error::from)?;
-        success.json::<T>().await.map_err(Error::from)
-    }
-}
-
-/// Returns an `Error` (with a description) if the `response` was not a 200-type success response.
-///
-/// Distinct from `Response::error_for_status` because it includes the body of the response as
-/// text. This ensures the error message from the server is not discarded.
-async fn error_for_status(response: Response) -> Result<Response, Error> {
-    let status = response.status();
-
-    if status.is_success() {
-        Ok(response)
-    } else {
-        let text_result = response.text().await;
-        match text_result {
-            Err(e) => Err(Error::ReqwestError(e)),
-            Ok(body) => Err(Error::DidNotSucceed { status, body }),
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum PublishStatus {
-    /// The object was valid and has been published to the network.
-    Valid,
-    /// The object was not valid and may or may not have been published to the network.
-    Invalid(String),
-    /// The server responded with an unknown status code. The object may or may not have been
-    /// published to the network.
-    Unknown,
-}
-
-impl PublishStatus {
-    /// Returns `true` if `*self == PublishStatus::Valid`.
-    pub fn is_valid(&self) -> bool {
-        *self == PublishStatus::Valid
-    }
-}
-
-/// Provides the functions on the `/validator` endpoint of the node.
-#[derive(Clone)]
-pub struct Validator<E>(HttpClient<E>);
-
-impl<E: EthSpec> Validator<E> {
-    fn url(&self, path: &str) -> Result<Url, Error> {
-        self.0
-            .url("validator/")
-            .and_then(move |url| url.join(path).map_err(Error::from))
-            .map_err(Into::into)
-    }
-
-    /// Produces an unsigned attestation.
-    pub async fn produce_attestation(
-        &self,
-        slot: Slot,
-        committee_index: CommitteeIndex,
-    ) -> Result<Attestation<E>, Error> {
-        let query_params = vec![
-            ("slot".into(), format!("{}", slot)),
-            ("committee_index".into(), format!("{}", committee_index)),
-        ];
-
-        let client = self.0.clone();
-        let url = self.url("attestation")?;
-        client.json_get(url, query_params).await
-    }
-
-    /// Produces an aggregate attestation.
-    pub async fn produce_aggregate_attestation(
-        &self,
-        attestation_data: &AttestationData,
-    ) -> Result<Attestation<E>, Error> {
-        let query_params = vec![(
-            "attestation_data".into(),
-            as_ssz_hex_string(attestation_data),
-        )];
-
-        let client = self.0.clone();
-        let url = self.url("aggregate_attestation")?;
-        client.json_get(url, query_params).await
-    }
-
-    /// Posts a list of attestations to the beacon node, expecting it to verify it and publish it to the network.
-    pub async fn publish_attestations(
-        &self,
-        attestation: Vec<(Attestation<E>, SubnetId)>,
-    ) -> Result<PublishStatus, Error> {
-        let client = self.0.clone();
-        let url = self.url("attestations")?;
-        let response = client.json_post::<_>(url, attestation).await?;
-
-        match response.status() {
-            StatusCode::OK => Ok(PublishStatus::Valid),
-            StatusCode::ACCEPTED => Ok(PublishStatus::Invalid(
-                response.text().await.map_err(Error::from)?,
-            )),
-            _ => response
-                .error_for_status()
-                .map_err(Error::from)
-                .map(|_| PublishStatus::Unknown),
-        }
-    }
-
-    /// Posts a list of signed aggregates and proofs to the beacon node, expecting it to verify it and publish it to the network.
-    pub async fn publish_aggregate_and_proof(
-        &self,
-        signed_aggregate_and_proofs: Vec<SignedAggregateAndProof<E>>,
-    ) -> Result<PublishStatus, Error> {
-        let client = self.0.clone();
-        let url = self.url("aggregate_and_proofs")?;
-        let response = client
-            .json_post::<_>(url, signed_aggregate_and_proofs)
-            .await?;
-
-        match response.status() {
-            StatusCode::OK => Ok(PublishStatus::Valid),
-            StatusCode::ACCEPTED => Ok(PublishStatus::Invalid(
-                response.text().await.map_err(Error::from)?,
-            )),
-            _ => response
-                .error_for_status()
-                .map_err(Error::from)
-                .map(|_| PublishStatus::Unknown),
-        }
-    }
-
-    /// Returns the duties required of the given validator pubkeys in the given epoch.
-    pub async fn get_duties(
-        &self,
-        epoch: Epoch,
-        validator_pubkeys: &[PublicKey],
-    ) -> Result<Vec<ValidatorDutyBytes>, Error> {
-        let client = self.0.clone();
-
-        let bulk_request = ValidatorDutiesRequest {
-            epoch,
-            pubkeys: validator_pubkeys
-                .iter()
-                .map(|pubkey| pubkey.clone().into())
-                .collect(),
-        };
-
-        let url = self.url("duties")?;
-        let response = client.json_post::<_>(url, bulk_request).await?;
-        let success = error_for_status(response).await.map_err(Error::from)?;
-        success.json().await.map_err(Error::from)
-    }
-
-    /// Posts a block to the beacon node, expecting it to verify it and publish it to the network.
-    pub async fn publish_block(&self, block: SignedBeaconBlock<E>) -> Result<PublishStatus, Error> {
-        let client = self.0.clone();
-        let url = self.url("block")?;
-        let response = client.json_post::<_>(url, block).await?;
-
-        match response.status() {
-            StatusCode::OK => Ok(PublishStatus::Valid),
-            StatusCode::ACCEPTED => Ok(PublishStatus::Invalid(
-                response.text().await.map_err(Error::from)?,
-            )),
-            _ => response
-                .error_for_status()
-                .map_err(Error::from)
-                .map(|_| PublishStatus::Unknown),
-        }
-    }
-
-    /// Requests a new (unsigned) block from the beacon node.
-    pub async fn produce_block(
-        &self,
-        slot: Slot,
-        randao_reveal: Signature,
-        graffiti: Option<Graffiti>,
-    ) -> Result<BeaconBlock<E>, Error> {
-        let client = self.0.clone();
-        let url = self.url("block")?;
-
-        let mut query_pairs = vec![
-            ("slot".into(), format!("{}", slot.as_u64())),
-            ("randao_reveal".into(), as_ssz_hex_string(&randao_reveal)),
-        ];
-
-        if let Some(graffiti_bytes) = graffiti {
-            query_pairs.push(("graffiti".into(), as_ssz_hex_string(&graffiti_bytes)));
-        }
-
-        client.json_get::<BeaconBlock<E>>(url, query_pairs).await
-    }
-
-    /// Subscribes a list of validators to particular slots for attestation production/publication.
-    pub async fn subscribe(
-        &self,
-        subscriptions: Vec<ValidatorSubscription>,
-    ) -> Result<PublishStatus, Error> {
-        let client = self.0.clone();
-        let url = self.url("subscribe")?;
-        let response = client.json_post::<_>(url, subscriptions).await?;
-
-        match response.status() {
-            StatusCode::OK => Ok(PublishStatus::Valid),
-            StatusCode::ACCEPTED => Ok(PublishStatus::Invalid(
-                response.text().await.map_err(Error::from)?,
-            )),
-            _ => response
-                .error_for_status()
-                .map_err(Error::from)
-                .map(|_| PublishStatus::Unknown),
-        }
-    }
-}
-
-/// Provides the functions on the `/beacon` endpoint of the node.
-#[derive(Clone)]
-pub struct Beacon<E>(HttpClient<E>);
-
-impl<E: EthSpec> Beacon<E> {
-    fn url(&self, path: &str) -> Result<Url, Error> {
-        self.0
-            .url("beacon/")
-            .and_then(move |url| url.join(path).map_err(Error::from))
-            .map_err(Into::into)
-    }
-
-    /// Returns the genesis time.
-    pub async fn get_genesis_time(&self) -> Result<u64, Error> {
-        let client = self.0.clone();
-        let url = self.url("genesis_time")?;
-        client.json_get(url, vec![]).await
-    }
-
-    /// Returns the genesis validators root.
-    pub async fn get_genesis_validators_root(&self) -> Result<Hash256, Error> {
-        let client = self.0.clone();
-        let url = self.url("genesis_validators_root")?;
-        client.json_get(url, vec![]).await
-    }
-
-    /// Returns the fork at the head of the beacon chain.
-    pub async fn get_fork(&self) -> Result<Fork, Error> {
-        let client = self.0.clone();
-        let url = self.url("fork")?;
-        client.json_get(url, vec![]).await
-    }
-
-    /// Returns info about the head of the canonical beacon chain.
-    pub async fn get_head(&self) -> Result<CanonicalHeadResponse, Error> {
-        let client = self.0.clone();
-        let url = self.url("head")?;
-        client.json_get::<CanonicalHeadResponse>(url, vec![]).await
-    }
-
-    /// Returns the set of known beacon chain head blocks. One of these will be the canonical head.
-    pub async fn get_heads(&self) -> Result<Vec<HeadBeaconBlock>, Error> {
-        let client = self.0.clone();
-        let url = self.url("heads")?;
-        client.json_get(url, vec![]).await
-    }
-
-    /// Returns the block and block root at the given slot.
-    pub async fn get_block_by_slot(
-        &self,
-        slot: Slot,
-    ) -> Result<(SignedBeaconBlock<E>, Hash256), Error> {
-        self.get_block("slot".to_string(), format!("{}", slot.as_u64()))
-            .await
-    }
-
-    /// Returns the block and block root at the given root.
-    pub async fn get_block_by_root(
-        &self,
-        root: Hash256,
-    ) -> Result<(SignedBeaconBlock<E>, Hash256), Error> {
-        self.get_block("root".to_string(), root_as_string(root))
-            .await
-    }
-
-    /// Returns the block and block root at the given slot.
-    async fn get_block(
-        &self,
-        query_key: String,
-        query_param: String,
-    ) -> Result<(SignedBeaconBlock<E>, Hash256), Error> {
-        let client = self.0.clone();
-        let url = self.url("block")?;
-        client
-            .json_get::<BlockResponse<E>>(url, vec![(query_key, query_param)])
-            .await
-            .map(|response| (response.beacon_block, response.root))
-    }
-
-    /// Returns the state and state root at the given slot.
-    pub async fn get_state_by_slot(&self, slot: Slot) -> Result<(BeaconState<E>, Hash256), Error> {
-        self.get_state("slot".to_string(), format!("{}", slot.as_u64()))
-            .await
-    }
-
-    /// Returns the state and state root at the given root.
-    pub async fn get_state_by_root(
-        &self,
-        root: Hash256,
-    ) -> Result<(BeaconState<E>, Hash256), Error> {
-        self.get_state("root".to_string(), root_as_string(root))
-            .await
-    }
-
-    /// Returns the root of the state at the given slot.
-    pub async fn get_state_root(&self, slot: Slot) -> Result<Hash256, Error> {
-        let client = self.0.clone();
-        let url = self.url("state_root")?;
-        client
-            .json_get(url, vec![("slot".into(), format!("{}", slot.as_u64()))])
-            .await
-    }
-
-    /// Returns the root of the block at the given slot.
-    pub async fn get_block_root(&self, slot: Slot) -> Result<Hash256, Error> {
-        let client = self.0.clone();
-        let url = self.url("block_root")?;
-        client
-            .json_get(url, vec![("slot".into(), format!("{}", slot.as_u64()))])
-            .await
-    }
-
-    /// Returns the state and state root at the given slot.
-    async fn get_state(
-        &self,
-        query_key: String,
-        query_param: String,
-    ) -> Result<(BeaconState<E>, Hash256), Error> {
-        let client = self.0.clone();
-        let url = self.url("state")?;
-        client
-            .json_get::<StateResponse<E>>(url, vec![(query_key, query_param)])
-            .await
-            .map(|response| (response.beacon_state, response.root))
-    }
-
-    /// Returns the block and block root at the given slot.
-    ///
-    /// If `state_root` is `Some`, the query will use the given state instead of the default
-    /// canonical head state.
-    pub async fn get_validators(
-        &self,
-        validator_pubkeys: Vec<PublicKey>,
-        state_root: Option<Hash256>,
-    ) -> Result<Vec<ValidatorResponse>, Error> {
-        let client = self.0.clone();
-
-        let bulk_request = ValidatorRequest {
-            state_root,
-            pubkeys: validator_pubkeys
-                .iter()
-                .map(|pubkey| pubkey.clone().into())
-                .collect(),
-        };
-
-        let url = self.url("validators")?;
-        let response = client.json_post::<_>(url, bulk_request).await?;
-        let success = error_for_status(response).await.map_err(Error::from)?;
-        success.json().await.map_err(Error::from)
-    }
-
-    /// Returns all validators.
-    ///
-    /// If `state_root` is `Some`, the query will use the given state instead of the default
-    /// canonical head state.
-    pub async fn get_all_validators(
-        &self,
-        state_root: Option<Hash256>,
-    ) -> Result<Vec<ValidatorResponse>, Error> {
-        let client = self.0.clone();
-
-        let query_params = if let Some(state_root) = state_root {
-            vec![("state_root".into(), root_as_string(state_root))]
-        } else {
-            vec![]
-        };
-
-        let url = self.url("validators/all")?;
-        client.json_get(url, query_params).await
-    }
-
-    /// Returns the active validators.
-    ///
-    /// If `state_root` is `Some`, the query will use the given state instead of the default
-    /// canonical head state.
-    pub async fn get_active_validators(
-        &self,
-        state_root: Option<Hash256>,
-    ) -> Result<Vec<ValidatorResponse>, Error> {
-        let client = self.0.clone();
-
-        let query_params = if let Some(state_root) = state_root {
-            vec![("state_root".into(), root_as_string(state_root))]
-        } else {
-            vec![]
-        };
-
-        let url = self.url("validators/active")?;
-        client.json_get(url, query_params).await
-    }
-
-    /// Returns committees at the given epoch.
-    pub async fn get_committees(&self, epoch: Epoch) -> Result<Vec<Committee>, Error> {
-        let client = self.0.clone();
-
-        let url = self.url("committees")?;
-        client
-            .json_get(url, vec![("epoch".into(), format!("{}", epoch.as_u64()))])
-            .await
-    }
-
-    pub async fn proposer_slashing(
-        &self,
-        proposer_slashing: ProposerSlashing,
-    ) -> Result<bool, Error> {
-        let client = self.0.clone();
-
-        let url = self.url("proposer_slashing")?;
-        let response = client.json_post::<_>(url, proposer_slashing).await?;
-        let success = error_for_status(response).await.map_err(Error::from)?;
-        success.json().await.map_err(Error::from)
-    }
-
-    pub async fn attester_slashing(
-        &self,
-        attester_slashing: AttesterSlashing<E>,
-    ) -> Result<bool, Error> {
-        let client = self.0.clone();
-
-        let url = self.url("attester_slashing")?;
-        let response = client.json_post::<_>(url, attester_slashing).await?;
-        let success = error_for_status(response).await.map_err(Error::from)?;
-        success.json().await.map_err(Error::from)
-    }
-}
-
-/// Provides the functions on the `/spec` endpoint of the node.
-#[derive(Clone)]
-pub struct Spec<E>(HttpClient<E>);
-
-impl<E: EthSpec> Spec<E> {
-    fn url(&self, path: &str) -> Result<Url, Error> {
-        self.0
-            .url("spec/")
-            .and_then(move |url| url.join(path).map_err(Error::from))
-            .map_err(Into::into)
-    }
-
-    pub async fn get_eth2_config(&self) -> Result<Eth2Config, Error> {
-        let client = self.0.clone();
-        let url = self.url("eth2_config")?;
-        client.json_get(url, vec![]).await
-    }
-}
-
-/// Provides the functions on the `/node` endpoint of the node.
-#[derive(Clone)]
-pub struct Node<E>(HttpClient<E>);
-
-impl<E: EthSpec> Node<E> {
-    fn url(&self, path: &str) -> Result<Url, Error> {
-        self.0
-            .url("node/")
-            .and_then(move |url| url.join(path).map_err(Error::from))
-            .map_err(Into::into)
-    }
-
-    pub async fn get_version(&self) -> Result<String, Error> {
-        let client = self.0.clone();
-        let url = self.url("version")?;
-        client.json_get(url, vec![]).await
-    }
-
-    pub async fn get_health(&self) -> Result<Health, Error> {
-        let client = self.0.clone();
-        let url = self.url("health")?;
-        client.json_get(url, vec![]).await
-    }
-
-    pub async fn syncing_status(&self) -> Result<SyncingResponse, Error> {
-        let client = self.0.clone();
-        let url = self.url("syncing")?;
-        client.json_get(url, vec![]).await
-    }
-}
-
-/// Provides the functions on the `/advanced` endpoint of the node.
-#[derive(Clone)]
-pub struct Advanced<E>(HttpClient<E>);
-
-impl<E: EthSpec> Advanced<E> {
-    fn url(&self, path: &str) -> Result<Url, Error> {
-        self.0
-            .url("advanced/")
-            .and_then(move |url| url.join(path).map_err(Error::from))
-            .map_err(Into::into)
-    }
-
-    /// Gets the core `ProtoArray` struct from the node.
-    pub async fn get_fork_choice(&self) -> Result<ProtoArray, Error> {
-        let client = self.0.clone();
-        let url = self.url("fork_choice")?;
-        client.json_get(url, vec![]).await
-    }
-
-    /// Gets the core `PersistedOperationPool` struct from the node.
-    pub async fn get_operation_pool(&self) -> Result<PersistedOperationPool<E>, Error> {
-        let client = self.0.clone();
-        let url = self.url("operation_pool")?;
-        client.json_get(url, vec![]).await
-    }
-}
-
-/// Provides the functions on the `/consensus` endpoint of the node.
-#[derive(Clone)]
-pub struct Consensus<E>(HttpClient<E>);
-
-impl<E: EthSpec> Consensus<E> {
-    fn url(&self, path: &str) -> Result<Url, Error> {
-        self.0
-            .url("consensus/")
-            .and_then(move |url| url.join(path).map_err(Error::from))
-            .map_err(Into::into)
-    }
-
-    /// Gets a `IndividualVote` for each of the given `pubkeys`.
-    pub async fn get_individual_votes(
-        &self,
-        epoch: Epoch,
-        pubkeys: Vec<PublicKeyBytes>,
-    ) -> Result<IndividualVotesResponse, Error> {
-        let client = self.0.clone();
-        let req_body = IndividualVotesRequest { epoch, pubkeys };
-
-        let url = self.url("individual_votes")?;
-        let response = client.json_post::<_>(url, req_body).await?;
-        let success = error_for_status(response).await.map_err(Error::from)?;
-        success.json().await.map_err(Error::from)
-    }
-
-    /// Gets a `VoteCount` for the given `epoch`.
-    pub async fn get_vote_count(&self, epoch: Epoch) -> Result<IndividualVotesResponse, Error> {
-        let client = self.0.clone();
-        let query_params = vec![("epoch".into(), format!("{}", epoch.as_u64()))];
-        let url = self.url("vote_count")?;
-        client.json_get(url, query_params).await
-    }
-}
-
-#[derive(Deserialize)]
-#[serde(bound = "T: EthSpec")]
-pub struct BlockResponse<T: EthSpec> {
-    pub beacon_block: SignedBeaconBlock<T>,
-    pub root: Hash256,
-}
-
-#[derive(Deserialize)]
-#[serde(bound = "T: EthSpec")]
-pub struct StateResponse<T: EthSpec> {
-    pub beacon_state: BeaconState<T>,
-    pub root: Hash256,
-}
-
-fn root_as_string(root: Hash256) -> String {
-    format!("0x{:?}", root)
-}
-
-fn as_ssz_hex_string<T: Encode>(item: &T) -> String {
-    format!("0x{}", hex::encode(item.as_ssz_bytes()))
-}
-
-impl From<reqwest::Error> for Error {
-    fn from(e: reqwest::Error) -> Error {
-        Error::ReqwestError(e)
-    }
-}
-
-impl From<url::ParseError> for Error {
-    fn from(e: url::ParseError) -> Error {
-        Error::UrlParseError(e)
-    }
-}
-
-impl From<serde_json::Error> for Error {
-    fn from(e: serde_json::Error) -> Error {
-        Error::SerdeJsonError(e)
-    }
-}

common/rest_types/Cargo.toml

@@ -1,27 +0,0 @@
-[package]
-name = "rest_types"
-version = "0.2.0"
-authors = ["Sigma Prime <contact@sigmaprime.io>"]
-edition = "2018"
-
-[dependencies]
-types = { path = "../../consensus/types" }
-eth2_ssz_derive = "0.1.0"
-eth2_ssz = "0.1.2"
-eth2_hashing = "0.1.0"
-tree_hash = "0.1.0"
-state_processing = { path = "../../consensus/state_processing" }
-bls = { path = "../../crypto/bls" }
-serde = { version = "1.0.110", features = ["derive"] }
-rayon = "1.3.0"
-hyper = "0.13.5"
-tokio = { version = "0.2.21", features = ["sync"] }
-environment = { path = "../../lighthouse/environment" }
-store = { path = "../../beacon_node/store" }
-beacon_chain = { path = "../../beacon_node/beacon_chain" }
-serde_json = "1.0.52"
-serde_yaml = "0.8.11"
-
-[target.'cfg(target_os = "linux")'.dependencies]
-psutil = "3.1.0"
-procinfo = "0.4.2"

common/rest_types/src/api_error.rs

@@ -1,99 +0,0 @@
-use hyper::{Body, Response, StatusCode};
-use std::error::Error as StdError;
-
-#[derive(PartialEq, Debug, Clone)]
-pub enum ApiError {
-    MethodNotAllowed(String),
-    ServerError(String),
-    NotImplemented(String),
-    BadRequest(String),
-    NotFound(String),
-    UnsupportedType(String),
-    ImATeapot(String),       // Just in case.
-    ProcessingError(String), // A 202 error, for when a block/attestation cannot be processed, but still transmitted.
-    InvalidHeaderValue(String),
-}
-
-pub type ApiResult = Result<Response<Body>, ApiError>;
-
-impl ApiError {
-    pub fn status_code(self) -> (StatusCode, String) {
-        match self {
-            ApiError::MethodNotAllowed(desc) => (StatusCode::METHOD_NOT_ALLOWED, desc),
-            ApiError::ServerError(desc) => (StatusCode::INTERNAL_SERVER_ERROR, desc),
-            ApiError::NotImplemented(desc) => (StatusCode::NOT_IMPLEMENTED, desc),
-            ApiError::BadRequest(desc) => (StatusCode::BAD_REQUEST, desc),
-            ApiError::NotFound(desc) => (StatusCode::NOT_FOUND, desc),
-            ApiError::UnsupportedType(desc) => (StatusCode::UNSUPPORTED_MEDIA_TYPE, desc),
-            ApiError::ImATeapot(desc) => (StatusCode::IM_A_TEAPOT, desc),
-            ApiError::ProcessingError(desc) => (StatusCode::ACCEPTED, desc),
-            ApiError::InvalidHeaderValue(desc) => (StatusCode::INTERNAL_SERVER_ERROR, desc),
-        }
-    }
-}
-
-impl Into<Response<Body>> for ApiError {
-    fn into(self) -> Response<Body> {
-        let (status_code, desc) = self.status_code();
-        Response::builder()
-            .status(status_code)
-            .header("content-type", "text/plain; charset=utf-8")
-            .body(Body::from(desc))
-            .expect("Response should always be created.")
-    }
-}
-
-impl From<store::Error> for ApiError {
-    fn from(e: store::Error) -> ApiError {
-        ApiError::ServerError(format!("Database error: {:?}", e))
-    }
-}
-
-impl From<types::BeaconStateError> for ApiError {
-    fn from(e: types::BeaconStateError) -> ApiError {
-        ApiError::ServerError(format!("BeaconState error: {:?}", e))
-    }
-}
-
-impl From<beacon_chain::BeaconChainError> for ApiError {
-    fn from(e: beacon_chain::BeaconChainError) -> ApiError {
-        ApiError::ServerError(format!("BeaconChainError error: {:?}", e))
-    }
-}
-
-impl From<state_processing::per_slot_processing::Error> for ApiError {
-    fn from(e: state_processing::per_slot_processing::Error) -> ApiError {
-        ApiError::ServerError(format!("PerSlotProcessing error: {:?}", e))
-    }
-}
-
-impl From<hyper::error::Error> for ApiError {
-    fn from(e: hyper::error::Error) -> ApiError {
-        ApiError::ServerError(format!("Networking error: {:?}", e))
-    }
-}
-
-impl From<std::io::Error> for ApiError {
-    fn from(e: std::io::Error) -> ApiError {
-        ApiError::ServerError(format!("IO error: {:?}", e))
-    }
-}
-
-impl From<hyper::header::InvalidHeaderValue> for ApiError {
-    fn from(e: hyper::header::InvalidHeaderValue) -> ApiError {
-        ApiError::InvalidHeaderValue(format!("Invalid CORS header value: {:?}", e))
-    }
-}
-
-impl StdError for ApiError {
-    fn cause(&self) -> Option<&dyn StdError> {
-        None
-    }
-}
-
-impl std::fmt::Display for ApiError {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        let status = self.clone().status_code();
-        write!(f, "{:?}: {:?}", status.0, status.1)
-    }
-}

common/rest_types/src/beacon.rs

@@ -1,65 +0,0 @@
-//! A collection of REST API types for interaction with the beacon node.
-
-use bls::PublicKeyBytes;
-use serde::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode};
-use types::beacon_state::EthSpec;
-use types::{BeaconState, CommitteeIndex, Hash256, SignedBeaconBlock, Slot, Validator};
-
-/// Information about a block that is at the head of a chain. May or may not represent the
-/// canonical head.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)]
-pub struct HeadBeaconBlock {
-    pub beacon_block_root: Hash256,
-    pub beacon_block_slot: Slot,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)]
-#[serde(bound = "T: EthSpec")]
-pub struct BlockResponse<T: EthSpec> {
-    pub root: Hash256,
-    pub beacon_block: SignedBeaconBlock<T>,
-}
-
-/// Information about the block and state that are at head of the beacon chain.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)]
-pub struct CanonicalHeadResponse {
-    pub slot: Slot,
-    pub block_root: Hash256,
-    pub state_root: Hash256,
-    pub finalized_slot: Slot,
-    pub finalized_block_root: Hash256,
-    pub justified_slot: Slot,
-    pub justified_block_root: Hash256,
-    pub previous_justified_slot: Slot,
-    pub previous_justified_block_root: Hash256,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)]
-pub struct ValidatorResponse {
-    pub pubkey: PublicKeyBytes,
-    pub validator_index: Option<usize>,
-    pub balance: Option<u64>,
-    pub validator: Option<Validator>,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)]
-pub struct ValidatorRequest {
-    /// If set to `None`, uses the canonical head state.
-    pub state_root: Option<Hash256>,
-    pub pubkeys: Vec<PublicKeyBytes>,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)]
-pub struct Committee {
-    pub slot: Slot,
-    pub index: CommitteeIndex,
-    pub committee: Vec<usize>,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)]
-#[serde(bound = "T: EthSpec")]
-pub struct StateResponse<T: EthSpec> {
-    pub root: Hash256,
-    pub beacon_state: BeaconState<T>,
-}

common/rest_types/src/consensus.rs

@@ -1,66 +0,0 @@
-use serde::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode};
-use state_processing::per_epoch_processing::ValidatorStatus;
-use types::{Epoch, PublicKeyBytes};
-
-#[derive(PartialEq, Debug, Serialize, Deserialize, Clone, Encode, Decode)]
-pub struct IndividualVotesRequest {
-    pub epoch: Epoch,
-    pub pubkeys: Vec<PublicKeyBytes>,
-}
-
-#[derive(PartialEq, Debug, Serialize, Deserialize, Clone, Encode, Decode)]
-pub struct IndividualVote {
-    /// True if the validator has been slashed, ever.
-    pub is_slashed: bool,
-    /// True if the validator can withdraw in the current epoch.
-    pub is_withdrawable_in_current_epoch: bool,
-    /// True if the validator was active in the state's _current_ epoch.
-    pub is_active_in_current_epoch: bool,
-    /// True if the validator was active in the state's _previous_ epoch.
-    pub is_active_in_previous_epoch: bool,
-    /// The validator's effective balance in the _current_ epoch.
-    pub current_epoch_effective_balance_gwei: u64,
-    /// True if the validator had an attestation included in the _current_ epoch.
-    pub is_current_epoch_attester: bool,
-    /// True if the validator's beacon block root attestation for the first slot of the _current_
-    /// epoch matches the block root known to the state.
-    pub is_current_epoch_target_attester: bool,
-    /// True if the validator had an attestation included in the _previous_ epoch.
-    pub is_previous_epoch_attester: bool,
-    /// True if the validator's beacon block root attestation for the first slot of the _previous_
-    /// epoch matches the block root known to the state.
-    pub is_previous_epoch_target_attester: bool,
-    /// True if the validator's beacon block root attestation in the _previous_ epoch at the
-    /// attestation's slot (`attestation_data.slot`) matches the block root known to the state.
-    pub is_previous_epoch_head_attester: bool,
-}
-
-impl Into<IndividualVote> for ValidatorStatus {
-    fn into(self) -> IndividualVote {
-        IndividualVote {
-            is_slashed: self.is_slashed,
-            is_withdrawable_in_current_epoch: self.is_withdrawable_in_current_epoch,
-            is_active_in_current_epoch: self.is_active_in_current_epoch,
-            is_active_in_previous_epoch: self.is_active_in_previous_epoch,
-            current_epoch_effective_balance_gwei: self.current_epoch_effective_balance,
-            is_current_epoch_attester: self.is_current_epoch_attester,
-            is_current_epoch_target_attester: self.is_current_epoch_target_attester,
-            is_previous_epoch_attester: self.is_previous_epoch_attester,
-            is_previous_epoch_target_attester: self.is_previous_epoch_target_attester,
-            is_previous_epoch_head_attester: self.is_previous_epoch_head_attester,
-        }
-    }
-}
-
-#[derive(PartialEq, Debug, Serialize, Deserialize, Clone, Encode, Decode)]
-pub struct IndividualVotesResponse {
-    /// The epoch which is considered the "current" epoch.
-    pub epoch: Epoch,
-    /// The validators public key.
-    pub pubkey: PublicKeyBytes,
-    /// The index of the validator in state.validators.
-    pub validator_index: Option<usize>,
-    /// Voting statistics for the validator, if they voted in the given epoch.
-    pub vote: Option<IndividualVote>,
-}

common/rest_types/src/handler.rs

@@ -1,247 +0,0 @@
-use crate::{ApiError, ApiResult};
-use environment::TaskExecutor;
-use hyper::header;
-use hyper::{Body, Request, Response, StatusCode};
-use serde::Deserialize;
-use serde::Serialize;
-use ssz::Encode;
-
-/// Defines the encoding for the API.
-#[derive(Clone, Serialize, Deserialize, Copy)]
-pub enum ApiEncodingFormat {
-    JSON,
-    YAML,
-    SSZ,
-}
-
-impl ApiEncodingFormat {
-    pub fn get_content_type(&self) -> &str {
-        match self {
-            ApiEncodingFormat::JSON => "application/json",
-            ApiEncodingFormat::YAML => "application/yaml",
-            ApiEncodingFormat::SSZ => "application/ssz",
-        }
-    }
-}
-
-impl From<&str> for ApiEncodingFormat {
-    fn from(f: &str) -> ApiEncodingFormat {
-        match f {
-            "application/yaml" => ApiEncodingFormat::YAML,
-            "application/ssz" => ApiEncodingFormat::SSZ,
-            _ => ApiEncodingFormat::JSON,
-        }
-    }
-}
-
-/// Provides a HTTP request handler with Lighthouse-specific functionality.
-pub struct Handler<T> {
-    executor: TaskExecutor,
-    req: Request<()>,
-    body: Body,
-    ctx: T,
-    encoding: ApiEncodingFormat,
-    allow_body: bool,
-}
-
-impl<T: Clone + Send + Sync + 'static> Handler<T> {
-    /// Start handling a new request.
-    pub fn new(req: Request<Body>, ctx: T, executor: TaskExecutor) -> Result<Self, ApiError> {
-        let (req_parts, body) = req.into_parts();
-        let req = Request::from_parts(req_parts, ());
-
-        let accept_header: String = req
-            .headers()
-            .get(header::ACCEPT)
-            .map_or(Ok(""), |h| h.to_str())
-            .map_err(|e| {
-                ApiError::BadRequest(format!(
-                    "The Accept header contains invalid characters: {:?}",
-                    e
-                ))
-            })
-            .map(String::from)?;
-
-        Ok(Self {
-            executor,
-            req,
-            body,
-            ctx,
-            allow_body: false,
-            encoding: ApiEncodingFormat::from(accept_header.as_str()),
-        })
-    }
-
-    /// The default behaviour is to return an error if any body is supplied in the request. Calling
-    /// this function disables that error.
-    pub fn allow_body(mut self) -> Self {
-        self.allow_body = true;
-        self
-    }
-
-    /// Return a simple static value.
-    ///
-    /// Does not use the blocking executor.
-    pub async fn static_value<V>(self, value: V) -> Result<HandledRequest<V>, ApiError> {
-        // Always check and disallow a body for a static value.
-        let _ = Self::get_body(self.body, false).await?;
-
-        Ok(HandledRequest {
-            value,
-            encoding: self.encoding,
-        })
-    }
-
-    /// Calls `func` in-line, on the core executor.
-    ///
-    /// This should only be used for very fast tasks.
-    pub async fn in_core_task<F, V>(self, func: F) -> Result<HandledRequest<V>, ApiError>
-    where
-        V: Send + Sync + 'static,
-        F: Fn(Request<Vec<u8>>, T) -> Result<V, ApiError> + Send + Sync + 'static,
-    {
-        let body = Self::get_body(self.body, self.allow_body).await?;
-        let (req_parts, _) = self.req.into_parts();
-        let req = Request::from_parts(req_parts, body);
-
-        let value = func(req, self.ctx)?;
-
-        Ok(HandledRequest {
-            value,
-            encoding: self.encoding,
-        })
-    }
-
-    /// Spawns `func` on the blocking executor.
-    ///
-    /// This method is suitable for handling long-running or intensive tasks.
-    pub async fn in_blocking_task<F, V>(self, func: F) -> Result<HandledRequest<V>, ApiError>
-    where
-        V: Send + Sync + 'static,
-        F: Fn(Request<Vec<u8>>, T) -> Result<V, ApiError> + Send + Sync + 'static,
-    {
-        let ctx = self.ctx;
-        let body = Self::get_body(self.body, self.allow_body).await?;
-        let (req_parts, _) = self.req.into_parts();
-        let req = Request::from_parts(req_parts, body);
-
-        let value = self
-            .executor
-            .clone()
-            .handle
-            .spawn_blocking(move || func(req, ctx))
-            .await
-            .map_err(|e| {
-                ApiError::ServerError(format!(
-                    "Failed to get blocking join handle: {}",
-                    e.to_string()
-                ))
-            })??;
-
-        Ok(HandledRequest {
-            value,
-            encoding: self.encoding,
-        })
-    }
-
-    /// Call `func`, then return a response that is suitable for an SSE stream.
-    pub async fn sse_stream<F>(self, func: F) -> ApiResult
-    where
-        F: Fn(Request<()>, T) -> Result<Body, ApiError>,
-    {
-        let body = func(self.req, self.ctx)?;
-
-        Response::builder()
-            .status(200)
-            .header("Content-Type", "text/event-stream")
-            .header("Connection", "Keep-Alive")
-            .header("Cache-Control", "no-cache")
-            .header("Access-Control-Allow-Origin", "*")
-            .body(body)
-            .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))
-    }
-
-    /// Downloads the bytes for `body`.
-    async fn get_body(body: Body, allow_body: bool) -> Result<Vec<u8>, ApiError> {
-        let bytes = hyper::body::to_bytes(body)
-            .await
-            .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-
-        if !allow_body && !bytes[..].is_empty() {
-            Err(ApiError::BadRequest(
-                "The request body must be empty".to_string(),
-            ))
-        } else {
-            Ok(bytes.into_iter().collect())
-        }
-    }
-}
-
-/// A request that has been "handled" and now a result (`value`) needs to be serialize and
-/// returned.
-pub struct HandledRequest<V> {
-    encoding: ApiEncodingFormat,
-    value: V,
-}
-
-impl HandledRequest<String> {
-    /// Simple encode a string as utf-8.
-    pub fn text_encoding(self) -> ApiResult {
-        Response::builder()
-            .status(StatusCode::OK)
-            .header("content-type", "text/plain; charset=utf-8")
-            .body(Body::from(self.value))
-            .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))
-    }
-}
-
-impl<V: Serialize + Encode> HandledRequest<V> {
-    /// Suitable for all items which implement `serde` and `ssz`.
-    pub fn all_encodings(self) -> ApiResult {
-        match self.encoding {
-            ApiEncodingFormat::SSZ => Response::builder()
-                .status(StatusCode::OK)
-                .header("content-type", "application/ssz")
-                .body(Body::from(self.value.as_ssz_bytes()))
-                .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))),
-            _ => self.serde_encodings(),
-        }
-    }
-}
-
-impl<V: Serialize> HandledRequest<V> {
-    /// Suitable for items which only implement `serde`.
-    pub fn serde_encodings(self) -> ApiResult {
-        let (body, content_type) = match self.encoding {
-            ApiEncodingFormat::JSON => (
-                Body::from(serde_json::to_string(&self.value).map_err(|e| {
-                    ApiError::ServerError(format!(
-                        "Unable to serialize response body as JSON: {:?}",
-                        e
-                    ))
-                })?),
-                "application/json",
-            ),
-            ApiEncodingFormat::SSZ => {
-                return Err(ApiError::UnsupportedType(
-                    "Response cannot be encoded as SSZ.".into(),
-                ));
-            }
-            ApiEncodingFormat::YAML => (
-                Body::from(serde_yaml::to_string(&self.value).map_err(|e| {
-                    ApiError::ServerError(format!(
-                        "Unable to serialize response body as YAML: {:?}",
-                        e
-                    ))
-                })?),
-                "application/yaml",
-            ),
-        };
-
-        Response::builder()
-            .status(StatusCode::OK)
-            .header("content-type", content_type)
-            .body(body)
-            .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))
-    }
-}

common/rest_types/src/lib.rs

@@ -1,22 +0,0 @@
-//! A collection of types used to pass data across the rest HTTP API.
-//!
-//! This is primarily used by the validator client and the beacon node rest API.
-
-mod api_error;
-mod beacon;
-mod consensus;
-mod handler;
-mod node;
-mod validator;
-
-pub use api_error::{ApiError, ApiResult};
-pub use beacon::{
-    BlockResponse, CanonicalHeadResponse, Committee, HeadBeaconBlock, StateResponse,
-    ValidatorRequest, ValidatorResponse,
-};
-pub use consensus::{IndividualVote, IndividualVotesRequest, IndividualVotesResponse};
-pub use handler::{ApiEncodingFormat, Handler};
-pub use node::{Health, SyncingResponse, SyncingStatus};
-pub use validator::{
-    ValidatorDutiesRequest, ValidatorDuty, ValidatorDutyBytes, ValidatorSubscription,
-};

common/rest_types/src/node.rs

@@ -1,103 +0,0 @@
-//! Collection of types for the /node HTTP
-use serde::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode};
-use types::Slot;
-
-#[cfg(target_os = "linux")]
-use {procinfo::pid, psutil::process::Process};
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)]
-/// The current syncing status of the node.
-pub struct SyncingStatus {
-    /// The starting slot of sync.
-    ///
-    /// For a finalized sync, this is the start slot of the current finalized syncing
-    /// chain.
-    ///
-    /// For head sync this is the last finalized slot.
-    pub starting_slot: Slot,
-    /// The current slot.
-    pub current_slot: Slot,
-    /// The highest known slot. For the current syncing chain.
-    ///
-    /// For a finalized sync, the target finalized slot.
-    /// For head sync, this is the highest known slot of all head chains.
-    pub highest_slot: Slot,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)]
-/// The response for the /node/syncing HTTP GET.
-pub struct SyncingResponse {
-    /// Is the node syncing.
-    pub is_syncing: bool,
-    /// The current sync status.
-    pub sync_status: SyncingStatus,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-/// Reports on the health of the Lighthouse instance.
-pub struct Health {
-    /// The pid of this process.
-    pub pid: u32,
-    /// The number of threads used by this pid.
-    pub pid_num_threads: i32,
-    /// The total resident memory used by this pid.
-    pub pid_mem_resident_set_size: u64,
-    /// The total virtual memory used by this pid.
-    pub pid_mem_virtual_memory_size: u64,
-    /// Total virtual memory on the system
-    pub sys_virt_mem_total: u64,
-    /// Total virtual memory available for new processes.
-    pub sys_virt_mem_available: u64,
-    /// Total virtual memory used on the system
-    pub sys_virt_mem_used: u64,
-    /// Total virtual memory not used on the system
-    pub sys_virt_mem_free: u64,
-    /// Percentage of virtual memory used on the system
-    pub sys_virt_mem_percent: f32,
-    /// System load average over 1 minute.
-    pub sys_loadavg_1: f64,
-    /// System load average over 5 minutes.
-    pub sys_loadavg_5: f64,
-    /// System load average over 15 minutes.
-    pub sys_loadavg_15: f64,
-}
-
-impl Health {
-    #[cfg(not(target_os = "linux"))]
-    pub fn observe() -> Result<Self, String> {
-        Err("Health is only available on Linux".into())
-    }
-
-    #[cfg(target_os = "linux")]
-    pub fn observe() -> Result<Self, String> {
-        let process =
-            Process::current().map_err(|e| format!("Unable to get current process: {:?}", e))?;
-
-        let process_mem = process
-            .memory_info()
-            .map_err(|e| format!("Unable to get process memory info: {:?}", e))?;
-
-        let stat = pid::stat_self().map_err(|e| format!("Unable to get stat: {:?}", e))?;
-
-        let vm = psutil::memory::virtual_memory()
-            .map_err(|e| format!("Unable to get virtual memory: {:?}", e))?;
-        let loadavg =
-            psutil::host::loadavg().map_err(|e| format!("Unable to get loadavg: {:?}", e))?;
-
-        Ok(Self {
-            pid: process.pid(),
-            pid_num_threads: stat.num_threads,
-            pid_mem_resident_set_size: process_mem.rss(),
-            pid_mem_virtual_memory_size: process_mem.vms(),
-            sys_virt_mem_total: vm.total(),
-            sys_virt_mem_available: vm.available(),
-            sys_virt_mem_used: vm.used(),
-            sys_virt_mem_free: vm.free(),
-            sys_virt_mem_percent: vm.percent(),
-            sys_loadavg_1: loadavg.one,
-            sys_loadavg_5: loadavg.five,
-            sys_loadavg_15: loadavg.fifteen,
-        })
-    }
-}

common/rest_types/src/validator.rs

@@ -1,103 +0,0 @@
-use bls::{PublicKey, PublicKeyBytes};
-use serde::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode};
-use types::{CommitteeIndex, Epoch, Slot};
-
-/// A Validator duty with the validator public key represented a `PublicKeyBytes`.
-pub type ValidatorDutyBytes = ValidatorDutyBase<PublicKeyBytes>;
-/// A validator duty with the pubkey represented as a `PublicKey`.
-pub type ValidatorDuty = ValidatorDutyBase<PublicKey>;
-
-// NOTE: if you add or remove fields, please adjust `eq_ignoring_proposal_slots`
-#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)]
-pub struct ValidatorDutyBase<T> {
-    /// The validator's BLS public key, uniquely identifying them.
-    pub validator_pubkey: T,
-    /// The validator's index in `state.validators`
-    pub validator_index: Option<u64>,
-    /// The slot at which the validator must attest.
-    pub attestation_slot: Option<Slot>,
-    /// The index of the committee within `slot` of which the validator is a member.
-    pub attestation_committee_index: Option<CommitteeIndex>,
-    /// The position of the validator in the committee.
-    pub attestation_committee_position: Option<usize>,
-    /// The committee count at `attestation_slot`.
-    pub committee_count_at_slot: Option<u64>,
-    /// The slots in which a validator must propose a block (can be empty).
-    ///
-    /// Should be set to `None` when duties are not yet known (before the current epoch).
-    pub block_proposal_slots: Option<Vec<Slot>>,
-    /// This provides the modulo: `max(1, len(committee) // TARGET_AGGREGATORS_PER_COMMITTEE)`
-    /// which allows the validator client to determine if this duty requires the validator to be
-    /// aggregate attestations.
-    pub aggregator_modulo: Option<u64>,
-}
-
-impl<T> ValidatorDutyBase<T> {
-    /// Return `true` if these validator duties are equal, ignoring their `block_proposal_slots`.
-    pub fn eq_ignoring_proposal_slots(&self, other: &Self) -> bool
-    where
-        T: PartialEq,
-    {
-        self.validator_pubkey == other.validator_pubkey
-            && self.validator_index == other.validator_index
-            && self.attestation_slot == other.attestation_slot
-            && self.attestation_committee_index == other.attestation_committee_index
-            && self.attestation_committee_position == other.attestation_committee_position
-            && self.committee_count_at_slot == other.committee_count_at_slot
-            && self.aggregator_modulo == other.aggregator_modulo
-    }
-}
-
-#[derive(PartialEq, Debug, Serialize, Deserialize, Clone, Encode, Decode)]
-pub struct ValidatorDutiesRequest {
-    pub epoch: Epoch,
-    pub pubkeys: Vec<PublicKeyBytes>,
-}
-
-/// A validator subscription, created when a validator subscribes to a slot to perform optional aggregation
-/// duties.
-#[derive(PartialEq, Debug, Serialize, Deserialize, Clone, Encode, Decode)]
-pub struct ValidatorSubscription {
-    /// The validators index.
-    pub validator_index: u64,
-    /// The index of the committee within `slot` of which the validator is a member. Used by the
-    /// beacon node to quickly evaluate the associated `SubnetId`.
-    pub attestation_committee_index: CommitteeIndex,
-    /// The slot in which to subscribe.
-    pub slot: Slot,
-    /// Committee count at slot to subscribe.
-    pub committee_count_at_slot: u64,
-    /// If true, the validator is an aggregator and the beacon node should aggregate attestations
-    /// for this slot.
-    pub is_aggregator: bool,
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use bls::SecretKey;
-
-    #[test]
-    fn eq_ignoring_proposal_slots() {
-        let validator_pubkey = SecretKey::deserialize(&[1; 32]).unwrap().public_key();
-
-        let duty1 = ValidatorDuty {
-            validator_pubkey,
-            validator_index: Some(10),
-            attestation_slot: Some(Slot::new(50)),
-            attestation_committee_index: Some(2),
-            attestation_committee_position: Some(6),
-            committee_count_at_slot: Some(4),
-            block_proposal_slots: None,
-            aggregator_modulo: Some(99),
-        };
-        let duty2 = ValidatorDuty {
-            block_proposal_slots: Some(vec![Slot::new(42), Slot::new(45)]),
-            ..duty1.clone()
-        };
-        assert_ne!(duty1, duty2);
-        assert!(duty1.eq_ignoring_proposal_slots(&duty2));
-        assert!(duty2.eq_ignoring_proposal_slots(&duty1));
-    }
-}

common/slot_clock/src/lib.rs

@@ -24,6 +24,16 @@ pub trait SlotClock: Send + Sync + Sized {
     /// Returns the slot at this present time.
     fn now(&self) -> Option<Slot>;
 
+    /// Returns the slot at this present time if genesis has happened. Otherwise, returns the
+    /// genesis slot. Returns `None` if there is an error reading the clock.
+    fn now_or_genesis(&self) -> Option<Slot> {
+        if self.is_prior_to_genesis()? {
+            Some(self.genesis_slot())
+        } else {
+            self.now()
+        }
+    }
+
     /// Indicates if the current time is prior to genesis time.
     ///
     /// Returns `None` if the system clock cannot be read.

common/warp_utils/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "warp_utils"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+warp = "0.2.5"
+eth2 = { path = "../eth2" }
+types = { path = "../../consensus/types" }
+beacon_chain = { path = "../../beacon_node/beacon_chain" }
+state_processing = { path = "../../consensus/state_processing" }
+safe_arith = { path = "../../consensus/safe_arith" }

common/warp_utils/src/lib.rs

@@ -0,0 +1,5 @@
+//! This crate contains functions that are common across multiple `warp` HTTP servers in the
+//! Lighthouse project. E.g., the `http_api` and `http_metrics` crates.
+
+pub mod reject;
+pub mod reply;

common/warp_utils/src/reject.rs

@@ -0,0 +1,168 @@
+use eth2::types::ErrorMessage;
+use std::convert::Infallible;
+use warp::{http::StatusCode, reject::Reject};
+
+#[derive(Debug)]
+pub struct BeaconChainError(pub beacon_chain::BeaconChainError);
+
+impl Reject for BeaconChainError {}
+
+pub fn beacon_chain_error(e: beacon_chain::BeaconChainError) -> warp::reject::Rejection {
+    warp::reject::custom(BeaconChainError(e))
+}
+
+#[derive(Debug)]
+pub struct BeaconStateError(pub types::BeaconStateError);
+
+impl Reject for BeaconStateError {}
+
+pub fn beacon_state_error(e: types::BeaconStateError) -> warp::reject::Rejection {
+    warp::reject::custom(BeaconStateError(e))
+}
+
+#[derive(Debug)]
+pub struct ArithError(pub safe_arith::ArithError);
+
+impl Reject for ArithError {}
+
+pub fn arith_error(e: safe_arith::ArithError) -> warp::reject::Rejection {
+    warp::reject::custom(ArithError(e))
+}
+
+#[derive(Debug)]
+pub struct SlotProcessingError(pub state_processing::SlotProcessingError);
+
+impl Reject for SlotProcessingError {}
+
+pub fn slot_processing_error(e: state_processing::SlotProcessingError) -> warp::reject::Rejection {
+    warp::reject::custom(SlotProcessingError(e))
+}
+
+#[derive(Debug)]
+pub struct BlockProductionError(pub beacon_chain::BlockProductionError);
+
+impl Reject for BlockProductionError {}
+
+pub fn block_production_error(e: beacon_chain::BlockProductionError) -> warp::reject::Rejection {
+    warp::reject::custom(BlockProductionError(e))
+}
+
+#[derive(Debug)]
+pub struct CustomNotFound(pub String);
+
+impl Reject for CustomNotFound {}
+
+pub fn custom_not_found(msg: String) -> warp::reject::Rejection {
+    warp::reject::custom(CustomNotFound(msg))
+}
+
+#[derive(Debug)]
+pub struct CustomBadRequest(pub String);
+
+impl Reject for CustomBadRequest {}
+
+pub fn custom_bad_request(msg: String) -> warp::reject::Rejection {
+    warp::reject::custom(CustomBadRequest(msg))
+}
+
+#[derive(Debug)]
+pub struct CustomServerError(pub String);
+
+impl Reject for CustomServerError {}
+
+pub fn custom_server_error(msg: String) -> warp::reject::Rejection {
+    warp::reject::custom(CustomServerError(msg))
+}
+
+#[derive(Debug)]
+pub struct BroadcastWithoutImport(pub String);
+
+impl Reject for BroadcastWithoutImport {}
+
+pub fn broadcast_without_import(msg: String) -> warp::reject::Rejection {
+    warp::reject::custom(BroadcastWithoutImport(msg))
+}
+
+#[derive(Debug)]
+pub struct ObjectInvalid(pub String);
+
+impl Reject for ObjectInvalid {}
+
+pub fn object_invalid(msg: String) -> warp::reject::Rejection {
+    warp::reject::custom(ObjectInvalid(msg))
+}
+
+#[derive(Debug)]
+pub struct NotSynced(pub String);
+
+impl Reject for NotSynced {}
+
+pub fn not_synced(msg: String) -> warp::reject::Rejection {
+    warp::reject::custom(NotSynced(msg))
+}
+
+/// This function receives a `Rejection` and tries to return a custom
+/// value, otherwise simply passes the rejection along.
+pub async fn handle_rejection(err: warp::Rejection) -> Result<impl warp::Reply, Infallible> {
+    let code;
+    let message;
+
+    if err.is_not_found() {
+        code = StatusCode::NOT_FOUND;
+        message = "NOT_FOUND".to_string();
+    } else if let Some(e) = err.find::<warp::filters::body::BodyDeserializeError>() {
+        message = format!("BAD_REQUEST: body deserialize error: {}", e);
+        code = StatusCode::BAD_REQUEST;
+    } else if let Some(e) = err.find::<warp::reject::InvalidQuery>() {
+        code = StatusCode::BAD_REQUEST;
+        message = format!("BAD_REQUEST: invalid query: {}", e);
+    } else if let Some(e) = err.find::<crate::reject::BeaconChainError>() {
+        code = StatusCode::INTERNAL_SERVER_ERROR;
+        message = format!("UNHANDLED_ERROR: {:?}", e.0);
+    } else if let Some(e) = err.find::<crate::reject::BeaconStateError>() {
+        code = StatusCode::INTERNAL_SERVER_ERROR;
+        message = format!("UNHANDLED_ERROR: {:?}", e.0);
+    } else if let Some(e) = err.find::<crate::reject::SlotProcessingError>() {
+        code = StatusCode::INTERNAL_SERVER_ERROR;
+        message = format!("UNHANDLED_ERROR: {:?}", e.0);
+    } else if let Some(e) = err.find::<crate::reject::BlockProductionError>() {
+        code = StatusCode::INTERNAL_SERVER_ERROR;
+        message = format!("UNHANDLED_ERROR: {:?}", e.0);
+    } else if let Some(e) = err.find::<crate::reject::CustomNotFound>() {
+        code = StatusCode::NOT_FOUND;
+        message = format!("NOT_FOUND: {}", e.0);
+    } else if let Some(e) = err.find::<crate::reject::CustomBadRequest>() {
+        code = StatusCode::BAD_REQUEST;
+        message = format!("BAD_REQUEST: {}", e.0);
+    } else if let Some(e) = err.find::<crate::reject::CustomServerError>() {
+        code = StatusCode::INTERNAL_SERVER_ERROR;
+        message = format!("INTERNAL_SERVER_ERROR: {}", e.0);
+    } else if let Some(e) = err.find::<crate::reject::BroadcastWithoutImport>() {
+        code = StatusCode::ACCEPTED;
+        message = format!(
+            "ACCEPTED: the object was broadcast to the network without being \
+            fully imported to the local database: {}",
+            e.0
+        );
+    } else if let Some(e) = err.find::<crate::reject::ObjectInvalid>() {
+        code = StatusCode::BAD_REQUEST;
+        message = format!("BAD_REQUEST: Invalid object: {}", e.0);
+    } else if let Some(e) = err.find::<crate::reject::NotSynced>() {
+        code = StatusCode::SERVICE_UNAVAILABLE;
+        message = format!("SERVICE_UNAVAILABLE: beacon node is syncing: {}", e.0);
+    } else if err.find::<warp::reject::MethodNotAllowed>().is_some() {
+        code = StatusCode::METHOD_NOT_ALLOWED;
+        message = "METHOD_NOT_ALLOWED".to_string();
+    } else {
+        code = StatusCode::INTERNAL_SERVER_ERROR;
+        message = "UNHANDLED_REJECTION".to_string();
+    }
+
+    let json = warp::reply::json(&ErrorMessage {
+        code: code.as_u16(),
+        message,
+        stacktraces: vec![],
+    });
+
+    Ok(warp::reply::with_status(json, code))
+}

common/warp_utils/src/reply.rs

@@ -0,0 +1,15 @@
+/// Add CORS headers to `reply` only if `allow_origin.is_some()`.
+pub fn maybe_cors<T: warp::Reply + 'static>(
+    reply: T,
+    allow_origin: Option<&String>,
+) -> Box<dyn warp::Reply> {
+    if let Some(allow_origin) = allow_origin {
+        Box::new(warp::reply::with_header(
+            reply,
+            "Access-Control-Allow-Origin",
+            allow_origin,
+        ))
+    } else {
+        Box::new(reply)
+    }
+}