diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e8e2f49dd2..5feefd8417 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -77,7 +77,7 @@ pub enum AttestationProcessingOutcome { Invalid(AttestationValidationError), } -pub trait BeaconChainTypes { +pub trait BeaconChainTypes: Send + Sync + 'static { type Store: store::Store; type SlotClock: slot_clock::SlotClock; type LmdGhost: LmdGhost; @@ -870,9 +870,16 @@ impl BeaconChain { let catchup_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CATCHUP_STATE); + // Keep a list of any states that were "skipped" (block-less) in between the parent state + // slot and the block slot. These will need to be stored in the database. + let mut intermediate_states = vec![]; + // Transition the parent state to the block slot. let mut state: BeaconState = parent_state; - for _ in state.slot.as_u64()..block.slot.as_u64() { + for i in state.slot.as_u64()..block.slot.as_u64() { + if i > 0 { + intermediate_states.push(state.clone()); + } per_slot_processing(&mut state, &self.spec)?; } @@ -911,6 +918,22 @@ impl BeaconChain { let db_write_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_WRITE); + // Store all the states between the parent block state and this blocks slot before storing + // the final state. + for (i, intermediate_state) in intermediate_states.iter().enumerate() { + // To avoid doing an unnecessary tree hash, use the following (slot + 1) state's + // state_roots field to find the root. + let following_state = match intermediate_states.get(i + 1) { + Some(following_state) => following_state, + None => &state, + }; + let intermediate_state_root = + following_state.get_state_root(intermediate_state.slot)?; + + self.store + .put(&intermediate_state_root, intermediate_state)?; + } + // Store the block and state. self.store.put(&block_root, &block)?; self.store.put(&state_root, &state)?; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 298c637dbd..09f4749ea3 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -54,7 +54,7 @@ where impl BeaconChainTypes for CommonTypes where - L: LmdGhost, + L: LmdGhost + 'static, E: EthSpec, { type Store = MemoryStore; @@ -69,7 +69,7 @@ where /// Used for testing. pub struct BeaconChainHarness where - L: LmdGhost, + L: LmdGhost + 'static, E: EthSpec, { pub chain: BeaconChain>, diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index b13f175a9b..9b5a9cf42c 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [dependencies] beacon_chain = { path = "../beacon_chain" } network = { path = "../network" } +eth2-libp2p = { path = "../eth2-libp2p" } rpc = { path = "../rpc" } rest_api = { path = "../rest_api" } prometheus = "^0.6" @@ -26,3 +27,5 @@ clap = "2.32.0" dirs = "1.0.3" exit-future = "0.1.3" futures = "0.1.25" +reqwest = "0.9" +url = "1.2" diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 0b86c95838..5168c067a9 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -1,3 +1,4 @@ +use crate::bootstrapper::Bootstrapper; use crate::error::Result; use crate::{config::GenesisState, ClientConfig}; use beacon_chain::{ @@ -35,7 +36,11 @@ pub struct ClientType { _phantom_u: PhantomData, } -impl BeaconChainTypes for ClientType { +impl BeaconChainTypes for ClientType +where + S: Store + 'static, + E: EthSpec, +{ type Store = S; type SlotClock = SystemTimeSlotClock; type LmdGhost = ThreadSafeReducedTree; @@ -74,6 +79,16 @@ where serde_yaml::from_reader(file) .map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))? } + GenesisState::HttpBootstrap { server } => { + let bootstrapper = Bootstrapper::from_server_string(server.to_string()) + .map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?; + + let (state, _block) = bootstrapper + .genesis() + .map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?; + + state + } }; let mut genesis_block = BeaconBlock::empty(&spec); diff --git a/beacon_node/client/src/bootstrapper.rs b/beacon_node/client/src/bootstrapper.rs new file mode 100644 index 0000000000..c94d9a51d8 --- /dev/null +++ b/beacon_node/client/src/bootstrapper.rs @@ -0,0 +1,210 @@ +use eth2_libp2p::{ + multiaddr::{Multiaddr, Protocol}, + Enr, +}; +use reqwest::{Error as HttpError, Url}; +use serde::Deserialize; +use std::borrow::Cow; +use std::net::Ipv4Addr; +use types::{BeaconBlock, BeaconState, Checkpoint, EthSpec, Hash256, Slot}; +use url::Host; + +#[derive(Debug)] +enum Error { + InvalidUrl, + HttpError(HttpError), +} + +impl From for Error { + fn from(e: HttpError) -> Error { + Error::HttpError(e) + } +} + +/// Used to load "bootstrap" information from the HTTP API of another Lighthouse beacon node. +/// +/// Bootstrapping information includes things like genesis and finalized states and blocks, and +/// libp2p connection details. +pub struct Bootstrapper { + url: Url, +} + +impl Bootstrapper { + /// Parses the given `server` as a URL, instantiating `Self`. + pub fn from_server_string(server: String) -> Result { + Ok(Self { + url: Url::parse(&server).map_err(|e| format!("Invalid bootstrap server url: {}", e))?, + }) + } + + /// Build a multiaddr using the HTTP server URL that is not guaranteed to be correct. + /// + /// The address is created by querying the HTTP server for its listening libp2p addresses. + /// Then, we find the first TCP port in those addresses and combine the port with the URL of + /// the server. + /// + /// For example, the server `http://192.168.0.1` might end up with a `best_effort_multiaddr` of + /// `/ipv4/192.168.0.1/tcp/9000` if the server advertises a listening address of + /// `/ipv4/172.0.0.1/tcp/9000`. + pub fn best_effort_multiaddr(&self) -> Option { + let tcp_port = self.listen_port().ok()?; + + let mut multiaddr = Multiaddr::with_capacity(2); + + match self.url.host()? { + Host::Ipv4(addr) => multiaddr.push(Protocol::Ip4(addr)), + Host::Domain(s) => multiaddr.push(Protocol::Dns4(Cow::Borrowed(s))), + _ => return None, + }; + + multiaddr.push(Protocol::Tcp(tcp_port)); + + Some(multiaddr) + } + + /// Returns the IPv4 address of the server URL, unless it contains a FQDN. + pub fn server_ipv4_addr(&self) -> Option { + match self.url.host()? { + Host::Ipv4(addr) => Some(addr), + _ => None, + } + } + + /// Returns the servers ENR address. + pub fn enr(&self) -> Result { + get_enr(self.url.clone()).map_err(|e| format!("Unable to get ENR: {:?}", e)) + } + + /// Returns the servers listening libp2p addresses. + pub fn listen_port(&self) -> Result { + get_listen_port(self.url.clone()).map_err(|e| format!("Unable to get listen port: {:?}", e)) + } + + /// Returns the genesis block and state. + pub fn genesis(&self) -> Result<(BeaconState, BeaconBlock), String> { + let genesis_slot = Slot::new(0); + + let block = get_block(self.url.clone(), genesis_slot) + .map_err(|e| format!("Unable to get genesis block: {:?}", e))? + .beacon_block; + let state = get_state(self.url.clone(), genesis_slot) + .map_err(|e| format!("Unable to get genesis state: {:?}", e))? + .beacon_state; + + Ok((state, block)) + } + + /// Returns the most recent finalized state and block. + pub fn finalized(&self) -> Result<(BeaconState, BeaconBlock), String> { + let slots_per_epoch = get_slots_per_epoch(self.url.clone()) + .map_err(|e| format!("Unable to get slots per epoch: {:?}", e))?; + let finalized_slot = get_finalized_slot(self.url.clone(), slots_per_epoch.as_u64()) + .map_err(|e| format!("Unable to get finalized slot: {:?}", e))?; + + let block = get_block(self.url.clone(), finalized_slot) + .map_err(|e| format!("Unable to get finalized block: {:?}", e))? + .beacon_block; + let state = get_state(self.url.clone(), finalized_slot) + .map_err(|e| format!("Unable to get finalized state: {:?}", e))? + .beacon_state; + + Ok((state, block)) + } +} + +fn get_slots_per_epoch(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("spec").push("slots_per_epoch"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_finalized_slot(mut url: Url, slots_per_epoch: u64) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("beacon").push("latest_finalized_checkpoint"); + }) + .map_err(|_| Error::InvalidUrl)?; + + let checkpoint: Checkpoint = reqwest::get(url)?.error_for_status()?.json()?; + + Ok(checkpoint.epoch.start_slot(slots_per_epoch)) +} + +#[derive(Deserialize)] +#[serde(bound = "T: EthSpec")] +pub struct StateResponse { + pub root: Hash256, + pub beacon_state: BeaconState, +} + +fn get_state(mut url: Url, slot: Slot) -> Result, Error> { + url.path_segments_mut() + .map(|mut url| { + url.push("beacon").push("state"); + }) + .map_err(|_| Error::InvalidUrl)?; + + url.query_pairs_mut() + .append_pair("slot", &format!("{}", slot.as_u64())); + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +#[derive(Deserialize)] +#[serde(bound = "T: EthSpec")] +pub struct BlockResponse { + pub root: Hash256, + pub beacon_block: BeaconBlock, +} + +fn get_block(mut url: Url, slot: Slot) -> Result, Error> { + url.path_segments_mut() + .map(|mut url| { + url.push("beacon").push("block"); + }) + .map_err(|_| Error::InvalidUrl)?; + + url.query_pairs_mut() + .append_pair("slot", &format!("{}", slot.as_u64())); + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_enr(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("network").push("enr"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_listen_port(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("network").push("listen_port"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index fcc2cc7dac..ea8186dbc9 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -1,8 +1,8 @@ -use crate::Eth2Config; +use crate::{Bootstrapper, Eth2Config}; use clap::ArgMatches; use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; -use slog::{info, o, Drain}; +use slog::{info, o, warn, Drain}; use std::fs::{self, OpenOptions}; use std::path::PathBuf; use std::sync::Mutex; @@ -46,6 +46,8 @@ pub enum GenesisState { }, /// Load a YAML-encoded genesis state from a file. Yaml { file: PathBuf }, + /// Use a HTTP server (running our REST-API) to load genesis and finalized states and blocks. + HttpBootstrap { server: String }, } impl Default for Config { @@ -147,6 +149,40 @@ impl Config { self.update_logger(log)?; }; + // If the `--bootstrap` flag is provided, overwrite the default configuration. + if let Some(server) = args.value_of("bootstrap") { + do_bootstrapping(self, server.to_string(), &log)?; + } + Ok(()) } } + +/// Perform the HTTP bootstrapping procedure, reading an ENR and multiaddr from the HTTP server and +/// adding them to the `config`. +fn do_bootstrapping(config: &mut Config, server: String, log: &slog::Logger) -> Result<(), String> { + // Set the genesis state source. + config.genesis_state = GenesisState::HttpBootstrap { + server: server.to_string(), + }; + + let bootstrapper = Bootstrapper::from_server_string(server.to_string())?; + + config.network.boot_nodes.push(bootstrapper.enr()?); + + if let Some(server_multiaddr) = bootstrapper.best_effort_multiaddr() { + info!( + log, + "Estimated bootstrapper libp2p address"; + "multiaddr" => format!("{:?}", server_multiaddr) + ); + config.network.libp2p_nodes.push(server_multiaddr); + } else { + warn!( + log, + "Unable to estimate a bootstrapper libp2p address, this node may not find any peers." + ); + } + + Ok(()) +} diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 5c37ac3e9c..6405e05e71 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -1,6 +1,7 @@ extern crate slog; mod beacon_chain_types; +mod bootstrapper; mod config; pub mod error; @@ -21,7 +22,8 @@ use tokio::timer::Interval; pub use beacon_chain::BeaconChainTypes; pub use beacon_chain_types::ClientType; pub use beacon_chain_types::InitialiseBeaconChain; -pub use config::Config as ClientConfig; +pub use bootstrapper::Bootstrapper; +pub use config::{Config as ClientConfig, GenesisState}; pub use eth2_config::Eth2Config; /// Main beacon node client service. This provides the connection and initialisation of the clients @@ -47,7 +49,7 @@ pub struct Client { impl Client where - T: BeaconChainTypes + InitialiseBeaconChain + Clone + 'static, + T: BeaconChainTypes + InitialiseBeaconChain + Clone, { /// Generate an instance of the client. Spawn and link all internal sub-processes. pub fn new( @@ -121,6 +123,7 @@ where &client_config.rest_api, executor, beacon_chain.clone(), + network.clone(), client_config.db_path().expect("unable to read datadir"), &log, ) { diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index 1c7cf38670..78e50ac79d 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -17,11 +17,7 @@ pub const WARN_PEER_COUNT: usize = 1; /// durations. /// /// Presently unused, but remains for future use. -pub fn run( - client: &Client, - executor: TaskExecutor, - exit: Exit, -) { +pub fn run(client: &Client, executor: TaskExecutor, exit: Exit) { // notification heartbeat let interval = Interval::new( Instant::now(), diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index b87f8a0613..9158fe4858 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -78,6 +78,10 @@ impl Behaviour { log: behaviour_log, }) } + + pub fn discovery(&self) -> &Discovery { + &self.discovery + } } // Implement the NetworkBehaviourEventProcess trait so that we can derive NetworkBehaviour for Behaviour diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index ca98db3246..87d5dd5581 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -103,6 +103,10 @@ impl Discovery { }) } + pub fn local_enr(&self) -> &Enr { + self.discovery.local_enr() + } + /// Manually search for peers. This restarts the discovery round, sparking multiple rapid /// queries. pub fn discover_peers(&mut self) { @@ -120,6 +124,11 @@ impl Discovery { self.connected_peers.len() } + /// The current number of connected libp2p peers. + pub fn connected_peer_set(&self) -> &HashSet { + &self.connected_peers + } + /// Search for new peers using the underlying discovery mechanism. fn find_peers(&mut self) { // pick a random NodeId diff --git a/beacon_node/eth2-libp2p/src/lib.rs b/beacon_node/eth2-libp2p/src/lib.rs index 33d5ba9ed9..4c84469cea 100644 --- a/beacon_node/eth2-libp2p/src/lib.rs +++ b/beacon_node/eth2-libp2p/src/lib.rs @@ -17,12 +17,13 @@ pub use behaviour::PubsubMessage; pub use config::{ Config as NetworkConfig, BEACON_ATTESTATION_TOPIC, BEACON_BLOCK_TOPIC, SHARD_TOPIC_PREFIX, }; +pub use libp2p::enr::Enr; pub use libp2p::gossipsub::{Topic, TopicHash}; pub use libp2p::multiaddr; pub use libp2p::Multiaddr; pub use libp2p::{ gossipsub::{GossipsubConfig, GossipsubConfigBuilder}, - PeerId, + PeerId, Swarm, }; pub use rpc::RPCEvent; pub use service::Libp2pEvent; diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 316aa05798..e208dbecac 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -16,7 +16,7 @@ use libp2p::core::{ upgrade::{InboundUpgradeExt, OutboundUpgradeExt}, }; use libp2p::{core, secio, PeerId, Swarm, Transport}; -use slog::{debug, info, trace, warn}; +use slog::{crit, debug, info, trace, warn}; use std::fs::File; use std::io::prelude::*; use std::io::{Error, ErrorKind}; @@ -33,7 +33,7 @@ pub struct Service { //TODO: Make this private pub swarm: Swarm, /// This node's PeerId. - _local_peer_id: PeerId, + pub local_peer_id: PeerId, /// The libp2p logger handle. pub log: slog::Logger, } @@ -69,10 +69,15 @@ impl Service { log_address.push(Protocol::P2p(local_peer_id.clone().into())); info!(log, "Listening on: {}", log_address); } - Err(err) => warn!( - log, - "Cannot listen on: {} because: {:?}", listen_multiaddr, err - ), + Err(err) => { + crit!( + log, + "Unable to listen on libp2p address"; + "error" => format!("{:?}", err), + "listen_multiaddr" => format!("{}", listen_multiaddr), + ); + return Err("Libp2p was unable to listen on the given listen address.".into()); + } }; // attempt to connect to user-input libp2p nodes @@ -113,7 +118,7 @@ impl Service { info!(log, "Subscribed to topics: {:?}", subscribed_topics); Ok(Service { - _local_peer_id: local_peer_id, + local_peer_id, swarm, log, }) diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index e5ca2a9175..152f4dc77d 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -5,7 +5,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use core::marker::PhantomData; use eth2_libp2p::Service as LibP2PService; use eth2_libp2p::Topic; -use eth2_libp2p::{Libp2pEvent, PeerId}; +use eth2_libp2p::{Enr, Libp2pEvent, Multiaddr, PeerId, Swarm}; use eth2_libp2p::{PubsubMessage, RPCEvent}; use futures::prelude::*; use futures::Stream; @@ -18,6 +18,7 @@ use tokio::sync::{mpsc, oneshot}; /// Service that handles communication between internal services and the eth2_libp2p network service. pub struct Service { libp2p_service: Arc>, + libp2p_port: u16, _libp2p_exit: oneshot::Sender<()>, _network_send: mpsc::UnboundedSender, _phantom: PhantomData, //message_handler: MessageHandler, @@ -56,6 +57,7 @@ impl Service { )?; let network_service = Service { libp2p_service, + libp2p_port: config.libp2p_port, _libp2p_exit: libp2p_exit, _network_send: network_send.clone(), _phantom: PhantomData, @@ -64,6 +66,52 @@ impl Service { Ok((Arc::new(network_service), network_send)) } + /// Returns the local ENR from the underlying Discv5 behaviour that external peers may connect + /// to. + pub fn local_enr(&self) -> Enr { + self.libp2p_service + .lock() + .swarm + .discovery() + .local_enr() + .clone() + } + + /// Returns the local libp2p PeerID. + pub fn local_peer_id(&self) -> PeerId { + self.libp2p_service.lock().local_peer_id.clone() + } + + /// Returns the list of `Multiaddr` that the underlying libp2p instance is listening on. + pub fn listen_multiaddrs(&self) -> Vec { + Swarm::listeners(&self.libp2p_service.lock().swarm) + .cloned() + .collect() + } + + /// Returns the libp2p port that this node has been configured to listen using. + pub fn listen_port(&self) -> u16 { + self.libp2p_port + } + + /// Returns the number of libp2p connected peers. + pub fn connected_peers(&self) -> usize { + self.libp2p_service.lock().swarm.connected_peers() + } + + /// Returns the set of `PeerId` that are connected via libp2p. + pub fn connected_peer_set(&self) -> Vec { + self.libp2p_service + .lock() + .swarm + .discovery() + .connected_peer_set() + .iter() + .cloned() + .collect() + } + + /// Provides a reference to the underlying libp2p service. pub fn libp2p_service(&self) -> Arc> { self.libp2p_service.clone() } diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml index c7026014c4..cac196d9cb 100644 --- a/beacon_node/rest_api/Cargo.toml +++ b/beacon_node/rest_api/Cargo.toml @@ -7,6 +7,8 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] beacon_chain = { path = "../beacon_chain" } +network = { path = "../network" } +eth2-libp2p = { path = "../eth2-libp2p" } store = { path = "../store" } version = { path = "../version" } serde = { version = "1.0", features = ["derive"] } diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index cef23abe81..1c66a2819f 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -2,9 +2,114 @@ use super::{success_response, ApiResult}; use crate::{helpers::*, ApiError, UrlQuery}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use hyper::{Body, Request}; +use serde::Serialize; use std::sync::Arc; use store::Store; -use types::BeaconState; +use types::{BeaconBlock, BeaconState, EthSpec, Hash256, Slot}; + +#[derive(Serialize)] +pub struct HeadResponse { + pub slot: Slot, + pub block_root: Hash256, + pub state_root: Hash256, +} + +/// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. +pub fn get_head(req: Request) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let head = HeadResponse { + slot: beacon_chain.head().beacon_state.slot, + block_root: beacon_chain.head().beacon_block_root, + state_root: beacon_chain.head().beacon_state_root, + }; + + let json: String = serde_json::to_string(&head) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize HeadResponse: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} + +#[derive(Serialize)] +#[serde(bound = "T: EthSpec")] +pub struct BlockResponse { + pub root: Hash256, + pub beacon_block: BeaconBlock, +} + +/// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. +pub fn get_block(req: Request) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let query_params = ["root", "slot"]; + let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?; + + let block_root = match (key.as_ref(), value) { + ("slot", value) => { + let target = parse_slot(&value)?; + + block_root_at_slot(&beacon_chain, target).ok_or_else(|| { + ApiError::NotFound(format!("Unable to find BeaconBlock for slot {}", target)) + })? + } + ("root", value) => parse_root(&value)?, + _ => return Err(ApiError::ServerError("Unexpected query parameter".into())), + }; + + let block = beacon_chain + .store + .get::>(&block_root)? + .ok_or_else(|| { + ApiError::NotFound(format!( + "Unable to find BeaconBlock for root {}", + block_root + )) + })?; + + let response = BlockResponse { + root: block_root, + beacon_block: block, + }; + + let json: String = serde_json::to_string(&response).map_err(|e| { + ApiError::ServerError(format!("Unable to serialize BlockResponse: {:?}", e)) + })?; + + Ok(success_response(Body::from(json))) +} + +/// HTTP handler to return a `BeaconBlock` root at a given `slot`. +pub fn get_block_root(req: Request) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; + let target = parse_slot(&slot_string)?; + + let root = block_root_at_slot(&beacon_chain, target).ok_or_else(|| { + ApiError::NotFound(format!("Unable to find BeaconBlock for slot {}", target)) + })?; + + let json: String = serde_json::to_string(&root) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} + +#[derive(Serialize)] +#[serde(bound = "T: EthSpec")] +pub struct StateResponse { + pub root: Hash256, + pub beacon_state: BeaconState, +} /// HTTP handler to return a `BeaconState` at a given `root` or `slot`. /// @@ -19,26 +124,34 @@ pub fn get_state(req: Request) -> ApiResult let query_params = ["root", "slot"]; let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?; - let state: BeaconState = match (key.as_ref(), value) { + let (root, state): (Hash256, BeaconState) = match (key.as_ref(), value) { ("slot", value) => state_at_slot(&beacon_chain, parse_slot(&value)?)?, ("root", value) => { let root = &parse_root(&value)?; - beacon_chain + let state = beacon_chain .store .get(root)? - .ok_or_else(|| ApiError::NotFound(format!("No state for root: {}", root)))? + .ok_or_else(|| ApiError::NotFound(format!("No state for root: {}", root)))?; + + (*root, state) } - _ => unreachable!("Guarded by UrlQuery::from_request()"), + _ => return Err(ApiError::ServerError("Unexpected query parameter".into())), }; - let json: String = serde_json::to_string(&state) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize BeaconState: {:?}", e)))?; + let response = StateResponse { + root, + beacon_state: state, + }; + + let json: String = serde_json::to_string(&response).map_err(|e| { + ApiError::ServerError(format!("Unable to serialize StateResponse: {:?}", e)) + })?; Ok(success_response(Body::from(json))) } -/// HTTP handler to return a `BeaconState` root at a given or `slot`. +/// HTTP handler to return a `BeaconState` root at a given `slot`. /// /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. @@ -58,3 +171,24 @@ pub fn get_state_root(req: Request) -> ApiR Ok(success_response(Body::from(json))) } + +/// HTTP handler to return the highest finalized slot. +pub fn get_latest_finalized_checkpoint( + req: Request, +) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let checkpoint = beacon_chain + .head() + .beacon_state + .finalized_checkpoint + .clone(); + + let json: String = serde_json::to_string(&checkpoint) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize checkpoint: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 2a429076c7..5365086df7 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -31,22 +31,40 @@ pub fn parse_root(string: &str) -> Result { } } -/// Returns a `BeaconState` in the canonical chain of `beacon_chain` at the given `slot`, if -/// possible. +/// Returns the root of the `BeaconBlock` in the canonical chain of `beacon_chain` at the given +/// `slot`, if possible. +/// +/// May return a root for a previous slot, in the case of skip slots. +pub fn block_root_at_slot( + beacon_chain: &BeaconChain, + target: Slot, +) -> Option { + beacon_chain + .rev_iter_block_roots() + .take_while(|(_root, slot)| *slot >= target) + .find(|(_root, slot)| *slot == target) + .map(|(root, _slot)| root) +} + +/// Returns a `BeaconState` and it's root in the canonical chain of `beacon_chain` at the given +/// `slot`, if possible. /// /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. pub fn state_at_slot( beacon_chain: &BeaconChain, slot: Slot, -) -> Result, ApiError> { +) -> Result<(Hash256, BeaconState), ApiError> { let head_state = &beacon_chain.head().beacon_state; if head_state.slot == slot { // The request slot is the same as the best block (head) slot. // I'm not sure if this `.clone()` will be optimized out. If not, it seems unnecessary. - Ok(beacon_chain.head().beacon_state.clone()) + Ok(( + beacon_chain.head().beacon_state_root, + beacon_chain.head().beacon_state.clone(), + )) } else { let root = state_root_at_slot(beacon_chain, slot)?; @@ -55,7 +73,7 @@ pub fn state_at_slot( .get(&root)? .ok_or_else(|| ApiError::NotFound(format!("Unable to find state at root {}", root)))?; - Ok(state) + Ok((root, state)) } } diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 57019deea0..964dd79982 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -1,15 +1,18 @@ #[macro_use] extern crate lazy_static; +extern crate network as client_network; mod beacon; mod config; mod helpers; mod metrics; +mod network; mod node; +mod spec; mod url_query; use beacon_chain::{BeaconChain, BeaconChainTypes}; -pub use config::Config as ApiConfig; +use client_network::Service as NetworkService; use hyper::rt::Future; use hyper::service::service_fn_ok; use hyper::{Body, Method, Response, Server, StatusCode}; @@ -20,6 +23,9 @@ use std::sync::Arc; use tokio::runtime::TaskExecutor; use url_query::UrlQuery; +pub use beacon::{BlockResponse, HeadResponse, StateResponse}; +pub use config::Config as ApiConfig; + #[derive(PartialEq, Debug)] pub enum ApiError { MethodNotAllowed(String), @@ -67,10 +73,11 @@ impl From for ApiError { } } -pub fn start_server( +pub fn start_server( config: &ApiConfig, executor: &TaskExecutor, beacon_chain: Arc>, + network_service: Arc>, db_path: PathBuf, log: &slog::Logger, ) -> Result { @@ -98,6 +105,7 @@ pub fn start_server( let log = server_log.clone(); let beacon_chain = server_bc.clone(); let db_path = db_path.clone(); + let network_service = network_service.clone(); // Create a simple handler for the router, inject our stateful objects into the request. service_fn_ok(move |mut req| { @@ -108,16 +116,34 @@ pub fn start_server( req.extensions_mut() .insert::>>(beacon_chain.clone()); req.extensions_mut().insert::(db_path.clone()); + req.extensions_mut() + .insert::>>(network_service.clone()); let path = req.uri().path().to_string(); // Route the request to the correct handler. let result = match (req.method(), path.as_ref()) { + (&Method::GET, "/beacon/head") => beacon::get_head::(req), + (&Method::GET, "/beacon/block") => beacon::get_block::(req), + (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), + (&Method::GET, "/beacon/latest_finalized_checkpoint") => { + beacon::get_latest_finalized_checkpoint::(req) + } (&Method::GET, "/beacon/state") => beacon::get_state::(req), (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), (&Method::GET, "/metrics") => metrics::get_prometheus::(req), + (&Method::GET, "/network/enr") => network::get_enr::(req), + (&Method::GET, "/network/peer_count") => network::get_peer_count::(req), + (&Method::GET, "/network/peer_id") => network::get_peer_id::(req), + (&Method::GET, "/network/peers") => network::get_peer_list::(req), + (&Method::GET, "/network/listen_port") => network::get_listen_port::(req), + (&Method::GET, "/network/listen_addresses") => { + network::get_listen_addresses::(req) + } (&Method::GET, "/node/version") => node::get_version(req), (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), + (&Method::GET, "/spec") => spec::get_spec::(req), + (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::(req), _ => Err(ApiError::MethodNotAllowed(path.clone())), }; diff --git a/beacon_node/rest_api/src/network.rs b/beacon_node/rest_api/src/network.rs new file mode 100644 index 0000000000..a3e4c5ee72 --- /dev/null +++ b/beacon_node/rest_api/src/network.rs @@ -0,0 +1,108 @@ +use crate::{success_response, ApiError, ApiResult, NetworkService}; +use beacon_chain::BeaconChainTypes; +use eth2_libp2p::{Enr, Multiaddr, PeerId}; +use hyper::{Body, Request}; +use std::sync::Arc; + +/// HTTP handle to return the list of libp2p multiaddr the client is listening on. +/// +/// Returns a list of `Multiaddr`, serialized according to their `serde` impl. +pub fn get_listen_addresses(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let multiaddresses: Vec = network.listen_multiaddrs(); + + Ok(success_response(Body::from( + serde_json::to_string(&multiaddresses) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the list of libp2p multiaddr the client is listening on. +/// +/// Returns a list of `Multiaddr`, serialized according to their `serde` impl. +pub fn get_listen_port(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + Ok(success_response(Body::from( + serde_json::to_string(&network.listen_port()) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize port: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the Discv5 ENR from the client's libp2p service. +/// +/// ENR is encoded as base64 string. +pub fn get_enr(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let enr: Enr = network.local_enr(); + + Ok(success_response(Body::from( + serde_json::to_string(&enr.to_base64()) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the `PeerId` from the client's libp2p service. +/// +/// PeerId is encoded as base58 string. +pub fn get_peer_id(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let peer_id: PeerId = network.local_peer_id(); + + Ok(success_response(Body::from( + serde_json::to_string(&peer_id.to_base58()) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the number of peers connected in the client's libp2p service. +pub fn get_peer_count(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let connected_peers: usize = network.connected_peers(); + + Ok(success_response(Body::from( + serde_json::to_string(&connected_peers) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the list of peers connected to the client's libp2p service. +/// +/// Peers are presented as a list of `PeerId::to_string()`. +pub fn get_peer_list(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let connected_peers: Vec = network + .connected_peer_set() + .iter() + .map(PeerId::to_string) + .collect(); + + Ok(success_response(Body::from( + serde_json::to_string(&connected_peers).map_err(|e| { + ApiError::ServerError(format!("Unable to serialize Vec: {:?}", e)) + })?, + ))) +} diff --git a/beacon_node/rest_api/src/spec.rs b/beacon_node/rest_api/src/spec.rs new file mode 100644 index 0000000000..d0c8e4368d --- /dev/null +++ b/beacon_node/rest_api/src/spec.rs @@ -0,0 +1,27 @@ +use super::{success_response, ApiResult}; +use crate::ApiError; +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use hyper::{Body, Request}; +use std::sync::Arc; +use types::EthSpec; + +/// HTTP handler to return the full spec object. +pub fn get_spec(req: Request) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let json: String = serde_json::to_string(&beacon_chain.spec) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize spec: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} + +/// HTTP handler to return the full spec object. +pub fn get_slots_per_epoch(_req: Request) -> ApiResult { + let json: String = serde_json::to_string(&T::EthSpec::slots_per_epoch()) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize epoch: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 9a52f2638c..04366baa7a 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -200,6 +200,16 @@ fn main() { .help("Sets the verbosity level") .takes_value(true), ) + /* + * Bootstrap. + */ + .arg( + Arg::with_name("bootstrap") + .long("bootstrap") + .value_name("HTTP_SERVER") + .help("Load the genesis state and libp2p address from the HTTP API of another Lighthouse node.") + .takes_value(true) + ) .get_matches(); // build the initial logger @@ -227,6 +237,11 @@ fn main() { let mut log = slog::Logger::root(drain.fuse(), o!()); + warn!( + log, + "Ethereum 2.0 is pre-release. This software is experimental." + ); + let data_dir = match matches .value_of("datadir") .and_then(|v| Some(PathBuf::from(v))) diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index c16d23e5f1..f88cb7460b 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -4,7 +4,7 @@ use client::{ }; use futures::sync::oneshot; use futures::Future; -use slog::{error, info, warn}; +use slog::{error, info}; use std::cell::RefCell; use std::path::Path; use std::path::PathBuf; @@ -42,11 +42,6 @@ pub fn run_beacon_node( let other_client_config = client_config.clone(); - warn!( - log, - "Ethereum 2.0 is pre-release. This software is experimental." - ); - info!( log, "BeaconNode init"; @@ -123,7 +118,7 @@ fn run( log: &slog::Logger, ) -> error::Result<()> where - T: BeaconChainTypes + InitialiseBeaconChain + Clone + Send + Sync + 'static, + T: BeaconChainTypes + InitialiseBeaconChain + Clone, T::Store: OpenDatabase, { let store = T::Store::open_database(&db_path)?; diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 64dc229ed8..c1f66b816c 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -1,5 +1,5 @@ use crate::test_utils::TestRandom; -use crate::utils::graffiti_from_hex_str; +use crate::utils::{graffiti_from_hex_str, graffiti_to_hex_str}; use crate::*; use serde_derive::{Deserialize, Serialize}; @@ -16,7 +16,10 @@ use tree_hash_derive::TreeHash; pub struct BeaconBlockBody { pub randao_reveal: Signature, pub eth1_data: Eth1Data, - #[serde(deserialize_with = "graffiti_from_hex_str")] + #[serde( + serialize_with = "graffiti_to_hex_str", + deserialize_with = "graffiti_from_hex_str" + )] pub graffiti: [u8; 32], pub proposer_slashings: VariableList, pub attester_slashings: VariableList, T::MaxAttesterSlashings>, diff --git a/eth2/types/src/utils/serde_utils.rs b/eth2/types/src/utils/serde_utils.rs index 4b46fc0dc6..a9b27d75b5 100644 --- a/eth2/types/src/utils/serde_utils.rs +++ b/eth2/types/src/utils/serde_utils.rs @@ -46,8 +46,20 @@ where Ok(array) } -// #[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `byte` to be a ref. -pub fn fork_to_hex_str(bytes: &[u8; 4], serializer: S) -> Result +pub fn fork_to_hex_str(bytes: &[u8; FORK_BYTES_LEN], serializer: S) -> Result +where + S: Serializer, +{ + let mut hex_string: String = "0x".to_string(); + hex_string.push_str(&hex::encode(&bytes)); + + serializer.serialize_str(&hex_string) +} + +pub fn graffiti_to_hex_str( + bytes: &[u8; GRAFFITI_BYTES_LEN], + serializer: S, +) -> Result where S: Serializer, {