Update to latest master

beacon_node/client/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 beacon_chain = { path = "../beacon_chain" }
 network = { path = "../network" }
-http_server = { path = "../http_server" }
+eth2-libp2p = { path = "../eth2-libp2p" }
 rpc = { path = "../rpc" }
 rest_api = { path = "../rest_api" }
 prometheus = "^0.6"
@@ -27,3 +27,5 @@ clap = "2.32.0"
 dirs = "1.0.3"
 exit-future = "0.1.3"
 futures = "0.1.25"
+reqwest = "0.9"
+url = "1.2"

beacon_node/client/src/beacon_chain_types.rs

@@ -1,3 +1,4 @@
+use crate::bootstrapper::Bootstrapper;
 use crate::error::Result;
 use crate::{config::GenesisState, ClientConfig};
 use beacon_chain::{
@@ -35,7 +36,11 @@ pub struct ClientType<S: Store, E: EthSpec> {
     _phantom_u: PhantomData<E>,
 }
 
-impl<S: Store, E: EthSpec + Clone> BeaconChainTypes for ClientType<S, E> {
+impl<S, E> BeaconChainTypes for ClientType<S, E>
+where
+    S: Store + 'static,
+    E: EthSpec,
+{
     type Store = S;
     type SlotClock = SystemTimeSlotClock;
     type LmdGhost = ThreadSafeReducedTree<S, E>;
@@ -74,6 +79,16 @@ where
             serde_yaml::from_reader(file)
                 .map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))?
         }
+        GenesisState::HttpBootstrap { server } => {
+            let bootstrapper = Bootstrapper::from_server_string(server.to_string())
+                .map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?;
+
+            let (state, _block) = bootstrapper
+                .genesis()
+                .map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?;
+
+            state
+        }
     };
 
     let mut genesis_block = BeaconBlock::empty(&spec);

beacon_node/client/src/bootstrapper.rs

@@ -0,0 +1,210 @@
+use eth2_libp2p::{
+    multiaddr::{Multiaddr, Protocol},
+    Enr,
+};
+use reqwest::{Error as HttpError, Url};
+use serde::Deserialize;
+use std::borrow::Cow;
+use std::net::Ipv4Addr;
+use types::{BeaconBlock, BeaconState, Checkpoint, EthSpec, Hash256, Slot};
+use url::Host;
+
+#[derive(Debug)]
+enum Error {
+    InvalidUrl,
+    HttpError(HttpError),
+}
+
+impl From<HttpError> for Error {
+    fn from(e: HttpError) -> Error {
+        Error::HttpError(e)
+    }
+}
+
+/// Used to load "bootstrap" information from the HTTP API of another Lighthouse beacon node.
+///
+/// Bootstrapping information includes things like genesis and finalized states and blocks, and
+/// libp2p connection details.
+pub struct Bootstrapper {
+    url: Url,
+}
+
+impl Bootstrapper {
+    /// Parses the given `server` as a URL, instantiating `Self`.
+    pub fn from_server_string(server: String) -> Result<Self, String> {
+        Ok(Self {
+            url: Url::parse(&server).map_err(|e| format!("Invalid bootstrap server url: {}", e))?,
+        })
+    }
+
+    /// Build a multiaddr using the HTTP server URL that is not guaranteed to be correct.
+    ///
+    /// The address is created by querying the HTTP server for its listening libp2p addresses.
+    /// Then, we find the first TCP port in those addresses and combine the port with the URL of
+    /// the server.
+    ///
+    /// For example, the server `http://192.168.0.1` might end up with a `best_effort_multiaddr` of
+    /// `/ipv4/192.168.0.1/tcp/9000` if the server advertises a listening address of
+    /// `/ipv4/172.0.0.1/tcp/9000`.
+    pub fn best_effort_multiaddr(&self) -> Option<Multiaddr> {
+        let tcp_port = self.listen_port().ok()?;
+
+        let mut multiaddr = Multiaddr::with_capacity(2);
+
+        match self.url.host()? {
+            Host::Ipv4(addr) => multiaddr.push(Protocol::Ip4(addr)),
+            Host::Domain(s) => multiaddr.push(Protocol::Dns4(Cow::Borrowed(s))),
+            _ => return None,
+        };
+
+        multiaddr.push(Protocol::Tcp(tcp_port));
+
+        Some(multiaddr)
+    }
+
+    /// Returns the IPv4 address of the server URL, unless it contains a FQDN.
+    pub fn server_ipv4_addr(&self) -> Option<Ipv4Addr> {
+        match self.url.host()? {
+            Host::Ipv4(addr) => Some(addr),
+            _ => None,
+        }
+    }
+
+    /// Returns the servers ENR address.
+    pub fn enr(&self) -> Result<Enr, String> {
+        get_enr(self.url.clone()).map_err(|e| format!("Unable to get ENR: {:?}", e))
+    }
+
+    /// Returns the servers listening libp2p addresses.
+    pub fn listen_port(&self) -> Result<u16, String> {
+        get_listen_port(self.url.clone()).map_err(|e| format!("Unable to get listen port: {:?}", e))
+    }
+
+    /// Returns the genesis block and state.
+    pub fn genesis<T: EthSpec>(&self) -> Result<(BeaconState<T>, BeaconBlock<T>), String> {
+        let genesis_slot = Slot::new(0);
+
+        let block = get_block(self.url.clone(), genesis_slot)
+            .map_err(|e| format!("Unable to get genesis block: {:?}", e))?
+            .beacon_block;
+        let state = get_state(self.url.clone(), genesis_slot)
+            .map_err(|e| format!("Unable to get genesis state: {:?}", e))?
+            .beacon_state;
+
+        Ok((state, block))
+    }
+
+    /// Returns the most recent finalized state and block.
+    pub fn finalized<T: EthSpec>(&self) -> Result<(BeaconState<T>, BeaconBlock<T>), String> {
+        let slots_per_epoch = get_slots_per_epoch(self.url.clone())
+            .map_err(|e| format!("Unable to get slots per epoch: {:?}", e))?;
+        let finalized_slot = get_finalized_slot(self.url.clone(), slots_per_epoch.as_u64())
+            .map_err(|e| format!("Unable to get finalized slot: {:?}", e))?;
+
+        let block = get_block(self.url.clone(), finalized_slot)
+            .map_err(|e| format!("Unable to get finalized block: {:?}", e))?
+            .beacon_block;
+        let state = get_state(self.url.clone(), finalized_slot)
+            .map_err(|e| format!("Unable to get finalized state: {:?}", e))?
+            .beacon_state;
+
+        Ok((state, block))
+    }
+}
+
+fn get_slots_per_epoch(mut url: Url) -> Result<Slot, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("spec").push("slots_per_epoch");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}
+
+fn get_finalized_slot(mut url: Url, slots_per_epoch: u64) -> Result<Slot, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("beacon").push("latest_finalized_checkpoint");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    let checkpoint: Checkpoint = reqwest::get(url)?.error_for_status()?.json()?;
+
+    Ok(checkpoint.epoch.start_slot(slots_per_epoch))
+}
+
+#[derive(Deserialize)]
+#[serde(bound = "T: EthSpec")]
+pub struct StateResponse<T: EthSpec> {
+    pub root: Hash256,
+    pub beacon_state: BeaconState<T>,
+}
+
+fn get_state<T: EthSpec>(mut url: Url, slot: Slot) -> Result<StateResponse<T>, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("beacon").push("state");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    url.query_pairs_mut()
+        .append_pair("slot", &format!("{}", slot.as_u64()));
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}
+
+#[derive(Deserialize)]
+#[serde(bound = "T: EthSpec")]
+pub struct BlockResponse<T: EthSpec> {
+    pub root: Hash256,
+    pub beacon_block: BeaconBlock<T>,
+}
+
+fn get_block<T: EthSpec>(mut url: Url, slot: Slot) -> Result<BlockResponse<T>, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("beacon").push("block");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    url.query_pairs_mut()
+        .append_pair("slot", &format!("{}", slot.as_u64()));
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}
+
+fn get_enr(mut url: Url) -> Result<Enr, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("network").push("enr");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}
+
+fn get_listen_port(mut url: Url) -> Result<u16, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("network").push("listen_port");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}

beacon_node/client/src/config.rs

@@ -1,9 +1,8 @@
-use crate::Eth2Config;
+use crate::{Bootstrapper, Eth2Config};
 use clap::ArgMatches;
-use http_server::HttpServerConfig;
 use network::NetworkConfig;
 use serde_derive::{Deserialize, Serialize};
-use slog::{info, o, Drain};
+use slog::{info, o, warn, Drain};
 use std::fs::{self, OpenOptions};
 use std::path::PathBuf;
 use std::sync::Mutex;
@@ -25,7 +24,6 @@ pub struct Config {
     pub genesis_state: GenesisState,
     pub network: network::NetworkConfig,
     pub rpc: rpc::RPCConfig,
-    pub http: HttpServerConfig,
     pub rest_api: rest_api::ApiConfig,
 }
 
@@ -48,6 +46,8 @@ pub enum GenesisState {
     },
     /// Load a YAML-encoded genesis state from a file.
     Yaml { file: PathBuf },
+    /// Use a HTTP server (running our REST-API) to load genesis and finalized states and blocks.
+    HttpBootstrap { server: String },
 }
 
 impl Default for Config {
@@ -59,7 +59,6 @@ impl Default for Config {
             db_name: "chain_db".to_string(),
             network: NetworkConfig::new(),
             rpc: rpc::RPCConfig::default(),
-            http: HttpServerConfig::default(),
             rest_api: rest_api::ApiConfig::default(),
             spec_constants: TESTNET_SPEC_CONSTANTS.into(),
             genesis_state: GenesisState::RecentGenesis {
@@ -143,7 +142,6 @@ impl Config {
 
         self.network.apply_cli_args(args)?;
         self.rpc.apply_cli_args(args)?;
-        self.http.apply_cli_args(args)?;
         self.rest_api.apply_cli_args(args)?;
 
         if let Some(log_file) = args.value_of("logfile") {
@@ -151,6 +149,40 @@ impl Config {
             self.update_logger(log)?;
         };
 
+        // If the `--bootstrap` flag is provided, overwrite the default configuration.
+        if let Some(server) = args.value_of("bootstrap") {
+            do_bootstrapping(self, server.to_string(), &log)?;
+        }
+
         Ok(())
     }
 }
+
+/// Perform the HTTP bootstrapping procedure, reading an ENR and multiaddr from the HTTP server and
+/// adding them to the `config`.
+fn do_bootstrapping(config: &mut Config, server: String, log: &slog::Logger) -> Result<(), String> {
+    // Set the genesis state source.
+    config.genesis_state = GenesisState::HttpBootstrap {
+        server: server.to_string(),
+    };
+
+    let bootstrapper = Bootstrapper::from_server_string(server.to_string())?;
+
+    config.network.boot_nodes.push(bootstrapper.enr()?);
+
+    if let Some(server_multiaddr) = bootstrapper.best_effort_multiaddr() {
+        info!(
+            log,
+            "Estimated bootstrapper libp2p address";
+            "multiaddr" => format!("{:?}", server_multiaddr)
+        );
+        config.network.libp2p_nodes.push(server_multiaddr);
+    } else {
+        warn!(
+            log,
+            "Unable to estimate a bootstrapper libp2p address, this node may not find any peers."
+        );
+    }
+
+    Ok(())
+}

beacon_node/client/src/lib.rs

@@ -1,6 +1,7 @@
 extern crate slog;
 
 mod beacon_chain_types;
+mod bootstrapper;
 mod config;
 
 pub mod error;
@@ -10,7 +11,6 @@ use beacon_chain::BeaconChain;
 use exit_future::Signal;
 use futures::{future::Future, Stream};
 use network::Service as NetworkService;
-use prometheus::Registry;
 use slog::{error, info, o};
 use slot_clock::SlotClock;
 use std::marker::PhantomData;
@@ -22,7 +22,8 @@ use tokio::timer::Interval;
 pub use beacon_chain::BeaconChainTypes;
 pub use beacon_chain_types::ClientType;
 pub use beacon_chain_types::InitialiseBeaconChain;
-pub use config::Config as ClientConfig;
+pub use bootstrapper::Bootstrapper;
+pub use config::{Config as ClientConfig, GenesisState};
 pub use eth2_config::Eth2Config;
 
 /// Main beacon node client service. This provides the connection and initialisation of the clients
@@ -36,8 +37,6 @@ pub struct Client<T: BeaconChainTypes> {
     pub network: Arc<NetworkService<T>>,
     /// Signal to terminate the RPC server.
     pub rpc_exit_signal: Option<Signal>,
-    /// Signal to terminate the HTTP server.
-    pub http_exit_signal: Option<Signal>,
     /// Signal to terminate the slot timer.
     pub slot_timer_exit_signal: Option<Signal>,
     /// Signal to terminate the API
@@ -50,7 +49,7 @@ pub struct Client<T: BeaconChainTypes> {
 
 impl<T> Client<T>
 where
-    T: BeaconChainTypes + InitialiseBeaconChain<T> + Clone + 'static,
+    T: BeaconChainTypes + InitialiseBeaconChain<T> + Clone,
 {
     /// Generate an instance of the client. Spawn and link all internal sub-processes.
     pub fn new(
@@ -60,7 +59,6 @@ where
         log: slog::Logger,
         executor: &TaskExecutor,
     ) -> error::Result<Self> {
-        let metrics_registry = Registry::new();
         let store = Arc::new(store);
         let seconds_per_slot = eth2_config.spec.seconds_per_slot;
 
@@ -71,11 +69,6 @@ where
             eth2_config.spec.clone(),
             log.clone(),
         )?);
-        // Registry all beacon chain metrics with the global registry.
-        beacon_chain
-            .metrics
-            .register(&metrics_registry)
-            .expect("Failed to registry metrics");
 
         if beacon_chain.read_slot_clock().is_none() {
             panic!("Cannot start client before genesis!")
@@ -117,29 +110,14 @@ where
             None
         };
 
-        // Start the `http_server` service.
-        //
-        // Note: presently we are ignoring the config and _always_ starting a HTTP server.
-        let http_exit_signal = if client_config.http.enabled {
-            Some(http_server::start_service(
-                &client_config.http,
-                executor,
-                network_send,
-                beacon_chain.clone(),
-                client_config.db_path().expect("unable to read datadir"),
-                metrics_registry,
-                &log,
-            ))
-        } else {
-            None
-        };
-
         // Start the `rest_api` service
         let api_exit_signal = if client_config.rest_api.enabled {
             match rest_api::start_server(
                 &client_config.rest_api,
                 executor,
                 beacon_chain.clone(),
+                network.clone(),
+                client_config.db_path().expect("unable to read datadir"),
                 &log,
             ) {
                 Ok(s) => Some(s),
@@ -181,7 +159,6 @@ where
         Ok(Client {
             _client_config: client_config,
             beacon_chain,
-            http_exit_signal,
             rpc_exit_signal,
             slot_timer_exit_signal: Some(slot_timer_exit_signal),
             api_exit_signal,

beacon_node/client/src/notifier.rs

@@ -17,11 +17,7 @@ pub const WARN_PEER_COUNT: usize = 1;
 /// durations.
 ///
 /// Presently unused, but remains for future use.
-pub fn run<T: BeaconChainTypes + Send + Sync + 'static>(
-    client: &Client<T>,
-    executor: TaskExecutor,
-    exit: Exit,
-) {
+pub fn run<T: BeaconChainTypes>(client: &Client<T>, executor: TaskExecutor, exit: Exit) {
     // notification heartbeat
     let interval = Interval::new(
         Instant::now(),