diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 32b7e92110..531c4615a1 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -6,11 +6,13 @@ edition = "2018" [dependencies] eth2_config = { path = "../eth2/utils/eth2_config" } +beacon_chain = { path = "beacon_chain" } types = { path = "../eth2/types" } store = { path = "./store" } client = { path = "client" } version = { path = "version" } clap = "2.32.0" +rand = "0.7" slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_trace"] } slog-term = "^2.4.0" slog-async = "^2.3.0" diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 1d3fc03b81..f6763d1671 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -5,14 +5,18 @@ authors = ["Paul Hauner ", "Age Manning BeaconChain { /// Instantiate a new Beacon Chain, from genesis. pub fn from_genesis( store: Arc, - slot_clock: T::SlotClock, mut genesis_state: BeaconState, mut genesis_block: BeaconBlock, spec: ChainSpec, @@ -148,6 +147,13 @@ impl BeaconChain { "genesis_block_root" => format!("{}", genesis_block_root), ); + // Slot clock + let slot_clock = T::SlotClock::new( + spec.genesis_slot, + genesis_state.genesis_time, + spec.seconds_per_slot, + ); + Ok(Self { spec, slot_clock, diff --git a/beacon_node/beacon_chain/src/beacon_chain_builder.rs b/beacon_node/beacon_chain/src/beacon_chain_builder.rs new file mode 100644 index 0000000000..79c74b0068 --- /dev/null +++ b/beacon_node/beacon_chain/src/beacon_chain_builder.rs @@ -0,0 +1,134 @@ +use super::bootstrapper::Bootstrapper; +use crate::{BeaconChain, BeaconChainTypes}; +use slog::Logger; +use std::fs::File; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::SystemTime; +use types::{test_utils::TestingBeaconStateBuilder, BeaconBlock, BeaconState, ChainSpec, EthSpec}; + +enum BuildStrategy { + FromGenesis { + genesis_state: Box>, + genesis_block: Box>, + }, + LoadFromStore, +} + +pub struct BeaconChainBuilder { + build_strategy: BuildStrategy, + spec: ChainSpec, + log: Logger, +} + +impl BeaconChainBuilder { + pub fn recent_genesis(validator_count: usize, spec: ChainSpec, log: Logger) -> Self { + Self::quick_start(recent_genesis_time(), validator_count, spec, log) + } + + pub fn quick_start( + genesis_time: u64, + validator_count: usize, + spec: ChainSpec, + log: Logger, + ) -> Self { + let (mut genesis_state, _keypairs) = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec) + .build(); + + genesis_state.genesis_time = genesis_time; + + Self::from_genesis_state(genesis_state, spec, log) + } + + pub fn yaml_state(file: &PathBuf, spec: ChainSpec, log: Logger) -> Result { + let file = File::open(file.clone()) + .map_err(|e| format!("Unable to open YAML genesis state file {:?}: {:?}", file, e))?; + + let genesis_state = serde_yaml::from_reader(file) + .map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))?; + + Ok(Self::from_genesis_state(genesis_state, spec, log)) + } + + pub fn http_bootstrap(server: &str, spec: ChainSpec, log: Logger) -> Result { + let bootstrapper = Bootstrapper::from_server_string(server.to_string()) + .map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?; + + let (genesis_state, genesis_block) = bootstrapper + .genesis() + .map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?; + + Ok(Self { + build_strategy: BuildStrategy::FromGenesis { + genesis_block: Box::new(genesis_block), + genesis_state: Box::new(genesis_state), + }, + spec, + log, + }) + } + + fn from_genesis_state( + genesis_state: BeaconState, + spec: ChainSpec, + log: Logger, + ) -> Self { + Self { + build_strategy: BuildStrategy::FromGenesis { + genesis_block: Box::new(genesis_block(&genesis_state, &spec)), + genesis_state: Box::new(genesis_state), + }, + spec, + log, + } + } + + pub fn from_store(spec: ChainSpec, log: Logger) -> Self { + Self { + build_strategy: BuildStrategy::LoadFromStore, + spec, + log, + } + } + + pub fn build(self, store: Arc) -> Result, String> { + Ok(match self.build_strategy { + BuildStrategy::LoadFromStore => BeaconChain::from_store(store, self.spec, self.log) + .map_err(|e| format!("Error loading BeaconChain from database: {:?}", e))? + .ok_or_else(|| format!("Unable to find exising BeaconChain in database."))?, + BuildStrategy::FromGenesis { + genesis_block, + genesis_state, + } => BeaconChain::from_genesis( + store, + genesis_state.as_ref().clone(), + genesis_block.as_ref().clone(), + self.spec, + self.log, + ) + .map_err(|e| format!("Failed to initialize new beacon chain: {:?}", e))?, + }) + } +} + +fn genesis_block(genesis_state: &BeaconState, spec: &ChainSpec) -> BeaconBlock { + let mut genesis_block = BeaconBlock::empty(&spec); + + genesis_block.state_root = genesis_state.canonical_root(); + + genesis_block +} + +/// Returns the system time, mod 30 minutes. +/// +/// Used for easily creating testnets. +fn recent_genesis_time() -> u64 { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let secs_after_last_period = now.checked_rem(30 * 60).unwrap_or(0); + // genesis is now the last 30 minute block. + now - secs_after_last_period +} diff --git a/beacon_node/beacon_chain/src/bootstrapper.rs b/beacon_node/beacon_chain/src/bootstrapper.rs new file mode 100644 index 0000000000..dc70c6d211 --- /dev/null +++ b/beacon_node/beacon_chain/src/bootstrapper.rs @@ -0,0 +1,233 @@ +use eth2_config::Eth2Config; +use eth2_libp2p::{ + multiaddr::{Multiaddr, Protocol}, + Enr, +}; +use reqwest::{Error as HttpError, Url}; +use serde::Deserialize; +use std::borrow::Cow; +use std::net::Ipv4Addr; +use types::{BeaconBlock, BeaconState, Checkpoint, EthSpec, Hash256, Slot}; +use url::Host; + +#[derive(Debug)] +enum Error { + InvalidUrl, + HttpError(HttpError), +} + +impl From for Error { + fn from(e: HttpError) -> Error { + Error::HttpError(e) + } +} + +/// Used to load "bootstrap" information from the HTTP API of another Lighthouse beacon node. +/// +/// Bootstrapping information includes things like genesis and finalized states and blocks, and +/// libp2p connection details. +pub struct Bootstrapper { + url: Url, +} + +impl Bootstrapper { + /// Parses the given `server` as a URL, instantiating `Self`. + pub fn from_server_string(server: String) -> Result { + Ok(Self { + url: Url::parse(&server).map_err(|e| format!("Invalid bootstrap server url: {}", e))?, + }) + } + + /// Build a multiaddr using the HTTP server URL that is not guaranteed to be correct. + /// + /// The address is created by querying the HTTP server for its listening libp2p addresses. + /// Then, we find the first TCP port in those addresses and combine the port with the URL of + /// the server. + /// + /// For example, the server `http://192.168.0.1` might end up with a `best_effort_multiaddr` of + /// `/ipv4/192.168.0.1/tcp/9000` if the server advertises a listening address of + /// `/ipv4/172.0.0.1/tcp/9000`. + pub fn best_effort_multiaddr(&self, port: Option) -> Option { + let tcp_port = if let Some(port) = port { + port + } else { + self.listen_port().ok()? + }; + + let mut multiaddr = Multiaddr::with_capacity(2); + + match self.url.host()? { + Host::Ipv4(addr) => multiaddr.push(Protocol::Ip4(addr)), + Host::Domain(s) => multiaddr.push(Protocol::Dns4(Cow::Borrowed(s))), + _ => return None, + }; + + multiaddr.push(Protocol::Tcp(tcp_port)); + + Some(multiaddr) + } + + /// Returns the IPv4 address of the server URL, unless it contains a FQDN. + pub fn server_ipv4_addr(&self) -> Option { + match self.url.host()? { + Host::Ipv4(addr) => Some(addr), + _ => None, + } + } + + /// Returns the servers Eth2Config. + pub fn eth2_config(&self) -> Result { + get_eth2_config(self.url.clone()).map_err(|e| format!("Unable to get Eth2Config: {:?}", e)) + } + + /// Returns the servers ENR address. + pub fn enr(&self) -> Result { + get_enr(self.url.clone()).map_err(|e| format!("Unable to get ENR: {:?}", e)) + } + + /// Returns the servers listening libp2p addresses. + pub fn listen_port(&self) -> Result { + get_listen_port(self.url.clone()).map_err(|e| format!("Unable to get listen port: {:?}", e)) + } + + /// Returns the genesis block and state. + pub fn genesis(&self) -> Result<(BeaconState, BeaconBlock), String> { + let genesis_slot = Slot::new(0); + + let block = get_block(self.url.clone(), genesis_slot) + .map_err(|e| format!("Unable to get genesis block: {:?}", e))? + .beacon_block; + let state = get_state(self.url.clone(), genesis_slot) + .map_err(|e| format!("Unable to get genesis state: {:?}", e))? + .beacon_state; + + Ok((state, block)) + } + + /// Returns the most recent finalized state and block. + pub fn finalized(&self) -> Result<(BeaconState, BeaconBlock), String> { + let slots_per_epoch = get_slots_per_epoch(self.url.clone()) + .map_err(|e| format!("Unable to get slots per epoch: {:?}", e))?; + let finalized_slot = get_finalized_slot(self.url.clone(), slots_per_epoch.as_u64()) + .map_err(|e| format!("Unable to get finalized slot: {:?}", e))?; + + let block = get_block(self.url.clone(), finalized_slot) + .map_err(|e| format!("Unable to get finalized block: {:?}", e))? + .beacon_block; + let state = get_state(self.url.clone(), finalized_slot) + .map_err(|e| format!("Unable to get finalized state: {:?}", e))? + .beacon_state; + + Ok((state, block)) + } +} + +fn get_slots_per_epoch(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("spec").push("slots_per_epoch"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_eth2_config(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("spec").push("eth2_config"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_finalized_slot(mut url: Url, slots_per_epoch: u64) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("beacon").push("latest_finalized_checkpoint"); + }) + .map_err(|_| Error::InvalidUrl)?; + + let checkpoint: Checkpoint = reqwest::get(url)?.error_for_status()?.json()?; + + Ok(checkpoint.epoch.start_slot(slots_per_epoch)) +} + +#[derive(Deserialize)] +#[serde(bound = "T: EthSpec")] +pub struct StateResponse { + pub root: Hash256, + pub beacon_state: BeaconState, +} + +fn get_state(mut url: Url, slot: Slot) -> Result, Error> { + url.path_segments_mut() + .map(|mut url| { + url.push("beacon").push("state"); + }) + .map_err(|_| Error::InvalidUrl)?; + + url.query_pairs_mut() + .append_pair("slot", &format!("{}", slot.as_u64())); + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +#[derive(Deserialize)] +#[serde(bound = "T: EthSpec")] +pub struct BlockResponse { + pub root: Hash256, + pub beacon_block: BeaconBlock, +} + +fn get_block(mut url: Url, slot: Slot) -> Result, Error> { + url.path_segments_mut() + .map(|mut url| { + url.push("beacon").push("block"); + }) + .map_err(|_| Error::InvalidUrl)?; + + url.query_pairs_mut() + .append_pair("slot", &format!("{}", slot.as_u64())); + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_enr(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("network").push("enr"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_listen_port(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("network").push("listen_port"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index cc7725dd83..560da65197 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -3,6 +3,8 @@ extern crate lazy_static; mod beacon_chain; +mod beacon_chain_builder; +mod bootstrapper; mod checkpoint; mod errors; mod fork_choice; @@ -16,6 +18,8 @@ pub use self::beacon_chain::{ }; pub use self::checkpoint::CheckPoint; pub use self::errors::{BeaconChainError, BlockProductionError}; +pub use beacon_chain_builder::BeaconChainBuilder; +pub use bootstrapper::Bootstrapper; pub use lmd_ghost; pub use metrics::scrape_for_metrics; pub use parking_lot; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 4d6e56b041..6ab657b087 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1,12 +1,10 @@ use crate::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use lmd_ghost::LmdGhost; use sloggers::{null::NullLoggerBuilder, Build}; -use slot_clock::SlotClock; use slot_clock::TestingSlotClock; use state_processing::per_slot_processing; use std::marker::PhantomData; use std::sync::Arc; -use std::time::Duration; use store::MemoryStore; use store::Store; use tree_hash::{SignedRoot, TreeHash}; @@ -115,23 +113,9 @@ where let builder = NullLoggerBuilder; let log = builder.build().expect("logger should build"); - // Slot clock - let slot_clock = TestingSlotClock::from_eth2_genesis( - spec.genesis_slot, - genesis_state.genesis_time, - Duration::from_secs(spec.seconds_per_slot), - ) - .expect("Slot clock should start"); - - let chain = BeaconChain::from_genesis( - store, - slot_clock, - genesis_state, - genesis_block, - spec.clone(), - log, - ) - .expect("Terminate if beacon chain generation fails"); + let chain = + BeaconChain::from_genesis(store, genesis_state, genesis_block, spec.clone(), log) + .expect("Terminate if beacon chain generation fails"); Self { chain, diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs deleted file mode 100644 index 5168c067a9..0000000000 --- a/beacon_node/client/src/beacon_chain_types.rs +++ /dev/null @@ -1,169 +0,0 @@ -use crate::bootstrapper::Bootstrapper; -use crate::error::Result; -use crate::{config::GenesisState, ClientConfig}; -use beacon_chain::{ - lmd_ghost::{LmdGhost, ThreadSafeReducedTree}, - slot_clock::SystemTimeSlotClock, - store::Store, - BeaconChain, BeaconChainTypes, -}; -use slog::{crit, info, Logger}; -use slot_clock::SlotClock; -use std::fs::File; -use std::marker::PhantomData; -use std::sync::Arc; -use std::time::SystemTime; -use tree_hash::TreeHash; -use types::{ - test_utils::TestingBeaconStateBuilder, BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, -}; - -/// Provides a new, initialized `BeaconChain` -pub trait InitialiseBeaconChain { - fn initialise_beacon_chain( - store: Arc, - config: &ClientConfig, - spec: ChainSpec, - log: Logger, - ) -> Result> { - maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, config, spec, log) - } -} - -#[derive(Clone)] -pub struct ClientType { - _phantom_t: PhantomData, - _phantom_u: PhantomData, -} - -impl BeaconChainTypes for ClientType -where - S: Store + 'static, - E: EthSpec, -{ - type Store = S; - type SlotClock = SystemTimeSlotClock; - type LmdGhost = ThreadSafeReducedTree; - type EthSpec = E; -} -impl InitialiseBeaconChain for ClientType {} - -/// Loads a `BeaconChain` from `store`, if it exists. Otherwise, create a new chain from genesis. -fn maybe_load_from_store_for_testnet( - store: Arc, - config: &ClientConfig, - spec: ChainSpec, - log: Logger, -) -> Result> -where - T: BeaconChainTypes, - T::LmdGhost: LmdGhost, -{ - let genesis_state = match &config.genesis_state { - GenesisState::Mainnet => { - crit!(log, "This release does not support mainnet genesis state."); - return Err("Mainnet is unsupported".into()); - } - GenesisState::RecentGenesis { validator_count } => { - generate_testnet_genesis_state(*validator_count, recent_genesis_time(), &spec) - } - GenesisState::Generated { - validator_count, - genesis_time, - } => generate_testnet_genesis_state(*validator_count, *genesis_time, &spec), - GenesisState::Yaml { file } => { - let file = File::open(file).map_err(|e| { - format!("Unable to open YAML genesis state file {:?}: {:?}", file, e) - })?; - - serde_yaml::from_reader(file) - .map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))? - } - GenesisState::HttpBootstrap { server } => { - let bootstrapper = Bootstrapper::from_server_string(server.to_string()) - .map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?; - - let (state, _block) = bootstrapper - .genesis() - .map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?; - - state - } - }; - - let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); - let genesis_block_root = genesis_block.canonical_root(); - - // Slot clock - let slot_clock = T::SlotClock::new( - spec.genesis_slot, - genesis_state.genesis_time, - spec.seconds_per_slot, - ); - - // Try load an existing `BeaconChain` from the store. If unable, create a new one. - if let Ok(Some(beacon_chain)) = - BeaconChain::from_store(store.clone(), spec.clone(), log.clone()) - { - // Here we check to ensure that the `BeaconChain` loaded from store has the expected - // genesis block. - // - // Without this check, it's possible that there will be an existing DB with a `BeaconChain` - // that has different parameters than provided to this executable. - if beacon_chain.genesis_block_root == genesis_block_root { - info!( - log, - "Loaded BeaconChain from store"; - "slot" => beacon_chain.head().beacon_state.slot, - "best_slot" => beacon_chain.best_slot(), - ); - - Ok(beacon_chain) - } else { - crit!( - log, - "The BeaconChain loaded from disk has an incorrect genesis root. \ - This may be caused by an old database in located in datadir." - ); - Err("Incorrect genesis root".into()) - } - } else { - BeaconChain::from_genesis( - store, - slot_clock, - genesis_state, - genesis_block, - spec, - log.clone(), - ) - .map_err(|e| format!("Failed to initialize new beacon chain: {:?}", e).into()) - } -} - -fn generate_testnet_genesis_state( - validator_count: usize, - genesis_time: u64, - spec: &ChainSpec, -) -> BeaconState { - let (mut genesis_state, _keypairs) = - TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, spec) - .build(); - - genesis_state.genesis_time = genesis_time; - - genesis_state -} - -/// Returns the system time, mod 30 minutes. -/// -/// Used for easily creating testnets. -fn recent_genesis_time() -> u64 { - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs(); - let secs_after_last_period = now.checked_rem(30 * 60).unwrap_or(0); - // genesis is now the last 30 minute block. - now - secs_after_last_period -} diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index ea8186dbc9..f2725b3e79 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -1,15 +1,11 @@ -use crate::{Bootstrapper, Eth2Config}; use clap::ArgMatches; use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; -use slog::{info, o, warn, Drain}; +use slog::{info, o, Drain}; use std::fs::{self, OpenOptions}; use std::path::PathBuf; use std::sync::Mutex; -/// The number initial validators when starting the `Minimal`. -const TESTNET_VALIDATOR_COUNT: usize = 16; - /// The number initial validators when starting the `Minimal`. const TESTNET_SPEC_CONSTANTS: &str = "minimal"; @@ -21,33 +17,49 @@ pub struct Config { db_name: String, pub log_file: PathBuf, pub spec_constants: String, - pub genesis_state: GenesisState, + /// Defines how we should initialize a BeaconChain instances. + /// + /// This field is not serialized, there for it will not be written to (or loaded from) config + /// files. It can only be configured via the CLI. + #[serde(skip)] + pub beacon_chain_start_method: BeaconChainStartMethod, pub network: network::NetworkConfig, pub rpc: rpc::RPCConfig, pub rest_api: rest_api::ApiConfig, } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum GenesisState { - /// Use the mainnet genesis state. - /// - /// Mainnet genesis state is not presently known, so this is a place-holder. +/// Defines how the client should initialize a BeaconChain. +/// +/// In general, there are two methods: +/// - resuming a new chain, or +/// - initializing a new one. +#[derive(Debug, Clone)] +pub enum BeaconChainStartMethod { + /// Resume from an existing BeaconChain, loaded from the existing local database. + Resume, + /// Resume from an existing BeaconChain, loaded from the existing local database. Mainnet, - /// Generate a state with `validator_count` validators, all with well-known secret keys. + /// Create a new beacon chain that can connect to mainnet. /// /// Set the genesis time to be the start of the previous 30-minute window. RecentGenesis { validator_count: usize }, - /// Generate a state with `genesis_time` and `validator_count` validators, all with well-known + /// Create a new beacon chain with `genesis_time` and `validator_count` validators, all with well-known /// secret keys. Generated { validator_count: usize, genesis_time: u64, }, - /// Load a YAML-encoded genesis state from a file. + /// Create a new beacon chain by loading a YAML-encoded genesis state from a file. Yaml { file: PathBuf }, - /// Use a HTTP server (running our REST-API) to load genesis and finalized states and blocks. - HttpBootstrap { server: String }, + /// Create a new beacon chain by using a HTTP server (running our REST-API) to load genesis and + /// finalized states and blocks. + HttpBootstrap { server: String, port: Option }, +} + +impl Default for BeaconChainStartMethod { + fn default() -> Self { + BeaconChainStartMethod::Resume + } } impl Default for Config { @@ -58,12 +70,10 @@ impl Default for Config { db_type: "disk".to_string(), db_name: "chain_db".to_string(), network: NetworkConfig::new(), - rpc: rpc::RPCConfig::default(), - rest_api: rest_api::ApiConfig::default(), + rpc: <_>::default(), + rest_api: <_>::default(), spec_constants: TESTNET_SPEC_CONSTANTS.into(), - genesis_state: GenesisState::RecentGenesis { - validator_count: TESTNET_VALIDATOR_COUNT, - }, + beacon_chain_start_method: <_>::default(), } } } @@ -127,15 +137,6 @@ impl Config { self.data_dir = PathBuf::from(dir); }; - if let Some(default_spec) = args.value_of("default-spec") { - match default_spec { - "mainnet" => self.spec_constants = Eth2Config::mainnet().spec_constants, - "minimal" => self.spec_constants = Eth2Config::minimal().spec_constants, - "interop" => self.spec_constants = Eth2Config::interop().spec_constants, - _ => {} // not supported - } - } - if let Some(dir) = args.value_of("db") { self.db_type = dir.to_string(); }; @@ -149,40 +150,6 @@ impl Config { self.update_logger(log)?; }; - // If the `--bootstrap` flag is provided, overwrite the default configuration. - if let Some(server) = args.value_of("bootstrap") { - do_bootstrapping(self, server.to_string(), &log)?; - } - Ok(()) } } - -/// Perform the HTTP bootstrapping procedure, reading an ENR and multiaddr from the HTTP server and -/// adding them to the `config`. -fn do_bootstrapping(config: &mut Config, server: String, log: &slog::Logger) -> Result<(), String> { - // Set the genesis state source. - config.genesis_state = GenesisState::HttpBootstrap { - server: server.to_string(), - }; - - let bootstrapper = Bootstrapper::from_server_string(server.to_string())?; - - config.network.boot_nodes.push(bootstrapper.enr()?); - - if let Some(server_multiaddr) = bootstrapper.best_effort_multiaddr() { - info!( - log, - "Estimated bootstrapper libp2p address"; - "multiaddr" => format!("{:?}", server_multiaddr) - ); - config.network.libp2p_nodes.push(server_multiaddr); - } else { - warn!( - log, - "Unable to estimate a bootstrapper libp2p address, this node may not find any peers." - ); - } - - Ok(()) -} diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 7dd7118a78..2612fd6489 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -1,31 +1,47 @@ extern crate slog; -mod beacon_chain_types; -mod bootstrapper; mod config; pub mod error; pub mod notifier; -use beacon_chain::BeaconChain; +use beacon_chain::{ + lmd_ghost::ThreadSafeReducedTree, slot_clock::SystemTimeSlotClock, store::Store, BeaconChain, + BeaconChainBuilder, +}; use exit_future::Signal; use futures::{future::Future, Stream}; use network::Service as NetworkService; -use slog::{error, info, o}; +use slog::{crit, error, info, o}; use slot_clock::SlotClock; use std::marker::PhantomData; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::runtime::TaskExecutor; use tokio::timer::Interval; +use types::EthSpec; pub use beacon_chain::BeaconChainTypes; -pub use beacon_chain_types::ClientType; -pub use beacon_chain_types::InitialiseBeaconChain; -pub use bootstrapper::Bootstrapper; -pub use config::{Config as ClientConfig, GenesisState}; +pub use config::{BeaconChainStartMethod, Config as ClientConfig}; pub use eth2_config::Eth2Config; +#[derive(Clone)] +pub struct ClientType { + _phantom_t: PhantomData, + _phantom_u: PhantomData, +} + +impl BeaconChainTypes for ClientType +where + S: Store + 'static, + E: EthSpec, +{ + type Store = S; + type SlotClock = SystemTimeSlotClock; + type LmdGhost = ThreadSafeReducedTree; + type EthSpec = E; +} + /// Main beacon node client service. This provides the connection and initialisation of the clients /// sub-services in multiple threads. pub struct Client { @@ -49,7 +65,7 @@ pub struct Client { impl Client where - T: BeaconChainTypes + InitialiseBeaconChain + Clone, + T: BeaconChainTypes + Clone, { /// Generate an instance of the client. Spawn and link all internal sub-processes. pub fn new( @@ -62,13 +78,41 @@ where let store = Arc::new(store); let seconds_per_slot = eth2_config.spec.seconds_per_slot; - // Load a `BeaconChain` from the store, or create a new one if it does not exist. - let beacon_chain = Arc::new(T::initialise_beacon_chain( - store, - &client_config, - eth2_config.spec.clone(), - log.clone(), - )?); + let spec = ð2_config.spec.clone(); + + let beacon_chain_builder = match &client_config.beacon_chain_start_method { + BeaconChainStartMethod::Resume => { + BeaconChainBuilder::from_store(spec.clone(), log.clone()) + } + BeaconChainStartMethod::Mainnet => { + crit!(log, "No mainnet beacon chain startup specification."); + return Err("Mainnet is not yet specified. We're working on it.".into()); + } + BeaconChainStartMethod::RecentGenesis { validator_count } => { + BeaconChainBuilder::recent_genesis(*validator_count, spec.clone(), log.clone()) + } + BeaconChainStartMethod::Generated { + validator_count, + genesis_time, + } => BeaconChainBuilder::quick_start( + *genesis_time, + *validator_count, + spec.clone(), + log.clone(), + ), + BeaconChainStartMethod::Yaml { file } => { + BeaconChainBuilder::yaml_state(file, spec.clone(), log.clone())? + } + BeaconChainStartMethod::HttpBootstrap { server, .. } => { + BeaconChainBuilder::http_bootstrap(server, spec.clone(), log.clone())? + } + }; + + let beacon_chain: Arc> = Arc::new( + beacon_chain_builder + .build(store) + .map_err(error::Error::from)?, + ); if beacon_chain.read_slot_clock().is_none() { panic!("Cannot start client before genesis!") @@ -118,6 +162,7 @@ where beacon_chain.clone(), network.clone(), client_config.db_path().expect("unable to read datadir"), + eth2_config.clone(), &log, ) { Ok(s) => Some(s), diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml index cac196d9cb..5303dc8bdc 100644 --- a/beacon_node/rest_api/Cargo.toml +++ b/beacon_node/rest_api/Cargo.toml @@ -27,5 +27,6 @@ exit-future = "0.1.3" tokio = "0.1.17" url = "2.0" lazy_static = "1.3.0" +eth2_config = { path = "../../eth2/utils/eth2_config" } lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } slot_clock = { path = "../../eth2/utils/slot_clock" } diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 964dd79982..b1137c2493 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -13,6 +13,7 @@ mod url_query; use beacon_chain::{BeaconChain, BeaconChainTypes}; use client_network::Service as NetworkService; +use eth2_config::Eth2Config; use hyper::rt::Future; use hyper::service::service_fn_ok; use hyper::{Body, Method, Response, Server, StatusCode}; @@ -79,6 +80,7 @@ pub fn start_server( beacon_chain: Arc>, network_service: Arc>, db_path: PathBuf, + eth2_config: Eth2Config, log: &slog::Logger, ) -> Result { let log = log.new(o!("Service" => "Api")); @@ -100,12 +102,14 @@ pub fn start_server( // Clone our stateful objects, for use in service closure. let server_log = log.clone(); let server_bc = beacon_chain.clone(); + let eth2_config = Arc::new(eth2_config); let service = move || { let log = server_log.clone(); let beacon_chain = server_bc.clone(); let db_path = db_path.clone(); let network_service = network_service.clone(); + let eth2_config = eth2_config.clone(); // Create a simple handler for the router, inject our stateful objects into the request. service_fn_ok(move |mut req| { @@ -118,6 +122,8 @@ pub fn start_server( req.extensions_mut().insert::(db_path.clone()); req.extensions_mut() .insert::>>(network_service.clone()); + req.extensions_mut() + .insert::>(eth2_config.clone()); let path = req.uri().path().to_string(); @@ -144,6 +150,7 @@ pub fn start_server( (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), (&Method::GET, "/spec") => spec::get_spec::(req), (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::(req), + (&Method::GET, "/spec/eth2_config") => spec::get_eth2_config::(req), _ => Err(ApiError::MethodNotAllowed(path.clone())), }; diff --git a/beacon_node/rest_api/src/spec.rs b/beacon_node/rest_api/src/spec.rs index d0c8e4368d..86d1c227d3 100644 --- a/beacon_node/rest_api/src/spec.rs +++ b/beacon_node/rest_api/src/spec.rs @@ -1,6 +1,7 @@ use super::{success_response, ApiResult}; use crate::ApiError; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use eth2_config::Eth2Config; use hyper::{Body, Request}; use std::sync::Arc; use types::EthSpec; @@ -18,6 +19,19 @@ pub fn get_spec(req: Request) -> ApiResult Ok(success_response(Body::from(json))) } +/// HTTP handler to return the full Eth2Config object. +pub fn get_eth2_config(req: Request) -> ApiResult { + let eth2_config = req + .extensions() + .get::>() + .ok_or_else(|| ApiError::ServerError("Eth2Config extension missing".to_string()))?; + + let json: String = serde_json::to_string(eth2_config.as_ref()) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize Eth2Config: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} + /// HTTP handler to return the full spec object. pub fn get_slots_per_epoch(_req: Request) -> ApiResult { let json: String = serde_json::to_string(&T::EthSpec::slots_per_epoch()) diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs new file mode 100644 index 0000000000..e76bd48fa7 --- /dev/null +++ b/beacon_node/src/config.rs @@ -0,0 +1,435 @@ +use beacon_chain::Bootstrapper; +use clap::ArgMatches; +use client::{BeaconChainStartMethod, ClientConfig, Eth2Config}; +use eth2_config::{read_from_file, write_to_file}; +use rand::{distributions::Alphanumeric, Rng}; +use slog::{crit, info, warn, Logger}; +use std::fs; +use std::path::{Path, PathBuf}; + +pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; +pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml"; +pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml"; + +type Result = std::result::Result; +type Config = (ClientConfig, Eth2Config); + +/// Gets the fully-initialized global client and eth2 configuration objects. +pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result { + let mut builder = ConfigBuilder::new(cli_args, log)?; + + match cli_args.subcommand() { + ("testnet", Some(sub_cmd_args)) => { + process_testnet_subcommand(&mut builder, sub_cmd_args, log)? + } + // No sub-command assumes a resume operation. + _ => { + info!( + log, + "Resuming from existing datadir"; + "path" => format!("{:?}", builder.data_dir) + ); + + // If no primary subcommand was given, start the beacon chain from an existing + // database. + builder.set_beacon_chain_start_method(BeaconChainStartMethod::Resume); + + // Whilst there is no large testnet or mainnet force the user to specify how they want + // to start a new chain (e.g., from a genesis YAML file, another node, etc). + if !builder.data_dir.exists() { + return Err( + "No datadir found. To start a new beacon chain, see `testnet --help`. \ + Use `--datadir` to specify a different directory" + .into(), + ); + } + + // If the `testnet` command was not provided, attempt to load an existing datadir and + // continue with an existing chain. + builder.load_from_datadir()?; + } + }; + + builder.build(cli_args) +} + +/// Process the `testnet` CLI subcommand arguments, updating the `builder`. +fn process_testnet_subcommand( + builder: &mut ConfigBuilder, + cli_args: &ArgMatches, + log: &Logger, +) -> Result<()> { + if cli_args.is_present("random-datadir") { + builder.set_random_datadir()?; + } + + let is_bootstrap = cli_args.subcommand_name() == Some("bootstrap"); + + if let Some(path_string) = cli_args.value_of("eth2-config") { + if is_bootstrap { + return Err("Cannot supply --eth2-config when using bootsrap".to_string()); + } + + let path = path_string + .parse::() + .map_err(|e| format!("Unable to parse eth2-config path: {:?}", e))?; + builder.load_eth2_config(path)?; + } else { + builder.update_spec_from_subcommand(&cli_args)?; + } + + if let Some(path_string) = cli_args.value_of("client-config") { + let path = path_string + .parse::() + .map_err(|e| format!("Unable to parse client config path: {:?}", e))?; + builder.load_client_config(path)?; + } + + if cli_args.is_present("force") { + builder.clean_datadir()?; + } + + info!( + log, + "Creating new datadir"; + "path" => format!("{:?}", builder.data_dir) + ); + + // Start matching on the second subcommand (e.g., `testnet bootstrap ...`) + match cli_args.subcommand() { + ("bootstrap", Some(cli_args)) => { + let server = cli_args + .value_of("server") + .ok_or_else(|| "No bootstrap server specified")?; + let port: Option = cli_args + .value_of("libp2p-port") + .and_then(|s| s.parse::().ok()); + + builder.import_bootstrap_libp2p_address(server, port)?; + builder.import_bootstrap_eth2_config(server)?; + + builder.set_beacon_chain_start_method(BeaconChainStartMethod::HttpBootstrap { + server: server.to_string(), + port, + }) + } + ("recent", Some(cli_args)) => { + let validator_count = cli_args + .value_of("validator_count") + .ok_or_else(|| "No validator_count specified")? + .parse::() + .map_err(|e| format!("Unable to parse validator_count: {:?}", e))?; + + builder.set_beacon_chain_start_method(BeaconChainStartMethod::RecentGenesis { + validator_count, + }) + } + _ => return Err("No testnet method specified. See 'testnet --help'.".into()), + }; + + builder.write_configs_to_new_datadir()?; + + Ok(()) +} + +/// Allows for building a set of configurations based upon `clap` arguments. +struct ConfigBuilder<'a> { + log: &'a Logger, + pub data_dir: PathBuf, + eth2_config: Eth2Config, + client_config: ClientConfig, +} + +impl<'a> ConfigBuilder<'a> { + /// Create a new builder with default settings. + pub fn new(cli_args: &'a ArgMatches, log: &'a Logger) -> Result { + // Read the `--datadir` flag. + // + // If it's not present, try and find the home directory (`~`) and push the default data + // directory onto it. + let data_dir: PathBuf = cli_args + .value_of("datadir") + .map(|string| PathBuf::from(string)) + .or_else(|| { + dirs::home_dir().map(|mut home| { + home.push(DEFAULT_DATA_DIR); + home + }) + }) + .ok_or_else(|| "Unable to find a home directory for the datadir".to_string())?; + + Ok(Self { + log, + data_dir, + eth2_config: Eth2Config::minimal(), + client_config: ClientConfig::default(), + }) + } + + /// Clears any configuration files that would interfere with writing new configs. + /// + /// Moves the following files in `data_dir` into a backup directory: + /// + /// - Client config + /// - Eth2 config + /// - The entire database directory + pub fn clean_datadir(&mut self) -> Result<()> { + let backup_dir = { + let mut s = String::from("backup_"); + s.push_str(&random_string(6)); + self.data_dir.join(s) + }; + + fs::create_dir_all(&backup_dir) + .map_err(|e| format!("Unable to create config backup dir: {:?}", e))?; + + let move_to_backup_dir = |path: &Path| -> Result<()> { + let file_name = path + .file_name() + .ok_or_else(|| "Invalid path found during datadir clean (no filename).")?; + + let mut new = path.to_path_buf(); + new.pop(); + new.push(backup_dir.clone()); + new.push(file_name); + + let _ = fs::rename(path, new); + + Ok(()) + }; + + move_to_backup_dir(&self.data_dir.join(CLIENT_CONFIG_FILENAME))?; + move_to_backup_dir(&self.data_dir.join(ETH2_CONFIG_FILENAME))?; + + if let Some(db_path) = self.client_config.db_path() { + move_to_backup_dir(&db_path)?; + } + + Ok(()) + } + + /// Sets the method for starting the beacon chain. + pub fn set_beacon_chain_start_method(&mut self, method: BeaconChainStartMethod) { + self.client_config.beacon_chain_start_method = method; + } + + /// Import the libp2p address for `server` into the list of bootnodes in `self`. + /// + /// If `port` is `Some`, it is used as the port for the `Multiaddr`. If `port` is `None`, + /// attempts to connect to the `server` via HTTP and retrieve it's libp2p listen port. + pub fn import_bootstrap_libp2p_address( + &mut self, + server: &str, + port: Option, + ) -> Result<()> { + let bootstrapper = Bootstrapper::from_server_string(server.to_string())?; + + if let Some(server_multiaddr) = bootstrapper.best_effort_multiaddr(port) { + info!( + self.log, + "Estimated bootstrapper libp2p address"; + "multiaddr" => format!("{:?}", server_multiaddr) + ); + + self.client_config + .network + .libp2p_nodes + .push(server_multiaddr); + } else { + warn!( + self.log, + "Unable to estimate a bootstrapper libp2p address, this node may not find any peers." + ); + }; + + Ok(()) + } + + /// Set the config data_dir to be an random directory. + /// + /// Useful for easily spinning up ephemeral testnets. + pub fn set_random_datadir(&mut self) -> Result<()> { + let mut s = DEFAULT_DATA_DIR.to_string(); + s.push_str("_random_"); + s.push_str(&random_string(6)); + + self.data_dir.pop(); + self.data_dir.push(s); + + Ok(()) + } + + /// Imports an `Eth2Config` from `server`, returning an error if this fails. + pub fn import_bootstrap_eth2_config(&mut self, server: &str) -> Result<()> { + let bootstrapper = Bootstrapper::from_server_string(server.to_string())?; + + self.update_eth2_config(bootstrapper.eth2_config()?); + + Ok(()) + } + + fn update_eth2_config(&mut self, eth2_config: Eth2Config) { + self.eth2_config = eth2_config; + } + + /// Reads the subcommand and tries to update `self.eth2_config` based up on the `--spec` flag. + /// + /// Returns an error if the `--spec` flag is not present in the given `cli_args`. + pub fn update_spec_from_subcommand(&mut self, cli_args: &ArgMatches) -> Result<()> { + // Re-initialise the `Eth2Config`. + // + // If a CLI parameter is set, overwrite any config file present. + // If a parameter is not set, use either the config file present or default to minimal. + let eth2_config = match cli_args.value_of("spec") { + Some("mainnet") => Eth2Config::mainnet(), + Some("minimal") => Eth2Config::minimal(), + Some("interop") => Eth2Config::interop(), + _ => return Err("Unable to determine specification type.".into()), + }; + + self.client_config.spec_constants = cli_args + .value_of("spec") + .expect("Guarded by prior match statement") + .to_string(); + self.eth2_config = eth2_config; + + Ok(()) + } + + /// Writes the configs in `self` to `self.data_dir`. + /// + /// Returns an error if `self.data_dir` already exists. + pub fn write_configs_to_new_datadir(&mut self) -> Result<()> { + let db_exists = self + .client_config + .db_path() + .map(|d| d.exists()) + .unwrap_or_else(|| false); + + // Do not permit creating a new config when the datadir exists. + if db_exists { + return Err("Database already exists. See `-f` in `testnet --help`".into()); + } + + // Create `datadir` and any non-existing parent directories. + fs::create_dir_all(&self.data_dir).map_err(|e| { + crit!(self.log, "Failed to initialize data dir"; "error" => format!("{}", e)); + format!("{}", e) + })?; + + let client_config_file = self.data_dir.join(CLIENT_CONFIG_FILENAME); + if client_config_file.exists() { + return Err(format!( + "Datadir is not clean, {} exists. See `-f` in `testnet --help`.", + CLIENT_CONFIG_FILENAME + )); + } else { + // Write the onfig to a TOML file in the datadir. + write_to_file( + self.data_dir.join(CLIENT_CONFIG_FILENAME), + &self.client_config, + ) + .map_err(|e| format!("Unable to write {} file: {:?}", CLIENT_CONFIG_FILENAME, e))?; + } + + let eth2_config_file = self.data_dir.join(ETH2_CONFIG_FILENAME); + if eth2_config_file.exists() { + return Err(format!( + "Datadir is not clean, {} exists. See `-f` in `testnet --help`.", + ETH2_CONFIG_FILENAME + )); + } else { + // Write the config to a TOML file in the datadir. + write_to_file(self.data_dir.join(ETH2_CONFIG_FILENAME), &self.eth2_config) + .map_err(|e| format!("Unable to write {} file: {:?}", ETH2_CONFIG_FILENAME, e))?; + } + + Ok(()) + } + + /// Attempts to load the client and eth2 configs from `self.data_dir`. + /// + /// Returns an error if any files are not found or are invalid. + pub fn load_from_datadir(&mut self) -> Result<()> { + // Check to ensure the datadir exists. + // + // For now we return an error. In the future we may decide to boot a default (e.g., + // public testnet or mainnet). + if !self.data_dir.exists() { + return Err( + "No datadir found. Either create a new testnet or specify a different `--datadir`." + .into(), + ); + } + + // If there is a path to a databse in the config, ensure it exists. + if !self + .client_config + .db_path() + .map(|path| path.exists()) + .unwrap_or_else(|| true) + { + return Err( + "No database found in datadir. Use 'testnet -f' to overwrite the existing \ + datadir, or specify a different `--datadir`." + .into(), + ); + } + + self.load_eth2_config(self.data_dir.join(ETH2_CONFIG_FILENAME))?; + self.load_client_config(self.data_dir.join(CLIENT_CONFIG_FILENAME))?; + + Ok(()) + } + + /// Attempts to load the client config from `path`. + /// + /// Returns an error if any files are not found or are invalid. + pub fn load_client_config(&mut self, path: PathBuf) -> Result<()> { + self.client_config = read_from_file::(path.clone()) + .map_err(|e| format!("Unable to parse {:?} file: {:?}", path, e))? + .ok_or_else(|| format!("{:?} file does not exist", path))?; + + Ok(()) + } + + /// Attempts to load the eth2 config from `path`. + /// + /// Returns an error if any files are not found or are invalid. + pub fn load_eth2_config(&mut self, path: PathBuf) -> Result<()> { + self.eth2_config = read_from_file::(path.clone()) + .map_err(|e| format!("Unable to parse {:?} file: {:?}", path, e))? + .ok_or_else(|| format!("{:?} file does not exist", path))?; + + Ok(()) + } + + /// Consumes self, returning the configs. + /// + /// The supplied `cli_args` should be the base-level `clap` cli_args (i.e., not a subcommand + /// cli_args). + pub fn build(mut self, cli_args: &ArgMatches) -> Result { + self.eth2_config.apply_cli_args(cli_args)?; + self.client_config + .apply_cli_args(cli_args, &mut self.log.clone())?; + + if self.eth2_config.spec_constants != self.client_config.spec_constants { + crit!(self.log, "Specification constants do not match."; + "client_config" => format!("{}", self.client_config.spec_constants), + "eth2_config" => format!("{}", self.eth2_config.spec_constants) + ); + return Err("Specification constant mismatch".into()); + } + + self.client_config.data_dir = self.data_dir; + + Ok((self.client_config, self.eth2_config)) + } +} + +fn random_string(len: usize) -> String { + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(len) + .collect::() +} diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 0eb5b83b43..aba44e6fe5 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -1,12 +1,10 @@ +mod config; mod run; -use clap::{App, Arg}; -use client::{ClientConfig, Eth2Config}; +use clap::{App, Arg, SubCommand}; +use config::get_configs; use env_logger::{Builder, Env}; -use eth2_config::{read_from_file, write_to_file}; use slog::{crit, o, warn, Drain, Level}; -use std::fs; -use std::path::PathBuf; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml"; @@ -30,6 +28,7 @@ fn main() { .value_name("DIR") .help("Data directory for keys and databases.") .takes_value(true) + .global(true) ) .arg( Arg::with_name("logfile") @@ -44,6 +43,7 @@ fn main() { .value_name("NETWORK-DIR") .help("Data directory for network keys.") .takes_value(true) + .global(true) ) /* * Network parameters. @@ -160,25 +160,7 @@ fn main() { .help("Type of database to use.") .takes_value(true) .possible_values(&["disk", "memory"]) - .default_value("memory"), - ) - /* - * Specification/testnet params. - */ - .arg( - Arg::with_name("default-spec") - .long("default-spec") - .value_name("TITLE") - .short("default-spec") - .help("Specifies the default eth2 spec to be used. This will override any spec written to disk and will therefore be used by default in future instances.") - .takes_value(true) - .possible_values(&["mainnet", "minimal", "interop"]) - ) - .arg( - Arg::with_name("recent-genesis") - .long("recent-genesis") - .short("r") - .help("When present, genesis will be within 30 minutes prior. Only for testing"), + .default_value("disk"), ) /* * Logging. @@ -200,14 +182,97 @@ fn main() { .takes_value(true), ) /* - * Bootstrap. + * The "testnet" sub-command. + * + * Allows for creating a new datadir with testnet-specific configs. */ - .arg( - Arg::with_name("bootstrap") - .long("bootstrap") - .value_name("HTTP_SERVER") - .help("Load the genesis state and libp2p address from the HTTP API of another Lighthouse node.") - .takes_value(true) + .subcommand(SubCommand::with_name("testnet") + .about("Create a new Lighthouse datadir using a testnet strategy.") + .arg( + Arg::with_name("spec") + .short("s") + .long("spec") + .value_name("TITLE") + .help("Specifies the default eth2 spec type. Only effective when creating a new datadir.") + .takes_value(true) + .required(true) + .possible_values(&["mainnet", "minimal", "interop"]) + .default_value("minimal") + ) + .arg( + Arg::with_name("eth2-config") + .long("eth2-config") + .value_name("TOML_FILE") + .help("A existing eth2_spec TOML file (e.g., eth2_spec.toml).") + .takes_value(true) + .conflicts_with("spec") + ) + .arg( + Arg::with_name("client-config") + .long("client-config") + .value_name("TOML_FILE") + .help("An existing beacon_node TOML file (e.g., beacon_node.toml).") + .takes_value(true) + ) + .arg( + Arg::with_name("random-datadir") + .long("random-datadir") + .short("r") + .help("If present, append a random string to the datadir path. Useful for fast development \ + iteration.") + ) + .arg( + Arg::with_name("force") + .long("force") + .short("f") + .help("If present, will create new config and database files and move the any existing to a \ + backup directory.") + .conflicts_with("random-datadir") + ) + /* + * Testnet sub-commands. + * + * `boostrap` + * + * Start a new node by downloading genesis and network info from another node via the + * HTTP API. + */ + .subcommand(SubCommand::with_name("bootstrap") + .about("Connects to the given HTTP server, downloads a genesis state and attempts to peer with it.") + .arg(Arg::with_name("server") + .value_name("HTTP_SERVER") + .required(true) + .help("A HTTP server, with a http:// prefix")) + .arg(Arg::with_name("libp2p-port") + .short("p") + .long("port") + .value_name("TCP_PORT") + .help("A libp2p listen port used to peer with the bootstrap server. This flag is useful \ + when port-fowarding is used: you may connect using a different port than \ + the one the server is immediately listening on.")) + ) + /* + * `recent` + * + * Start a new node, with a specified number of validators with a genesis time in the last + * 30-minutes. + */ + .subcommand(SubCommand::with_name("recent") + .about("Creates a new genesis state where the genesis time was at the previous \ + 30-minute boundary (e.g., 12:00, 12:30, 13:00, etc.)") + .arg(Arg::with_name("validator_count") + .value_name("VALIDATOR_COUNT") + .required(true) + .help("The number of validators in the genesis state")) + ) + .subcommand(SubCommand::with_name("yaml-genesis-state") + .about("Creates a new datadir where the genesis state is read from YAML. Will fail to parse \ + a YAML state that was generated to a different spec than that specified by --spec.") + .arg(Arg::with_name("file") + .value_name("YAML_FILE") + .required(true) + .help("A YAML file from which to read the state")) + ) ) .get_matches(); @@ -227,143 +292,31 @@ fn main() { _ => unreachable!("guarded by clap"), }; - let mut log = slog::Logger::root(drain.fuse(), o!()); + let drain = match matches.occurrences_of("verbosity") { + 0 => drain.filter_level(Level::Info), + 1 => drain.filter_level(Level::Debug), + 2 => drain.filter_level(Level::Trace), + _ => drain.filter_level(Level::Trace), + }; + + let log = slog::Logger::root(drain.fuse(), o!()); warn!( log, "Ethereum 2.0 is pre-release. This software is experimental." ); - let data_dir = match matches - .value_of("datadir") - .and_then(|v| Some(PathBuf::from(v))) - { - Some(v) => v, - None => { - // use the default - let mut default_dir = match dirs::home_dir() { - Some(v) => v, - None => { - crit!(log, "Failed to find a home directory"); - return; - } - }; - default_dir.push(DEFAULT_DATA_DIR); - default_dir - } - }; - - // create the directory if needed - match fs::create_dir_all(&data_dir) { - Ok(_) => {} - Err(e) => { - crit!(log, "Failed to initialize data dir"; "error" => format!("{}", e)); - return; - } - } - - let client_config_path = data_dir.join(CLIENT_CONFIG_FILENAME); - - // Attempt to load the `ClientConfig` from disk. + // Load the process-wide configuration. // - // If file doesn't exist, create a new, default one. - let mut client_config = match read_from_file::(client_config_path.clone()) { - Ok(Some(c)) => c, - Ok(None) => { - let default = ClientConfig::default(); - if let Err(e) = write_to_file(client_config_path, &default) { - crit!(log, "Failed to write default ClientConfig to file"; "error" => format!("{:?}", e)); - return; - } - default - } + // May load this from disk or create a new configuration, depending on the CLI flags supplied. + let (client_config, eth2_config) = match get_configs(&matches, &log) { + Ok(configs) => configs, Err(e) => { - crit!(log, "Failed to load a ChainConfig file"; "error" => format!("{:?}", e)); + crit!(log, "Failed to load configuration"; "error" => e); return; } }; - // Ensure the `data_dir` in the config matches that supplied to the CLI. - client_config.data_dir = data_dir.clone(); - - // Update the client config with any CLI args. - match client_config.apply_cli_args(&matches, &mut log) { - Ok(()) => (), - Err(s) => { - crit!(log, "Failed to parse ClientConfig CLI arguments"; "error" => s); - return; - } - }; - - let eth2_config_path = data_dir.join(ETH2_CONFIG_FILENAME); - - // Initialise the `Eth2Config`. - // - // If a CLI parameter is set, overwrite any config file present. - // If a parameter is not set, use either the config file present or default to minimal. - let cli_config = match matches.value_of("default-spec") { - Some("mainnet") => Some(Eth2Config::mainnet()), - Some("minimal") => Some(Eth2Config::minimal()), - Some("interop") => Some(Eth2Config::interop()), - _ => None, - }; - // if a CLI flag is specified, write the new config if it doesn't exist, - // otherwise notify the user that the file will not be written. - let eth2_config_from_file = match read_from_file::(eth2_config_path.clone()) { - Ok(config) => config, - Err(e) => { - crit!(log, "Failed to read the Eth2Config from file"; "error" => format!("{:?}", e)); - return; - } - }; - - let mut eth2_config = { - if let Some(cli_config) = cli_config { - if eth2_config_from_file.is_none() { - // write to file if one doesn't exist - if let Err(e) = write_to_file(eth2_config_path, &cli_config) { - crit!(log, "Failed to write default Eth2Config to file"; "error" => format!("{:?}", e)); - return; - } - } else { - warn!( - log, - "Eth2Config file exists. Configuration file is ignored, using default" - ); - } - cli_config - } else { - // CLI config not specified, read from disk - match eth2_config_from_file { - Some(config) => config, - None => { - // set default to minimal - let eth2_config = Eth2Config::minimal(); - if let Err(e) = write_to_file(eth2_config_path, ð2_config) { - crit!(log, "Failed to write default Eth2Config to file"; "error" => format!("{:?}", e)); - return; - } - eth2_config - } - } - } - }; - - // Update the eth2 config with any CLI flags. - match eth2_config.apply_cli_args(&matches) { - Ok(()) => (), - Err(s) => { - crit!(log, "Failed to parse Eth2Config CLI arguments"; "error" => s); - return; - } - }; - - // check to ensure the spec constants between the client and eth2_config match - if eth2_config.spec_constants != client_config.spec_constants { - crit!(log, "Specification constants do not match."; "client_config" => format!("{}", client_config.spec_constants), "eth2_config" => format!("{}", eth2_config.spec_constants)); - return; - } - // Start the node using a `tokio` executor. match run::run_beacon_node(client_config, eth2_config, &log) { Ok(_) => {} diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index f88cb7460b..620cb64bb5 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -1,7 +1,4 @@ -use client::{ - error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, Eth2Config, - InitialiseBeaconChain, -}; +use client::{error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, Eth2Config}; use futures::sync::oneshot; use futures::Future; use slog::{error, info}; @@ -46,7 +43,6 @@ pub fn run_beacon_node( log, "BeaconNode init"; "p2p_listen_address" => format!("{:?}", &other_client_config.network.listen_address), - "data_dir" => format!("{:?}", other_client_config.data_dir()), "network_dir" => format!("{:?}", other_client_config.network.network_dir), "spec_constants" => &spec_constants, "db_type" => &other_client_config.db_type, @@ -118,7 +114,7 @@ fn run( log: &slog::Logger, ) -> error::Result<()> where - T: BeaconChainTypes + InitialiseBeaconChain + Clone, + T: BeaconChainTypes + Clone, T::Store: OpenDatabase, { let store = T::Store::open_database(&db_path)?;