diff --git a/Cargo.lock b/Cargo.lock index 9c2ab8bc2f..c3034cefce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -217,7 +217,6 @@ dependencies = [ "genesis 0.1.0", "integer-sqrt 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lighthouse_bootstrap 0.1.0", "lighthouse_metrics 0.1.0", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "lru 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -260,16 +259,17 @@ dependencies = [ "exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "genesis 0.1.0", - "lighthouse_bootstrap 0.1.0", "logging 0.1.0", "node_test_rig 0.1.0", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "slog-async 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "slog-term 2.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "store 0.1.0", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "types 0.1.0", "version 0.1.0", ] @@ -521,7 +521,6 @@ dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "genesis 0.1.0", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lighthouse_bootstrap 0.1.0", "lighthouse_metrics 0.1.0", "network 0.1.0", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2384,19 +2383,6 @@ dependencies = [ "validator_client 0.1.0", ] -[[package]] -name = "lighthouse_bootstrap" -version = "0.1.0" -dependencies = [ - "eth2-libp2p 0.1.0", - "eth2_config 0.1.0", - "reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "types 0.1.0", - "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "lighthouse_metrics" version = "0.1.0" @@ -4704,7 +4690,6 @@ dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", - "lighthouse_bootstrap 0.1.0", "logging 0.1.0", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 172fcb0365..0713dfebb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ members = [ "eth2/utils/logging", "eth2/utils/eth2_hashing", "eth2/utils/lighthouse_metrics", - "eth2/utils/lighthouse_bootstrap", "eth2/utils/merkle_proof", "eth2/utils/int_to_bytes", "eth2/utils/serde_hex", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 83d77a3c95..9ed5a27585 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -13,7 +13,6 @@ node_test_rig = { path = "../tests/node_test_rig" } [dependencies] eth2_config = { path = "../eth2/utils/eth2_config" } -lighthouse_bootstrap = { path = "../eth2/utils/lighthouse_bootstrap" } beacon_chain = { path = "beacon_chain" } types = { path = "../eth2/types" } store = { path = "./store" } @@ -37,3 +36,5 @@ genesis = { path = "genesis" } eth2_testnet_config = { path = "../eth2/utils/eth2_testnet_config" } eth2-libp2p = { path = "./eth2-libp2p" } eth2_ssz = { path = "../eth2/utils/ssz" } +toml = "0.5.4" +serde = "1.0.102" diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index bd8fcb5e97..61fee2a894 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -15,7 +15,6 @@ store = { path = "../store" } parking_lot = "0.9.0" lazy_static = "1.4.0" lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } -lighthouse_bootstrap = { path = "../../eth2/utils/lighthouse_bootstrap" } log = "0.4.8" operation_pool = { path = "../../eth2/operation_pool" } rayon = "1.2.0" diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index a53ce5455c..ea119557e0 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -36,7 +36,6 @@ url = "2.1.0" eth1 = { path = "../eth1" } genesis = { path = "../genesis" } environment = { path = "../../lighthouse/environment" } -lighthouse_bootstrap = { path = "../../eth2/utils/lighthouse_bootstrap" } eth2_ssz = { path = "../../eth2/utils/ssz" } lazy_static = "1.4.0" lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index d66cf60c23..b1c50dca23 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -16,10 +16,7 @@ use eth1::{Config as Eth1Config, Service as Eth1Service}; use eth2_config::Eth2Config; use exit_future::Signal; use futures::{future, Future, IntoFuture}; -use genesis::{ - generate_deterministic_keypairs, interop_genesis_state, state_from_ssz_file, Eth1GenesisService, -}; -use lighthouse_bootstrap::Bootstrapper; +use genesis::{interop_genesis_state, Eth1GenesisService}; use network::{NetworkConfig, NetworkMessage, Service as NetworkService}; use slog::info; use ssz::Decode; @@ -28,7 +25,7 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; -use types::{BeaconState, ChainSpec, EthSpec}; +use types::{test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec}; use websocket_server::{Config as WebSocketConfig, WebSocketSender}; /// Interval between polling the eth1 node for genesis information. @@ -161,12 +158,13 @@ where // // Alternatively, if there's a beacon chain in the database then always resume // using it. - let client_genesis = if client_genesis == ClientGenesis::Resume && !chain_exists { + let client_genesis = if client_genesis == ClientGenesis::FromStore && !chain_exists + { info!(context.log, "Defaulting to deposit contract genesis"); ClientGenesis::DepositContract } else if chain_exists { - ClientGenesis::Resume + ClientGenesis::FromStore } else { client_genesis }; @@ -187,16 +185,6 @@ where Box::new(future) } - ClientGenesis::SszFile { path } => { - let result = state_from_ssz_file(path); - - let future = result - .and_then(move |genesis_state| builder.genesis_state(genesis_state)) - .into_future() - .map(|v| (v, None)); - - Box::new(future) - } ClientGenesis::SszBytes { genesis_state_bytes, } => { @@ -235,25 +223,7 @@ where Box::new(future) } - ClientGenesis::RemoteNode { server, .. } => { - let future = Bootstrapper::connect(server, &context.log) - .map_err(|e| { - format!("Failed to initialize bootstrap client: {}", e) - }) - .into_future() - .and_then(|bootstrapper| { - let (genesis_state, _genesis_block) = - bootstrapper.genesis().map_err(|e| { - format!("Failed to bootstrap genesis state: {}", e) - })?; - - builder.genesis_state(genesis_state) - }) - .map(|v| (v, None)); - - Box::new(future) - } - ClientGenesis::Resume => { + ClientGenesis::FromStore => { let future = builder.resume_from_db().into_future().map(|v| (v, None)); Box::new(future) diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index d779eb7926..9f3bd598f8 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -1,4 +1,3 @@ -use beacon_chain::builder::PUBKEY_CACHE_FILENAME; use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; use std::fs; @@ -12,32 +11,24 @@ const TESTNET_SPEC_CONSTANTS: &str = "minimal"; /// Default directory name for the freezer database under the top-level data dir. const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db"; -/// Trap file indicating if chain_db was purged -const CHAIN_DB_PURGED_TRAP_FILE: &str = ".db_purged"; - /// Defines how the client should initialize the `BeaconChain` and other components. #[derive(PartialEq, Debug, Clone, Serialize, Deserialize)] pub enum ClientGenesis { - /// Reads the genesis state and other persisted data from the `Store`. - Resume, /// Creates a genesis state as per the 2019 Canada interop specifications. Interop { validator_count: usize, genesis_time: u64, }, + /// Reads the genesis state and other persisted data from the `Store`. + FromStore, /// Connects to an eth1 node and waits until it can create the genesis state from the deposit /// contract. DepositContract, - /// Loads the genesis state from a SSZ-encoded `BeaconState` file. - SszFile { path: PathBuf }, /// Loads the genesis state from SSZ-encoded `BeaconState` bytes. /// /// We include the bytes instead of the `BeaconState` because the `EthSpec` type /// parameter would be very annoying. SszBytes { genesis_state_bytes: Vec }, - /// Connects to another Lighthouse instance and reads the genesis state and other data via the - /// HTTP API. - RemoteNode { server: String, port: Option }, } impl Default for ClientGenesis { @@ -101,68 +92,6 @@ impl Config { .map(|data_dir| data_dir.join(&self.db_name)) } - /// Get the path of the chain db purged trap file - pub fn get_db_purged_trap_file_path(&self) -> Option { - self.get_data_dir() - .map(|data_dir| data_dir.join(CHAIN_DB_PURGED_TRAP_FILE)) - } - - /// returns whether chain_db was recently purged - pub fn chain_db_was_purged(&self) -> bool { - self.get_db_purged_trap_file_path() - .map_or(false, |trap_file| trap_file.exists()) - } - - /// purges the chain_db and creates trap file - pub fn purge_chain_db(&self) -> Result<(), String> { - // create the trap file - let trap_file = self - .get_db_purged_trap_file_path() - .ok_or("Failed to get trap file path".to_string())?; - fs::File::create(trap_file) - .map_err(|err| format!("Failed to create trap file: {}", err))?; - - // remove the chain_db - fs::remove_dir_all( - self.get_db_path() - .ok_or("Failed to get db_path".to_string())?, - ) - .map_err(|err| format!("Failed to remove chain_db: {}", err))?; - - // remove the freezer db - fs::remove_dir_all( - self.get_freezer_db_path() - .ok_or("Failed to get freezer db path".to_string())?, - ) - .map_err(|err| format!("Failed to remove chain_db: {}", err))?; - - // also need to remove pubkey cache file if it exists - let pubkey_cache_file = self - .get_data_dir() - .map(|data_dir| data_dir.join(PUBKEY_CACHE_FILENAME)) - .ok_or("Failed to get pubkey cache file path".to_string())?; - if !pubkey_cache_file.exists() { - return Ok(()); - } - fs::remove_file(pubkey_cache_file) - .map_err(|err| format!("Failed to remove pubkey cache: {}", err))?; - - Ok(()) - } - - /// cleans up purge_db trap file - pub fn cleanup_after_purge_db(&self) -> Result<(), String> { - let trap_file = self - .get_db_purged_trap_file_path() - .ok_or("Failed to get trap file path".to_string())?; - if !trap_file.exists() { - return Ok(()); - } - fs::remove_file(trap_file).map_err(|err| format!("Failed to remove trap file: {}", err))?; - - Ok(()) - } - /// Get the database path, creating it if necessary. pub fn create_db_path(&self) -> Result { let db_path = self diff --git a/beacon_node/genesis/src/interop.rs b/beacon_node/genesis/src/interop.rs index 5a91967fbe..c70ba4b29b 100644 --- a/beacon_node/genesis/src/interop.rs +++ b/beacon_node/genesis/src/interop.rs @@ -3,7 +3,6 @@ use eth2_hashing::hash; use rayon::prelude::*; use ssz::Encode; use state_processing::initialize_beacon_state_from_eth1; -use std::time::SystemTime; use types::{BeaconState, ChainSpec, DepositData, EthSpec, Hash256, Keypair, PublicKey, Signature}; /// Builds a genesis state as defined by the Eth2 interop procedure (see below). @@ -57,18 +56,6 @@ pub fn interop_genesis_state( Ok(state) } -/// Returns the system time, mod 30 minutes. -/// -/// Used for easily creating testnets. -pub fn recent_genesis_time(minutes: u64) -> u64 { - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs(); - let secs_after_last_period = now.checked_rem(minutes * 60).unwrap_or(0); - now - secs_after_last_period -} - #[cfg(test)] mod test { use super::*; diff --git a/beacon_node/genesis/src/lib.rs b/beacon_node/genesis/src/lib.rs index d6b3606f7f..80574bbb0c 100644 --- a/beacon_node/genesis/src/lib.rs +++ b/beacon_node/genesis/src/lib.rs @@ -4,28 +4,5 @@ mod interop; pub use eth1::Config as Eth1Config; pub use eth1_genesis_service::Eth1GenesisService; -pub use interop::{interop_genesis_state, recent_genesis_time}; +pub use interop::interop_genesis_state; pub use types::test_utils::generate_deterministic_keypairs; - -use ssz::Decode; -use std::fs::File; -use std::io::prelude::*; -use std::path::PathBuf; -use types::{BeaconState, EthSpec}; - -/// Load a `BeaconState` from the given `path`. The file should contain raw SSZ bytes (i.e., no -/// ASCII encoding or schema). -pub fn state_from_ssz_file(path: PathBuf) -> Result, String> { - File::open(path.clone()) - .map_err(move |e| format!("Unable to open SSZ genesis state file {:?}: {:?}", path, e)) - .and_then(|mut file| { - let mut bytes = vec![]; - file.read_to_end(&mut bytes) - .map_err(|e| format!("Failed to read SSZ file: {:?}", e))?; - Ok(bytes) - }) - .and_then(|bytes| { - BeaconState::from_ssz_bytes(&bytes) - .map_err(|e| format!("Unable to parse SSZ genesis state file: {:?}", e)) - }) -} diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 23b6b939da..8cfbca0434 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1,4 +1,4 @@ -use clap::{App, Arg, SubCommand}; +use clap::{App, Arg}; pub fn cli_app<'a, 'b>() -> App<'a, 'b> { App::new("beacon_node") @@ -26,15 +26,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("Data directory for the freezer database.") .takes_value(true) ) - .arg( - Arg::with_name("testnet-dir") - .long("testnet-dir") - .value_name("DIR") - .help("Path to directory containing eth2_testnet specs. Defaults to \ - a hard-coded Lighthouse testnet. Only effective if there is no \ - existing database.") - .takes_value(true) - ) /* * Network parameters. */ @@ -119,6 +110,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { automatically.") .takes_value(true), ) + .arg( + Arg::with_name("random-propagation") + .long("random-propagation") + .value_name("INTEGER") + .takes_value(true) + .help("Specifies (as a percentage) the likelihood of propagating blocks and \ + attestations. This should only be used for testing networking elements. The \ + value must like in the range 1-100. Default is 100.") + ) /* REST API related arguments */ .arg( Arg::with_name("http") @@ -214,107 +214,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true) ) /* - * The "testnet" sub-command. - * - * Allows for creating a new datadir with testnet-specific configs. + * Purge. */ - .subcommand(SubCommand::with_name("testnet") - .about("Create a new Lighthouse datadir using a testnet strategy.") - .arg( - Arg::with_name("random-datadir") - .long("random-datadir") - .short("r") - .help("If present, append a random string to the datadir path. Useful for fast development \ - iteration.") - ) - .arg( - Arg::with_name("force") - .long("force") - .short("f") - .help("If present, will create new config and database files and move the any existing to a \ - backup directory.") - .conflicts_with("random-datadir") - ) - .arg( - Arg::with_name("random-propagation") - .long("random-propagation") - .value_name("INTEGER") - .takes_value(true) - .help("Specifies (as a percentage) the likelihood of propagating blocks and \ - attestations. This should only be used for testing networking elements. The \ - value must like in the range 1-100. Default is 100.") - ) - .arg( - Arg::with_name("slot-time") - .long("slot-time") - .short("t") - .value_name("MILLISECONDS") - .help("Defines the slot time when creating a new testnet. The default is \ - specified by the spec.") - ) - /* - * `recent` - * - * Start a new node, with a specified number of validators with a genesis time in the last - * 30-minutes. - */ - .subcommand(SubCommand::with_name("recent") - .about("Creates a new genesis state where the genesis time was at the previous \ - MINUTES boundary (e.g., when MINUTES == 30; 12:00, 12:30, 13:00, etc.)") - .arg(Arg::with_name("validator_count") - .value_name("VALIDATOR_COUNT") - .required(true) - .help("The number of validators in the genesis state")) - .arg(Arg::with_name("minutes") - .long("minutes") - .short("m") - .value_name("MINUTES") - .required(true) - .default_value("30") - .help("The maximum number of minutes that will have elapsed before genesis")) - ) - /* - * `quick` - * - * Start a new node, specifying the number of validators and genesis time - */ - .subcommand(SubCommand::with_name("quick") - .about("Creates a new genesis state from the specified validator count and genesis time. \ - Compatible with the `quick-start genesis` defined in the eth2.0-pm repo.") - .arg(Arg::with_name("validator_count") - .value_name("VALIDATOR_COUNT") - .required(true) - .help("The number of validators in the genesis state")) - .arg(Arg::with_name("genesis_time") - .value_name("UNIX_EPOCH_SECONDS") - .required(true) - .help("The genesis time for the given state.")) - ) - /* - * `yaml` - * - * Start a new node, using a genesis state loaded from a YAML file - */ - .subcommand(SubCommand::with_name("file") - .about("Creates a new datadir where the genesis state is read from file. May fail to parse \ - a file that was generated to a different spec than that specified by --spec.") - .arg(Arg::with_name("format") - .value_name("FORMAT") - .required(true) - .possible_values(&["ssz"]) - .help("The encoding of the state in the file.")) - .arg(Arg::with_name("file") - .value_name("FILE") - .required(true) - .help("A file from which to read the state")) - ) - ) - /* - * The "purge" sub-command. - * - * Allows user to purge beacon database - */ - .subcommand(SubCommand::with_name("purge") - .about("Purge the beacon chain database.") + .arg( + Arg::with_name("purge-db") + .long("purge-db") + .help("If present, the chain database will be deleted. Use with caution.") ) } diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 62bf55af05..86f34fe70b 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1,14 +1,13 @@ +use beacon_chain::builder::PUBKEY_CACHE_FILENAME; use clap::ArgMatches; -use client::{config::DEFAULT_DATADIR, ClientConfig, ClientGenesis, Eth2Config}; -use environment::ETH2_CONFIG_FILENAME; -use eth2_config::{read_from_file, write_to_file}; +use client::{config::DEFAULT_DATADIR, ClientConfig, ClientGenesis}; use eth2_libp2p::{Enr, Multiaddr}; use eth2_testnet_config::Eth2TestnetConfig; -use genesis::recent_genesis_time; -use rand::{distributions::Alphanumeric, Rng}; -use slog::{crit, info, warn, Logger}; +use slog::{crit, warn, Logger}; use ssz::Encode; use std::fs; +use std::fs::File; +use std::io::prelude::*; use std::net::{IpAddr, Ipv4Addr}; use std::net::{TcpListener, UdpSocket}; use std::path::PathBuf; @@ -18,8 +17,6 @@ pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml"; pub const BEACON_NODE_DIR: &str = "beacon"; pub const NETWORK_DIR: &str = "network"; -type Result = std::result::Result; - /// Gets the fully-initialized global client. /// /// The top-level `clap` arguments should be provided as `cli_args`. @@ -30,22 +27,52 @@ type Result = std::result::Result; #[allow(clippy::cognitive_complexity)] pub fn get_config( cli_args: &ArgMatches, - eth2_config: Eth2Config, - core_log: Logger, -) -> Result { - let log = core_log.clone(); - + spec_constants: &str, + log: Logger, +) -> Result { let mut client_config = ClientConfig::default(); - client_config.spec_constants = eth2_config.spec_constants.clone(); client_config.data_dir = get_data_dir(cli_args); + // If necessary, remove any existing database and configuration + if client_config.data_dir.exists() && cli_args.is_present("purge-db") { + // Remove the chain_db. + fs::remove_dir_all( + client_config + .get_db_path() + .ok_or("Failed to get db_path".to_string())?, + ) + .map_err(|err| format!("Failed to remove chain_db: {}", err))?; + + // Remove the freezer db. + fs::remove_dir_all( + client_config + .get_freezer_db_path() + .ok_or("Failed to get freezer db path".to_string())?, + ) + .map_err(|err| format!("Failed to remove chain_db: {}", err))?; + + // Remove the pubkey cache file if it exists + let pubkey_cache_file = client_config.data_dir.join(PUBKEY_CACHE_FILENAME); + if pubkey_cache_file.exists() { + fs::remove_file(&pubkey_cache_file) + .map_err(|e| format!("Failed to remove {:?}: {:?}", pubkey_cache_file, e))?; + } + } + + // Create `datadir` and any non-existing parent directories. + fs::create_dir_all(&client_config.data_dir) + .map_err(|e| format!("Failed to create data dir: {}", e))?; + // Load the client config, if it exists . - let path = client_config.data_dir.join(CLIENT_CONFIG_FILENAME); - if path.exists() { - client_config = read_from_file(path.clone()) - .map_err(|e| format!("Unable to parse {:?} file: {:?}", path, e))? - .ok_or_else(|| format!("{:?} file does not exist", path))?; + let config_file_path = client_config.data_dir.join(CLIENT_CONFIG_FILENAME); + let config_file_existed = config_file_path.exists(); + if config_file_existed { + client_config = read_from_file(config_file_path.clone()) + .map_err(|e| format!("Unable to parse {:?} file: {:?}", config_file_path, e))? + .ok_or_else(|| format!("{:?} file does not exist", config_file_path))?; + } else { + client_config.spec_constants = spec_constants.into(); } client_config.testnet_dir = get_testnet_dir(cli_args); @@ -85,7 +112,7 @@ pub fn get_config( client_config.network.boot_nodes = boot_enr_str .split(',') .map(|enr| enr.parse().map_err(|_| format!("Invalid ENR: {}", enr))) - .collect::>>()?; + .collect::, _>>()?; } if let Some(libp2p_addresses_str) = cli_args.value_of("libp2p-addresses") { @@ -96,7 +123,7 @@ pub fn get_config( .parse() .map_err(|_| format!("Invalid Multiaddr: {}", multiaddr)) }) - .collect::>>()?; + .collect::, _>>()?; } if let Some(topics_str) = cli_args.value_of("topics") { @@ -121,6 +148,20 @@ pub fn get_config( client_config.network.secret_key_hex = Some(p2p_priv_key.to_string()); } + // Define a percentage of messages that should be propogated, useful for simulating bad network + // conditions. + // + // WARNING: setting this to anything less than 100 will cause bad behaviour. + if let Some(propagation_percentage_string) = cli_args.value_of("random-propagation") { + let percentage = propagation_percentage_string + .parse::() + .map_err(|_| "Unable to parse the propagation percentage".to_string())?; + if percentage > 100 { + return Err("Propagation percentage greater than 100".to_string()); + } + client_config.network.propagation_percentage = Some(percentage); + } + /* * Http server */ @@ -185,50 +226,6 @@ pub fn get_config( client_config.eth1.endpoint = val.to_string(); } - match cli_args.subcommand() { - ("testnet", Some(sub_cmd_args)) => { - process_testnet_subcommand(&mut client_config, ð2_config, sub_cmd_args)? - } - ("purge", _) => { - client_config.purge_chain_db()?; - println!("Successfully purged chain db"); - std::process::exit(0); - } - // No sub-command assumes a resume operation. - _ => { - // If no primary subcommand was given, start the beacon chain from an existing - // database. - client_config.genesis = ClientGenesis::Resume; - - let db_path_exists: bool = match client_config.get_db_path() { - Some(path) => path.exists(), - None => false, - }; - - // Whilst there is no large testnet or mainnet force the user to specify how they want - // to start a new chain (e.g., from a genesis YAML file, another node, etc). - if !client_config.data_dir.exists() - || (!db_path_exists && client_config.chain_db_was_purged()) - { - info!( - log, - "Starting from an empty database"; - "data_dir" => format!("{:?}", client_config.data_dir) - ); - init_new_client::(&mut client_config, ð2_config)? - } else { - info!( - log, - "Resuming from existing datadir"; - "data_dir" => format!("{:?}", client_config.data_dir) - ); - // If the `testnet` command was not provided, attempt to load an existing datadir and - // continue with an existing chain. - load_from_datadir(&mut client_config)? - } - } - }; - if let Some(freezer_dir) = cli_args.value_of("freezer-dir") { client_config.freezer_db_path = Some(PathBuf::from(freezer_dir)); } @@ -256,10 +253,10 @@ pub fn get_config( .map_err(|_| "block-cache-size is not a valid integer".to_string())?; } - if eth2_config.spec_constants != client_config.spec_constants { + if spec_constants != client_config.spec_constants { crit!(log, "Specification constants do not match."; "client_config" => client_config.spec_constants.to_string(), - "eth2_config" => eth2_config.spec_constants + "eth2_config" => spec_constants ); return Err("Specification constant mismatch".into()); } @@ -294,6 +291,39 @@ pub fn get_config( client_config.network.discovery_address = Some("127.0.0.1".parse().expect("Valid IP address")) } + + /* + * Load the eth2 testnet dir to obtain some additional config values. + */ + let eth2_testnet_config: Eth2TestnetConfig = + get_eth2_testnet_config(&client_config.testnet_dir)?; + + client_config.eth1.deposit_contract_address = + format!("{:?}", eth2_testnet_config.deposit_contract_address()?); + client_config.eth1.deposit_contract_deploy_block = + eth2_testnet_config.deposit_contract_deploy_block; + client_config.eth1.lowest_cached_block_number = + client_config.eth1.deposit_contract_deploy_block; + + if let Some(mut boot_nodes) = eth2_testnet_config.boot_enr { + client_config.network.boot_nodes.append(&mut boot_nodes) + } + + if let Some(genesis_state) = eth2_testnet_config.genesis_state { + // Note: re-serializing the genesis state is not so efficient, however it avoids adding + // trait bounds to the `ClientGenesis` enum. This would have significant flow-on + // effects. + client_config.genesis = ClientGenesis::SszBytes { + genesis_state_bytes: genesis_state.as_ssz_bytes(), + }; + } else { + client_config.genesis = ClientGenesis::DepositContract; + } + + if !config_file_existed { + write_to_file(config_file_path, &client_config)?; + } + Ok(client_config) } @@ -320,239 +350,26 @@ pub fn get_testnet_dir(cli_args: &ArgMatches) -> Option { } } +/// If `testnet_dir` is `Some`, returns the `Eth2TestnetConfig` at that path or returns an error. +/// If it is `None`, returns the "hard coded" config. pub fn get_eth2_testnet_config( testnet_dir: &Option, -) -> Result> { +) -> Result, String> { Ok(if let Some(testnet_dir) = testnet_dir { Eth2TestnetConfig::load(testnet_dir.clone()) .map_err(|e| format!("Unable to open testnet dir at {:?}: {}", testnet_dir, e))? } else { - Eth2TestnetConfig::hard_coded() - .map_err(|e| format!("Unable to load hard-coded testnet dir: {}", e))? + Eth2TestnetConfig::hard_coded().map_err(|e| { + format!( + "The hard-coded testnet directory was invalid. \ + This happens when Lighthouse is migrating between spec versions. \ + Error : {}", + e + ) + })? }) } -/// Load from an existing database. -fn load_from_datadir(client_config: &mut ClientConfig) -> Result<()> { - // Check to ensure the datadir exists. - // - // For now we return an error. In the future we may decide to boot a default (e.g., - // public testnet or mainnet). - if !client_config.get_data_dir().map_or(false, |d| d.exists()) { - return Err( - "No datadir found. Either create a new testnet or specify a different `--datadir`." - .into(), - ); - } - - // If there is a path to a database in the config, ensure it exists. - if !client_config - .get_db_path() - .map_or(false, |path| path.exists()) - { - return Err( - "No database found in datadir. Please make sure the directory provided is valid, or specify a different `--datadir`." - .into(), - ); - } - - client_config.genesis = ClientGenesis::Resume; - - Ok(()) -} - -/// Create a new client with the default configuration. -fn init_new_client( - client_config: &mut ClientConfig, - eth2_config: &Eth2Config, -) -> Result<()> { - let eth2_testnet_config: Eth2TestnetConfig = - get_eth2_testnet_config(&client_config.testnet_dir)?; - - client_config.eth1.deposit_contract_address = - format!("{:?}", eth2_testnet_config.deposit_contract_address()?); - client_config.eth1.deposit_contract_deploy_block = - eth2_testnet_config.deposit_contract_deploy_block; - - client_config.eth1.follow_distance = eth2_config.spec.eth1_follow_distance / 2; - client_config.eth1.lowest_cached_block_number = client_config - .eth1 - .deposit_contract_deploy_block - .saturating_sub(client_config.eth1.follow_distance * 2); - - if let Some(mut boot_nodes) = eth2_testnet_config.boot_enr { - client_config.network.boot_nodes.append(&mut boot_nodes) - } - - if let Some(genesis_state) = eth2_testnet_config.genesis_state { - // Note: re-serializing the genesis state is not so efficient, however it avoids adding - // trait bounds to the `ClientGenesis` enum. This would have significant flow-on - // effects. - client_config.genesis = ClientGenesis::SszBytes { - genesis_state_bytes: genesis_state.as_ssz_bytes(), - }; - } else { - client_config.genesis = ClientGenesis::DepositContract; - } - - create_new_datadir(&client_config, ð2_config)?; - - Ok(()) -} - -/// Writes the configs in `self` to `self.data_dir`. -/// -/// Returns an error if `self.data_dir` already exists. -pub fn create_new_datadir(client_config: &ClientConfig, eth2_config: &Eth2Config) -> Result<()> { - let rebuild_db = client_config.chain_db_was_purged(); - if client_config.data_dir.exists() && !rebuild_db { - return Err(format!( - "Data dir already exists at {:?}", - client_config.data_dir - )); - } - - // Create `datadir` and any non-existing parent directories. - fs::create_dir_all(&client_config.data_dir) - .map_err(|e| format!("Failed to create data dir: {}", e))?; - - macro_rules! write_to_file { - ($file: ident, $variable: ident) => { - let file = client_config.data_dir.join($file); - if file.exists() { - if !rebuild_db { - return Err(format!("Datadir is not clean, {} exists.", $file)); - } - } else { - // Write the onfig to a TOML file in the datadir. - write_to_file(client_config.data_dir.join($file), $variable) - .map_err(|e| format!("Unable to write {} file: {:?}", $file, e))?; - } - }; - } - - write_to_file!(CLIENT_CONFIG_FILENAME, client_config); - write_to_file!(ETH2_CONFIG_FILENAME, eth2_config); - client_config.cleanup_after_purge_db()?; - - Ok(()) -} - -/// Process the `testnet` CLI subcommand arguments, updating the `builder`. -fn process_testnet_subcommand( - client_config: &mut ClientConfig, - eth2_config: &Eth2Config, - cli_args: &ArgMatches, -) -> Result<()> { - // Specifies that a random datadir should be used. - if cli_args.is_present("random-datadir") { - client_config - .data_dir - .push(format!("random_{}", random_string(6))); - client_config.network.network_dir = client_config.data_dir.join("network"); - } - - // Deletes the existing datadir. - if cli_args.is_present("force") && client_config.data_dir.exists() { - fs::remove_dir_all(&client_config.data_dir) - .map_err(|e| format!("Unable to delete existing datadir: {:?}", e))?; - } - - // Define a percentage of messages that should be propogated, useful for simulating bad network - // conditions. - // - // WARNING: setting this to anything less than 100 will cause bad behaviour. - if let Some(propagation_percentage_string) = cli_args.value_of("random-propagation") { - let percentage = propagation_percentage_string - .parse::() - .map_err(|_| "Unable to parse the propagation percentage".to_string())?; - if percentage > 100 { - return Err("Propagation percentage greater than 100".to_string()); - } - client_config.network.propagation_percentage = Some(percentage); - } - - // Start matching on the second subcommand (e.g., `testnet bootstrap ...`). - match cli_args.subcommand() { - ("recent", Some(cli_args)) => { - let validator_count = cli_args - .value_of("validator_count") - .ok_or_else(|| "No validator_count specified")? - .parse::() - .map_err(|e| format!("Unable to parse validator_count: {:?}", e))?; - - let minutes = cli_args - .value_of("minutes") - .ok_or_else(|| "No recent genesis minutes supplied")? - .parse::() - .map_err(|e| format!("Unable to parse minutes: {:?}", e))?; - - client_config.dummy_eth1_backend = true; - - client_config.genesis = ClientGenesis::Interop { - validator_count, - genesis_time: recent_genesis_time(minutes), - }; - } - ("quick", Some(cli_args)) => { - let validator_count = cli_args - .value_of("validator_count") - .ok_or_else(|| "No validator_count specified")? - .parse::() - .map_err(|e| format!("Unable to parse validator_count: {:?}", e))?; - - let genesis_time = cli_args - .value_of("genesis_time") - .ok_or_else(|| "No genesis time supplied")? - .parse::() - .map_err(|e| format!("Unable to parse genesis time: {:?}", e))?; - - client_config.dummy_eth1_backend = true; - - client_config.genesis = ClientGenesis::Interop { - validator_count, - genesis_time, - }; - } - ("file", Some(cli_args)) => { - let path = cli_args - .value_of("file") - .ok_or_else(|| "No filename specified")? - .parse::() - .map_err(|e| format!("Unable to parse filename: {:?}", e))?; - - let format = cli_args - .value_of("format") - .ok_or_else(|| "No file format specified")?; - - let start_method = match format { - "ssz" => ClientGenesis::SszFile { path }, - other => return Err(format!("Unknown genesis file format: {}", other)), - }; - - client_config.genesis = start_method; - } - (cmd, Some(_)) => { - return Err(format!( - "Invalid valid method specified: {}. See 'testnet --help'.", - cmd - )) - } - _ => return Err("No testnet method specified. See 'testnet --help'.".into()), - }; - - create_new_datadir(&client_config, ð2_config)?; - - Ok(()) -} - -fn random_string(len: usize) -> String { - rand::thread_rng() - .sample_iter(&Alphanumeric) - .take(len) - .collect::() -} - /// A bit of hack to find an unused port. /// /// Does not guarantee that the given port is unused after the function exists, just that it was @@ -566,7 +383,7 @@ fn random_string(len: usize) -> String { /// it doesn't allow binding to the same port even after the socket is closed. /// We might have to use SO_REUSEADDR socket option from `std::net2` crate in /// that case. -pub fn unused_port(transport: &str) -> Result { +pub fn unused_port(transport: &str) -> Result { let local_addr = match transport { "tcp" => { let listener = TcpListener::bind("127.0.0.1:0").map_err(|e| { @@ -593,3 +410,42 @@ pub fn unused_port(transport: &str) -> Result { }; Ok(local_addr.port()) } + +/// Write a configuration to file. +pub fn write_to_file(path: PathBuf, config: &T) -> Result<(), String> +where + T: Default + serde::de::DeserializeOwned + serde::Serialize, +{ + if let Ok(mut file) = File::create(path.clone()) { + let toml_encoded = toml::to_string(&config).map_err(|e| { + format!( + "Failed to write configuration to {:?}. Error: {:?}", + path, e + ) + })?; + file.write_all(toml_encoded.as_bytes()) + .unwrap_or_else(|_| panic!("Unable to write to {:?}", path)); + } + + Ok(()) +} + +/// Loads a `ClientConfig` from file. If unable to load from file, generates a default +/// configuration and saves that as a sample file. +pub fn read_from_file(path: PathBuf) -> Result, String> +where + T: Default + serde::de::DeserializeOwned + serde::Serialize, +{ + if let Ok(mut file) = File::open(path.clone()) { + let mut contents = String::new(); + file.read_to_string(&mut contents) + .map_err(|e| format!("Unable to read {:?}. Error: {:?}", path, e))?; + + let config = toml::from_str(&contents) + .map_err(|e| format!("Unable to parse {:?}: {:?}", path, e))?; + + Ok(Some(config)) + } else { + Ok(None) + } +} diff --git a/beacon_node/src/lib.rs b/beacon_node/src/lib.rs index 2bafa55233..d281b8d2bd 100644 --- a/beacon_node/src/lib.rs +++ b/beacon_node/src/lib.rs @@ -55,9 +55,13 @@ impl ProductionBeaconNode { context: RuntimeContext, matches: &ArgMatches<'b>, ) -> impl Future + 'a { - get_config::(&matches, context.eth2_config.clone(), context.log.clone()) - .into_future() - .and_then(move |client_config| Self::new(context, client_config)) + get_config::( + &matches, + &context.eth2_config.spec_constants, + context.log.clone(), + ) + .into_future() + .and_then(move |client_config| Self::new(context, client_config)) } /// Starts a new beacon node `Client` in the given `environment`. diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 54546c3bc1..6b072e6be0 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -6,9 +6,7 @@ * [Building from Source](./become-a-validator-source.md) * [Installation](./installation.md) * [Docker](./docker.md) -* [CLI](./cli.md) - * [Simple Local Testnet](./simple-testnet.md) - * [Local Testnets](./local-testnets.md) +* [Local Testnets](./local-testnets.md) * [API](./api.md) * [HTTP (RESTful JSON)](./http.md) * [/beacon](./http_beacon.md) diff --git a/book/src/intro.md b/book/src/intro.md index 8198c9886e..47a1ce6e95 100644 --- a/book/src/intro.md +++ b/book/src/intro.md @@ -21,7 +21,7 @@ You may read this book from start to finish, or jump to some of these topics: - Follow the [Installation Guide](./installation.md) to install Lighthouse. - Get hacking with the [Development Environment Guide](./setup.md). -- Utilize the whole stack by starting a [simple local testnet](./simple-testnet.md). +- Utilize the whole stack by starting a [local testnet](./local-testnets.md). - Query the [RESTful HTTP API](./http.md) using `curl`. - Listen to events with the [JSON WebSocket API](./websockets.md). - View the [Rust code docs](http://lighthouse-docs.sigmaprime.io/). diff --git a/book/src/local-testnets.md b/book/src/local-testnets.md index 39a5d024b7..1c269a66a9 100644 --- a/book/src/local-testnets.md +++ b/book/src/local-testnets.md @@ -3,143 +3,104 @@ > This section is about running your own private local testnets. > - If you wish to join the ongoing public testnet, please read [become a validator](./become-a-validator.md). -The `beacon_node` and `validator` commands have a `testnet` sub-command to -allow creating or connecting to Eth2 beacon chain testnets. +It is possible to create local, short-lived Lighthouse testnets that _don't_ +require a deposit contract and Eth1 connection. There are two components +required for this: -For detailed documentation, use the `--help` flag on the CLI: +1. Creating a "testnet directory", containing the configuration of your new + testnet. +1. Using the `--dummy-eth1` flag on your beacon node to avoid needing an Eth1 + node for block production. + +There is a TL;DR (too long; didn't read), followed by detailed steps if the +TL;DR isn't adequate. + +## TL;DR ```bash -$ lighthouse bn testnet --help +lcli new-testnet +lcli interop-genesis 128 +lighthouse bn --testnet-dir ~/.lighthouse/testnet --dummy-eth1 --http +lighthouse vc --testnet-dir ~/.lighthouse/testnet --allow-unsynced testnet insecure 0 128 ``` +Optionally update the genesis time to now: + ```bash -$ lighthouse vc testnet --help +lcli change-genesis-time ~/.lighthouse/testnet/genesis.ssz $(date +%s) ``` -## Examples +## 1. Creating a testnet directory -All examples assume a working [development environment](./setup.md) and -commands are based in the `target/release` directory (this is the build dir for -`cargo`). +### 1.1 Install `lcli` -### Start a beacon node given a validator count and genesis_time +This guide requires `lcli`, the "Lighthouse CLI tool". It is a development tool +used for starting testnets and debugging. - -To start a brand-new beacon node (with no history) use: +Install `lcli` from the root directory of this repository with: ```bash -$ lighthouse bn testnet -f quick 8 +cargo install --path lcli --force ``` -Where `GENESIS_TIME` is in [unix time](https://duckduckgo.com/?q=unix+time&t=ffab&ia=answer). +### 1.2 Create a testnet directory -> Notes: -> -> - This method conforms the ["Quick-start -genesis"](https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#quick-start-genesis) -method in the `ethereum/eth2.0-pm` repository. -> - The `-f` flag ignores any existing database or configuration, backing them -> up before re-initializing. -> - `8` is the validator count and `1567222226` is the genesis time. -> - See `$ lighthouse bn testnet quick --help` for more configuration options. +The default location for a testnet directory is `~/.lighthouse/testnet`. We'll +use this directory to keep the examples simple, however you can always specify +a different directory using the `--testnet-dir` flag. -### Start a beacon node given a genesis state file - -A genesis state can be read from file using the `testnet file` subcommand. -There are three supported formats: - -- `ssz` (default) -- `json` -- `yaml` - -Start a new node using `/tmp/genesis.ssz` as the genesis state: +Once you have `lcli` installed, create a new testnet directory with: ```bash -$ lighthouse bn testnet --spec minimal -f file ssz /tmp/genesis.ssz +lcli new-testnet ``` -> Notes: -> -> - The `-f` flag ignores any existing database or configuration, backing them -> up before re-initializing. -> - See `$ lighthouse bn testnet file --help` for more configuration options. -> - The `--spec` flag is required to allow SSZ parsing of fixed-length lists. -> Here the `minimal` eth2 specification is chosen, allowing for lower -> validator counts. See -> [eth2.0-specs/configs](https://github.com/ethereum/eth2.0-specs/tree/dev/configs) -> for more info. +> - This will create a "mainnet" spec testnet. To create a minimal spec use `lcli --spec minim new-testnet`. +> - The `lcli new-testnet` command has many options, use `lcli new-testnet --help` to see them. -### Start an auto-configured validator client +### 1.3 Create a genesis state -To start a brand-new validator client (with no history) use: +Your new testnet directory at `~/.lighthouse/testnet` doesn't yet have a +genesis state (`genesis.ssz`). Since there's no deposit contract in this +testnet, there's no way for nodes to find genesis themselves. + +Manually create an "interop" genesis state with `128` validators: ```bash -$ lighthouse vc testnet insecure 0 8 +lcli interop-genesis 128 ``` -> Notes: -> -> - The `insecure` command dictates that the [interop keypairs](https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#pubkeyprivkey-generation) -> will be used. -> - The `0 8` indicates that this validator client should manage 8 validators, -> starting at validator 0 (the first deposited validator). -> - The validator client will try to connect to the beacon node at `localhost`. -> See `--help` to configure that address and other features. -> - The validator client will operate very unsafely in `testnet` mode, happily -> swapping between chains and creating double-votes. +> - A custom genesis time can be provided with `-t`. +> - See `lcli interop-genesis --help` for more info. -### Exporting a genesis file +## 2. Start the beacon nodes and validator clients -Genesis states can downloaded from a running Lighthouse node via the HTTP API. Three content-types are supported: +Now the testnet has been specified in `~/.lighthouse/testnet`, it's time to +start a beacon node and validator client. -- `application/json` -- `application/yaml` -- `application/ssz` +### 2.1 Start a beacon node -Using `curl`, a genesis state can be downloaded to `/tmp/genesis.ssz`: +Start a beacon node: ```bash -$ curl --header "Content-Type: application/ssz" "localhost:5052/beacon/state/genesis" -o /tmp/genesis.ssz +lighthouse bn --testnet-dir ~/.lighthouse/testnet --dummy-eth1 --http ``` -## Advanced +> - `--testnet-dir` instructs the beacon node to use the spec we generated earlier. +> - `--dummy-eth1` uses deterministic "junk data" for linking to the eth1 chain, avoiding the requirement for an eth1 node. The downside is that new validators cannot be on-boarded after genesis. +> - `--http` starts the REST API so the validator client can produce blocks. -Below are some CLI commands useful when working with testnets. +### 2.2 Start a validator client -### Specify a boot node by multiaddr - -You can specify a static list of multiaddrs when booting Lighthouse using -the `--libp2p-addresses` command. - -#### Example: +Once the beacon node has started and begun trying to sync, start a validator +client: ```bash -$ lighthouse bn --libp2p-addresses /ip4/192.168.0.1/tcp/9000 +lighthouse vc --testnet-dir ~/.lighthouse/testnet --allow-unsynced testnet insecure 0 128 ``` -### Specify a boot node by ENR (Ethereum Name Record) - -You can specify a static list of Discv5 addresses when booting Lighthouse using -the `--boot-nodes` command. - -#### Example: - -```bash -$ lighthouse bn --boot-nodes -IW4QB2Hi8TPuEzQ41Cdf1r2AUU1FFVFDBJdJyOkWk2qXpZfFZQy2YnJIyoT_5fnbtrXUouoskmydZl4pIg90clIkYUDgmlwhH8AAAGDdGNwgiMog3VkcIIjKIlzZWNwMjU2azGhAjg0-DsTkQynhJCRnLLttBK1RS78lmUkLa-wgzAi-Ob5 -``` - -### Start a testnet with a custom slot time - -Lighthouse can run at quite low slot times when there are few validators (e.g., -`500 ms` slot times should be fine for 8 validators). - -#### Example: - -The `-t` (`--slot-time`) flag specifies the milliseconds per slot. - -```bash -$ lighthouse bn testnet -t 500 recent 8 -``` - -> Note: `bootstrap` loads the slot time via HTTP and therefore conflicts with -> this flag. +> - `--testnet-dir` instructs the validator client to use the spec we generated earlier. +> - `--allow-unsynced` stops the validator client checking to see if the beacon node is synced prior to producing blocks. +> - `testnet insecure 0 128` instructs the validator client to use insecure +> testnet private keys and that it should control validators from `0` to +> `127` (inclusive). diff --git a/book/src/simple-testnet.md b/book/src/simple-testnet.md deleted file mode 100644 index 06cc744b53..0000000000 --- a/book/src/simple-testnet.md +++ /dev/null @@ -1,70 +0,0 @@ -# Simple Local Testnet - -> This guide is about running your own private local testnet. -> - If you wish to join the ongoing public testnet, please read [become a validator](./become-a-validator.md). - -This guide will help you setup your own private local testnet. - -First, [install Lighthouse](./installation.md). - -Then, get the current unix time in seconds; you can use -[epochconverter.com](https://www.epochconverter.com/) or `$ date +%s`. It -should look like this `1576803034` and you should use it wherever we put -`