From 140c677a38d16a8f51f4b6521ee0f74f1cd1ddca Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 25 Aug 2019 12:14:04 +1000 Subject: [PATCH] Add much more progress to new CLI setup --- beacon_node/client/src/beacon_chain_types.rs | 7 +- beacon_node/client/src/config.rs | 9 +- beacon_node/client/src/lib.rs | 2 +- beacon_node/src/config.rs | 258 ++++++++++++++----- beacon_node/src/main.rs | 19 +- 5 files changed, 214 insertions(+), 81 deletions(-) diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 37e4a055e8..7a57aa4757 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -60,9 +60,10 @@ where T::LmdGhost: LmdGhost, { let genesis_state = match &config.beacon_chain_start_method { - BeaconChainStartMethod::Resume => { - crit!(log, "This release does not support mainnet genesis state."); - return Err("Mainnet is unsupported".into()); + BeaconChainStartMethod::Resume => unimplemented!("No resume code yet"), + BeaconChainStartMethod::Mainnet => { + crit!(log, "No mainnet beacon chain startup specification."); + return Err("Mainnet is not yet specified. We're working on it.".into()); } BeaconChainStartMethod::RecentGenesis { validator_count } => { generate_testnet_genesis_state(*validator_count, recent_genesis_time(), &spec) diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 1e8f60f6ef..f2725b3e79 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -37,7 +37,9 @@ pub struct Config { pub enum BeaconChainStartMethod { /// Resume from an existing BeaconChain, loaded from the existing local database. Resume, - /// Create a new beacon chain with `validator_count` validators, all with well-known secret keys. + /// Resume from an existing BeaconChain, loaded from the existing local database. + Mainnet, + /// Create a new beacon chain that can connect to mainnet. /// /// Set the genesis time to be the start of the previous 30-minute window. RecentGenesis { validator_count: usize }, @@ -51,10 +53,7 @@ pub enum BeaconChainStartMethod { Yaml { file: PathBuf }, /// Create a new beacon chain by using a HTTP server (running our REST-API) to load genesis and /// finalized states and blocks. - HttpBootstrap { - server: String, - port: Option, - }, + HttpBootstrap { server: String, port: Option }, } impl Default for BeaconChainStartMethod { diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 3eb5553696..9d3e001faf 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -23,7 +23,7 @@ pub use beacon_chain::BeaconChainTypes; pub use beacon_chain_types::ClientType; pub use beacon_chain_types::InitialiseBeaconChain; pub use bootstrapper::Bootstrapper; -pub use config::Config as ClientConfig; +pub use config::{BeaconChainStartMethod, Config as ClientConfig}; pub use eth2_config::Eth2Config; /// Main beacon node client service. This provides the connection and initialisation of the clients diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index c1074da03d..68d905ed22 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1,10 +1,10 @@ use clap::ArgMatches; -use client::{Bootstrapper, ClientConfig, Eth2Config}; +use client::{BeaconChainStartMethod, Bootstrapper, ClientConfig, Eth2Config}; use eth2_config::{read_from_file, write_to_file}; use rand::{distributions::Alphanumeric, Rng}; use slog::{crit, info, warn, Logger}; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml"; @@ -19,29 +19,9 @@ pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result { match cli_args.subcommand() { ("testnet", Some(sub_cmd_args)) => { - if sub_cmd_args.is_present("random-datadir") { - builder.set_random_datadir()?; - } - - info!( - log, - "Creating new datadir"; - "path" => format!("{:?}", builder.data_dir) - ); - - builder.update_spec_from_subcommand(&sub_cmd_args)?; - - match sub_cmd_args.subcommand() { - // The bootstrap testnet method requires inserting a libp2p address into the - // network config. - ("bootstrap", Some(sub_cmd_args)) => { - builder.import_bootstrap_libp2p_address(&sub_cmd_args)?; - } - _ => (), - }; - - builder.write_configs_to_new_datadir()?; + process_testnet_subcommand(&mut builder, sub_cmd_args, log)? } + // No sub-command assumes a resume operation. _ => { info!( log, @@ -49,6 +29,20 @@ pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result { "path" => format!("{:?}", builder.data_dir) ); + // If no primary subcommand was given, start the beacon chain from an existing + // database. + builder.set_beacon_chain_start_method(BeaconChainStartMethod::Resume); + + // Whilst there is no large testnet or mainnet force the user to specify how they want + // to start a new chain (e.g., from a genesis YAML file, another node, etc). + if !builder.data_dir.exists() { + return Err( + "No datadir found. To start a new beacon chain, see `testnet --help`. \ + Use `--datadir` to specify a different directory" + .into(), + ); + } + // If the `testnet` command was not provided, attempt to load an existing datadir and // continue with an existing chain. builder.load_from_datadir()?; @@ -58,9 +52,62 @@ pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result { builder.build(cli_args) } -/// Decodes an optional string into an optional u16. -fn parse_port_option(o: Option<&str>) -> Option { - o.and_then(|s| s.parse::().ok()) +/// Process the `testnet` CLI subcommand arguments, updating the `builder`. +fn process_testnet_subcommand( + builder: &mut ConfigBuilder, + cli_args: &ArgMatches, + log: &Logger, +) -> Result<()> { + if cli_args.is_present("random-datadir") { + builder.set_random_datadir()?; + } + + if cli_args.is_present("force") { + builder.clean_datadir()?; + } + + info!( + log, + "Creating new datadir"; + "path" => format!("{:?}", builder.data_dir) + ); + + builder.update_spec_from_subcommand(&cli_args)?; + + // Start matching on the second subcommand (e.g., `testnet bootstrap ...`) + match cli_args.subcommand() { + ("bootstrap", Some(cli_args)) => { + let server = cli_args + .value_of("server") + .ok_or_else(|| "No bootstrap server specified")?; + let port: Option = cli_args + .value_of("port") + .and_then(|s| s.parse::().ok()); + + builder.import_bootstrap_libp2p_address(server, port)?; + + builder.set_beacon_chain_start_method(BeaconChainStartMethod::HttpBootstrap { + server: server.to_string(), + port, + }) + } + ("recent", Some(cli_args)) => { + let validator_count = cli_args + .value_of("validator_count") + .ok_or_else(|| "No validator_count specified")? + .parse::() + .map_err(|e| format!("Unable to parse validator_count: {:?}", e))?; + + builder.set_beacon_chain_start_method(BeaconChainStartMethod::RecentGenesis { + validator_count, + }) + } + _ => return Err("No testnet method specified. See 'testnet --help'.".into()), + }; + + builder.write_configs_to_new_datadir()?; + + Ok(()) } /// Allows for building a set of configurations based upon `clap` arguments. @@ -97,29 +144,65 @@ impl<'a> ConfigBuilder<'a> { }) } - pub fn set_beacon_chain_start_method(&mut self, cli_args: &ArgMatches) -> Result<()> { - // + /// Clears any configuration files that would interfere with writing new configs. + /// + /// Moves the following files in `data_dir` into a backup directory: + /// + /// - Client config + /// - Eth2 config + /// - The entire database directory + pub fn clean_datadir(&mut self) -> Result<()> { + let backup_dir = { + let mut s = String::from("backup_"); + s.push_str(&random_string(6)); + self.data_dir.join(s) + }; + + fs::create_dir_all(&backup_dir) + .map_err(|e| format!("Unable to create config backup dir: {:?}", e))?; + + let move_to_backup_dir = |path: &Path| -> Result<()> { + let file_name = path + .file_name() + .ok_or_else(|| "Invalid path found during datadir clean (no filename).")?; + + let mut new = path.to_path_buf(); + new.pop(); + new.push(backup_dir.clone()); + new.push(file_name); + + let _ = fs::rename(path, new); + + Ok(()) + }; + + move_to_backup_dir(&self.data_dir.join(CLIENT_CONFIG_FILENAME))?; + move_to_backup_dir(&self.data_dir.join(ETH2_CONFIG_FILENAME))?; + + if let Some(db_path) = self.client_config.db_path() { + move_to_backup_dir(&db_path)?; + } + + Ok(()) } - /// Reads a `server` flag from `cli_args` and attempts to generate a libp2p `Multiaddr` that - /// this client can use to connect to the given `server`. - /// - /// Also reads for a `libp2p_port` flag in `cli_args`, using that as the port for the - /// `Multiaddr`. If `libp2p_port` is not in `cli_args`, attempts to connect to `server` via HTTP - /// and retrieve it's libp2p listen port. - /// - /// Returns an error if the `server` flag is not present in `cli_args`. - pub fn import_bootstrap_libp2p_address(&mut self, cli_args: &ArgMatches) -> Result<()> { - let server: String = cli_args - .value_of("server") - .ok_or_else(|| "No bootstrap server specified")? - .to_string(); + /// Sets the method for starting the beacon chain. + pub fn set_beacon_chain_start_method(&mut self, method: BeaconChainStartMethod) { + self.client_config.beacon_chain_start_method = method; + } + /// Import the libp2p address for `server` into the list of bootnodes in `self`. + /// + /// If `port` is `Some`, it is used as the port for the `Multiaddr`. If `port` is `None`, + /// attempts to connect to the `server` via HTTP and retrieve it's libp2p listen port. + pub fn import_bootstrap_libp2p_address( + &mut self, + server: &str, + port: Option, + ) -> Result<()> { let bootstrapper = Bootstrapper::from_server_string(server.to_string())?; - if let Some(server_multiaddr) = - bootstrapper.best_effort_multiaddr(parse_port_option(cli_args.value_of("libp2p_port"))) - { + if let Some(server_multiaddr) = bootstrapper.best_effort_multiaddr(port) { info!( self.log, "Estimated bootstrapper libp2p address"; @@ -132,9 +215,9 @@ impl<'a> ConfigBuilder<'a> { .push(server_multiaddr); } else { warn!( - self.log, - "Unable to estimate a bootstrapper libp2p address, this node may not find any peers." - ); + self.log, + "Unable to estimate a bootstrapper libp2p address, this node may not find any peers." + ); }; Ok(()) @@ -144,14 +227,9 @@ impl<'a> ConfigBuilder<'a> { /// /// Useful for easily spinning up ephemeral testnets. pub fn set_random_datadir(&mut self) -> Result<()> { - let random = rand::thread_rng() - .sample_iter(&Alphanumeric) - .take(10) - .collect::(); - let mut s = DEFAULT_DATA_DIR.to_string(); s.push_str("_random_"); - s.push_str(&random); + s.push_str(&random_string(6)); self.data_dir.pop(); self.data_dir.push(s); @@ -187,12 +265,15 @@ impl<'a> ConfigBuilder<'a> { /// /// Returns an error if `self.data_dir` already exists. pub fn write_configs_to_new_datadir(&mut self) -> Result<()> { + let db_exists = self + .client_config + .db_path() + .map(|d| d.exists()) + .unwrap_or_else(|| false); + // Do not permit creating a new config when the datadir exists. - if self.data_dir.exists() { - return Err( - "Datadir already exists, will not overwrite. Remove the directory or use --datadir." - .into(), - ); + if db_exists { + return Err("Database already exists. See `-f` in `testnet --help`".into()); } // Create `datadir` and any non-existing parent directories. @@ -201,16 +282,35 @@ impl<'a> ConfigBuilder<'a> { format!("{}", e) })?; - // Write the client config to a TOML file in the datadir. - write_to_file( - self.data_dir.join(CLIENT_CONFIG_FILENAME), - &self.client_config, - ) - .map_err(|e| format!("Unable to write {} file: {:?}", CLIENT_CONFIG_FILENAME, e))?; + let client_config_file = self.data_dir.join(CLIENT_CONFIG_FILENAME); + if client_config_file.exists() { + return Err(format!( + "Datadir is not clean, {} exists. See `-f` in `testnet --help`.", + CLIENT_CONFIG_FILENAME + )); + } else { + // Write the onfig to a TOML file in the datadir. + write_to_file( + self.data_dir.join(CLIENT_CONFIG_FILENAME), + &self.client_config, + ) + .map_err(|e| format!("Unable to write {} file: {:?}", CLIENT_CONFIG_FILENAME, e))?; + } - // Write the eth2 config to a TOML file in the datadir. - write_to_file(self.data_dir.join(ETH2_CONFIG_FILENAME), &self.eth2_config) + let eth2_config_file = self.data_dir.join(ETH2_CONFIG_FILENAME); + if eth2_config_file.exists() { + return Err(format!( + "Datadir is not clean, {} exists. See `-f` in `testnet --help`.", + ETH2_CONFIG_FILENAME + )); + } else { + // Write the config to a TOML file in the datadir. + write_to_file( + self.data_dir.join(ETH2_CONFIG_FILENAME), + &self.client_config, + ) .map_err(|e| format!("Unable to write {} file: {:?}", ETH2_CONFIG_FILENAME, e))?; + } Ok(()) } @@ -225,7 +325,22 @@ impl<'a> ConfigBuilder<'a> { // public testnet or mainnet). if !self.data_dir.exists() { return Err( - "No datadir found. Use the 'testnet' sub-command to select a testnet type.".into(), + "No datadir found. Either create a new testnet or specify a different `--datadir`." + .into(), + ); + } + + // If there is a path to a databse in the config, ensure it exists. + if !self + .client_config + .db_path() + .map(|path| path.exists()) + .unwrap_or_else(|| true) + { + return Err( + "No database found in datadir. Use the 'testnet -f' sub-command to overwrite the \ + existing datadir, or specify a different `--datadir`." + .into(), ); } @@ -263,3 +378,10 @@ impl<'a> ConfigBuilder<'a> { Ok((self.client_config, self.eth2_config)) } } + +fn random_string(len: usize) -> String { + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(len) + .collect::() +} diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index d7a4bae795..4430db1287 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -161,7 +161,7 @@ fn main() { .help("Type of database to use.") .takes_value(true) .possible_values(&["disk", "memory"]) - .default_value("memory"), + .default_value("disk"), ) /* * Logging. @@ -207,15 +207,20 @@ fn main() { iteration.") ) .arg( - Arg::with_name("force-create") - .long("force-create") + Arg::with_name("force") + .long("force") .short("f") - .help("If present, will delete any existing datadir before creating a new one. Cannot be \ + .help("If present, will backup any existing config files before creating new ones. Cannot be \ used when specifying --random-datadir (logic error).") .conflicts_with("random-datadir") ) /* * Testnet sub-commands. + * + * `boostrap` + * + * Start a new node by downloading genesis and network info from another node via the + * HTTP API. */ .subcommand(SubCommand::with_name("bootstrap") .about("Connects to the given HTTP server, downloads a genesis state and attempts to peer with it.") @@ -231,6 +236,12 @@ fn main() { when port-fowarding is used: you may connect using a different port than \ the one the server is immediately listening on.")) ) + /* + * `recent` + * + * Start a new node, with a specified number of validators with a genesis time in the last + * 30-minutes. + */ .subcommand(SubCommand::with_name("recent") .about("Creates a new genesis state where the genesis time was at the previous \ 30-minute boundary (e.g., 12:00, 12:30, 13:00, etc.)")