gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-06 10:11:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix issues with testnet dir, update docs (#992)

Browse Source 
* Fix issues with testnet dir, update docs

* Remove "simple testnet" docs

* Tear out old "bn testnet" stuff

* Add back ClientGenesis::Interop

* Tidy

* Remove lighthouse-bootstrap module

* Fix bug with spec constant mismatch

* Ensure beacon-node.toml is written to correct dir

* Add -t alias for --testnet-dir

* Update book/src/local-testnets.md

Co-Authored-By: Age Manning <Age@AgeManning.com>

* Add --purge CLI flag

* Update purge docs

* Perform manual delete of files in purge

* Rename --purge to --purge-db

* Address Michael's comments

Co-authored-by: Age Manning <Age@AgeManning.com>
This commit is contained in:
Paul Hauner
2020-04-17 17:49:29 +10:00
committed by GitHub
parent a8ee3389c2
commit 1a3d1b3077
23 changed files with 280 additions and 1139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

440
beacon_node/src/config.rs
View File

@@ -1,14 +1,13 @@
use beacon_chain::builder::PUBKEY_CACHE_FILENAME;
use clap::ArgMatches;
use client::{config::DEFAULT_DATADIR, ClientConfig, ClientGenesis, Eth2Config};
use environment::ETH2_CONFIG_FILENAME;
use eth2_config::{read_from_file, write_to_file};
use client::{config::DEFAULT_DATADIR, ClientConfig, ClientGenesis};
use eth2_libp2p::{Enr, Multiaddr};
use eth2_testnet_config::Eth2TestnetConfig;
use genesis::recent_genesis_time;
use rand::{distributions::Alphanumeric, Rng};
use slog::{crit, info, warn, Logger};
use slog::{crit, warn, Logger};
use ssz::Encode;
use std::fs;
use std::fs::File;
use std::io::prelude::*;
use std::net::{IpAddr, Ipv4Addr};
use std::net::{TcpListener, UdpSocket};
use std::path::PathBuf;
@@ -18,8 +17,6 @@ pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml";
pub const BEACON_NODE_DIR: &str = "beacon";
pub const NETWORK_DIR: &str = "network";
type Result<T> = std::result::Result<T, String>;
/// Gets the fully-initialized global client.
///
/// The top-level `clap` arguments should be provided as `cli_args`.
@@ -30,22 +27,52 @@ type Result<T> = std::result::Result<T, String>;
#[allow(clippy::cognitive_complexity)]
pub fn get_config<E: EthSpec>(
cli_args: &ArgMatches,
eth2_config: Eth2Config,
core_log: Logger,
) -> Result<ClientConfig> {
let log = core_log.clone();
spec_constants: &str,
log: Logger,
) -> Result<ClientConfig, String> {
let mut client_config = ClientConfig::default();
client_config.spec_constants = eth2_config.spec_constants.clone();
client_config.data_dir = get_data_dir(cli_args);
// If necessary, remove any existing database and configuration
if client_config.data_dir.exists() && cli_args.is_present("purge-db") {
// Remove the chain_db.
fs::remove_dir_all(
client_config
.get_db_path()
.ok_or("Failed to get db_path".to_string())?,
)
.map_err(|err| format!("Failed to remove chain_db: {}", err))?;
// Remove the freezer db.
fs::remove_dir_all(
client_config
.get_freezer_db_path()
.ok_or("Failed to get freezer db path".to_string())?,
)
.map_err(|err| format!("Failed to remove chain_db: {}", err))?;
// Remove the pubkey cache file if it exists
let pubkey_cache_file = client_config.data_dir.join(PUBKEY_CACHE_FILENAME);
if pubkey_cache_file.exists() {
fs::remove_file(&pubkey_cache_file)
.map_err(|e| format!("Failed to remove {:?}: {:?}", pubkey_cache_file, e))?;
}
}
// Create `datadir` and any non-existing parent directories.
fs::create_dir_all(&client_config.data_dir)
.map_err(|e| format!("Failed to create data dir: {}", e))?;
// Load the client config, if it exists .
let path = client_config.data_dir.join(CLIENT_CONFIG_FILENAME);
if path.exists() {
client_config = read_from_file(path.clone())
.map_err(|e| format!("Unable to parse {:?} file: {:?}", path, e))?
.ok_or_else(|| format!("{:?} file does not exist", path))?;
let config_file_path = client_config.data_dir.join(CLIENT_CONFIG_FILENAME);
let config_file_existed = config_file_path.exists();
if config_file_existed {
client_config = read_from_file(config_file_path.clone())
.map_err(|e| format!("Unable to parse {:?} file: {:?}", config_file_path, e))?
.ok_or_else(|| format!("{:?} file does not exist", config_file_path))?;
} else {
client_config.spec_constants = spec_constants.into();
}
client_config.testnet_dir = get_testnet_dir(cli_args);
@@ -85,7 +112,7 @@ pub fn get_config<E: EthSpec>(
client_config.network.boot_nodes = boot_enr_str
.split(',')
.map(|enr| enr.parse().map_err(|_| format!("Invalid ENR: {}", enr)))
.collect::<Result<Vec<Enr>>>()?;
.collect::<Result<Vec<Enr>, _>>()?;
}
if let Some(libp2p_addresses_str) = cli_args.value_of("libp2p-addresses") {
@@ -96,7 +123,7 @@ pub fn get_config<E: EthSpec>(
.parse()
.map_err(|_| format!("Invalid Multiaddr: {}", multiaddr))
})
.collect::<Result<Vec<Multiaddr>>>()?;
.collect::<Result<Vec<Multiaddr>, _>>()?;
}
if let Some(topics_str) = cli_args.value_of("topics") {
@@ -121,6 +148,20 @@ pub fn get_config<E: EthSpec>(
client_config.network.secret_key_hex = Some(p2p_priv_key.to_string());
}
// Define a percentage of messages that should be propogated, useful for simulating bad network
// conditions.
//
// WARNING: setting this to anything less than 100 will cause bad behaviour.
if let Some(propagation_percentage_string) = cli_args.value_of("random-propagation") {
let percentage = propagation_percentage_string
.parse::<u8>()
.map_err(|_| "Unable to parse the propagation percentage".to_string())?;
if percentage > 100 {
return Err("Propagation percentage greater than 100".to_string());
}
client_config.network.propagation_percentage = Some(percentage);
}
/*
* Http server
*/
@@ -185,50 +226,6 @@ pub fn get_config<E: EthSpec>(
client_config.eth1.endpoint = val.to_string();
}
match cli_args.subcommand() {
("testnet", Some(sub_cmd_args)) => {
process_testnet_subcommand(&mut client_config, &eth2_config, sub_cmd_args)?
}
("purge", _) => {
client_config.purge_chain_db()?;
println!("Successfully purged chain db");
std::process::exit(0);
}
// No sub-command assumes a resume operation.
_ => {
// If no primary subcommand was given, start the beacon chain from an existing
// database.
client_config.genesis = ClientGenesis::Resume;
let db_path_exists: bool = match client_config.get_db_path() {
Some(path) => path.exists(),
None => false,
};
// Whilst there is no large testnet or mainnet force the user to specify how they want
// to start a new chain (e.g., from a genesis YAML file, another node, etc).
if !client_config.data_dir.exists()
|| (!db_path_exists && client_config.chain_db_was_purged())
{
info!(
log,
"Starting from an empty database";
"data_dir" => format!("{:?}", client_config.data_dir)
);
init_new_client::<E>(&mut client_config, &eth2_config)?
} else {
info!(
log,
"Resuming from existing datadir";
"data_dir" => format!("{:?}", client_config.data_dir)
);
// If the `testnet` command was not provided, attempt to load an existing datadir and
// continue with an existing chain.
load_from_datadir(&mut client_config)?
}
}
};
if let Some(freezer_dir) = cli_args.value_of("freezer-dir") {
client_config.freezer_db_path = Some(PathBuf::from(freezer_dir));
}
@@ -256,10 +253,10 @@ pub fn get_config<E: EthSpec>(
.map_err(|_| "block-cache-size is not a valid integer".to_string())?;
}
if eth2_config.spec_constants != client_config.spec_constants {
if spec_constants != client_config.spec_constants {
crit!(log, "Specification constants do not match.";
"client_config" => client_config.spec_constants.to_string(),
"eth2_config" => eth2_config.spec_constants
"eth2_config" => spec_constants
);
return Err("Specification constant mismatch".into());
}
@@ -294,6 +291,39 @@ pub fn get_config<E: EthSpec>(
client_config.network.discovery_address =
Some("127.0.0.1".parse().expect("Valid IP address"))
}
/*
* Load the eth2 testnet dir to obtain some additional config values.
*/
let eth2_testnet_config: Eth2TestnetConfig<E> =
get_eth2_testnet_config(&client_config.testnet_dir)?;
client_config.eth1.deposit_contract_address =
format!("{:?}", eth2_testnet_config.deposit_contract_address()?);
client_config.eth1.deposit_contract_deploy_block =
eth2_testnet_config.deposit_contract_deploy_block;
client_config.eth1.lowest_cached_block_number =
client_config.eth1.deposit_contract_deploy_block;
if let Some(mut boot_nodes) = eth2_testnet_config.boot_enr {
client_config.network.boot_nodes.append(&mut boot_nodes)
}
if let Some(genesis_state) = eth2_testnet_config.genesis_state {
// Note: re-serializing the genesis state is not so efficient, however it avoids adding
// trait bounds to the `ClientGenesis` enum. This would have significant flow-on
// effects.
client_config.genesis = ClientGenesis::SszBytes {
genesis_state_bytes: genesis_state.as_ssz_bytes(),
};
} else {
client_config.genesis = ClientGenesis::DepositContract;
}
if !config_file_existed {
write_to_file(config_file_path, &client_config)?;
}
Ok(client_config)
}
@@ -320,239 +350,26 @@ pub fn get_testnet_dir(cli_args: &ArgMatches) -> Option<PathBuf> {
}
}
/// If `testnet_dir` is `Some`, returns the `Eth2TestnetConfig` at that path or returns an error.
/// If it is `None`, returns the "hard coded" config.
pub fn get_eth2_testnet_config<E: EthSpec>(
testnet_dir: &Option<PathBuf>,
) -> Result<Eth2TestnetConfig<E>> {
) -> Result<Eth2TestnetConfig<E>, String> {
Ok(if let Some(testnet_dir) = testnet_dir {
Eth2TestnetConfig::load(testnet_dir.clone())
.map_err(|e| format!("Unable to open testnet dir at {:?}: {}", testnet_dir, e))?
} else {
Eth2TestnetConfig::hard_coded()
.map_err(|e| format!("Unable to load hard-coded testnet dir: {}", e))?
Eth2TestnetConfig::hard_coded().map_err(|e| {
format!(
"The hard-coded testnet directory was invalid. \
This happens when Lighthouse is migrating between spec versions. \
Error : {}",
e
)
})?
})
}
/// Load from an existing database.
fn load_from_datadir(client_config: &mut ClientConfig) -> Result<()> {
// Check to ensure the datadir exists.
//
// For now we return an error. In the future we may decide to boot a default (e.g.,
// public testnet or mainnet).
if !client_config.get_data_dir().map_or(false, |d| d.exists()) {
return Err(
"No datadir found. Either create a new testnet or specify a different `--datadir`."
.into(),
);
}
// If there is a path to a database in the config, ensure it exists.
if !client_config
.get_db_path()
.map_or(false, |path| path.exists())
{
return Err(
"No database found in datadir. Please make sure the directory provided is valid, or specify a different `--datadir`."
.into(),
);
}
client_config.genesis = ClientGenesis::Resume;
Ok(())
}
/// Create a new client with the default configuration.
fn init_new_client<E: EthSpec>(
client_config: &mut ClientConfig,
eth2_config: &Eth2Config,
) -> Result<()> {
let eth2_testnet_config: Eth2TestnetConfig<E> =
get_eth2_testnet_config(&client_config.testnet_dir)?;
client_config.eth1.deposit_contract_address =
format!("{:?}", eth2_testnet_config.deposit_contract_address()?);
client_config.eth1.deposit_contract_deploy_block =
eth2_testnet_config.deposit_contract_deploy_block;
client_config.eth1.follow_distance = eth2_config.spec.eth1_follow_distance / 2;
client_config.eth1.lowest_cached_block_number = client_config
.eth1
.deposit_contract_deploy_block
.saturating_sub(client_config.eth1.follow_distance * 2);
if let Some(mut boot_nodes) = eth2_testnet_config.boot_enr {
client_config.network.boot_nodes.append(&mut boot_nodes)
}
if let Some(genesis_state) = eth2_testnet_config.genesis_state {
// Note: re-serializing the genesis state is not so efficient, however it avoids adding
// trait bounds to the `ClientGenesis` enum. This would have significant flow-on
// effects.
client_config.genesis = ClientGenesis::SszBytes {
genesis_state_bytes: genesis_state.as_ssz_bytes(),
};
} else {
client_config.genesis = ClientGenesis::DepositContract;
}
create_new_datadir(&client_config, &eth2_config)?;
Ok(())
}
/// Writes the configs in `self` to `self.data_dir`.
///
/// Returns an error if `self.data_dir` already exists.
pub fn create_new_datadir(client_config: &ClientConfig, eth2_config: &Eth2Config) -> Result<()> {
let rebuild_db = client_config.chain_db_was_purged();
if client_config.data_dir.exists() && !rebuild_db {
return Err(format!(
"Data dir already exists at {:?}",
client_config.data_dir
));
}
// Create `datadir` and any non-existing parent directories.
fs::create_dir_all(&client_config.data_dir)
.map_err(|e| format!("Failed to create data dir: {}", e))?;
macro_rules! write_to_file {
($file: ident, $variable: ident) => {
let file = client_config.data_dir.join($file);
if file.exists() {
if !rebuild_db {
return Err(format!("Datadir is not clean, {} exists.", $file));
}
} else {
// Write the onfig to a TOML file in the datadir.
write_to_file(client_config.data_dir.join($file), $variable)
.map_err(|e| format!("Unable to write {} file: {:?}", $file, e))?;
}
};
}
write_to_file!(CLIENT_CONFIG_FILENAME, client_config);
write_to_file!(ETH2_CONFIG_FILENAME, eth2_config);
client_config.cleanup_after_purge_db()?;
Ok(())
}
/// Process the `testnet` CLI subcommand arguments, updating the `builder`.
fn process_testnet_subcommand(
client_config: &mut ClientConfig,
eth2_config: &Eth2Config,
cli_args: &ArgMatches,
) -> Result<()> {
// Specifies that a random datadir should be used.
if cli_args.is_present("random-datadir") {
client_config
.data_dir
.push(format!("random_{}", random_string(6)));
client_config.network.network_dir = client_config.data_dir.join("network");
}
// Deletes the existing datadir.
if cli_args.is_present("force") && client_config.data_dir.exists() {
fs::remove_dir_all(&client_config.data_dir)
.map_err(|e| format!("Unable to delete existing datadir: {:?}", e))?;
}
// Define a percentage of messages that should be propogated, useful for simulating bad network
// conditions.
//
// WARNING: setting this to anything less than 100 will cause bad behaviour.
if let Some(propagation_percentage_string) = cli_args.value_of("random-propagation") {
let percentage = propagation_percentage_string
.parse::<u8>()
.map_err(|_| "Unable to parse the propagation percentage".to_string())?;
if percentage > 100 {
return Err("Propagation percentage greater than 100".to_string());
}
client_config.network.propagation_percentage = Some(percentage);
}
// Start matching on the second subcommand (e.g., `testnet bootstrap ...`).
match cli_args.subcommand() {
("recent", Some(cli_args)) => {
let validator_count = cli_args
.value_of("validator_count")
.ok_or_else(|| "No validator_count specified")?
.parse::<usize>()
.map_err(|e| format!("Unable to parse validator_count: {:?}", e))?;
let minutes = cli_args
.value_of("minutes")
.ok_or_else(|| "No recent genesis minutes supplied")?
.parse::<u64>()
.map_err(|e| format!("Unable to parse minutes: {:?}", e))?;
client_config.dummy_eth1_backend = true;
client_config.genesis = ClientGenesis::Interop {
validator_count,
genesis_time: recent_genesis_time(minutes),
};
}
("quick", Some(cli_args)) => {
let validator_count = cli_args
.value_of("validator_count")
.ok_or_else(|| "No validator_count specified")?
.parse::<usize>()
.map_err(|e| format!("Unable to parse validator_count: {:?}", e))?;
let genesis_time = cli_args
.value_of("genesis_time")
.ok_or_else(|| "No genesis time supplied")?
.parse::<u64>()
.map_err(|e| format!("Unable to parse genesis time: {:?}", e))?;
client_config.dummy_eth1_backend = true;
client_config.genesis = ClientGenesis::Interop {
validator_count,
genesis_time,
};
}
("file", Some(cli_args)) => {
let path = cli_args
.value_of("file")
.ok_or_else(|| "No filename specified")?
.parse::<PathBuf>()
.map_err(|e| format!("Unable to parse filename: {:?}", e))?;
let format = cli_args
.value_of("format")
.ok_or_else(|| "No file format specified")?;
let start_method = match format {
"ssz" => ClientGenesis::SszFile { path },
other => return Err(format!("Unknown genesis file format: {}", other)),
};
client_config.genesis = start_method;
}
(cmd, Some(_)) => {
return Err(format!(
"Invalid valid method specified: {}. See 'testnet --help'.",
cmd
))
}
_ => return Err("No testnet method specified. See 'testnet --help'.".into()),
};
create_new_datadir(&client_config, &eth2_config)?;
Ok(())
}
fn random_string(len: usize) -> String {
rand::thread_rng()
.sample_iter(&Alphanumeric)
.take(len)
.collect::<String>()
}
/// A bit of hack to find an unused port.
///
/// Does not guarantee that the given port is unused after the function exists, just that it was
@@ -566,7 +383,7 @@ fn random_string(len: usize) -> String {
/// it doesn't allow binding to the same port even after the socket is closed.
/// We might have to use SO_REUSEADDR socket option from `std::net2` crate in
/// that case.
pub fn unused_port(transport: &str) -> Result<u16> {
pub fn unused_port(transport: &str) -> Result<u16, String> {
let local_addr = match transport {
"tcp" => {
let listener = TcpListener::bind("127.0.0.1:0").map_err(|e| {
@@ -593,3 +410,42 @@ pub fn unused_port(transport: &str) -> Result<u16> {
};
Ok(local_addr.port())
}
/// Write a configuration to file.
pub fn write_to_file<T>(path: PathBuf, config: &T) -> Result<(), String>
where
T: Default + serde::de::DeserializeOwned + serde::Serialize,
{
if let Ok(mut file) = File::create(path.clone()) {
let toml_encoded = toml::to_string(&config).map_err(|e| {
format!(
"Failed to write configuration to {:?}. Error: {:?}",
path, e
)
})?;
file.write_all(toml_encoded.as_bytes())
.unwrap_or_else(|_| panic!("Unable to write to {:?}", path));
}
Ok(())
}
/// Loads a `ClientConfig` from file. If unable to load from file, generates a default
/// configuration and saves that as a sample file.
pub fn read_from_file<T>(path: PathBuf) -> Result<Option<T>, String>
where
T: Default + serde::de::DeserializeOwned + serde::Serialize,
{
if let Ok(mut file) = File::open(path.clone()) {
let mut contents = String::new();
file.read_to_string(&mut contents)
.map_err(|e| format!("Unable to read {:?}. Error: {:?}", path, e))?;
let config = toml::from_str(&contents)
.map_err(|e| format!("Unable to parse {:?}: {:?}", path, e))?;
Ok(Some(config))
} else {
Ok(None)
}
}