From 2577136ba71648a15eb1760f1d02e9c9c7762598 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 28 Nov 2019 14:49:15 +1100 Subject: [PATCH] Add support for YamlConfig in Eth2TestnetDir --- beacon_node/Cargo.toml | 1 + beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/builder.rs | 21 ++++++++- beacon_node/client/src/config.rs | 5 +++ beacon_node/src/config.rs | 69 ++++++++++++++++------------- eth2/types/src/chain_spec.rs | 2 +- eth2/types/src/lib.rs | 2 +- eth2/utils/eth2_testnet/Cargo.toml | 2 +- eth2/utils/eth2_testnet/src/lib.rs | 29 ++++++------ lcli/src/deploy_deposit_contract.rs | 22 ++++++++- lcli/src/eth1_genesis.rs | 36 +++++++-------- 11 files changed, 118 insertions(+), 72 deletions(-) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 8fc0d8f659..a451178cdf 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -36,3 +36,4 @@ environment = { path = "../lighthouse/environment" } genesis = { path = "genesis" } eth2_testnet = { path = "../eth2/utils/eth2_testnet" } eth2-libp2p = { path = "./eth2-libp2p" } +eth2_ssz = { path = "../eth2/utils/ssz" } diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 5802fe98cb..e5293cd196 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -38,3 +38,4 @@ eth1 = { path = "../eth1" } genesis = { path = "../genesis" } environment = { path = "../../lighthouse/environment" } lighthouse_bootstrap = { path = "../../eth2/utils/lighthouse_bootstrap" } +eth2_ssz = { path = "../../eth2/utils/ssz" } diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 630832cdd4..1ebf96d769 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -23,13 +23,14 @@ use lighthouse_bootstrap::Bootstrapper; use lmd_ghost::LmdGhost; use network::{NetworkConfig, NetworkMessage, Service as NetworkService}; use slog::{debug, error, info, warn}; +use ssz::Decode; use std::net::SocketAddr; use std::path::Path; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::mpsc::UnboundedSender; use tokio::timer::Interval; -use types::{ChainSpec, EthSpec}; +use types::{BeaconState, ChainSpec, EthSpec}; use websocket_server::{Config as WebSocketConfig, WebSocketSender}; /// The interval between notifier events. @@ -187,6 +188,24 @@ where Box::new(future) } + ClientGenesis::SszBytes { + genesis_state_bytes, + } => { + info!( + context.log, + "Using trusted genesis state"; + ); + + let result = BeaconState::from_ssz_bytes(&genesis_state_bytes) + .map_err(|e| format!("Unable to parse genesis state SSZ: {:?}", e)); + + let future = result + .and_then(move |genesis_state| builder.genesis_state(genesis_state)) + .into_future() + .map(|v| (v, None)); + + Box::new(future) + } ClientGenesis::DepositContract => { info!( context.log, diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 661240358a..dc01a25d6a 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -24,6 +24,11 @@ pub enum ClientGenesis { DepositContract, /// Loads the genesis state from a SSZ-encoded `BeaconState` file. SszFile { path: PathBuf }, + /// Loads the genesis state from SSZ-encoded `BeaconState` bytes. + /// + /// We include the bytes instead of the `BeaconState` because the `EthSpec` type + /// parameter would be very annoying. + SszBytes { genesis_state_bytes: Vec }, /// Connects to another Lighthouse instance and reads the genesis state and other data via the /// HTTP API. RemoteNode { server: String, port: Option }, diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 475a6475d6..a4d23e93b9 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -6,6 +6,7 @@ use eth2_testnet::Eth2TestnetDir; use genesis::recent_genesis_time; use rand::{distributions::Alphanumeric, Rng}; use slog::{crit, info, Logger}; +use ssz::Encode; use std::fs; use std::net::Ipv4Addr; use std::path::PathBuf; @@ -17,8 +18,6 @@ pub const ETH2_TESTNET_DIR: &str = "testnet"; pub const BEACON_NODE_DIR: &str = "beacon"; pub const NETWORK_DIR: &str = "network"; -pub const SECONDS_PER_ETH1_BLOCK: u64 = 15; - type Result = std::result::Result; type Config = (ClientConfig, Eth2Config, Logger); @@ -207,12 +206,6 @@ pub fn get_configs( } // No sub-command assumes a resume operation. _ => { - info!( - log, - "Resuming from existing datadir"; - "path" => format!("{:?}", client_config.data_dir) - ); - // If no primary subcommand was given, start the beacon chain from an existing // database. client_config.genesis = ClientGenesis::Resume; @@ -220,8 +213,18 @@ pub fn get_configs( // Whilst there is no large testnet or mainnet force the user to specify how they want // to start a new chain (e.g., from a genesis YAML file, another node, etc). if !client_config.data_dir.exists() { + info!( + log, + "Starting from an empty database"; + "data_dir" => format!("{:?}", client_config.data_dir) + ); init_new_client::(&mut client_config, &mut eth2_config)? } else { + info!( + log, + "Resuming from existing datadir"; + "data_dir" => format!("{:?}", client_config.data_dir) + ); // If the `testnet` command was not provided, attempt to load an existing datadir and // continue with an existing chain. load_from_datadir(&mut client_config, &mut eth2_config)? @@ -291,6 +294,8 @@ fn load_from_datadir(client_config: &mut ClientConfig, eth2_config: &mut Eth2Con .map_err(|e| format!("Unable to parse {:?} file: {:?}", path, e))? .ok_or_else(|| format!("{:?} file does not exist", path))?; + client_config.genesis = ClientGenesis::Resume; + Ok(()) } @@ -299,44 +304,46 @@ fn init_new_client( client_config: &mut ClientConfig, eth2_config: &mut Eth2Config, ) -> Result<()> { - let spec = &mut eth2_config.spec; - - spec.min_deposit_amount = 100; - spec.max_effective_balance = 3_200_000_000; - spec.ejection_balance = 1_600_000_000; - spec.effective_balance_increment = 100_000_000; - let testnet_dir = client_config.testnet_dir.clone(); let eth2_testnet_dir: Eth2TestnetDir = Eth2TestnetDir::load(testnet_dir.clone()) .map_err(|e| format!("Unable to open testnet dir at {:?}: {}", testnet_dir, e))?; + eth2_config.spec = eth2_testnet_dir + .yaml_config + .as_ref() + .ok_or_else(|| "The testnet directory must contain a spec config".to_string())? + .apply_to_chain_spec::(ð2_config.spec) + .ok_or_else(|| { + format!( + "The loaded config is not compatible with the {} spec", + ð2_config.spec_constants + ) + })?; + + let spec = &mut eth2_config.spec; + client_config.eth1.deposit_contract_address = format!("{:?}", eth2_testnet_dir.deposit_contract_address()?); client_config.eth1.deposit_contract_deploy_block = eth2_testnet_dir.deposit_contract_deploy_block; - spec.min_genesis_time = eth2_testnet_dir.min_genesis_time; - client_config.eth1.follow_distance = 16; + client_config.eth1.follow_distance = spec.eth1_follow_distance / 2; client_config.dummy_eth1_backend = false; - client_config.sync_eth1_chain = true; client_config.eth1.lowest_cached_block_number = client_config .eth1 .deposit_contract_deploy_block .saturating_sub(client_config.eth1.follow_distance * 2); - // Note: these constants _should_ only be used during genesis to determine the genesis - // time. This allows the testnet to start shortly after the time + validator count - // conditions are satisfied, not 1-2 days. - // - // This value must be at least 2x the `ETH1_FOLLOW_DISTANCE` otherwise `all_eth1_data` - // can become a subset of `new_eth1_data`. - // - // See: - // https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/validator/0_beacon-chain-validator.md#eth1-data - // TODO: set from 10 back to 2 - spec.seconds_per_day = SECONDS_PER_ETH1_BLOCK * spec.eth1_follow_distance * 2; - - client_config.genesis = ClientGenesis::DepositContract; + if let Some(genesis_state) = eth2_testnet_dir.genesis_state { + // Note: re-serializing the genesis state is not so efficient, however it avoids adding + // trait bounds to the `ClientGenesis` enum. This would have significant flow-on + // effects. + client_config.genesis = ClientGenesis::SszBytes { + genesis_state_bytes: genesis_state.as_ssz_bytes(), + }; + } else { + client_config.genesis = ClientGenesis::DepositContract; + } create_new_datadir(&client_config, ð2_config)?; diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 00efdbfe3e..2811fbd9bb 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -319,7 +319,7 @@ mod tests { } // Yaml Config is declared here in order to access domain fields of ChainSpec which are private fields. -#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] #[serde(rename_all = "UPPERCASE")] #[serde(default)] #[serde(deny_unknown_fields)] diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index d6d7dae7a8..a82d398f91 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -48,7 +48,7 @@ pub use crate::beacon_block_body::BeaconBlockBody; pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{Error as BeaconStateError, *}; -pub use crate::chain_spec::{ChainSpec, Domain}; +pub use crate::chain_spec::{ChainSpec, Domain, YamlConfig}; pub use crate::checkpoint::Checkpoint; pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH}; pub use crate::deposit_data::DepositData; diff --git a/eth2/utils/eth2_testnet/Cargo.toml b/eth2/utils/eth2_testnet/Cargo.toml index 543b23fc12..967c28880b 100644 --- a/eth2/utils/eth2_testnet/Cargo.toml +++ b/eth2/utils/eth2_testnet/Cargo.toml @@ -11,6 +11,6 @@ tempdir = "0.3" [dependencies] serde = "1.0" -serde_json = "^1.0" +serde_yaml = "0.8" types = { path = "../../types"} eth2-libp2p = { path = "../../../beacon_node/eth2-libp2p"} diff --git a/eth2/utils/eth2_testnet/src/lib.rs b/eth2/utils/eth2_testnet/src/lib.rs index e0701099cb..e37e11d044 100644 --- a/eth2/utils/eth2_testnet/src/lib.rs +++ b/eth2/utils/eth2_testnet/src/lib.rs @@ -10,21 +10,21 @@ use eth2_libp2p::Enr; use std::fs::{create_dir_all, File}; use std::path::PathBuf; -use types::{Address, BeaconState, EthSpec}; +use types::{Address, BeaconState, EthSpec, YamlConfig}; pub const ADDRESS_FILE: &str = "deposit_contract.txt"; pub const DEPLOY_BLOCK_FILE: &str = "deploy_block.txt"; -pub const MIN_GENESIS_TIME_FILE: &str = "min_genesis_time.txt"; -pub const BOOT_NODES_FILE: &str = "boot_enr.json"; +pub const BOOT_NODES_FILE: &str = "boot_enr.yaml"; pub const GENESIS_STATE_FILE: &str = "genesis.ssz"; +pub const YAML_CONFIG_FILE: &str = "config.yaml"; #[derive(Clone, PartialEq, Debug)] pub struct Eth2TestnetDir { pub deposit_contract_address: String, pub deposit_contract_deploy_block: u64, - pub min_genesis_time: u64, pub boot_enr: Option>, pub genesis_state: Option>, + pub yaml_config: Option, } impl Eth2TestnetDir { @@ -47,7 +47,7 @@ impl Eth2TestnetDir { File::create(base_dir.join($file)) .map_err(|e| format!("Unable to create {}: {:?}", $file, e)) .and_then(|file| { - serde_json::to_writer(file, &$variable) + serde_yaml::to_writer(file, &$variable) .map_err(|e| format!("Unable to write {}: {:?}", $file, e)) })?; }; @@ -55,7 +55,6 @@ impl Eth2TestnetDir { write_to_file!(ADDRESS_FILE, self.deposit_contract_address); write_to_file!(DEPLOY_BLOCK_FILE, self.deposit_contract_deploy_block); - write_to_file!(MIN_GENESIS_TIME_FILE, self.min_genesis_time); if let Some(boot_enr) = &self.boot_enr { write_to_file!(BOOT_NODES_FILE, boot_enr); @@ -65,6 +64,10 @@ impl Eth2TestnetDir { write_to_file!(GENESIS_STATE_FILE, genesis_state); } + if let Some(yaml_config) = &self.yaml_config { + write_to_file!(YAML_CONFIG_FILE, yaml_config); + } + Ok(()) } @@ -74,7 +77,7 @@ impl Eth2TestnetDir { File::open(base_dir.join($file)) .map_err(|e| format!("Unable to open {}: {:?}", $file, e)) .and_then(|file| { - serde_json::from_reader(file) + serde_yaml::from_reader(file) .map_err(|e| format!("Unable to parse {}: {:?}", $file, e)) })?; }; @@ -92,16 +95,16 @@ impl Eth2TestnetDir { let deposit_contract_address = load_from_file!(ADDRESS_FILE); let deposit_contract_deploy_block = load_from_file!(DEPLOY_BLOCK_FILE); - let min_genesis_time = load_from_file!(MIN_GENESIS_TIME_FILE); let boot_enr = optional_load_from_file!(BOOT_NODES_FILE); let genesis_state = optional_load_from_file!(GENESIS_STATE_FILE); + let yaml_config = optional_load_from_file!(YAML_CONFIG_FILE); Ok(Self { deposit_contract_address, deposit_contract_deploy_block, - min_genesis_time, boot_enr, genesis_state, + yaml_config, }) } @@ -130,16 +133,16 @@ mod tests { let base_dir = PathBuf::from(temp_dir.path().join("my_testnet")); let deposit_contract_address = "0xBB9bc244D798123fDe783fCc1C72d3Bb8C189413".to_string(); let deposit_contract_deploy_block = 42; - let min_genesis_time = 1337; let testnet: Eth2TestnetDir = Eth2TestnetDir { deposit_contract_address: deposit_contract_address.clone(), deposit_contract_deploy_block: deposit_contract_deploy_block, - min_genesis_time, // TODO: add some Enr for testing. boot_enr: None, // TODO: add a genesis state for testing. genesis_state: None, + // TODO: add a yaml config for testing. + yaml_config: None, }; testnet @@ -156,10 +159,6 @@ mod tests { decoded.deposit_contract_deploy_block, deposit_contract_deploy_block, "deposit_contract_deploy_block" ); - assert_eq!( - decoded.min_genesis_time, min_genesis_time, - "min_genesis_time" - ); assert_eq!(testnet, decoded, "should decode as encoded"); } diff --git a/lcli/src/deploy_deposit_contract.rs b/lcli/src/deploy_deposit_contract.rs index 4db463f400..ecfcc21f41 100644 --- a/lcli/src/deploy_deposit_contract.rs +++ b/lcli/src/deploy_deposit_contract.rs @@ -5,9 +5,11 @@ use eth2_testnet::Eth2TestnetDir; use std::fs::File; use std::io::Read; use std::path::PathBuf; -use types::EthSpec; +use types::{EthSpec, YamlConfig}; use web3::{transports::Http, Web3}; +pub const SECONDS_PER_ETH1_BLOCK: u64 = 15; + pub fn run(mut env: Environment, matches: &ArgMatches) -> Result<(), String> { let min_genesis_time = matches .value_of("min-genesis-time") @@ -86,12 +88,28 @@ pub fn run(mut env: Environment, matches: &ArgMatches) -> Result< info!("Writing config to {:?}", output_dir); + let mut spec = env.core_context().eth2_config.spec.clone(); + + spec.min_deposit_amount = 100; + spec.max_effective_balance = 3_200_000_000; + spec.ejection_balance = 1_600_000_000; + spec.effective_balance_increment = 100_000_000; + spec.min_genesis_time = min_genesis_time; + + // This value must be at least 2x the `ETH1_FOLLOW_DISTANCE` otherwise `all_eth1_data` + // can become a subset of `new_eth1_data`. + // + // See: + // https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/validator/0_beacon-chain-validator.md#eth1-data + // TODO: set from 10 back to 2 + spec.seconds_per_day = SECONDS_PER_ETH1_BLOCK * spec.eth1_follow_distance * 2; + let testnet_dir: Eth2TestnetDir = Eth2TestnetDir { deposit_contract_address: format!("{}", deposit_contract.address()), deposit_contract_deploy_block: deploy_block.as_u64(), - min_genesis_time, boot_enr: None, genesis_state: None, + yaml_config: Some(YamlConfig::from_spec::(&spec)), }; testnet_dir.write_to_file(output_dir)?; diff --git a/lcli/src/eth1_genesis.rs b/lcli/src/eth1_genesis.rs index c2e32286cd..07e541e3c2 100644 --- a/lcli/src/eth1_genesis.rs +++ b/lcli/src/eth1_genesis.rs @@ -5,11 +5,10 @@ use futures::Future; use genesis::{Eth1Config, Eth1GenesisService}; use std::path::PathBuf; use std::time::Duration; -use types::{EthSpec, Fork}; +use types::EthSpec; /// Interval between polling the eth1 node for genesis information. -pub const ETH1_GENESIS_UPDATE_INTERVAL_MILLIS: u64 = 7_000; -pub const SECONDS_PER_ETH1_BLOCK: u64 = 15; +pub const ETH1_GENESIS_UPDATE_INTERVAL: Duration = Duration::from_millis(7_000); pub fn run(mut env: Environment, matches: &ArgMatches) -> Result<(), String> { let endpoint = matches @@ -28,32 +27,29 @@ pub fn run(mut env: Environment, matches: &ArgMatches) -> Result< let mut eth2_testnet_dir: Eth2TestnetDir = Eth2TestnetDir::load(testnet_dir.clone())?; + let spec = eth2_testnet_dir + .yaml_config + .as_ref() + .ok_or_else(|| "The testnet directory must contain a spec config".to_string())? + .apply_to_chain_spec::(&env.core_context().eth2_config.spec) + .ok_or_else(|| { + format!( + "The loaded config is not compatible with the {} spec", + &env.core_context().eth2_config.spec_constants + ) + })?; + let mut config = Eth1Config::default(); config.endpoint = endpoint.to_string(); config.deposit_contract_address = eth2_testnet_dir.deposit_contract_address.clone(); config.deposit_contract_deploy_block = eth2_testnet_dir.deposit_contract_deploy_block; config.lowest_cached_block_number = eth2_testnet_dir.deposit_contract_deploy_block; + config.follow_distance = spec.eth1_follow_distance / 2; let genesis_service = Eth1GenesisService::new(config, env.core_context().log.clone()); - let mut spec = env.core_context().eth2_config.spec.clone(); - - spec.min_genesis_time = eth2_testnet_dir.min_genesis_time; - - spec.min_deposit_amount = 100; - spec.max_effective_balance = 3_200_000_000; - spec.ejection_balance = 1_600_000_000; - spec.effective_balance_increment = 100_000_000; - - // Note: these are hard-coded hacky values. This should be fixed when we can load a testnet - // dir from the `Eth2TestnetDir`. - spec.eth1_follow_distance = 16; - spec.seconds_per_day = SECONDS_PER_ETH1_BLOCK * spec.eth1_follow_distance * 2; let future = genesis_service - .wait_for_genesis_state( - Duration::from_millis(ETH1_GENESIS_UPDATE_INTERVAL_MILLIS), - spec, - ) + .wait_for_genesis_state(ETH1_GENESIS_UPDATE_INTERVAL, spec) .map(move |genesis_state| { eth2_testnet_dir.genesis_state = Some(genesis_state); eth2_testnet_dir.force_write_to_file(testnet_dir)