From 93afdac904f37afc81973e93590c337ea2575ce7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 19 Nov 2019 10:52:29 +1100 Subject: [PATCH 1/5] Integrate ValidatorDirectory with validator_client --- account_manager/Cargo.toml | 1 - validator_client/Cargo.toml | 3 +- validator_client/src/cli.rs | 24 +++++-- validator_client/src/config.rs | 115 +++++++++++--------------------- validator_client/src/service.rs | 3 +- 5 files changed, 61 insertions(+), 85 deletions(-) diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index 55221ba522..b9b4f8b4bd 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -13,7 +13,6 @@ clap = "2.33.0" slog = "2.5.2" slog-term = "2.4.2" slog-async = "2.3.0" -validator_client = { path = "../validator_client" } types = { path = "../eth2/types" } dirs = "2.0.2" environment = { path = "../lighthouse/environment" } diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index c3e9cb8561..f824b9975a 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -9,7 +9,6 @@ name = "validator_client" path = "src/lib.rs" [dependencies] -bls = { path = "../eth2/utils/bls" } eth2_ssz = "0.1.2" eth2_config = { path = "../eth2/utils/eth2_config" } tree_hash = "0.1.0" @@ -37,3 +36,5 @@ environment = { path = "../lighthouse/environment" } parking_lot = "0.7" exit-future = "0.1.4" libc = "0.2.65" +account_manager = { path = "../account_manager" } +lazy_static = "1.4.0" diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 6b6956f786..1eb71e68d2 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -1,5 +1,21 @@ -use crate::config::{DEFAULT_SERVER, DEFAULT_SERVER_GRPC_PORT, DEFAULT_SERVER_HTTP_PORT}; +use crate::config::Config; use clap::{App, Arg, SubCommand}; +use lazy_static::lazy_static; + +lazy_static! { + /// The default configuration. Is in lazy_static because clap requires references, therefore we + /// can't initialize the defaults in the `cli_app` function + static ref DEFAULTS: Config = { + Config::default() + }; + + static ref DEFAULT_SERVER_GRPC_PORT: String = { + format!("{}", DEFAULTS.server_grpc_port) + }; + static ref DEFAULT_SERVER_HTTP_PORT: String = { + format!("{}", DEFAULTS.server_http_port) + }; +} pub fn cli_app<'a, 'b>() -> App<'a, 'b> { App::new("Validator Client") @@ -35,7 +51,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("server") .value_name("NETWORK_ADDRESS") .help("Address to connect to BeaconNode.") - .default_value(DEFAULT_SERVER) + .default_value(&DEFAULTS.server) .takes_value(true), ) .arg( @@ -44,7 +60,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .short("g") .value_name("PORT") .help("Port to use for gRPC API connection to the server.") - .default_value(DEFAULT_SERVER_GRPC_PORT) + .default_value(&DEFAULT_SERVER_GRPC_PORT) .takes_value(true), ) .arg( @@ -53,7 +69,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .short("h") .value_name("PORT") .help("Port to use for HTTP API connection to the server.") - .default_value(DEFAULT_SERVER_HTTP_PORT) + .default_value(&DEFAULT_SERVER_HTTP_PORT) .takes_value(true), ) /* diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 749a5813cd..539e95bef5 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -1,5 +1,5 @@ +use account_manager::validator::ValidatorDirectory; use bincode; -use bls::Keypair; use clap::ArgMatches; use serde_derive::{Deserialize, Serialize}; use slog::{error, warn}; @@ -9,13 +9,9 @@ use std::ops::Range; use std::path::PathBuf; use types::{ test_utils::{generate_deterministic_keypair, load_keypairs_from_yaml}, - EthSpec, MainnetEthSpec, + EthSpec, Keypair, MainnetEthSpec, }; -pub const DEFAULT_SERVER: &str = "localhost"; -pub const DEFAULT_SERVER_GRPC_PORT: &str = "5051"; -pub const DEFAULT_SERVER_HTTP_PORT: &str = "5052"; - #[derive(Clone)] pub enum KeySource { /// Load the keypairs from disk. @@ -58,16 +54,12 @@ impl Default for Config { /// Build a new configuration from defaults. fn default() -> Self { Self { - data_dir: PathBuf::from(".lighthouse-validator"), + data_dir: PathBuf::from(".lighthouse/validators"), key_source: <_>::default(), log_file: PathBuf::from(""), - server: DEFAULT_SERVER.into(), - server_grpc_port: DEFAULT_SERVER_GRPC_PORT - .parse::() - .expect("gRPC port constant should be valid"), - server_http_port: DEFAULT_SERVER_GRPC_PORT - .parse::() - .expect("HTTP port constant should be valid"), + server: "localhost".into(), + server_grpc_port: 5051, + server_http_port: 5052, slots_per_epoch: MainnetEthSpec::slots_per_epoch(), } } @@ -106,75 +98,44 @@ impl Config { Ok(()) } - /// Reads a single keypair from the given `path`. + /// Loads the validator keys from disk. /// - /// `path` should be the path to a directory containing a private key. The file name of `path` - /// must align with the public key loaded from it, otherwise an error is returned. + /// ## Errors /// - /// An error will be returned if `path` is a file (not a directory). - fn read_keypair_file(&self, path: PathBuf) -> Result { - if !path.is_dir() { - return Err("Is not a directory".into()); - } - - let key_filename: PathBuf = path.join(DEFAULT_PRIVATE_KEY_FILENAME); - - if !key_filename.is_file() { - return Err(format!( - "Private key is not a file: {:?}", - key_filename.to_str() - )); - } - - let mut key_file = File::open(key_filename.clone()) - .map_err(|e| format!("Unable to open private key file: {}", e))?; - - let key: Keypair = bincode::deserialize_from(&mut key_file) - .map_err(|e| format!("Unable to deserialize private key: {:?}", e))?; - - let ki = key.identifier(); - if ki - != path - .file_name() - .ok_or_else(|| "Invalid path".to_string())? - .to_string_lossy() - { - Err(format!( - "The validator key ({:?}) did not match the directory filename {:?}.", - ki, - path.to_str() - )) - } else { - Ok(key) - } - } - + /// Returns an error if the base directory does not exist, however it does not return for any + /// invalid directories/files. Instead, it just filters out failures and logs errors. This + /// behaviour is intended to avoid the scenario where a single invalid file can stop all + /// validators. pub fn fetch_keys_from_disk(&self, log: &slog::Logger) -> Result, String> { - Ok( - fs::read_dir(&self.full_data_dir().expect("Data dir must exist")) - .map_err(|e| format!("Failed to read datadir: {:?}", e))? - .filter_map(|validator_dir| { - let path = validator_dir.ok()?.path(); + let base_dir = self + .full_data_dir() + .ok_or_else(|| format!("Base directory does not exist: {:?}", self.full_data_dir()))?; - if path.is_dir() { - match self.read_keypair_file(path.clone()) { - Ok(keypair) => Some(keypair), - Err(e) => { - error!( - log, - "Failed to parse a validator keypair"; - "error" => e, - "path" => path.to_str(), - ); - None - } + let keypairs = fs::read_dir(&base_dir) + .map_err(|e| format!("Failed to read base directory: {:?}", e))? + .filter_map(|validator_dir| { + let path = validator_dir.ok()?.path(); + + if path.is_dir() { + match ValidatorDirectory::load_for_signing(path.clone()) { + Ok(validator_directory) => validator_directory.voting_keypair, + Err(e) => { + error!( + log, + "Failed to load a validator directory"; + "error" => e, + "path" => path.to_str(), + ); + None } - } else { - None } - }) - .collect(), - ) + } else { + None + } + }) + .collect(); + + Ok(keypairs) } pub fn fetch_testing_keypairs( diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index b193941147..4d304ddd2c 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -14,7 +14,6 @@ use crate::config::Config as ValidatorConfig; use crate::duties::{BeaconNodeDuties, DutiesManager, EpochDutiesMap}; use crate::error as error_chain; use crate::signer::Signer; -use bls::Keypair; use eth2_config::Eth2Config; use grpcio::{ChannelBuilder, EnvBuilder}; use parking_lot::RwLock; @@ -28,7 +27,7 @@ use slot_clock::{SlotClock, SystemTimeSlotClock}; use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; -use types::{ChainSpec, Epoch, EthSpec, Fork, Slot}; +use types::{ChainSpec, Epoch, EthSpec, Fork, Keypair, Slot}; /// The validator service. This is the main thread that executes and maintains validator /// duties. From c5071afdbca0881a7daa4d969c38e67e118bdf15 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 19 Nov 2019 11:01:49 +1100 Subject: [PATCH 2/5] Move ValidatorDirectory into validator_client --- account_manager/Cargo.toml | 1 + account_manager/src/lib.rs | 3 +-- validator_client/Cargo.toml | 8 +++++++- validator_client/src/config.rs | 2 +- validator_client/src/lib.rs | 1 + .../src/validator_directory.rs | 0 6 files changed, 11 insertions(+), 4 deletions(-) rename account_manager/src/validator.rs => validator_client/src/validator_directory.rs (100%) diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index b9b4f8b4bd..efbf9beb00 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -21,3 +21,4 @@ libc = "0.2.65" eth2_ssz = { path = "../eth2/utils/ssz" } eth2_ssz_derive = { path = "../eth2/utils/ssz_derive" } hex = "0.4" +validator_client = { path = "../validator_client" } diff --git a/account_manager/src/lib.rs b/account_manager/src/lib.rs index 58d2427f2d..7c19e953c0 100644 --- a/account_manager/src/lib.rs +++ b/account_manager/src/lib.rs @@ -1,5 +1,4 @@ mod cli; -pub mod validator; use clap::ArgMatches; use environment::RuntimeContext; @@ -7,7 +6,7 @@ use slog::{crit, info}; use std::fs; use std::path::PathBuf; use types::{ChainSpec, EthSpec}; -use validator::{ValidatorDirectory, ValidatorDirectoryBuilder}; +use validator_client::validator_directory::{ValidatorDirectory, ValidatorDirectoryBuilder}; pub use cli::cli_app; diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index f824b9975a..1d6dadb4a0 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -8,6 +8,9 @@ edition = "2018" name = "validator_client" path = "src/lib.rs" +[dev-dependencies] +tempdir = "0.3" + [dependencies] eth2_ssz = "0.1.2" eth2_config = { path = "../eth2/utils/eth2_config" } @@ -36,5 +39,8 @@ environment = { path = "../lighthouse/environment" } parking_lot = "0.7" exit-future = "0.1.4" libc = "0.2.65" -account_manager = { path = "../account_manager" } lazy_static = "1.4.0" +eth2_ssz_derive = { path = "../eth2/utils/ssz_derive" } +hex = "0.4" +deposit_contract = { path = "../eth2/utils/deposit_contract" } +bls = { path = "../eth2/utils/bls" } diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 539e95bef5..336585788f 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -1,4 +1,4 @@ -use account_manager::validator::ValidatorDirectory; +use crate::validator_directory::ValidatorDirectory; use bincode; use clap::ArgMatches; use serde_derive::{Deserialize, Serialize}; diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 175ee4793e..930f35c32a 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -6,6 +6,7 @@ mod duties; mod error; mod service; mod signer; +pub mod validator_directory; pub use cli::cli_app; pub use config::Config; diff --git a/account_manager/src/validator.rs b/validator_client/src/validator_directory.rs similarity index 100% rename from account_manager/src/validator.rs rename to validator_client/src/validator_directory.rs From 90d63a46c7674ece2dc5f6465c5300af900e33cd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 19 Nov 2019 13:29:29 +1100 Subject: [PATCH 3/5] Add beacon_chain_sim --- Cargo.toml | 1 + beacon_node/client/src/lib.rs | 11 +++++ lighthouse/environment/src/lib.rs | 2 +- tests/beacon_chain_sim/Cargo.toml | 11 +++++ tests/beacon_chain_sim/src/main.rs | 77 ++++++++++++++++++++++++++++++ tests/node_test_rig/src/lib.rs | 26 +++++----- 6 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 tests/beacon_chain_sim/Cargo.toml create mode 100644 tests/beacon_chain_sim/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 6fc81d267f..e4c029951c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ members = [ "beacon_node/eth1", "beacon_node/beacon_chain", "beacon_node/websocket_server", + "tests/beacon_chain_sim", "tests/ef_tests", "tests/eth1_test_rig", "tests/node_test_rig", diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 5da442bb10..f0ed63e485 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -6,6 +6,7 @@ pub mod builder; pub mod error; use beacon_chain::BeaconChain; +use eth2_libp2p::{Enr, Multiaddr}; use exit_future::Signal; use network::Service as NetworkService; use std::net::SocketAddr; @@ -48,6 +49,16 @@ impl Client { pub fn libp2p_listen_port(&self) -> Option { self.libp2p_network.as_ref().map(|n| n.listen_port()) } + + /// Returns the list of libp2p addresses the client is listening to. + pub fn libp2p_listen_addresses(&self) -> Option> { + self.libp2p_network.as_ref().map(|n| n.listen_multiaddrs()) + } + + /// Returns the local libp2p ENR of this node, for network discovery. + pub fn enr(&self) -> Option { + self.libp2p_network.as_ref().map(|n| n.local_enr()) + } } impl Drop for Client { diff --git a/lighthouse/environment/src/lib.rs b/lighthouse/environment/src/lib.rs index 69002682a3..631e799271 100644 --- a/lighthouse/environment/src/lib.rs +++ b/lighthouse/environment/src/lib.rs @@ -193,7 +193,7 @@ impl Environment { } /// Returns a `Context` where the `service_name` is added to the logger output. - pub fn service_context(&mut self, service_name: &'static str) -> RuntimeContext { + pub fn service_context(&mut self, service_name: String) -> RuntimeContext { RuntimeContext { executor: self.runtime.executor(), log: self.log.new(o!("service" => service_name)), diff --git a/tests/beacon_chain_sim/Cargo.toml b/tests/beacon_chain_sim/Cargo.toml new file mode 100644 index 0000000000..8ce05e2d66 --- /dev/null +++ b/tests/beacon_chain_sim/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "beacon_chain_sim" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +node_test_rig = { path = "../node_test_rig" } +types = { path = "../../eth2/types" } diff --git a/tests/beacon_chain_sim/src/main.rs b/tests/beacon_chain_sim/src/main.rs new file mode 100644 index 0000000000..0915bd372f --- /dev/null +++ b/tests/beacon_chain_sim/src/main.rs @@ -0,0 +1,77 @@ +use node_test_rig::{ + environment::{EnvironmentBuilder, RuntimeContext}, + testing_client_config, ClientConfig, LocalBeaconNode, ProductionClient, +}; +use types::EthSpec; + +pub type BeaconNode = LocalBeaconNode>; + +fn main() { + match simulation(4) { + Ok(()) => println!("Simulation exited successfully"), + Err(e) => println!("Simulation exited with error: {}", e), + } +} + +fn simulation(num_nodes: usize) -> Result<(), String> { + if num_nodes < 1 { + return Err("Must have at least one node".into()); + } + + let mut env = EnvironmentBuilder::minimal() + .async_logger("debug")? + .multi_threaded_tokio_runtime()? + .build()?; + + let base_config = testing_client_config(); + + let boot_node = + BeaconNode::production(env.service_context("boot_node".into()), base_config.clone()); + + let nodes = (1..num_nodes) + .map(|i| { + let context = env.service_context(format!("node_{}", i)); + new_with_bootnode_via_enr(context, &boot_node, base_config.clone()) + }) + .collect::>(); + + env.block_until_ctrl_c()?; + + Ok(()) +} + +// TODO: this function does not result in nodes connecting to each other. Age to investigate? +fn new_with_bootnode_via_enr( + context: RuntimeContext, + boot_node: &BeaconNode, + base_config: ClientConfig, +) -> BeaconNode { + let mut config = base_config; + config.network.boot_nodes.push( + boot_node + .client + .enr() + .expect("bootnode must have a network"), + ); + + BeaconNode::production(context, config) +} + +fn new_with_bootnode_via_multiaddr( + context: RuntimeContext, + boot_node: &BeaconNode, + base_config: ClientConfig, +) -> BeaconNode { + let mut config = base_config; + config.network.libp2p_nodes.push( + boot_node + .client + .libp2p_listen_addresses() + .expect("bootnode must have a network") + .first() + .expect("bootnode must have at least one listen addr") + .clone(), + ); + + BeaconNode::production(context, config) +} diff --git a/tests/node_test_rig/src/lib.rs b/tests/node_test_rig/src/lib.rs index 5a0f21e097..060215df29 100644 --- a/tests/node_test_rig/src/lib.rs +++ b/tests/node_test_rig/src/lib.rs @@ -1,13 +1,12 @@ -use beacon_node::{ - beacon_chain::BeaconChainTypes, Client, ClientConfig, ClientGenesis, ProductionBeaconNode, - ProductionClient, -}; +use beacon_node::{beacon_chain::BeaconChainTypes, Client, ClientGenesis, ProductionBeaconNode}; use environment::RuntimeContext; use futures::Future; use remote_beacon_node::RemoteBeaconNode; +use std::path::PathBuf; use tempdir::TempDir; use types::EthSpec; +pub use beacon_node::{ClientConfig, ProductionClient}; pub use environment; /// Provides a beacon node that is running in the current process. Useful for testing purposes. @@ -18,8 +17,13 @@ pub struct LocalBeaconNode { impl LocalBeaconNode> { /// Starts a new, production beacon node. - pub fn production(context: RuntimeContext) -> Self { - let (client_config, datadir) = testing_client_config(); + pub fn production(context: RuntimeContext, mut client_config: ClientConfig) -> Self { + // Creates a temporary directory that will be deleted once this `TempDir` is dropped. + let datadir = TempDir::new("lighthouse_node_test_rig") + .expect("should create temp directory for client datadir"); + + client_config.data_dir = datadir.path().into(); + client_config.network.network_dir = PathBuf::from(datadir.path()).join("network"); let client = ProductionBeaconNode::new(context, client_config) .wait() @@ -42,15 +46,9 @@ impl LocalBeaconNode> { } } -fn testing_client_config() -> (ClientConfig, TempDir) { - // Creates a temporary directory that will be deleted once this `TempDir` is dropped. - let tempdir = TempDir::new("lighthouse_node_test_rig") - .expect("should create temp directory for client datadir"); - +pub fn testing_client_config() -> ClientConfig { let mut client_config = ClientConfig::default(); - client_config.data_dir = tempdir.path().into(); - // Setting ports to `0` means that the OS will choose some available port. client_config.network.libp2p_port = 0; client_config.network.discovery_port = 0; @@ -63,5 +61,5 @@ fn testing_client_config() -> (ClientConfig, TempDir) { genesis_time: 13_371_337, }; - (client_config, tempdir) + client_config } From 40a0bd05448b73414c4105894bc115605cec82a9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 20 Nov 2019 14:40:28 +1100 Subject: [PATCH 4/5] Expand `beacon_chain_sim` --- beacon_node/client/src/builder.rs | 9 ++++- beacon_node/client/src/lib.rs | 6 +++ beacon_node/rpc/src/lib.rs | 31 +++++++++----- tests/beacon_chain_sim/Cargo.toml | 1 + tests/beacon_chain_sim/src/main.rs | 62 +++++++++++++++++++++++++--- tests/node_test_rig/Cargo.toml | 1 + tests/node_test_rig/src/lib.rs | 65 +++++++++++++++++++++++++++++- validator_client/src/lib.rs | 24 +++++++++-- validator_client/src/service.rs | 6 +++ 9 files changed, 181 insertions(+), 24 deletions(-) diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index c160081456..f8375c3d2d 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -63,6 +63,7 @@ pub struct ClientBuilder { libp2p_network: Option>>, libp2p_network_send: Option>, http_listen_addr: Option, + grpc_listen_addr: Option<(String, u16)>, websocket_listen_addr: Option, eth_spec_instance: T::EthSpec, } @@ -94,6 +95,7 @@ where libp2p_network: None, libp2p_network_send: None, http_listen_addr: None, + grpc_listen_addr: None, websocket_listen_addr: None, eth_spec_instance, } @@ -283,15 +285,17 @@ where .clone() .ok_or_else(|| "grpc_server requires a libp2p network")?; - let exit_signal = rpc::start_server( + let (exit_signal, listen_addr) = rpc::start_server( config, &context.executor, network_send, beacon_chain, context.log, - ); + ) + .map_err(|e| format!("Failed to start gRPC server: {}", e))?; self.exit_signals.push(exit_signal); + self.grpc_listen_addr = Some(listen_addr); Ok(self) } @@ -455,6 +459,7 @@ where beacon_chain: self.beacon_chain, libp2p_network: self.libp2p_network, http_listen_addr: self.http_listen_addr, + grpc_listen_addr: self.grpc_listen_addr, websocket_listen_addr: self.websocket_listen_addr, _exit_signals: self.exit_signals, } diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index f0ed63e485..4da474ea1a 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -25,6 +25,7 @@ pub struct Client { libp2p_network: Option>>, http_listen_addr: Option, websocket_listen_addr: Option, + grpc_listen_addr: Option<(String, u16)>, /// Exit signals will "fire" when dropped, causing each service to exit gracefully. _exit_signals: Vec, } @@ -40,6 +41,11 @@ impl Client { self.http_listen_addr } + /// Returns the address of the client's gRPC API server, if it was started. + pub fn grpc_listen_addr(&self) -> Option<(String, u16)> { + self.grpc_listen_addr.clone() + } + /// Returns the address of the client's WebSocket API server, if it was started. pub fn websocket_listen_addr(&self) -> Option { self.websocket_listen_addr diff --git a/beacon_node/rpc/src/lib.rs b/beacon_node/rpc/src/lib.rs index 3425eeeac2..b08701df67 100644 --- a/beacon_node/rpc/src/lib.rs +++ b/beacon_node/rpc/src/lib.rs @@ -28,7 +28,7 @@ pub fn start_server( network_chan: mpsc::UnboundedSender, beacon_chain: Arc>, log: slog::Logger, -) -> exit_future::Signal { +) -> Result<(exit_future::Signal, (String, u16)), String> { let env = Arc::new(Environment::new(1)); // build a channel to kill the rpc server @@ -76,16 +76,24 @@ pub fn start_server( .build() .unwrap(); + server.start(); + + for &(ref host, port) in server.bind_addrs() { + info!( + log, + "gRPC API started"; + "port" => port, + "host" => host, + ); + } + + let listen_addr = server + .bind_addrs() + .first() + .ok_or_else(|| "gRPC server is not listening on any ports".to_string())? + .clone(); + let spawn_rpc = { - server.start(); - for &(ref host, port) in server.bind_addrs() { - info!( - log, - "gRPC API started"; - "port" => port, - "host" => host, - ); - } rpc_exit.and_then(move |_| { info!(log, "RPC Server shutting down"); server @@ -97,5 +105,6 @@ pub fn start_server( }) }; executor.spawn(spawn_rpc); - rpc_exit_signal + + Ok((rpc_exit_signal, listen_addr)) } diff --git a/tests/beacon_chain_sim/Cargo.toml b/tests/beacon_chain_sim/Cargo.toml index 8ce05e2d66..37734a0305 100644 --- a/tests/beacon_chain_sim/Cargo.toml +++ b/tests/beacon_chain_sim/Cargo.toml @@ -9,3 +9,4 @@ edition = "2018" [dependencies] node_test_rig = { path = "../node_test_rig" } types = { path = "../../eth2/types" } +validator_client = { path = "../../validator_client" } diff --git a/tests/beacon_chain_sim/src/main.rs b/tests/beacon_chain_sim/src/main.rs index 0915bd372f..fba4995f78 100644 --- a/tests/beacon_chain_sim/src/main.rs +++ b/tests/beacon_chain_sim/src/main.rs @@ -1,19 +1,23 @@ use node_test_rig::{ environment::{EnvironmentBuilder, RuntimeContext}, - testing_client_config, ClientConfig, LocalBeaconNode, ProductionClient, + testing_client_config, ClientConfig, LocalBeaconNode, LocalValidatorClient, ProductionClient, + ValidatorConfig, }; use types::EthSpec; pub type BeaconNode = LocalBeaconNode>; fn main() { - match simulation(4) { + let nodes = 4; + let validators_per_node = 64 / nodes; + + match simulation(nodes, validators_per_node) { Ok(()) => println!("Simulation exited successfully"), Err(e) => println!("Simulation exited with error: {}", e), } } -fn simulation(num_nodes: usize) -> Result<(), String> { +fn simulation(num_nodes: usize, validators_per_node: usize) -> Result<(), String> { if num_nodes < 1 { return Err("Must have at least one node".into()); } @@ -28,19 +32,48 @@ fn simulation(num_nodes: usize) -> Result<(), String> { let boot_node = BeaconNode::production(env.service_context("boot_node".into()), base_config.clone()); - let nodes = (1..num_nodes) + let mut nodes = (1..num_nodes) .map(|i| { let context = env.service_context(format!("node_{}", i)); new_with_bootnode_via_enr(context, &boot_node, base_config.clone()) }) .collect::>(); + let validators = nodes + .iter() + .enumerate() + .map(|(i, node)| { + let mut context = env.service_context(format!("validator_{}", i)); + + // Pull the spec from the beacon node's beacon chain, in case there were some changes + // to the spec after the node booted. + context.eth2_config.spec = node + .client + .beacon_chain() + .expect("should have beacon chain") + .spec + .clone(); + + let indices = + (i * validators_per_node..(i + 1) * validators_per_node).collect::>(); + new_validator_client( + env.service_context(format!("validator_{}", i)), + node, + ValidatorConfig::default(), + &indices, + ) + }) + .collect::>(); + + nodes.insert(0, boot_node); + env.block_until_ctrl_c()?; Ok(()) } -// TODO: this function does not result in nodes connecting to each other. Age to investigate? +// TODO: this function does not result in nodes connecting to each other. This is a bug due to +// using a 0 port for discovery. Age is fixing it. fn new_with_bootnode_via_enr( context: RuntimeContext, boot_node: &BeaconNode, @@ -75,3 +108,22 @@ fn new_with_bootnode_via_multiaddr( BeaconNode::production(context, config) } + +fn new_validator_client( + context: RuntimeContext, + beacon_node: &BeaconNode, + base_config: ValidatorConfig, + keypair_indices: &[usize], +) -> LocalValidatorClient { + let mut config = base_config; + + let (grpc_endpoint, grpc_port) = beacon_node + .client + .grpc_listen_addr() + .expect("Must have gRPC started"); + + config.server = grpc_endpoint; + config.server_grpc_port = grpc_port; + + LocalValidatorClient::production_with_insecure_keypairs(context, config, keypair_indices) +} diff --git a/tests/node_test_rig/Cargo.toml b/tests/node_test_rig/Cargo.toml index 7bb19db9c6..c8e1ebcc68 100644 --- a/tests/node_test_rig/Cargo.toml +++ b/tests/node_test_rig/Cargo.toml @@ -16,3 +16,4 @@ serde = "1.0" futures = "0.1.25" genesis = { path = "../../beacon_node/genesis" } remote_beacon_node = { path = "../../eth2/utils/remote_beacon_node" } +validator_client = { path = "../../validator_client" } diff --git a/tests/node_test_rig/src/lib.rs b/tests/node_test_rig/src/lib.rs index 060215df29..1499cff31d 100644 --- a/tests/node_test_rig/src/lib.rs +++ b/tests/node_test_rig/src/lib.rs @@ -3,13 +3,17 @@ use environment::RuntimeContext; use futures::Future; use remote_beacon_node::RemoteBeaconNode; use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; use tempdir::TempDir; use types::EthSpec; +use validator_client::{validator_directory::ValidatorDirectoryBuilder, ProductionValidatorClient}; pub use beacon_node::{ClientConfig, ProductionClient}; pub use environment; +pub use validator_client::Config as ValidatorConfig; -/// Provides a beacon node that is running in the current process. Useful for testing purposes. +/// Provides a beacon node that is running in the current process (i.e., local). Useful for testing +/// purposes. pub struct LocalBeaconNode { pub client: T, pub datadir: TempDir, @@ -56,10 +60,67 @@ pub fn testing_client_config() -> ClientConfig { client_config.rest_api.port = 0; client_config.websocket_server.port = 0; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should get system time") + .as_secs(); + client_config.genesis = ClientGenesis::Interop { validator_count: 8, - genesis_time: 13_371_337, + genesis_time: now, }; client_config } + +pub struct LocalValidatorClient { + pub client: ProductionValidatorClient, + pub datadir: TempDir, +} + +impl LocalValidatorClient { + pub fn production_with_insecure_keypairs( + context: RuntimeContext, + config: ValidatorConfig, + keypair_indices: &[usize], + ) -> Self { + // Creates a temporary directory that will be deleted once this `TempDir` is dropped. + let datadir = TempDir::new("lighthouse-beacon-node") + .expect("should create temp directory for client datadir"); + + keypair_indices.iter().for_each(|i| { + ValidatorDirectoryBuilder::default() + .spec(context.eth2_config.spec.clone()) + .full_deposit_amount() + .expect("should set full deposit amount") + .insecure_keypairs(*i) + .create_directory(PathBuf::from(datadir.path())) + .expect("should create directory") + .write_keypair_files() + .expect("should write keypair files") + .write_eth1_data_file() + .expect("should write eth1 data file") + .build() + .expect("should build dir"); + }); + + Self::new(context, config, datadir) + } + + pub fn production(context: RuntimeContext, config: ValidatorConfig) -> Self { + // Creates a temporary directory that will be deleted once this `TempDir` is dropped. + let datadir = TempDir::new("lighthouse-validator") + .expect("should create temp directory for client datadir"); + + Self::new(context, config, datadir) + } + + fn new(context: RuntimeContext, mut config: ValidatorConfig, datadir: TempDir) -> Self { + config.data_dir = datadir.path().into(); + + let client = + ProductionValidatorClient::new(context, config).expect("should start validator client"); + + Self { client, datadir } + } +} diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 930f35c32a..d9b6f40443 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -43,20 +43,36 @@ pub struct ProductionValidatorClient { impl ProductionValidatorClient { /// Instantiates the validator client, _without_ starting the timers to trigger block /// and attestation production. - pub fn new_from_cli(context: RuntimeContext, matches: &ArgMatches) -> Result { + pub fn new_from_cli( + mut context: RuntimeContext, + matches: &ArgMatches, + ) -> Result { let mut log = context.log.clone(); - let (client_config, eth2_config) = get_configs(&matches, &mut log) + let (config, eth2_config) = get_configs(&matches, &mut log) .map_err(|e| format!("Unable to initialize config: {}", e))?; + // TODO: the eth2 config in the env is being completely ignored. + // + // See https://github.com/sigp/lighthouse/issues/602 + context.eth2_config = eth2_config; + + Self::new(context, config) + } + + /// Instantiates the validator client, _without_ starting the timers to trigger block + /// and attestation production. + pub fn new(context: RuntimeContext, config: Config) -> Result { + let log = context.log.clone(); + info!( log, "Starting validator client"; - "datadir" => client_config.full_data_dir().expect("Unable to find datadir").to_str(), + "datadir" => config.full_data_dir().expect("Unable to find datadir").to_str(), ); let service: Service = - Service::initialize_service(client_config, eth2_config, log.clone()) + Service::initialize_service(config, context.eth2_config.clone(), log.clone()) .map_err(|e| e.to_string())?; Ok(Self { diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index 4d304ddd2c..c733ef31ed 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -161,6 +161,12 @@ impl Service { // Load generated keypairs let keypairs = Arc::new(client_config.fetch_keys(&log)?); + info!( + log, + "Keypairs loaded"; + "local_validator_count" => keypairs.len() + ); + let slots_per_epoch = E::slots_per_epoch(); // TODO: keypairs are randomly generated; they should be loaded from a file or generated. From 444db0257ed2715783cfe5cba1a28e5158ece816 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 20 Nov 2019 15:08:39 +1100 Subject: [PATCH 5/5] Fix spec for `beacon_chain_sim --- tests/beacon_chain_sim/src/main.rs | 37 +++++++++++------------------- tests/node_test_rig/src/lib.rs | 10 ++++++-- validator_client/src/service.rs | 4 ---- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/tests/beacon_chain_sim/src/main.rs b/tests/beacon_chain_sim/src/main.rs index fba4995f78..f7dfddedc3 100644 --- a/tests/beacon_chain_sim/src/main.rs +++ b/tests/beacon_chain_sim/src/main.rs @@ -1,8 +1,9 @@ use node_test_rig::{ environment::{EnvironmentBuilder, RuntimeContext}, - testing_client_config, ClientConfig, LocalBeaconNode, LocalValidatorClient, ProductionClient, - ValidatorConfig, + testing_client_config, ClientConfig, ClientGenesis, LocalBeaconNode, LocalValidatorClient, + ProductionClient, ValidatorConfig, }; +use std::time::{SystemTime, UNIX_EPOCH}; use types::EthSpec; pub type BeaconNode = LocalBeaconNode>; @@ -27,7 +28,16 @@ fn simulation(num_nodes: usize, validators_per_node: usize) -> Result<(), String .multi_threaded_tokio_runtime()? .build()?; - let base_config = testing_client_config(); + let mut base_config = testing_client_config(); + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should get system time") + .as_secs(); + base_config.genesis = ClientGenesis::Interop { + genesis_time: now, + validator_count: num_nodes * validators_per_node, + }; let boot_node = BeaconNode::production(env.service_context("boot_node".into()), base_config.clone()); @@ -39,7 +49,7 @@ fn simulation(num_nodes: usize, validators_per_node: usize) -> Result<(), String }) .collect::>(); - let validators = nodes + let _validators = nodes .iter() .enumerate() .map(|(i, node)| { @@ -90,25 +100,6 @@ fn new_with_bootnode_via_enr( BeaconNode::production(context, config) } -fn new_with_bootnode_via_multiaddr( - context: RuntimeContext, - boot_node: &BeaconNode, - base_config: ClientConfig, -) -> BeaconNode { - let mut config = base_config; - config.network.libp2p_nodes.push( - boot_node - .client - .libp2p_listen_addresses() - .expect("bootnode must have a network") - .first() - .expect("bootnode must have at least one listen addr") - .clone(), - ); - - BeaconNode::production(context, config) -} - fn new_validator_client( context: RuntimeContext, beacon_node: &BeaconNode, diff --git a/tests/node_test_rig/src/lib.rs b/tests/node_test_rig/src/lib.rs index 1499cff31d..37e84c2057 100644 --- a/tests/node_test_rig/src/lib.rs +++ b/tests/node_test_rig/src/lib.rs @@ -1,4 +1,4 @@ -use beacon_node::{beacon_chain::BeaconChainTypes, Client, ClientGenesis, ProductionBeaconNode}; +use beacon_node::{beacon_chain::BeaconChainTypes, Client, ProductionBeaconNode}; use environment::RuntimeContext; use futures::Future; use remote_beacon_node::RemoteBeaconNode; @@ -8,7 +8,7 @@ use tempdir::TempDir; use types::EthSpec; use validator_client::{validator_directory::ValidatorDirectoryBuilder, ProductionValidatorClient}; -pub use beacon_node::{ClientConfig, ProductionClient}; +pub use beacon_node::{ClientConfig, ClientGenesis, ProductionClient}; pub use environment; pub use validator_client::Config as ValidatorConfig; @@ -60,6 +60,8 @@ pub fn testing_client_config() -> ClientConfig { client_config.rest_api.port = 0; client_config.websocket_server.port = 0; + client_config.dummy_eth1_backend = true; + let now = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("should get system time") @@ -121,6 +123,10 @@ impl LocalValidatorClient { let client = ProductionValidatorClient::new(context, config).expect("should start validator client"); + client + .start_service() + .expect("should start validator client"); + Self { client, datadir } } } diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index c733ef31ed..60d1fad018 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -169,10 +169,6 @@ impl Service { let slots_per_epoch = E::slots_per_epoch(); - // TODO: keypairs are randomly generated; they should be loaded from a file or generated. - // https://github.com/sigp/lighthouse/issues/160 - //let keypairs = Arc::new(generate_deterministic_keypairs(8)); - // Builds a mapping of Epoch -> Map(PublicKey, EpochDuty) // where EpochDuty contains slot numbers and attestation data that each validator needs to // produce work on.