From 2fdd130f4c0bfb02475e48dcee318a7bc3c73b24 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 24 Nov 2019 18:51:37 +1100 Subject: [PATCH] Add efforts to automate eth1 testnet deployment --- Cargo.toml | 1 + account_manager/Cargo.toml | 1 + account_manager/src/cli.rs | 2 +- beacon_node/rest_api/tests/test.rs | 2 +- beacon_node/src/cli.rs | 12 +++ eth2/utils/deposit_contract/src/lib.rs | 2 +- eth2/utils/eth2_testnet/Cargo.toml | 14 +++ eth2/utils/eth2_testnet/src/lib.rs | 134 +++++++++++++++++++++++++ lcli/Cargo.toml | 1 + lcli/src/main.rs | 52 +++++++--- lcli/src/testnet.rs | 84 ++++++++++++++++ tests/eth1_test_rig/src/lib.rs | 26 ++++- 12 files changed, 312 insertions(+), 19 deletions(-) create mode 100644 eth2/utils/eth2_testnet/Cargo.toml create mode 100644 eth2/utils/eth2_testnet/src/lib.rs create mode 100644 lcli/src/testnet.rs diff --git a/Cargo.toml b/Cargo.toml index 23ae186571..cb5b55e658 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "eth2/utils/deposit_contract", "eth2/utils/eth2_config", "eth2/utils/eth2_interop_keypairs", + "eth2/utils/eth2_testnet", "eth2/utils/logging", "eth2/utils/eth2_hashing", "eth2/utils/lighthouse_metrics", diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index 729adb3353..9a4fe112c6 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -23,3 +23,4 @@ eth2_ssz_derive = { path = "../eth2/utils/ssz_derive" } hex = "0.4" validator_client = { path = "../validator_client" } rayon = "1.2.0" +eth2_testnet = { path = "../eth2/utils/eth2_testnet" } diff --git a/account_manager/src/cli.rs b/account_manager/src/cli.rs index 01d6376cdb..ef69dd1227 100644 --- a/account_manager/src/cli.rs +++ b/account_manager/src/cli.rs @@ -25,7 +25,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("last") .index(2) .value_name("INDEX") - .help("Index of the first validator") + .help("Index of the last validator") .takes_value(true) .required(true), ), diff --git a/beacon_node/rest_api/tests/test.rs b/beacon_node/rest_api/tests/test.rs index 3d029f0966..dd9bb9f33f 100644 --- a/beacon_node/rest_api/tests/test.rs +++ b/beacon_node/rest_api/tests/test.rs @@ -7,7 +7,7 @@ use node_test_rig::{ }; use remote_beacon_node::{PublishStatus, ValidatorDuty}; use std::sync::Arc; -use tree_hash::{SignedRoot, TreeHash}; +use tree_hash::TreeHash; use types::{ test_utils::generate_deterministic_keypair, BeaconBlock, ChainSpec, Domain, Epoch, EthSpec, MinimalEthSpec, PublicKey, RelativeEpoch, Signature, Slot, diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 2384092f3c..1589f3e164 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -305,6 +305,18 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .required(true) .help("A file from which to read the state")) ) + /* + * `sigp` + * + * Connect to a Sigma Prime testnet. + */ + .subcommand(SubCommand::with_name("sigp") + .about("Connect to a Sigma Prime testnet on Goerli.") + .arg(Arg::with_name("file") + .value_name("JSON_FILE") + .required(true) + .help("A sigma_prime_testnet.json file")) + ) /* * `prysm` * diff --git a/eth2/utils/deposit_contract/src/lib.rs b/eth2/utils/deposit_contract/src/lib.rs index b0c052ede6..a235d0ff71 100644 --- a/eth2/utils/deposit_contract/src/lib.rs +++ b/eth2/utils/deposit_contract/src/lib.rs @@ -62,6 +62,6 @@ mod tests { let data = eth1_tx_data(&deposit).expect("should produce tx data"); - assert_eq!(data.len(), 388, "bytes should be correct length"); + assert_eq!(data.len(), 420, "bytes should be correct length"); } } diff --git a/eth2/utils/eth2_testnet/Cargo.toml b/eth2/utils/eth2_testnet/Cargo.toml new file mode 100644 index 0000000000..dc97e1327c --- /dev/null +++ b/eth2/utils/eth2_testnet/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "eth2_testnet" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dev-dependencies] +tempdir = "0.3" + +[dependencies] +serde = "1.0" +serde_json = "^1.0" diff --git a/eth2/utils/eth2_testnet/src/lib.rs b/eth2/utils/eth2_testnet/src/lib.rs new file mode 100644 index 0000000000..9883d45ade --- /dev/null +++ b/eth2/utils/eth2_testnet/src/lib.rs @@ -0,0 +1,134 @@ +//! This crate should eventually represent the structure at this repo: +//! +//! https://github.com/eth2-clients/eth2-testnets/tree/master/nimbus/testnet1 +//! +//! It is not accurate at the moment, we include extra files and we also don't support a few +//! others. We are unable to confirm to the repo until we have the following PR merged: +//! +//! https://github.com/sigp/lighthouse/pull/605 + +use std::fs::{create_dir_all, File}; +use std::path::PathBuf; + +pub const ADDRESS_FILE: &str = "deposit_contract.txt"; +pub const DEPLOY_BLOCK_FILE: &str = "deploy_block.txt"; +pub const MIN_GENESIS_TIME_FILE: &str = "min_genesis_time.txt"; + +#[derive(Clone, PartialEq, Debug)] +pub struct Eth2TestnetDir { + pub deposit_contract_address: String, + pub deposit_contract_deploy_block: u64, + pub min_genesis_time: u64, +} + +impl Eth2TestnetDir { + pub fn new( + base_dir: PathBuf, + deposit_contract_address: String, + deposit_contract_deploy_block: u64, + min_genesis_time: u64, + ) -> Result { + if base_dir.exists() { + return Err("Testnet directory already exists".to_string()); + } + + create_dir_all(&base_dir) + .map_err(|e| format!("Unable to create testnet directory: {:?}", e))?; + + File::create(base_dir.join(ADDRESS_FILE)) + .map_err(|e| format!("Unable to create {}: {:?}", ADDRESS_FILE, e)) + .and_then(|file| { + serde_json::to_writer(file, &deposit_contract_address) + .map_err(|e| format!("Unable to write {}: {:?}", ADDRESS_FILE, e)) + })?; + + File::create(base_dir.join(DEPLOY_BLOCK_FILE)) + .map_err(|e| format!("Unable to create {}: {:?}", DEPLOY_BLOCK_FILE, e)) + .and_then(|file| { + serde_json::to_writer(file, &deposit_contract_deploy_block) + .map_err(|e| format!("Unable to write {}: {:?}", DEPLOY_BLOCK_FILE, e)) + })?; + + File::create(base_dir.join(MIN_GENESIS_TIME_FILE)) + .map_err(|e| format!("Unable to create {}: {:?}", MIN_GENESIS_TIME_FILE, e)) + .and_then(|file| { + serde_json::to_writer(file, &min_genesis_time) + .map_err(|e| format!("Unable to write {}: {:?}", MIN_GENESIS_TIME_FILE, e)) + })?; + + Ok(Self { + deposit_contract_address, + deposit_contract_deploy_block, + min_genesis_time, + }) + } + + pub fn load(base_dir: PathBuf) -> Result { + let deposit_contract_address = File::open(base_dir.join(ADDRESS_FILE)) + .map_err(|e| format!("Unable to open {}: {:?}", ADDRESS_FILE, e)) + .and_then(|file| { + serde_json::from_reader(file) + .map_err(|e| format!("Unable to parse {}: {:?}", ADDRESS_FILE, e)) + })?; + + let deposit_contract_deploy_block = File::open(base_dir.join(DEPLOY_BLOCK_FILE)) + .map_err(|e| format!("Unable to open {}: {:?}", DEPLOY_BLOCK_FILE, e)) + .and_then(|file| { + serde_json::from_reader(file) + .map_err(|e| format!("Unable to parse {}: {:?}", DEPLOY_BLOCK_FILE, e)) + })?; + + let min_genesis_time = File::open(base_dir.join(MIN_GENESIS_TIME_FILE)) + .map_err(|e| format!("Unable to open {}: {:?}", MIN_GENESIS_TIME_FILE, e)) + .and_then(|file| { + serde_json::from_reader(file) + .map_err(|e| format!("Unable to parse {}: {:?}", MIN_GENESIS_TIME_FILE, e)) + })?; + + Ok(Self { + deposit_contract_address, + deposit_contract_deploy_block, + min_genesis_time, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempdir::TempDir; + + #[test] + fn round_trip() { + let temp_dir = TempDir::new("eth2_testnet_test").expect("should create temp dir"); + let base_dir = PathBuf::from(temp_dir.path().join("my_testnet")); + let deposit_contract_address = "0xBB9bc244D798123fDe783fCc1C72d3Bb8C189413".to_string(); + let deposit_contract_deploy_block = 42; + let min_genesis_time = 1337; + + let testnet = Eth2TestnetDir::new( + base_dir.clone(), + deposit_contract_address.clone(), + deposit_contract_deploy_block, + min_genesis_time, + ) + .expect("should create struct"); + + let decoded = Eth2TestnetDir::load(base_dir).expect("should load struct"); + + assert_eq!( + decoded.deposit_contract_address, deposit_contract_address, + "deposit_contract_address" + ); + assert_eq!( + decoded.deposit_contract_deploy_block, deposit_contract_deploy_block, + "deposit_contract_deploy_block" + ); + assert_eq!( + decoded.min_genesis_time, min_genesis_time, + "min_genesis_time" + ); + + assert_eq!(testnet, decoded, "should decode as encoded"); + } +} diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index b8bd0e16f2..812b365832 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -22,3 +22,4 @@ eth1_test_rig = { path = "../tests/eth1_test_rig" } futures = "0.1.25" environment = { path = "../lighthouse/environment" } web3 = "0.8.0" +eth2_testnet = { path = "../eth2/utils/eth2_testnet" } diff --git a/lcli/src/main.rs b/lcli/src/main.rs index 85af9f21e8..9eb57d6e38 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -4,6 +4,7 @@ extern crate log; mod deposit_contract; mod parse_hex; mod pycli; +mod testnet; mod transition_blocks; use clap::{App, Arg, SubCommand}; @@ -24,8 +25,6 @@ fn main() { simple_logger::init_with_level(Level::Info).expect("logger should initialize"); let matches = App::new("Lighthouse CLI Tool") - .version("0.1.0") - .author("Paul Hauner ") .about( "Performs various testing-related tasks, modelled after zcli. \ by @protolambda.", @@ -33,8 +32,6 @@ fn main() { .subcommand( SubCommand::with_name("genesis_yaml") .about("Generates a genesis YAML file") - .version("0.1.0") - .author("Paul Hauner ") .arg( Arg::with_name("num_validators") .short("n") @@ -73,8 +70,6 @@ fn main() { .subcommand( SubCommand::with_name("transition-blocks") .about("Performs a state transition given a pre-state and block") - .version("0.1.0") - .author("Paul Hauner ") .arg( Arg::with_name("pre-state") .value_name("BEACON_STATE") @@ -101,8 +96,6 @@ fn main() { .subcommand( SubCommand::with_name("pretty-hex") .about("Parses SSZ encoded as ASCII 0x-prefixed hex") - .version("0.1.0") - .author("Paul Hauner ") .arg( Arg::with_name("type") .value_name("TYPE") @@ -124,8 +117,6 @@ fn main() { .about( "Uses an eth1 test rpc (e.g., ganache-cli) to simulate the deposit contract.", ) - .version("0.1.0") - .author("Paul Hauner ") .arg( Arg::with_name("count") .short("c") @@ -158,11 +149,46 @@ fn main() { .help("The number of block confirmations before declaring the contract deployed."), ) ) + .subcommand( + SubCommand::with_name("testnet") + .about( + "Deploy an eth1 deposit contract and create files with testnet details.", + ) + .arg( + Arg::with_name("output") + .short("o") + .value_name("PATH") + .takes_value(true) + .default_value("~/.lighthouse/testnet") + .help("The output directory."), + ) + .arg( + Arg::with_name("min_genesis_time") + .short("t") + .value_name("UNIX_EPOCH_SECONDS") + .takes_value(true) + .default_value("0") + .help("The MIN_GENESIS_TIME constant."), + ) + .arg( + Arg::with_name("endpoint") + .short("e") + .value_name("HTTP_SERVER") + .takes_value(true) + .default_value("http://localhost:8545") + .help("The URL to the eth1 JSON-RPC http API."), + ) + .arg( + Arg::with_name("confirmations") + .value_name("INTEGER") + .takes_value(true) + .default_value("3") + .help("The number of block confirmations before declaring the contract deployed."), + ) + ) .subcommand( SubCommand::with_name("pycli") .about("TODO") - .version("0.1.0") - .author("Paul Hauner ") .arg( Arg::with_name("pycli-path") .long("pycli-path") @@ -231,6 +257,8 @@ fn main() { .unwrap_or_else(|e| error!("Failed to run pycli: {}", e)), ("deposit-contract", Some(matches)) => run_deposit_contract::(env, matches) .unwrap_or_else(|e| error!("Failed to run deposit contract sim: {}", e)), + ("testnet", Some(matches)) => testnet::new_testnet::(env, matches) + .unwrap_or_else(|e| error!("Failed to run testnet commmand: {}", e)), (other, _) => error!("Unknown subcommand {}. See --help.", other), } } diff --git a/lcli/src/testnet.rs b/lcli/src/testnet.rs new file mode 100644 index 0000000000..4c10e9012b --- /dev/null +++ b/lcli/src/testnet.rs @@ -0,0 +1,84 @@ +use clap::ArgMatches; +use environment::Environment; +use eth1_test_rig::DepositContract; +use eth2_testnet::Eth2TestnetDir; +use std::path::PathBuf; +use types::EthSpec; +use web3::{transports::Http, Web3}; + +pub fn new_testnet( + mut env: Environment, + matches: &ArgMatches, +) -> Result<(), String> { + let min_genesis_time = matches + .value_of("min_genesis_time") + .ok_or_else(|| "min_genesis_time not specified")? + .parse::() + .map_err(|e| format!("Failed to parse min_genesis_time: {}", e))?; + + let confirmations = matches + .value_of("confirmations") + .ok_or_else(|| "Confirmations not specified")? + .parse::() + .map_err(|e| format!("Failed to parse confirmations: {}", e))?; + + let output_dir = matches + .value_of("output") + .ok_or_else(|| "Output directory not specified")? + .parse::() + .map_err(|e| format!("Failed to parse output directory: {}", e))?; + + let endpoint = matches + .value_of("endpoint") + .ok_or_else(|| "Endpoint not specified")?; + + let (_event_loop, transport) = Http::new(&endpoint).map_err(|e| { + format!( + "Failed to start HTTP transport connected to ganache: {:?}", + e + ) + })?; + let web3 = Web3::new(transport); + + if output_dir.exists() { + return Err("Output directory already exists".to_string()); + } + + // It's unlikely that this will be the _actual_ deployment block, however it'll be close + // enough to serve our purposes. + // + // We only need the deposit block to put a lower bound on the block number we need to search + // for deposit logs. + let deploy_block = env + .runtime() + .block_on(web3.eth().block_number()) + .map_err(|e| format!("Failed to get block number: {}", e))?; + + info!("Present eth1 block number is {}", deploy_block); + + info!( + "Submitting deployment transaction, waiting for {} confirmations", + confirmations + ); + + let deposit_contract = env + .runtime() + .block_on(DepositContract::deploy_testnet(web3, confirmations)) + .map_err(|e| format!("Failed to deploy contract: {}", e))?; + + info!( + "Deposit contract deployed. address: {}, min_genesis_time: {}, deploy_block: {}", + deposit_contract.address(), + min_genesis_time, + deploy_block + ); + + Eth2TestnetDir::new( + output_dir, + format!("0x{}", deposit_contract.address()), + deploy_block.as_u64(), + min_genesis_time, + )?; + + Ok(()) +} diff --git a/tests/eth1_test_rig/src/lib.rs b/tests/eth1_test_rig/src/lib.rs index 1c404e2d5f..a144683c40 100644 --- a/tests/eth1_test_rig/src/lib.rs +++ b/tests/eth1_test_rig/src/lib.rs @@ -7,7 +7,7 @@ //! some initial issues. mod ganache; -use deposit_contract::{eth1_tx_data, ABI, BYTECODE, CONTRACT_DEPLOY_GAS, DEPOSIT_GAS}; +use deposit_contract::{eth1_tx_data, testnet, ABI, BYTECODE, CONTRACT_DEPLOY_GAS, DEPOSIT_GAS}; use futures::{stream, Future, IntoFuture, Stream}; use ganache::GanacheInstance; use std::time::{Duration, Instant}; @@ -58,10 +58,26 @@ impl DepositContract { pub fn deploy( web3: Web3, confirmations: usize, + ) -> impl Future { + Self::deploy_bytecode(web3, confirmations, BYTECODE, ABI) + } + + pub fn deploy_testnet( + web3: Web3, + confirmations: usize, + ) -> impl Future { + Self::deploy_bytecode(web3, confirmations, testnet::BYTECODE, testnet::ABI) + } + + fn deploy_bytecode( + web3: Web3, + confirmations: usize, + bytecode: &[u8], + abi: &[u8], ) -> impl Future { let web3_1 = web3.clone(); - deploy_deposit_contract(web3.clone(), confirmations) + deploy_deposit_contract(web3.clone(), confirmations, bytecode.to_vec(), abi.to_vec()) .map_err(|e| { format!( "Failed to deploy contract: {}. Is scripts/ganache_tests_node.sh running?.", @@ -211,8 +227,10 @@ fn from_gwei(gwei: u64) -> U256 { fn deploy_deposit_contract( web3: Web3, confirmations: usize, + bytecode: Vec, + abi: Vec, ) -> impl Future { - let bytecode = String::from_utf8_lossy(&BYTECODE); + let bytecode = String::from_utf8(bytecode).expect("bytecode must be valid utf8"); web3.eth() .accounts() @@ -224,7 +242,7 @@ fn deploy_deposit_contract( .ok_or_else(|| "Insufficient accounts for deployer".to_string()) }) .and_then(move |deploy_address| { - Contract::deploy(web3.eth(), &ABI) + Contract::deploy(web3.eth(), &abi) .map_err(|e| format!("Unable to build contract deployer: {:?}", e))? .confirmations(confirmations) .options(Options {