From 7b86c9a08fd80889df07961ed9ae28148dca0c63 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 Apr 2020 12:20:43 +1000 Subject: [PATCH] Update testnet tooling (#1001) * Add progress on new deposits * Add deposited command to account manager * Remove old lcli::helpers mod * Clean clap_utils * Refactor lcli deposit contract commands to use IPC * Make testnet optional for environment * Use dbg formatting for deploy address * Add command to generate bootnode enr * Ensure lcli returns with 1 on error * Ensure account manager returns 1 on error * Disallow deposits to the zero address * Update web3 in eth1 crate * Ensure correct lighthouse dir is created * Reduce deposit gas requirement * Update cargo.lock * Add progress on new deposits * Add deposited command to account manager * Remove old lcli::helpers mod * Clean clap_utils * Refactor lcli deposit contract commands to use IPC * Add command to generate bootnode enr * Ensure lcli returns with 1 on error * Ensure account manager returns 1 on error * Update web3 in eth1 crate * Update Cargo.lock * Move lcli out of main install script * Change --limit to --at-least * Change --datadir to --validator-dir * Remove duplication in docs --- .github/workflows/test-suite.yml | 9 + Cargo.lock | 91 ++++++++- Cargo.toml | 1 + Makefile | 5 +- account_manager/Cargo.toml | 3 +- account_manager/src/cli.rs | 2 + account_manager/src/deposits.rs | 129 +++++++++++++ account_manager/src/lib.rs | 18 +- beacon_node/eth1/Cargo.toml | 2 +- beacon_node/eth2-libp2p/src/discovery/enr.rs | 5 +- beacon_node/eth2-libp2p/src/discovery/mod.rs | 6 +- beacon_node/eth2-libp2p/src/lib.rs | 2 +- beacon_node/eth2-libp2p/src/service.rs | 2 +- book/src/local-testnets.md | 4 +- eth2/utils/clap_utils/Cargo.toml | 15 ++ eth2/utils/clap_utils/src/lib.rs | 116 ++++++++++++ eth2/utils/deposit_contract/src/lib.rs | 2 +- lcli/Cargo.toml | 2 + lcli/src/check_deposit_data.rs | 6 +- lcli/src/deploy_deposit_contract.rs | 104 ++++------- lcli/src/generate_bootnode_enr.rs | 60 ++++++ lcli/src/main.rs | 185 +++++++++++-------- lcli/src/new_testnet.rs | 36 ++-- lcli/src/refund_deposit_contract.rs | 106 ++--------- lighthouse/Cargo.toml | 1 + lighthouse/environment/src/lib.rs | 10 +- lighthouse/src/main.rs | 10 +- tests/eth1_test_rig/Cargo.toml | 2 +- validator_client/Cargo.toml | 1 + validator_client/src/validator_directory.rs | 80 ++++++-- 30 files changed, 711 insertions(+), 304 deletions(-) create mode 100644 account_manager/src/deposits.rs create mode 100644 eth2/utils/clap_utils/Cargo.toml create mode 100644 eth2/utils/clap_utils/src/lib.rs create mode 100644 lcli/src/generate_bootnode_enr.rs diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 51665d86d0..772a75461e 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -74,3 +74,12 @@ jobs: - uses: actions/checkout@v1 - name: Typecheck benchmark code without running it run: make check-benches + install-lcli: + runs-on: ubuntu-latest + needs: cargo-fmt + steps: + - uses: actions/checkout@v1 + - name: Get latest version of stable Rust + run: rustup update stable + - name: Build lcli via Makefile + run: make install-lcli diff --git a/Cargo.lock b/Cargo.lock index 5f08d407a0..8cff5ccf68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,6 +6,7 @@ version = "0.0.1" dependencies = [ "bls 0.2.0", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "clap_utils 0.1.0", "deposit_contract 0.2.0", "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "environment 0.2.0", @@ -22,7 +23,7 @@ dependencies = [ "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "types 0.2.0", "validator_client 0.2.0", - "web3 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "web3 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -493,6 +494,18 @@ dependencies = [ "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "clap_utils" +version = "0.1.0" +dependencies = [ + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "eth2_ssz 0.1.2", + "eth2_testnet_config 0.2.0", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "types 0.2.0", +] + [[package]] name = "clear_on_drop" version = "0.2.3" @@ -890,6 +903,16 @@ dependencies = [ "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "derive_more" +version = "0.99.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "digest" version = "0.8.1" @@ -1079,7 +1102,7 @@ dependencies = [ "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "tree_hash 0.1.1", "types 0.2.0", - "web3 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "web3 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1091,7 +1114,7 @@ dependencies = [ "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "types 0.2.0", - "web3 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "web3 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1224,6 +1247,20 @@ dependencies = [ "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ethabi" +version = "9.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "error-chain 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ethabi" version = "11.0.0" @@ -1790,6 +1827,18 @@ dependencies = [ "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "jsonrpc-core" +version = "14.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "keccak" version = "0.1.0" @@ -1824,10 +1873,12 @@ name = "lcli" version = "0.2.0" dependencies = [ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "clap_utils 0.1.0", "deposit_contract 0.2.0", "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "environment 0.2.0", "eth1_test_rig 0.2.0", + "eth2-libp2p 0.2.0", "eth2_ssz 0.1.2", "eth2_testnet_config 0.2.0", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2339,6 +2390,7 @@ dependencies = [ "account_manager 0.0.1", "beacon_node 0.2.0", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "clap_utils 0.1.0", "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "environment 0.2.0", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4706,6 +4758,7 @@ dependencies = [ "tokio-timer 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", "tree_hash 0.1.1", "types 0.2.0", + "web3 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4909,6 +4962,34 @@ dependencies = [ "websocket 0.21.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "web3" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.5 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-core 14.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-uds 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "websocket 0.21.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "webpki" version = "0.21.2" @@ -5173,6 +5254,7 @@ dependencies = [ "checksum db-key 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b72465f46d518f6015d9cf07f7f3013a95dd6b9c2747c3d65ae0cce43929d14f" "checksum derivative 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3c6d883546668a3e2011b6a716a7330b82eabb0151b138217f632c8243e17135" "checksum derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a141330240c921ec6d074a3e188a7c7ef95668bb95e7d44fa0e5778ec2a7afe" +"checksum derive_more 0.99.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e2323f3f47db9a0e77ce7a300605d8d2098597fc451ed1a97bb1f6411bb550a7" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" "checksum dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" "checksum dirs-sys 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "afa0b23de8fd801745c471deffa6e12d248f962c9fd4b4c33787b055599bde7b" @@ -5187,6 +5269,7 @@ dependencies = [ "checksum error-chain 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d371106cc88ffdfb1eabd7111e432da544f16f3e2d7bf1dfe8bf575f1df045cd" "checksum ethabi 11.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97652a7d1f2504d6c885c87e242a06ccef5bd3054093d3fb742d8fb64806231a" "checksum ethabi 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ebdeeea85a6d217b9fcc862906d7e283c047e04114165c433756baf5dce00a6c" +"checksum ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "965126c64662832991f5a748893577630b558e47fa94e7f35aefcd20d737cef7" "checksum ethbloom 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3932e82d64d347a045208924002930dc105a138995ccdc1479d0f05f0359f17c" "checksum ethbloom 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "32cfe1c169414b709cf28aa30c74060bdb830a03a8ba473314d079ac79d80a5f" "checksum ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "62d1bc682337e2c5ec98930853674dd2b4bd5d0d246933a9e98e5280f7c76c5f" @@ -5243,6 +5326,7 @@ dependencies = [ "checksum itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" "checksum js-sys 0.3.37 (registry+https://github.com/rust-lang/crates.io-index)" = "6a27d435371a2fa5b6d2b028a74bbdb1234f308da363226a2854ca3ff8ba7055" "checksum jsonrpc-core 11.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97b83fdc5e0218128d0d270f2f2e7a5ea716f3240c8518a58bc89e6716ba8581" +"checksum jsonrpc-core 14.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "25525f6002338fb4debb5167a89a0b47f727a5a48418417545ad3429758b7fec" "checksum keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" @@ -5509,6 +5593,7 @@ dependencies = [ "checksum wasm-bindgen-test-macro 0.3.10 (registry+https://github.com/rust-lang/crates.io-index)" = "cf2f86cd78a2aa7b1fb4bb6ed854eccb7f9263089c79542dca1576a1518a8467" "checksum wasm-timer 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aa3e01d234bb71760e685cfafa5e2c96f8ad877c161a721646356651069e26ac" "checksum web-sys 0.3.37 (registry+https://github.com/rust-lang/crates.io-index)" = "2d6f51648d8c56c366144378a33290049eafdd784071077f6fe37dae64c1c4cb" +"checksum web3 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a0631c83208cf420eeb2ed9b6cb2d5fc853aa76a43619ccec2a3d52d741f1261" "checksum web3 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "076f34ed252d74a8521e3b013254b1a39f94a98f23aae7cfc85cda6e7b395664" "checksum webpki 0.21.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f1f50e1972865d6b1adb54167d1c8ed48606004c2c9d0ea5f1eeb34d95e863ef" "checksum webpki-roots 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91cd5736df7f12a964a5067a12c62fa38e1bd8080aff1f80bc29be7c80d19ab4" diff --git a/Cargo.toml b/Cargo.toml index 6912e33882..2e24a902d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "eth2/state_processing", "eth2/types", "eth2/utils/bls", + "eth2/utils/clap_utils", "eth2/utils/compare_fields", "eth2/utils/compare_fields_derive", "eth2/utils/deposit_contract", diff --git a/Makefile b/Makefile index d84e3237b3..e451a502be 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,14 @@ EF_TESTS = "tests/ef_tests" -# Builds the entire workspace in release (optimized). +# Builds the Lighthouse binary in release (optimized). # # Binaries will most likely be found in `./target/release` install: cargo install --path lighthouse --force --locked + +# Builds the lcli binary in release (optimized). +install-lcli: cargo install --path lcli --force --locked # Runs the full workspace tests in **release**, without downloading any additional diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index ad91ad4efe..eac3ceb06b 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -24,5 +24,6 @@ hex = "0.3" validator_client = { path = "../validator_client" } rayon = "1.2.0" eth2_testnet_config = { path = "../eth2/utils/eth2_testnet_config" } -web3 = "0.8.0" +web3 = "0.10.0" futures = "0.1.25" +clap_utils = { path = "../eth2/utils/clap_utils" } diff --git a/account_manager/src/cli.rs b/account_manager/src/cli.rs index 07685fb706..51df84bce9 100644 --- a/account_manager/src/cli.rs +++ b/account_manager/src/cli.rs @@ -1,3 +1,4 @@ +use crate::deposits; use clap::{App, Arg, SubCommand}; pub fn cli_app<'a, 'b>() -> App<'a, 'b> { @@ -7,6 +8,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .subcommand( SubCommand::with_name("validator") .about("Generate or manage Etheruem 2.0 validators.") + .subcommand(deposits::cli_app()) .subcommand( SubCommand::with_name("new") .about("Create a new Ethereum 2.0 validator.") diff --git a/account_manager/src/deposits.rs b/account_manager/src/deposits.rs new file mode 100644 index 0000000000..c29bb5a214 --- /dev/null +++ b/account_manager/src/deposits.rs @@ -0,0 +1,129 @@ +use clap::{App, Arg, ArgMatches}; +use clap_utils; +use environment::Environment; +use std::fs; +use std::path::PathBuf; +use types::EthSpec; +use validator_client::validator_directory::ValidatorDirectoryBuilder; +use web3::{transports::Ipc, types::Address, Web3}; + +pub fn cli_app<'a, 'b>() -> App<'a, 'b> { + App::new("deposited") + .about("Creates new Lighthouse validator keys and directories. Each newly-created validator + will have a deposit transaction formed and submitted to the deposit contract via + --eth1-ipc. Will only write each validator keys to disk if the deposit transaction returns + successfully from the eth1 node. The process exits immediately if any Eth1 tx fails. Does + not wait for Eth1 confirmation blocks, so there is no guarantee that a deposit will be + accepted in the Eth1 chain.") + .arg( + Arg::with_name("validator-dir") + .long("validator-dir") + .value_name("VALIDATOR_DIRECTORY") + .help("The path where the validator directories will be created. Defaults to ~/.lighthouse/validators") + .takes_value(true), + ) + .arg( + Arg::with_name("eth1-ipc") + .long("eth1-ipc") + .value_name("ETH1_IPC_PATH") + .help("Path to an Eth1 JSON-RPC IPC endpoint") + .takes_value(true) + .required(true) + ) + .arg( + Arg::with_name("from-address") + .long("from-address") + .value_name("FROM_ETH1_ADDRESS") + .help("The address that will submit the eth1 deposit. Must be unlocked on the node + at --eth1-ipc.") + .takes_value(true) + .required(true) + ) + .arg( + Arg::with_name("deposit-gwei") + .long("deposit-gwei") + .value_name("DEPOSIT_GWEI") + .help("The GWEI value of the deposit amount. Defaults to the minimum amount + required for an active validator (MAX_EFFECTIVE_BALANCE.") + .takes_value(true), + ) + .arg( + Arg::with_name("count") + .long("count") + .value_name("DEPOSIT_COUNT") + .help("The number of deposits to create, regardless of how many already exist") + .conflicts_with("limit") + .takes_value(true), + ) + .arg( + Arg::with_name("at-least") + .long("at-least") + .value_name("VALIDATOR_COUNT") + .help("Observe the number of validators in --validator-dir, only creating enough to + ensure reach the given count. Never deletes an existing validator.") + .conflicts_with("count") + .takes_value(true), + ) +} + +pub fn cli_run(matches: &ArgMatches, mut env: Environment) -> Result<(), String> { + let spec = env.core_context().eth2_config.spec; + + let validator_dir = clap_utils::parse_path_with_default_in_home_dir( + matches, + "validator_dir", + PathBuf::new().join(".lighthouse").join("validators"), + )?; + let eth1_ipc_path: PathBuf = clap_utils::parse_required(matches, "eth1-ipc")?; + let from_address: Address = clap_utils::parse_required(matches, "from-address")?; + let deposit_gwei = clap_utils::parse_optional(matches, "deposit-gwei")? + .unwrap_or_else(|| spec.max_effective_balance); + let count: Option = clap_utils::parse_optional(matches, "count")?; + let at_least: Option = clap_utils::parse_optional(matches, "at-least")?; + + let n = match (count, at_least) { + (Some(_), Some(_)) => Err("Cannot supply --count and --at-least".to_string()), + (None, None) => Err("Must supply either --count or --at-least".to_string()), + (Some(count), None) => Ok(count), + (None, Some(at_least)) => fs::read_dir(&validator_dir) + .map(|iter| at_least.saturating_sub(iter.count())) + .map_err(|e| format!("Unable to read {:?}: {}", validator_dir, e)), + }?; + + let deposit_contract = env + .testnet + .as_ref() + .ok_or_else(|| "Unable to run account manager without a testnet dir".to_string())? + .deposit_contract_address() + .map_err(|e| format!("Unable to parse deposit contract address: {}", e))?; + + if deposit_contract == Address::zero() { + return Err("Refusing to deposit to the zero address. Check testnet configuration.".into()); + } + + let (_event_loop_handle, transport) = + Ipc::new(eth1_ipc_path).map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?; + let web3 = Web3::new(transport); + + for _ in 0..n { + let validator = env + .runtime() + .block_on( + ValidatorDirectoryBuilder::default() + .spec(spec.clone()) + .custom_deposit_amount(deposit_gwei) + .thread_random_keypairs() + .submit_eth1_deposit(web3.clone(), from_address, deposit_contract), + )? + .create_directory(validator_dir.clone())? + .write_keypair_files()? + .write_eth1_data_file()? + .build()?; + + if let Some(voting_keypair) = validator.voting_keypair { + println!("{:?}", voting_keypair.pk) + } + } + + Ok(()) +} diff --git a/account_manager/src/lib.rs b/account_manager/src/lib.rs index 101c7634ec..4f3c80ec76 100644 --- a/account_manager/src/lib.rs +++ b/account_manager/src/lib.rs @@ -1,4 +1,5 @@ mod cli; +mod deposits; use clap::ArgMatches; use deposit_contract::DEPOSIT_GAS; @@ -6,7 +7,7 @@ use environment::{Environment, RuntimeContext}; use eth2_testnet_config::Eth2TestnetConfig; use futures::{future, Future, IntoFuture, Stream}; use rayon::prelude::*; -use slog::{crit, error, info, Logger}; +use slog::{error, info, Logger}; use std::fs; use std::fs::File; use std::io::Read; @@ -21,20 +22,8 @@ use web3::{ pub use cli::cli_app; -/// Run the account manager, logging an error if the operation did not succeed. -pub fn run(matches: &ArgMatches, mut env: Environment) { - let log = env.core_context().log.clone(); - match run_account_manager(matches, env) { - Ok(()) => (), - Err(e) => crit!(log, "Account manager failed"; "error" => e), - } -} - /// Run the account manager, returning an error if the operation did not succeed. -fn run_account_manager( - matches: &ArgMatches, - mut env: Environment, -) -> Result<(), String> { +pub fn run(matches: &ArgMatches, mut env: Environment) -> Result<(), String> { let context = env.core_context(); let log = context.log.clone(); @@ -60,6 +49,7 @@ fn run_account_manager( match matches.subcommand() { ("validator", Some(matches)) => match matches.subcommand() { + ("deposited", Some(matches)) => deposits::cli_run(matches, env)?, ("new", Some(matches)) => run_new_validator_subcommand(matches, datadir, env)?, _ => { return Err("Invalid 'validator new' command. See --help.".to_string()); diff --git a/beacon_node/eth1/Cargo.toml b/beacon_node/eth1/Cargo.toml index fa945752ba..a4bb724315 100644 --- a/beacon_node/eth1/Cargo.toml +++ b/beacon_node/eth1/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" eth1_test_rig = { path = "../../tests/eth1_test_rig" } environment = { path = "../../lighthouse/environment" } toml = "^0.5" -web3 = "0.8.0" +web3 = "0.10.0" [dependencies] reqwest = "0.9" diff --git a/beacon_node/eth2-libp2p/src/discovery/enr.rs b/beacon_node/eth2-libp2p/src/discovery/enr.rs index 6cd3beac41..edd08bc9ac 100644 --- a/beacon_node/eth2-libp2p/src/discovery/enr.rs +++ b/beacon_node/eth2-libp2p/src/discovery/enr.rs @@ -1,10 +1,11 @@ //! Helper functions and an extension trait for Ethereum 2 ENRs. +pub use libp2p::{core::identity::Keypair, discv5::enr::CombinedKey}; + use super::ENR_FILENAME; use crate::types::{Enr, EnrBitfield}; use crate::NetworkConfig; -use libp2p::core::identity::Keypair; -use libp2p::discv5::enr::{CombinedKey, EnrBuilder}; +use libp2p::discv5::enr::EnrBuilder; use slog::{debug, warn}; use ssz::{Decode, Encode}; use ssz_types::BitVector; diff --git a/beacon_node/eth2-libp2p/src/discovery/mod.rs b/beacon_node/eth2-libp2p/src/discovery/mod.rs index a30f860921..13b37c6e67 100644 --- a/beacon_node/eth2-libp2p/src/discovery/mod.rs +++ b/beacon_node/eth2-libp2p/src/discovery/mod.rs @@ -2,13 +2,13 @@ pub(crate) mod enr; // Allow external use of the lighthouse ENR builder -pub use enr::build_enr; +pub use enr::{build_enr, CombinedKey, Keypair}; use crate::metrics; use crate::{error, Enr, NetworkConfig, NetworkGlobals}; use enr::{Eth2Enr, BITFIELD_ENR_KEY, ETH2_ENR_KEY}; use futures::prelude::*; -use libp2p::core::{identity::Keypair, ConnectedPoint, Multiaddr, PeerId}; +use libp2p::core::{ConnectedPoint, Multiaddr, PeerId}; use libp2p::discv5::enr::NodeId; use libp2p::discv5::{Discv5, Discv5Event}; use libp2p::multiaddr::Protocol; @@ -30,7 +30,7 @@ const MAX_TIME_BETWEEN_PEER_SEARCHES: u64 = 120; /// Initial delay between peer searches. const INITIAL_SEARCH_DELAY: u64 = 5; /// Local ENR storage filename. -const ENR_FILENAME: &str = "enr.dat"; +pub const ENR_FILENAME: &str = "enr.dat"; /// Number of peers we'd like to have connected to a given long-lived subnet. const TARGET_SUBNET_PEERS: u64 = 3; diff --git a/beacon_node/eth2-libp2p/src/lib.rs b/beacon_node/eth2-libp2p/src/lib.rs index 98f3ffc9f3..a0d6af1b95 100644 --- a/beacon_node/eth2-libp2p/src/lib.rs +++ b/beacon_node/eth2-libp2p/src/lib.rs @@ -22,4 +22,4 @@ pub use libp2p::{multiaddr, Multiaddr}; pub use libp2p::{PeerId, Swarm}; pub use peer_manager::{PeerDB, PeerInfo, PeerSyncStatus}; pub use rpc::RPCEvent; -pub use service::Service; +pub use service::{Service, NETWORK_KEY_FILENAME}; diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 49b3909105..49b8747faf 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -27,7 +27,7 @@ use types::{EnrForkId, EthSpec}; type Libp2pStream = Boxed<(PeerId, StreamMuxerBox), Error>; type Libp2pBehaviour = Behaviour, TSpec>; -const NETWORK_KEY_FILENAME: &str = "key"; +pub const NETWORK_KEY_FILENAME: &str = "key"; /// The time in milliseconds to wait before banning a peer. This allows for any Goodbye messages to be /// flushed and protocols to be negotiated. const BAN_PEER_WAIT_TIMEOUT: u64 = 200; diff --git a/book/src/local-testnets.md b/book/src/local-testnets.md index b9b351bab5..3b15cefdd6 100644 --- a/book/src/local-testnets.md +++ b/book/src/local-testnets.md @@ -18,6 +18,7 @@ TL;DR isn't adequate. ## TL;DR ```bash +make install-lcli lcli new-testnet lcli interop-genesis 128 lighthouse bn --testnet-dir ~/.lighthouse/testnet --dummy-eth1 --http --enr-match @@ -27,7 +28,6 @@ lighthouse vc --testnet-dir ~/.lighthouse/testnet --allow-unsynced testnet insec Optionally update the genesis time to now: ```bash -<<<<<<< HEAD lcli change-genesis-time ~/.lighthouse/testnet/genesis.ssz $(date +%s) ``` @@ -41,7 +41,7 @@ used for starting testnets and debugging. Install `lcli` from the root directory of this repository with: ```bash -cargo install --path lcli --force +make install-lcli ``` ### 1.2 Create a testnet directory diff --git a/eth2/utils/clap_utils/Cargo.toml b/eth2/utils/clap_utils/Cargo.toml new file mode 100644 index 0000000000..f1916c4ba4 --- /dev/null +++ b/eth2/utils/clap_utils/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "clap_utils" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = "2.33.0" +hex = "0.3" +dirs = "2.0" +types = { path = "../../types" } +eth2_testnet_config = { path = "../eth2_testnet_config" } +eth2_ssz = { path = "../ssz" } diff --git a/eth2/utils/clap_utils/src/lib.rs b/eth2/utils/clap_utils/src/lib.rs new file mode 100644 index 0000000000..ecd1b698b6 --- /dev/null +++ b/eth2/utils/clap_utils/src/lib.rs @@ -0,0 +1,116 @@ +//! A helper library for parsing values from `clap::ArgMatches`. + +use clap::ArgMatches; +use eth2_testnet_config::Eth2TestnetConfig; +use hex; +use ssz::Decode; +use std::path::PathBuf; +use std::str::FromStr; +use types::EthSpec; + +/// Attempts to load the testnet dir at the path if `name` is in `matches`, returning an error if +/// the path cannot be found or the testnet dir is invalid. +/// +/// If `name` is not in `matches`, attempts to return the "hard coded" testnet dir. +pub fn parse_testnet_dir_with_hardcoded_default( + matches: &ArgMatches, + name: &'static str, +) -> Result, String> { + parse_required::(matches, name) + .and_then(|path| { + Eth2TestnetConfig::load(path.clone()) + .map_err(|e| format!("Unable to open testnet dir at {:?}: {}", path, e)) + }) + .map(Result::Ok) + .unwrap_or_else(|_| { + Eth2TestnetConfig::hard_coded().map_err(|e| { + format!( + "The hard-coded testnet directory was invalid. \ + This happens when Lighthouse is migrating between spec versions. \ + Error : {}", + e + ) + }) + }) +} + +/// If `name` is in `matches`, parses the value as a path. Otherwise, attempts to find the user's +/// home directory and appends `default` to it. +pub fn parse_path_with_default_in_home_dir( + matches: &ArgMatches, + name: &'static str, + default: PathBuf, +) -> Result { + matches + .value_of(name) + .map(|dir| { + dir.parse::() + .map_err(|e| format!("Unable to parse {}: {}", name, e)) + }) + .unwrap_or_else(|| { + dirs::home_dir() + .map(|home| home.join(default)) + .ok_or_else(|| format!("Unable to locate home directory. Try specifying {}", name)) + }) +} + +/// Returns the value of `name` or an error if it is not in `matches` or does not parse +/// successfully using `std::string::FromStr`. +pub fn parse_required(matches: &ArgMatches, name: &'static str) -> Result +where + T: FromStr, + ::Err: std::fmt::Display, +{ + parse_optional(matches, name)?.ok_or_else(|| format!("{} not specified", name)) +} + +/// Returns the value of `name` (if present) or an error if it does not parse successfully using +/// `std::string::FromStr`. +pub fn parse_optional(matches: &ArgMatches, name: &'static str) -> Result, String> +where + T: FromStr, + ::Err: std::fmt::Display, +{ + matches + .value_of(name) + .map(|val| { + val.parse() + .map_err(|e| format!("Unable to parse {}: {}", name, e)) + }) + .transpose() +} + +/// Returns the value of `name` or an error if it is not in `matches` or does not parse +/// successfully using `ssz::Decode`. +/// +/// Expects the value of `name` to be 0x-prefixed ASCII-hex. +pub fn parse_ssz_required( + matches: &ArgMatches, + name: &'static str, +) -> Result { + parse_ssz_optional(matches, name)?.ok_or_else(|| format!("{} not specified", name)) +} + +/// Returns the value of `name` (if present) or an error if it does not parse successfully using +/// `ssz::Decode`. +/// +/// Expects the value of `name` (if any) to be 0x-prefixed ASCII-hex. +pub fn parse_ssz_optional( + matches: &ArgMatches, + name: &'static str, +) -> Result, String> { + matches + .value_of(name) + .map(|val| { + if val.starts_with("0x") { + let vec = hex::decode(&val[2..]) + .map_err(|e| format!("Unable to parse {} as hex: {:?}", name, e))?; + + T::from_ssz_bytes(&vec) + .map_err(|e| format!("Unable to parse {} as SSZ: {:?}", name, e)) + } else { + Err(format!("Unable to parse {}, must have 0x prefix", name)) + } + }) + .transpose() +} diff --git a/eth2/utils/deposit_contract/src/lib.rs b/eth2/utils/deposit_contract/src/lib.rs index 11a831c0fa..2a5ea514e9 100644 --- a/eth2/utils/deposit_contract/src/lib.rs +++ b/eth2/utils/deposit_contract/src/lib.rs @@ -22,7 +22,7 @@ impl From for DecodeError { } pub const CONTRACT_DEPLOY_GAS: usize = 4_000_000; -pub const DEPOSIT_GAS: usize = 4_000_000; +pub const DEPOSIT_GAS: usize = 400_000; pub const ABI: &[u8] = include_bytes!("../contracts/v0.11.1_validator_registration.json"); pub const BYTECODE: &[u8] = include_bytes!("../contracts/v0.11.1_validator_registration.bytecode"); pub const DEPOSIT_DATA_LEN: usize = 420; // lol diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index 507390f262..61eba90c67 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -27,3 +27,5 @@ dirs = "2.0" genesis = { path = "../beacon_node/genesis" } deposit_contract = { path = "../eth2/utils/deposit_contract" } tree_hash = { path = "../eth2/utils/tree_hash" } +clap_utils = { path = "../eth2/utils/clap_utils" } +eth2-libp2p = { path = "../beacon_node/eth2-libp2p" } diff --git a/lcli/src/check_deposit_data.rs b/lcli/src/check_deposit_data.rs index af2b6fb478..56f18f9988 100644 --- a/lcli/src/check_deposit_data.rs +++ b/lcli/src/check_deposit_data.rs @@ -1,12 +1,12 @@ -use crate::helpers::{parse_hex_bytes, parse_u64}; use clap::ArgMatches; +use clap_utils::{parse_required, parse_ssz_required}; use deposit_contract::{decode_eth1_tx_data, DEPOSIT_DATA_LEN}; use tree_hash::TreeHash; use types::EthSpec; pub fn run(matches: &ArgMatches) -> Result<(), String> { - let rlp_bytes = parse_hex_bytes(matches, "deposit-data")?; - let amount = parse_u64(matches, "deposit-amount")?; + let rlp_bytes = parse_ssz_required::>(matches, "deposit-data")?; + let amount = parse_required(matches, "deposit-amount")?; if rlp_bytes.len() != DEPOSIT_DATA_LEN { return Err(format!( diff --git a/lcli/src/deploy_deposit_contract.rs b/lcli/src/deploy_deposit_contract.rs index 095da6b66d..22fd84fc65 100644 --- a/lcli/src/deploy_deposit_contract.rs +++ b/lcli/src/deploy_deposit_contract.rs @@ -1,31 +1,35 @@ use clap::ArgMatches; +use clap_utils; +use deposit_contract::{ + testnet::{ABI, BYTECODE}, + CONTRACT_DEPLOY_GAS, +}; use environment::Environment; -use eth1_test_rig::DepositContract; -use std::fs::File; -use std::io::Read; +use futures::{Future, IntoFuture}; +use std::path::PathBuf; use types::EthSpec; -use web3::{transports::Http, Web3}; +use web3::{ + contract::{Contract, Options}, + transports::Ipc, + types::{Address, U256}, + Web3, +}; pub fn run(mut env: Environment, matches: &ArgMatches) -> Result<(), String> { - let confirmations = matches - .value_of("confirmations") - .ok_or_else(|| "Confirmations not specified")? - .parse::() - .map_err(|e| format!("Failed to parse confirmations: {}", e))?; + let eth1_ipc_path: PathBuf = clap_utils::parse_required(matches, "eth1-ipc")?; + let from_address: Address = clap_utils::parse_required(matches, "from-address")?; + let confirmations: usize = clap_utils::parse_required(matches, "confirmations")?; - let password = parse_password(matches)?; + let (_event_loop_handle, transport) = + Ipc::new(eth1_ipc_path).map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?; + let web3 = Web3::new(transport); - let endpoint = matches - .value_of("eth1-endpoint") - .ok_or_else(|| "eth1-endpoint not specified")?; - - let (_event_loop, transport) = Http::new(&endpoint).map_err(|e| { + let bytecode = String::from_utf8(BYTECODE.to_vec()).map_err(|e| { format!( - "Failed to start HTTP transport connected to ganache: {:?}", + "Unable to parse deposit contract bytecode as utf-8: {:?}", e ) })?; - let web3 = Web3::new(transport); // It's unlikely that this will be the _actual_ deployment block, however it'll be close // enough to serve our purposes. @@ -37,54 +41,26 @@ pub fn run(mut env: Environment, matches: &ArgMatches) -> Result< .block_on(web3.eth().block_number()) .map_err(|e| format!("Failed to get block number: {}", e))?; - info!("Present eth1 block number is {}", deploy_block); + let address = env.runtime().block_on( + Contract::deploy(web3.eth(), &ABI) + .map_err(|e| format!("Unable to build contract deployer: {:?}", e))? + .confirmations(confirmations) + .options(Options { + gas: Some(U256::from(CONTRACT_DEPLOY_GAS)), + ..Options::default() + }) + .execute(bytecode, (), from_address) + .into_future() + .map_err(|e| format!("Unable to execute deployment: {:?}", e)) + .and_then(|pending| { + pending.map_err(|e| format!("Unable to await pending contract: {:?}", e)) + }) + .map(|tx_receipt| tx_receipt.address()) + .map_err(|e| format!("Failed to execute deployment: {:?}", e)), + )?; - info!("Deploying the bytecode at https://github.com/sigp/unsafe-eth2-deposit-contract",); - - info!( - "Submitting deployment transaction, waiting for {} confirmations", - confirmations - ); - - let deposit_contract = env - .runtime() - .block_on(DepositContract::deploy_testnet( - web3, - confirmations, - password, - )) - .map_err(|e| format!("Failed to deploy contract: {}", e))?; - - info!( - "Deposit contract deployed. address: {}, deploy_block: {}", - deposit_contract.address(), - deploy_block - ); + println!("deposit_contract_address: {:?}", address); + println!("deposit_contract_deploy_block: {}", deploy_block); Ok(()) } - -pub fn parse_password(matches: &ArgMatches) -> Result, String> { - if let Some(password_path) = matches.value_of("password") { - Ok(Some( - File::open(password_path) - .map_err(|e| format!("Unable to open password file: {:?}", e)) - .and_then(|mut file| { - let mut password = String::new(); - file.read_to_string(&mut password) - .map_err(|e| format!("Unable to read password file to string: {:?}", e)) - .map(|_| password) - }) - .map(|password| { - // Trim the linefeed from the end. - if password.ends_with('\n') { - password[0..password.len() - 1].to_string() - } else { - password - } - })?, - )) - } else { - Ok(None) - } -} diff --git a/lcli/src/generate_bootnode_enr.rs b/lcli/src/generate_bootnode_enr.rs new file mode 100644 index 0000000000..2d6e685a62 --- /dev/null +++ b/lcli/src/generate_bootnode_enr.rs @@ -0,0 +1,60 @@ +use clap::ArgMatches; +use eth2_libp2p::{ + discovery::{build_enr, CombinedKey, Keypair, ENR_FILENAME}, + NetworkConfig, NETWORK_KEY_FILENAME, +}; +use std::convert::TryInto; +use std::fs; +use std::fs::File; +use std::io::Write; +use std::net::IpAddr; +use std::path::PathBuf; +use types::{EnrForkId, EthSpec}; + +pub fn run(matches: &ArgMatches) -> Result<(), String> { + let ip: IpAddr = clap_utils::parse_required(matches, "ip")?; + let udp_port: u16 = clap_utils::parse_required(matches, "udp-port")?; + let tcp_port: u16 = clap_utils::parse_required(matches, "tcp-port")?; + let output_dir: PathBuf = clap_utils::parse_required(matches, "output-dir")?; + + if output_dir.exists() { + return Err(format!( + "{:?} already exists, will not override", + output_dir + )); + } + + let mut config = NetworkConfig::default(); + config.enr_address = Some(ip); + config.enr_udp_port = Some(udp_port); + config.enr_tcp_port = Some(tcp_port); + + let local_keypair = Keypair::generate_secp256k1(); + let enr_key: CombinedKey = local_keypair + .clone() + .try_into() + .map_err(|e| format!("Unable to convert keypair: {:?}", e))?; + let enr = build_enr::(&enr_key, &config, EnrForkId::default()) + .map_err(|e| format!("Unable to create ENR: {:?}", e))?; + + fs::create_dir_all(&output_dir).map_err(|e| format!("Unable to create output-dir: {:?}", e))?; + + let mut enr_file = File::create(output_dir.join(ENR_FILENAME)) + .map_err(|e| format!("Unable to create {}: {:?}", ENR_FILENAME, e))?; + enr_file + .write_all(&enr.to_base64().as_bytes()) + .map_err(|e| format!("Unable to write ENR to {}: {:?}", ENR_FILENAME, e))?; + + let secret_bytes = match local_keypair { + Keypair::Secp256k1(key) => key.secret().to_bytes(), + _ => return Err("Key is not a secp256k1 key".into()), + }; + + let mut key_file = File::create(output_dir.join(NETWORK_KEY_FILENAME)) + .map_err(|e| format!("Unable to create {}: {:?}", NETWORK_KEY_FILENAME, e))?; + key_file + .write_all(&secret_bytes) + .map_err(|e| format!("Unable to write key to {}: {:?}", NETWORK_KEY_FILENAME, e))?; + + Ok(()) +} diff --git a/lcli/src/main.rs b/lcli/src/main.rs index cc28832591..7abd4b44a6 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -5,7 +5,7 @@ mod change_genesis_time; mod check_deposit_data; mod deploy_deposit_contract; mod eth1_genesis; -mod helpers; +mod generate_bootnode_enr; mod interop_genesis; mod new_testnet; mod parse_hex; @@ -18,6 +18,7 @@ use log::Level; use parse_hex::run_parse_hex; use std::fs::File; use std::path::PathBuf; +use std::process; use std::time::{SystemTime, UNIX_EPOCH}; use transition_blocks::run_transition_blocks; use types::{test_utils::TestingBeaconStateBuilder, EthSpec, MainnetEthSpec, MinimalEthSpec}; @@ -27,8 +28,7 @@ fn main() { let matches = App::new("Lighthouse CLI Tool") .about( - "Performs various testing-related tasks, modelled after zcli. \ - by @protolambda.", + "Performs various testing-related tasks, including defining testnets.", ) .arg( Arg::with_name("spec") @@ -40,6 +40,15 @@ fn main() { .possible_values(&["minimal", "mainnet"]) .default_value("mainnet") ) + .arg( + Arg::with_name("testnet-dir") + .short("d") + .long("testnet-dir") + .value_name("PATH") + .takes_value(true) + .global(true) + .help("The testnet dir. Defaults to ~/.lighthouse/testnet"), + ) .subcommand( SubCommand::with_name("genesis_yaml") .about("Generates a genesis YAML file") @@ -119,13 +128,22 @@ fn main() { "Deploy a testing eth1 deposit contract.", ) .arg( - Arg::with_name("eth1-endpoint") + Arg::with_name("eth1-ipc") + .long("eth1-ipc") .short("e") - .long("eth1-endpoint") - .value_name("HTTP_SERVER") + .value_name("ETH1_IPC_PATH") + .help("Path to an Eth1 JSON-RPC IPC endpoint") .takes_value(true) - .default_value("http://localhost:8545") - .help("The URL to the eth1 JSON-RPC http API."), + .required(true) + ) + .arg( + Arg::with_name("from-address") + .long("from-address") + .short("f") + .value_name("FROM_ETH1_ADDRESS") + .help("The address that will submit the contract creation. Must be unlocked.") + .takes_value(true) + .required(true) ) .arg( Arg::with_name("confirmations") @@ -135,13 +153,6 @@ fn main() { .default_value("3") .help("The number of block confirmations before declaring the contract deployed."), ) - .arg( - Arg::with_name("password") - .long("password") - .value_name("FILE") - .takes_value(true) - .help("The password file to unlock the eth1 account (see --index)"), - ) ) .subcommand( SubCommand::with_name("refund-deposit-contract") @@ -149,37 +160,32 @@ fn main() { "Calls the steal() function on a testnet eth1 contract.", ) .arg( - Arg::with_name("testnet-dir") - .short("d") - .long("testnet-dir") - .value_name("PATH") - .takes_value(true) - .help("The testnet dir. Defaults to ~/.lighthouse/testnet"), - ) - .arg( - Arg::with_name("eth1-endpoint") + Arg::with_name("eth1-ipc") + .long("eth1-ipc") .short("e") - .long("eth1-endpoint") - .value_name("HTTP_SERVER") + .value_name("ETH1_IPC_PATH") + .help("Path to an Eth1 JSON-RPC IPC endpoint") .takes_value(true) - .default_value("http://localhost:8545") - .help("The URL to the eth1 JSON-RPC http API."), + .required(true) ) .arg( - Arg::with_name("password") - .long("password") - .value_name("FILE") + Arg::with_name("from-address") + .long("from-address") + .short("f") + .value_name("FROM_ETH1_ADDRESS") + .help("The address that will submit the contract creation. Must be unlocked.") .takes_value(true) - .help("The password file to unlock the eth1 account (see --index)"), + .required(true) ) .arg( - Arg::with_name("account-index") - .short("i") - .long("account-index") - .value_name("INDEX") + Arg::with_name("contract-address") + .long("contract-address") + .short("c") + .value_name("CONTRACT_ETH1_ADDRESS") + .help("The address of the contract to be refunded. Its owner must match + --from-address.") .takes_value(true) - .default_value("0") - .help("The eth1 accounts[] index which will send the transaction"), + .required(true) ) ) .subcommand( @@ -187,14 +193,6 @@ fn main() { .about( "Listens to the eth1 chain and finds the genesis beacon state", ) - .arg( - Arg::with_name("testnet-dir") - .short("d") - .long("testnet-dir") - .value_name("PATH") - .takes_value(true) - .help("The testnet dir. Defaults to ~/.lighthouse/testnet"), - ) .arg( Arg::with_name("eth1-endpoint") .short("e") @@ -210,14 +208,6 @@ fn main() { .about( "Produces an interop-compatible genesis state using deterministic keypairs", ) - .arg( - Arg::with_name("testnet-dir") - .short("d") - .long("testnet-dir") - .value_name("PATH") - .takes_value(true) - .help("The testnet dir. Defaults to ~/.lighthouse/testnet"), - ) .arg( Arg::with_name("validator-count") .long("validator-count") @@ -263,13 +253,6 @@ fn main() { .about( "Produce a new testnet directory.", ) - .arg( - Arg::with_name("testnet-dir") - .long("testnet-dir") - .value_name("DIRECTORY") - .takes_value(true) - .help("The output path for the new testnet directory. Defaults to ~/.lighthouse/testnet"), - ) .arg( Arg::with_name("min-genesis-time") .long("min-genesis-time") @@ -384,11 +367,55 @@ fn main() { function signature."), ) ) + .subcommand( + SubCommand::with_name("generate-bootnode-enr") + .about( + "Generates an ENR address to be used as a pre-genesis boot node..", + ) + .arg( + Arg::with_name("ip") + .long("ip") + .value_name("IP_ADDRESS") + .takes_value(true) + .required(true) + .help("The IP address to be included in the ENR and used for discovery"), + ) + .arg( + Arg::with_name("udp-port") + .long("udp-port") + .value_name("UDP_PORT") + .takes_value(true) + .required(true) + .help("The UDP port to be included in the ENR and used for discovery"), + ) + .arg( + Arg::with_name("tcp-port") + .long("tcp-port") + .value_name("TCP_PORT") + .takes_value(true) + .required(true) + .help("The TCP port to be included in the ENR and used for application comms"), + ) + .arg( + Arg::with_name("output-dir") + .long("output-dir") + .value_name("OUTPUT_DIRECTORY") + .takes_value(true) + .required(true) + .help("The directory in which to create the network dir"), + ) + ) .get_matches(); macro_rules! run_with_spec { ($env_builder: expr) => { - run($env_builder, &matches) + match run($env_builder, &matches) { + Ok(()) => process::exit(0), + Err(e) => { + println!("Failed to run lcli: {}", e); + process::exit(1) + } + } }; } @@ -403,14 +430,14 @@ fn main() { } } -fn run(env_builder: EnvironmentBuilder, matches: &ArgMatches) { +fn run(env_builder: EnvironmentBuilder, matches: &ArgMatches) -> Result<(), String> { let env = env_builder .multi_threaded_tokio_runtime() - .expect("should start tokio runtime") + .map_err(|e| format!("should start tokio runtime: {:?}", e))? .async_logger("trace", None) - .expect("should start null logger") + .map_err(|e| format!("should start null logger: {:?}", e))? .build() - .expect("should build env"); + .map_err(|e| format!("should build env: {:?}", e))?; match matches.subcommand() { ("genesis_yaml", Some(matches)) => { @@ -449,30 +476,34 @@ fn run(env_builder: EnvironmentBuilder, matches: &ArgMatches) { _ => unreachable!("guarded by slog possible_values"), }; info!("Genesis state YAML file created. Exiting successfully."); + Ok(()) } ("transition-blocks", Some(matches)) => run_transition_blocks::(matches) - .unwrap_or_else(|e| error!("Failed to transition blocks: {}", e)), - ("pretty-hex", Some(matches)) => run_parse_hex::(matches) - .unwrap_or_else(|e| error!("Failed to pretty print hex: {}", e)), + .map_err(|e| format!("Failed to transition blocks: {}", e)), + ("pretty-hex", Some(matches)) => { + run_parse_hex::(matches).map_err(|e| format!("Failed to pretty print hex: {}", e)) + } ("deploy-deposit-contract", Some(matches)) => { deploy_deposit_contract::run::(env, matches) - .unwrap_or_else(|e| error!("Failed to run deploy-deposit-contract command: {}", e)) + .map_err(|e| format!("Failed to run deploy-deposit-contract command: {}", e)) } ("refund-deposit-contract", Some(matches)) => { refund_deposit_contract::run::(env, matches) - .unwrap_or_else(|e| error!("Failed to run refund-deposit-contract command: {}", e)) + .map_err(|e| format!("Failed to run refund-deposit-contract command: {}", e)) } ("eth1-genesis", Some(matches)) => eth1_genesis::run::(env, matches) - .unwrap_or_else(|e| error!("Failed to run eth1-genesis command: {}", e)), + .map_err(|e| format!("Failed to run eth1-genesis command: {}", e)), ("interop-genesis", Some(matches)) => interop_genesis::run::(env, matches) - .unwrap_or_else(|e| error!("Failed to run interop-genesis command: {}", e)), + .map_err(|e| format!("Failed to run interop-genesis command: {}", e)), ("change-genesis-time", Some(matches)) => change_genesis_time::run::(matches) - .unwrap_or_else(|e| error!("Failed to run change-genesis-time command: {}", e)), + .map_err(|e| format!("Failed to run change-genesis-time command: {}", e)), ("new-testnet", Some(matches)) => new_testnet::run::(matches) - .unwrap_or_else(|e| error!("Failed to run new_testnet command: {}", e)), + .map_err(|e| format!("Failed to run new_testnet command: {}", e)), ("check-deposit-data", Some(matches)) => check_deposit_data::run::(matches) - .unwrap_or_else(|e| error!("Failed to run check-deposit-data command: {}", e)), - (other, _) => error!("Unknown subcommand {}. See --help.", other), + .map_err(|e| format!("Failed to run check-deposit-data command: {}", e)), + ("generate-bootnode-enr", Some(matches)) => generate_bootnode_enr::run::(matches) + .map_err(|e| format!("Failed to run generate-bootnode-enr command: {}", e)), + (other, _) => Err(format!("Unknown subcommand {}. See --help.", other)), } } diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index 447bffdfc3..7050defa2f 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -1,8 +1,11 @@ -use crate::helpers::*; use clap::ArgMatches; +use clap_utils::{ + parse_optional, parse_path_with_default_in_home_dir, parse_required, parse_ssz_optional, +}; use eth2_testnet_config::Eth2TestnetConfig; use std::path::PathBuf; -use types::{EthSpec, YamlConfig}; +use std::time::{SystemTime, UNIX_EPOCH}; +use types::{Address, EthSpec, YamlConfig}; pub fn run(matches: &ArgMatches) -> Result<(), String> { let testnet_dir_path = parse_path_with_default_in_home_dir( @@ -10,18 +13,18 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { "testnet-dir", PathBuf::from(".lighthouse/testnet"), )?; - let min_genesis_time = parse_u64_opt(matches, "min-genesis-time")?; - let min_genesis_delay = parse_u64(matches, "min-genesis-delay")?; + let min_genesis_time = parse_optional(matches, "min-genesis-time")?; + let min_genesis_delay = parse_required(matches, "min-genesis-delay")?; let min_genesis_active_validator_count = - parse_u64(matches, "min-genesis-active-validator-count")?; - let min_deposit_amount = parse_u64(matches, "min-deposit-amount")?; - let max_effective_balance = parse_u64(matches, "max-effective-balance")?; - let effective_balance_increment = parse_u64(matches, "effective-balance-increment")?; - let ejection_balance = parse_u64(matches, "ejection-balance")?; - let eth1_follow_distance = parse_u64(matches, "eth1-follow-distance")?; - let deposit_contract_deploy_block = parse_u64(matches, "deposit-contract-deploy-block")?; - let genesis_fork_version = parse_fork_opt(matches, "genesis-fork-version")?; - let deposit_contract_address = parse_address(matches, "deposit-contract-address")?; + parse_required(matches, "min-genesis-active-validator-count")?; + let min_deposit_amount = parse_required(matches, "min-deposit-amount")?; + let max_effective_balance = clap_utils::parse_required(matches, "max-effective-balance")?; + let effective_balance_increment = parse_required(matches, "effective-balance-increment")?; + let ejection_balance = parse_required(matches, "ejection-balance")?; + let eth1_follow_distance = parse_required(matches, "eth1-follow-distance")?; + let deposit_contract_deploy_block = parse_required(matches, "deposit-contract-deploy-block")?; + let genesis_fork_version = parse_ssz_optional::<[u8; 4]>(matches, "genesis-fork-version")?; + let deposit_contract_address: Address = parse_required(matches, "deposit-contract-address")?; if testnet_dir_path.exists() { return Err(format!( @@ -57,3 +60,10 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { testnet.write_to_file(testnet_dir_path) } + +pub fn time_now() -> Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .map_err(|e| format!("Unable to get time: {:?}", e)) +} diff --git a/lcli/src/refund_deposit_contract.rs b/lcli/src/refund_deposit_contract.rs index d413b7f5ce..719a8ef1b0 100644 --- a/lcli/src/refund_deposit_contract.rs +++ b/lcli/src/refund_deposit_contract.rs @@ -1,12 +1,10 @@ -use crate::deploy_deposit_contract::parse_password; use clap::ArgMatches; use environment::Environment; -use eth2_testnet_config::Eth2TestnetConfig; -use futures::{future, Future}; +use futures::Future; use std::path::PathBuf; use types::EthSpec; use web3::{ - transports::Http, + transports::Ipc, types::{Address, TransactionRequest, U256}, Web3, }; @@ -15,102 +13,28 @@ use web3::{ pub const STEAL_FN_SIGNATURE: &[u8] = &[0xcf, 0x7a, 0x89, 0x65]; pub fn run(mut env: Environment, matches: &ArgMatches) -> Result<(), String> { - let endpoint = matches - .value_of("eth1-endpoint") - .ok_or_else(|| "eth1-endpoint not specified")?; + let eth1_ipc_path: PathBuf = clap_utils::parse_required(matches, "eth1-ipc")?; + let from: Address = clap_utils::parse_required(matches, "from-address")?; + let contract_address: Address = clap_utils::parse_required(matches, "contract-address")?; - let account_index = matches - .value_of("account-index") - .ok_or_else(|| "No account-index".to_string())? - .parse::() - .map_err(|e| format!("Unable to parse account-index: {}", e))?; + let (_event_loop_handle, transport) = + Ipc::new(eth1_ipc_path).map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?; + let web3 = Web3::new(transport); - let password_opt = parse_password(matches)?; - - let testnet_dir = matches - .value_of("testnet-dir") - .ok_or_else(|| ()) - .and_then(|dir| dir.parse::().map_err(|_| ())) - .unwrap_or_else(|_| { - dirs::home_dir() - .map(|home| home.join(".lighthouse").join("testnet")) - .expect("should locate home directory") - }); - - let eth2_testnet_config: Eth2TestnetConfig = Eth2TestnetConfig::load(testnet_dir)?; - - let (_event_loop, transport) = Http::new(&endpoint).map_err(|e| { - format!( - "Failed to start HTTP transport connected to ganache: {:?}", - e - ) - })?; - - let web3_1 = Web3::new(transport); - let web3_2 = web3_1.clone(); - - // Convert from `types::Address` to `web3::types::Address`. - let deposit_contract = Address::from_slice( - eth2_testnet_config - .deposit_contract_address()? - .as_fixed_bytes(), - ); - - let future = web3_1 - .eth() - .accounts() - .map_err(|e| format!("Failed to get accounts: {:?}", e)) - .and_then(move |accounts| { - accounts - .get(account_index) - .cloned() - .ok_or_else(|| "Insufficient accounts for deposit".to_string()) - }) - .and_then(move |from_address| { - let future: Box + Send> = - if let Some(password) = password_opt { - // Unlock for only a single transaction. - let duration = None; - - let future = web3_1 - .personal() - .unlock_account(from_address, &password, duration) - .then(move |result| match result { - Ok(true) => Ok(from_address), - Ok(false) => Err("Eth1 node refused to unlock account".to_string()), - Err(e) => Err(format!("Eth1 unlock request failed: {:?}", e)), - }); - - Box::new(future) - } else { - Box::new(future::ok(from_address)) - }; - - future - }) - .and_then(move |from| { - let tx_request = TransactionRequest { + env.runtime().block_on( + web3.eth() + .send_transaction(TransactionRequest { from, - to: Some(deposit_contract), + to: Some(contract_address), gas: Some(U256::from(400_000)), gas_price: None, value: Some(U256::zero()), data: Some(STEAL_FN_SIGNATURE.into()), nonce: None, condition: None, - }; - - web3_2 - .eth() - .send_transaction(tx_request) - .map_err(|e| format!("Failed to call deposit fn: {:?}", e)) - }) - .map(move |tx| info!("Refund transaction submitted: eth1_tx_hash: {:?}", tx)) - .map_err(move |e| error!("Unable to submit refund transaction: error: {}", e)); - - env.runtime() - .block_on(future) - .map_err(|()| "Failed to send transaction".to_string())?; + }) + .map_err(|e| format!("Failed to call deposit fn: {:?}", e)), + )?; Ok(()) } diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index f22f79097a..6f7527694a 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -19,3 +19,4 @@ environment = { path = "./environment" } futures = "0.1.25" validator_client = { "path" = "../validator_client" } account_manager = { "path" = "../account_manager" } +clap_utils = { path = "../eth2/utils/clap_utils" } diff --git a/lighthouse/environment/src/lib.rs b/lighthouse/environment/src/lib.rs index 08c6d0a5d2..20e54b0979 100644 --- a/lighthouse/environment/src/lib.rs +++ b/lighthouse/environment/src/lib.rs @@ -28,6 +28,7 @@ pub struct EnvironmentBuilder { log: Option, eth_spec_instance: E, eth2_config: Eth2Config, + testnet: Option>, } impl EnvironmentBuilder { @@ -38,6 +39,7 @@ impl EnvironmentBuilder { log: None, eth_spec_instance: MinimalEthSpec, eth2_config: Eth2Config::minimal(), + testnet: None, } } } @@ -50,6 +52,7 @@ impl EnvironmentBuilder { log: None, eth_spec_instance: MainnetEthSpec, eth2_config: Eth2Config::mainnet(), + testnet: None, } } } @@ -62,6 +65,7 @@ impl EnvironmentBuilder { log: None, eth_spec_instance: InteropEthSpec, eth2_config: Eth2Config::interop(), + testnet: None, } } } @@ -140,7 +144,7 @@ impl EnvironmentBuilder { /// Setups eth2 config using the CLI arguments. pub fn eth2_testnet_config( mut self, - eth2_testnet_config: &Eth2TestnetConfig, + eth2_testnet_config: Eth2TestnetConfig, ) -> Result { // Create a new chain spec from the default configuration. self.eth2_config.spec = eth2_testnet_config @@ -155,6 +159,8 @@ impl EnvironmentBuilder { ) })?; + self.testnet = Some(eth2_testnet_config); + Ok(self) } @@ -169,6 +175,7 @@ impl EnvironmentBuilder { .ok_or_else(|| "Cannot build environment without log".to_string())?, eth_spec_instance: self.eth_spec_instance, eth2_config: self.eth2_config, + testnet: self.testnet, }) } } @@ -211,6 +218,7 @@ pub struct Environment { log: Logger, eth_spec_instance: E, pub eth2_config: Eth2Config, + pub testnet: Option>, } impl Environment { diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 3c473db2d1..dbb3c90395 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -1,8 +1,9 @@ #[macro_use] extern crate clap; -use beacon_node::{get_eth2_testnet_config, get_testnet_dir, ProductionBeaconNode}; +use beacon_node::ProductionBeaconNode; use clap::{App, Arg, ArgMatches}; +use clap_utils; use env_logger::{Builder, Env}; use environment::EnvironmentBuilder; use slog::{crit, info, warn}; @@ -123,12 +124,13 @@ fn run( .ok_or_else(|| "Expected --debug-level flag".to_string())?; let log_format = matches.value_of("log-format"); - let eth2_testnet_config = get_eth2_testnet_config(&get_testnet_dir(matches))?; + let eth2_testnet_config = + clap_utils::parse_testnet_dir_with_hardcoded_default(matches, "testnet-dir")?; let mut environment = environment_builder .async_logger(debug_level, log_format)? .multi_threaded_tokio_runtime()? - .eth2_testnet_config(ð2_testnet_config)? + .eth2_testnet_config(eth2_testnet_config)? .build()?; let log = environment.core_context().log; @@ -164,7 +166,7 @@ fn run( if let Some(sub_matches) = matches.subcommand_matches("account_manager") { // Pass the entire `environment` to the account manager so it can run blocking operations. - account_manager::run(sub_matches, environment); + account_manager::run(sub_matches, environment)?; // Exit as soon as account manager returns control. return Ok(()); diff --git a/tests/eth1_test_rig/Cargo.toml b/tests/eth1_test_rig/Cargo.toml index a949a9d8bc..11552f4633 100644 --- a/tests/eth1_test_rig/Cargo.toml +++ b/tests/eth1_test_rig/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -web3 = "0.8.0" +web3 = "0.10.0" tokio = "0.1.22" futures = "0.1.25" types = { path = "../../eth2/types"} diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 11db63302e..68dfc906d0 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -41,3 +41,4 @@ bls = { path = "../eth2/utils/bls" } remote_beacon_node = { path = "../eth2/utils/remote_beacon_node" } tempdir = "0.3" rayon = "1.2.0" +web3 = "0.10.0" diff --git a/validator_client/src/validator_directory.rs b/validator_client/src/validator_directory.rs index f904e7f678..82f02cf34f 100644 --- a/validator_client/src/validator_directory.rs +++ b/validator_client/src/validator_directory.rs @@ -1,5 +1,6 @@ use bls::get_withdrawal_credentials; -use deposit_contract::encode_eth1_tx_data; +use deposit_contract::{encode_eth1_tx_data, DEPOSIT_GAS}; +use futures::{Future, IntoFuture}; use hex; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; @@ -12,6 +13,10 @@ use types::{ test_utils::generate_deterministic_keypair, ChainSpec, DepositData, Hash256, Keypair, PublicKey, SecretKey, Signature, }; +use web3::{ + types::{Address, TransactionRequest, U256}, + Transport, Web3, +}; const VOTING_KEY_PREFIX: &str = "voting"; const WITHDRAWAL_KEY_PREFIX: &str = "withdrawal"; @@ -241,7 +246,7 @@ impl ValidatorDirectoryBuilder { Ok(()) } - pub fn write_eth1_data_file(mut self) -> Result { + fn get_deposit_data(&self) -> Result<(Vec, u64), String> { let voting_keypair = self .voting_keypair .as_ref() @@ -254,30 +259,35 @@ impl ValidatorDirectoryBuilder { .amount .ok_or_else(|| "write_eth1_data_file requires an amount")?; let spec = self.spec.as_ref().ok_or_else(|| "build requires a spec")?; + + let withdrawal_credentials = Hash256::from_slice(&get_withdrawal_credentials( + &withdrawal_keypair.pk, + spec.bls_withdrawal_prefix_byte, + )); + + let mut deposit_data = DepositData { + pubkey: voting_keypair.pk.clone().into(), + withdrawal_credentials, + amount, + signature: Signature::empty_signature().into(), + }; + + deposit_data.signature = deposit_data.create_signature(&voting_keypair.sk, &spec); + + let deposit_data = encode_eth1_tx_data(&deposit_data) + .map_err(|e| format!("Unable to encode eth1 deposit tx data: {:?}", e))?; + + Ok((deposit_data, amount)) + } + + pub fn write_eth1_data_file(mut self) -> Result { let path = self .directory .as_ref() .map(|directory| directory.join(ETH1_DEPOSIT_DATA_FILE)) .ok_or_else(|| "write_eth1_data_filer requires a directory")?; - let deposit_data = { - let withdrawal_credentials = Hash256::from_slice(&get_withdrawal_credentials( - &withdrawal_keypair.pk, - spec.bls_withdrawal_prefix_byte, - )); - - let mut deposit_data = DepositData { - pubkey: voting_keypair.pk.clone().into(), - withdrawal_credentials, - amount, - signature: Signature::empty_signature().into(), - }; - - deposit_data.signature = deposit_data.create_signature(&voting_keypair.sk, &spec); - - encode_eth1_tx_data(&deposit_data) - .map_err(|e| format!("Unable to encode eth1 deposit tx data: {:?}", e))? - }; + let (deposit_data, _) = self.get_deposit_data()?; if path.exists() { return Err(format!("Eth1 data file already exists at: {:?}", path)); @@ -293,6 +303,31 @@ impl ValidatorDirectoryBuilder { Ok(self) } + pub fn submit_eth1_deposit( + self, + web3: Web3, + from: Address, + deposit_contract: Address, + ) -> impl Future { + self.get_deposit_data() + .into_future() + .and_then(move |(deposit_data, deposit_amount)| { + web3.eth() + .send_transaction(TransactionRequest { + from, + to: Some(deposit_contract), + gas: Some(DEPOSIT_GAS.into()), + gas_price: None, + value: Some(from_gwei(deposit_amount)), + data: Some(deposit_data.into()), + nonce: None, + condition: None, + }) + .map_err(|e| format!("Failed to send transaction: {:?}", e)) + }) + .map(|_tx| self) + } + pub fn build(self) -> Result { Ok(ValidatorDirectory { directory: self.directory.ok_or_else(|| "build requires a directory")?, @@ -303,6 +338,11 @@ impl ValidatorDirectoryBuilder { } } +/// Converts gwei to wei. +fn from_gwei(gwei: u64) -> U256 { + U256::from(gwei) * U256::exp10(9) +} + #[cfg(test)] mod tests { use super::*;