From e463518df695eab204ab78fa1397df9c56aa3fa5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 17 Aug 2022 11:12:08 +1000 Subject: [PATCH] Start refactoring into separate commands --- common/eth2/src/lighthouse_vc/std_types.rs | 3 +- testing/ef_tests/src/cases/fork_choice.rs | 2 +- validator_manager/src/validator/common/mod.rs | 77 +++- validator_manager/src/validator/create.rs | 195 ++++------ .../src/validator/create_validators.rs | 344 ++++++++++++++++++ .../src/validator/import_validators.rs | 322 ++++++++++++++++ validator_manager/src/validator/mod.rs | 2 + 7 files changed, 819 insertions(+), 126 deletions(-) create mode 100644 validator_manager/src/validator/create_validators.rs create mode 100644 validator_manager/src/validator/import_validators.rs diff --git a/common/eth2/src/lighthouse_vc/std_types.rs b/common/eth2/src/lighthouse_vc/std_types.rs index 887bcb99ea..101d71c3b3 100644 --- a/common/eth2/src/lighthouse_vc/std_types.rs +++ b/common/eth2/src/lighthouse_vc/std_types.rs @@ -1,9 +1,10 @@ use account_utils::ZeroizeString; use eth2_keystore::Keystore; use serde::{Deserialize, Serialize}; -use slashing_protection::interchange::Interchange; use types::{Address, PublicKeyBytes}; +pub use slashing_protection::interchange::Interchange; + #[derive(Debug, Deserialize, Serialize, PartialEq)] pub struct GetFeeRecipientResponse { pub pubkey: PublicKeyBytes, diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 650452d783..2d22d9cb53 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -153,7 +153,7 @@ impl Case for ForkChoiceTest { self.description.clone() } - fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> { + fn result(&self, case_index: usize, fork_name: ForkName) -> Result<(), Error> { let tester = Tester::new(self, testing_spec::(fork_name))?; for step in &self.steps { diff --git a/validator_manager/src/validator/common/mod.rs b/validator_manager/src/validator/common/mod.rs index 58bab59ba7..d0431248e1 100644 --- a/validator_manager/src/validator/common/mod.rs +++ b/validator_manager/src/validator/common/mod.rs @@ -1,15 +1,15 @@ use account_utils::ZeroizeString; +use eth2::lighthouse_vc::std_types::KeystoreJsonStr; use eth2::SensitiveUrl; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use tree_hash::TreeHash; use types::*; #[derive(Serialize, Deserialize)] -pub struct CreateValidatorSpec { - pub derivation_index: u32, - pub voting_keystore_password: Option, - pub deposit_gwei: u64, - pub eth1_withdrawal_address: Option
, +pub struct ValidatorSpecification { + pub voting_keystore: KeystoreJsonStr, + pub voting_keystore_password: ZeroizeString, pub fee_recipient: Option
, pub gas_limit: Option, pub builder_proposals: Option, @@ -23,5 +23,70 @@ pub struct CreateSpec { pub validator_client_token_path: Option, pub json_deposit_data_path: Option, pub ignore_duplicates: bool, - pub validators: Vec, + pub validators: Vec, +} + +/// The structure generated by the `staking-deposit-cli` which has become a quasi-standard for +/// browser-based deposit submission tools (e.g., the Ethereum Launchpad and Lido). +/// +/// We assume this code as the canonical definition: +/// +/// https://github.com/ethereum/staking-deposit-cli/blob/76ed78224fdfe3daca788d12442b3d1a37978296/staking_deposit/credentials.py#L131-L144 +#[derive(Debug, PartialEq, Serialize)] +pub struct StandardDepositDataJson { + pub pubkey: PublicKeyBytes, + pub withdrawal_credentials: Hash256, + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub amount: u64, + pub signature: SignatureBytes, + #[serde(with = "eth2_serde_utils::bytes_4_hex")] + pub fork_version: [u8; 4], + pub eth2_network_name: String, + pub deposit_message_root: Hash256, + pub deposit_data_root: Hash256, +} + +impl StandardDepositDataJson { + pub fn new( + keypair: &Keypair, + withdrawal_credentials: Hash256, + amount: u64, + spec: &ChainSpec, + ) -> Result { + let deposit_data = { + let mut deposit_data = DepositData { + pubkey: keypair.pk.clone().into(), + withdrawal_credentials, + amount, + signature: SignatureBytes::empty(), + }; + deposit_data.signature = deposit_data.create_signature(&keypair.sk, spec); + deposit_data + }; + + let domain = spec.get_deposit_domain(); + let deposit_message_root = deposit_data.as_deposit_message().signing_root(domain); + let deposit_data_root = deposit_data.tree_hash_root(); + + let DepositData { + pubkey, + withdrawal_credentials, + amount, + signature, + } = deposit_data; + + Ok(Self { + pubkey, + withdrawal_credentials, + amount, + signature, + fork_version: spec.genesis_fork_version, + eth2_network_name: spec + .config_name + .clone() + .ok_or("The network specification does not have a CONFIG_NAME set")?, + deposit_message_root, + deposit_data_root, + }) + } } diff --git a/validator_manager/src/validator/create.rs b/validator_manager/src/validator/create.rs index db8de37d57..40725e9c4e 100644 --- a/validator_manager/src/validator/create.rs +++ b/validator_manager/src/validator/create.rs @@ -20,7 +20,6 @@ use eth2_wallet::{ use serde::Serialize; use std::fs; use std::path::PathBuf; -use tree_hash::TreeHash; use types::*; pub const CMD: &str = "create"; @@ -49,69 +48,9 @@ struct ValidatorKeystore { enabled: Option, } -/// The structure generated by the `staking-deposit-cli` which has become a quasi-standard for -/// browser-based deposit submission tools (e.g., the Ethereum Launchpad and Lido). -/// -/// We assume this code as the canonical definition: -/// -/// https://github.com/ethereum/staking-deposit-cli/blob/76ed78224fdfe3daca788d12442b3d1a37978296/staking_deposit/credentials.py#L131-L144 -#[derive(Debug, PartialEq, Serialize)] -pub struct StandardDepositDataJson { - pub pubkey: PublicKeyBytes, - pub withdrawal_credentials: Hash256, - #[serde(with = "eth2_serde_utils::quoted_u64")] - pub amount: u64, - pub signature: SignatureBytes, - #[serde(with = "eth2_serde_utils::bytes_4_hex")] - pub fork_version: [u8; 4], - pub eth2_network_name: String, - pub deposit_message_root: Hash256, - pub deposit_data_root: Hash256, -} - -impl StandardDepositDataJson { - fn new( - keypair: &Keypair, - withdrawal_credentials: Hash256, - amount: u64, - spec: &ChainSpec, - ) -> Result { - let deposit_data = { - let mut deposit_data = DepositData { - pubkey: keypair.pk.clone().into(), - withdrawal_credentials, - amount, - signature: SignatureBytes::empty(), - }; - deposit_data.signature = deposit_data.create_signature(&keypair.sk, spec); - deposit_data - }; - - let domain = spec.get_deposit_domain(); - let deposit_message_root = deposit_data.as_deposit_message().signing_root(domain); - let deposit_data_root = deposit_data.tree_hash_root(); - - let DepositData { - pubkey, - withdrawal_credentials, - amount, - signature, - } = deposit_data; - - Ok(Self { - pubkey, - withdrawal_credentials, - amount, - signature, - fork_version: spec.genesis_fork_version, - eth2_network_name: spec - .config_name - .clone() - .ok_or("The network specification does not have a CONFIG_NAME set")?, - deposit_message_root, - deposit_data_root, - }) - } +struct ValidatorsAndDeposits { + validators: Vec, + deposits: Option>, } pub fn cli_app<'a, 'b>() -> App<'a, 'b> { @@ -258,20 +197,21 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .required(false), ) } + pub async fn cli_run<'a, T: EthSpec>( matches: &'a ArgMatches<'a>, mut env: Environment, ) -> Result<(), String> { let spec = &env.core_context().eth2_config.spec; - let create_spec = build_spec_from_cli(matches, spec)?; + let create_spec = build_validator_spec_from_cli(matches, spec)?; enact_spec(create_spec, spec).await } -pub fn build_spec_from_cli<'a>( +pub fn build_validator_spec_from_cli<'a>( matches: &'a ArgMatches<'a>, spec: &ChainSpec, -) -> Result { +) -> Result { let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)? .unwrap_or(spec.max_effective_balance); let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?; @@ -301,13 +241,74 @@ pub fn build_spec_from_cli<'a>( None }; + /* + * Generate a wallet to be used for HD key generation. + */ + + // A random password is always appropriate for the wallet since it is ephemeral. + let wallet_password = random_password_string(); + // A random password is always appropriate for the withdrawal keystore since we don't ever store + // it anywhere. + let withdrawal_keystore_password = random_password_string(); + let mut wallet = + WalletBuilder::from_mnemonic(&mnemonic, wallet_password.as_ref(), "".to_string()) + .map_err(|e| format!("Unable create seed from mnemonic: {:?}", e))? + .build() + .map_err(|e| format!("Unable to create wallet: {:?}", e))?; + + /* + * Start deriving individual validators. + */ + let mut validators = Vec::with_capacity(count as usize); - for derivation_index in first_index..first_index + count { - let validator = CreateValidatorSpec { - derivation_index, + let mut deposits = Some(vec![]).filter(|_| json_deposit_data_path.is_some()); + + for (i, derivation_index) in (first_index..first_index + count).enumerate() { + let voting_keystore_password = + voting_keystore_password.unwrap_or_else(|| random_password_string()); + + wallet + .set_nextaccount(derivation_index) + .map_err(|e| format!("Failure to set validator derivation index: {:?}", e))?; + + let keystores = wallet + .next_validator( + wallet_password.as_ref(), + voting_keystore_password.as_ref(), + withdrawal_keystore_password.as_ref(), + ) + .map_err(|e| format!("Failed to derive keystore {}: {:?}", i, e))?; + let voting_keystore = keystores.voting; + let voting_keypair = voting_keystore + .decrypt_keypair(voting_keystore_password.as_ref()) + .map_err(|e| format!("Failed to decrypt voting keystore {}: {:?}", i, e))?; + + if let Some(deposits) = &mut deposits { + let withdrawal_credentials = if let Some(eth1_withdrawal_address) = + eth1_withdrawal_address + { + WithdrawalCredentials::eth1(eth1_withdrawal_address, &spec) + } else { + let withdrawal_keypair = keystores + .withdrawal + .decrypt_keypair(withdrawal_keystore_password.as_ref()) + .map_err(|e| format!("Failed to decrypt withdrawal keystore {}: {:?}", i, e))?; + WithdrawalCredentials::bls(&withdrawal_keypair.pk, &spec) + }; + + let json_deposit = StandardDepositDataJson::new( + &voting_keypair, + withdrawal_credentials.into(), + deposit_gwei, + &spec, + )?; + + deposits.push(json_deposit); + } + + let validator = ValidatorSpecification { + voting_keystore: KeystoreJsonStr(voting_keystore), voting_keystore_password: voting_keystore_password.clone(), - deposit_gwei, - eth1_withdrawal_address, fee_recipient, gas_limit, builder_proposals: Some(builder_proposals), @@ -316,13 +317,9 @@ pub fn build_spec_from_cli<'a>( validators.push(validator); } - Ok(CreateSpec { - mnemonic: mnemonic.to_string(), - validator_client_url: vc_url, - validator_client_token_path: vc_token_path, - json_deposit_data_path, - ignore_duplicates, + Ok(ValidatorsAndDeposits { validators, + deposits, }) } @@ -390,37 +387,21 @@ pub async fn enact_spec<'a>(create_spec: CreateSpec, spec: &ChainSpec) -> Result .map_err(|e| format!("Unable to create wallet: {:?}", e))?; let mut validator_keystores = Vec::with_capacity(count); - let mut json_deposits = Some(vec![]).filter(|_| json_deposit_data_path.is_some()); eprintln!("Starting key generation. Each validator may take several seconds."); for (i, validator) in validators.into_iter().enumerate() { let CreateValidatorSpec { - derivation_index, + voting_keystore, voting_keystore_password, - deposit_gwei, - eth1_withdrawal_address, fee_recipient, gas_limit, builder_proposals, enabled, } = validator; - let voting_keystore_password = - voting_keystore_password.unwrap_or_else(|| random_password_string()); + let voting_keystore = voting_keystore.0; - wallet - .set_nextaccount(derivation_index) - .map_err(|e| format!("Failure to set validator derivation index: {:?}", e))?; - - let keystores = wallet - .next_validator( - wallet_password.as_ref(), - voting_keystore_password.as_ref(), - withdrawal_keystore_password.as_ref(), - ) - .map_err(|e| format!("Failed to derive keystore {}: {:?}", i, e))?; - let voting_keystore = keystores.voting; let voting_keypair = voting_keystore .decrypt_keypair(voting_keystore_password.as_ref()) .map_err(|e| format!("Failed to decrypt voting keystore {}: {:?}", i, e))?; @@ -454,28 +435,6 @@ pub async fn enact_spec<'a>(create_spec: CreateSpec, spec: &ChainSpec) -> Result } } - let withdrawal_credentials = if let Some(eth1_withdrawal_address) = eth1_withdrawal_address - { - WithdrawalCredentials::eth1(eth1_withdrawal_address, &spec) - } else { - let withdrawal_keypair = keystores - .withdrawal - .decrypt_keypair(withdrawal_keystore_password.as_ref()) - .map_err(|e| format!("Failed to decrypt withdrawal keystore {}: {:?}", i, e))?; - WithdrawalCredentials::bls(&withdrawal_keypair.pk, &spec) - }; - - if let Some(json_deposits) = &mut json_deposits { - let json_deposit = StandardDepositDataJson::new( - &voting_keypair, - withdrawal_credentials.into(), - deposit_gwei, - &spec, - )?; - - json_deposits.push(json_deposit); - } - eprintln!( "{}/{}: {:?}", i.saturating_add(1), diff --git a/validator_manager/src/validator/create_validators.rs b/validator_manager/src/validator/create_validators.rs new file mode 100644 index 0000000000..3c66c09481 --- /dev/null +++ b/validator_manager/src/validator/create_validators.rs @@ -0,0 +1,344 @@ +use super::common::*; +use account_utils::{ + random_password_string, read_mnemonic_from_cli, read_password_from_user, ZeroizeString, +}; +use clap::{App, Arg, ArgMatches}; +use environment::Environment; +use eth2::lighthouse_vc::std_types::KeystoreJsonStr; +use eth2_keystore::Keystore; +use eth2_wallet::WalletBuilder; +use serde::Serialize; +use std::fs; +use std::path::{Path, PathBuf}; +use types::*; + +pub const CMD: &str = "create"; +pub const OUTPUT_PATH_FLAG: &str = "output-path"; +pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei"; +pub const DISABLE_DEPOSITS_FLAG: &str = "disable-deposits"; +pub const COUNT_FLAG: &str = "count"; +pub const STDIN_INPUTS_FLAG: &str = "stdin-inputs"; +pub const FIRST_INDEX_FLAG: &str = "first-index"; +pub const MNEMONIC_FLAG: &str = "mnemonic-path"; +pub const SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG: &str = "specify-voting-keystore-password"; +pub const ETH1_WITHDRAWAL_ADDRESS_FLAG: &str = "eth1-withdrawal-address"; +pub const IGNORE_DUPLICATES_FLAG: &str = "ignore-duplicates"; +pub const GAS_LIMIT_FLAG: &str = "gas-limit"; +pub const FEE_RECIPIENT_FLAG: &str = "suggested-fee-recipient"; +pub const BUILDER_PROPOSALS_FLAG: &str = "builder-proposals"; + +pub const VALIDATORS_FILENAME: &str = "validators.json"; +pub const DEPOSITS_FILENAME: &str = "deposits.json"; + +struct ValidatorKeystore { + voting_keystore: Keystore, + voting_keystore_password: ZeroizeString, + voting_pubkey_bytes: PublicKeyBytes, + fee_recipient: Option
, + gas_limit: Option, + builder_proposals: Option, + enabled: Option, +} + +struct ValidatorsAndDeposits { + validators: Vec, + deposits: Option>, +} + +pub fn cli_app<'a, 'b>() -> App<'a, 'b> { + App::new(CMD) + .about("Creates new validators from BIP-39 mnemonic.") + .arg( + Arg::with_name(OUTPUT_PATH_FLAG) + .long(OUTPUT_PATH_FLAG) + .value_name("DIRECTORY") + .help( + "The path to a directory where the validator and (optionally) deposits \ + files will be created. The directory will be created if it does not exist.", + ) + .conflicts_with(DISABLE_DEPOSITS_FLAG) + .required(true) + .takes_value(true), + ) + .arg( + Arg::with_name(DEPOSIT_GWEI_FLAG) + .long(DEPOSIT_GWEI_FLAG) + .value_name("DEPOSIT_GWEI") + .help( + "The GWEI value of the deposit amount. Defaults to the minimum amount \ + required for an active validator (MAX_EFFECTIVE_BALANCE)", + ) + .conflicts_with(DISABLE_DEPOSITS_FLAG) + .takes_value(true), + ) + .arg( + Arg::with_name(FIRST_INDEX_FLAG) + .long(FIRST_INDEX_FLAG) + .value_name("FIRST_INDEX") + .help("The first of consecutive key indexes you wish to recover.") + .takes_value(true) + .required(false) + .default_value("0"), + ) + .arg( + Arg::with_name(COUNT_FLAG) + .long(COUNT_FLAG) + .value_name("VALIDATOR_COUNT") + .help("The number of validators to create, regardless of how many already exist") + .conflicts_with("at-most") + .takes_value(true), + ) + .arg( + Arg::with_name(MNEMONIC_FLAG) + .long(MNEMONIC_FLAG) + .value_name("MNEMONIC_PATH") + .help("If present, the mnemonic will be read in from this file.") + .takes_value(true), + ) + .arg( + Arg::with_name(STDIN_INPUTS_FLAG) + .takes_value(false) + .hidden(cfg!(windows)) + .long(STDIN_INPUTS_FLAG) + .help("If present, read all user inputs from stdin instead of tty."), + ) + .arg( + Arg::with_name(DISABLE_DEPOSITS_FLAG) + .long(DISABLE_DEPOSITS_FLAG) + .value_name("PATH") + .help( + "When provided don't generate the deposits JSON file that is \ + commonly used for submitting validator deposits via a web UI. \ + Using this flag will save several seconds per validator if the \ + user has an alternate strategy for submitting deposits.", + ), + ) + .arg( + Arg::with_name(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG) + .long(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG) + .value_name("STRING") + .takes_value(true) + .help( + "If present, the user will be prompted to enter the voting keystore \ + password that will be used to encrypt the voting keystores. If this \ + flag is not provided, a random password will be used. It is not \ + necessary to keep backups of voting keystore passwords if the \ + mnemonic is safely backed up.", + ), + ) + .arg( + Arg::with_name(ETH1_WITHDRAWAL_ADDRESS_FLAG) + .long(ETH1_WITHDRAWAL_ADDRESS_FLAG) + .value_name("ETH1_ADDRESS") + .help( + "If this field is set, the given eth1 address will be used to create the \ + withdrawal credentials. Otherwise, it will generate withdrawal credentials \ + with the mnemonic-derived withdrawal public key in EIP-2334 format.", + ) + .conflicts_with(DISABLE_DEPOSITS_FLAG) + .takes_value(true), + ) + .arg( + Arg::with_name(GAS_LIMIT_FLAG) + .long(GAS_LIMIT_FLAG) + .value_name("UINT64") + .help( + "All created validators will use this gas limit. It is recommended \ + to leave this as the default value by not specifying this flag.", + ) + .required(false) + .takes_value(true), + ) + .arg( + Arg::with_name(FEE_RECIPIENT_FLAG) + .long(FEE_RECIPIENT_FLAG) + .value_name("ETH1_ADDRESS") + .help( + "All created validators will use this value for the suggested \ + fee recipient. Omit this flag to use the default value from the VC.", + ) + .required(false) + .takes_value(true), + ) + .arg( + Arg::with_name(BUILDER_PROPOSALS_FLAG) + .long(BUILDER_PROPOSALS_FLAG) + .help( + "When provided, all created validators will attempt to create \ + blocks via builder rather than the local EL.", + ) + .required(false), + ) +} + +pub async fn cli_run<'a, T: EthSpec>( + matches: &'a ArgMatches<'a>, + mut env: Environment, +) -> Result<(), String> { + let spec = &env.core_context().eth2_config.spec; + let output_path: PathBuf = clap_utils::parse_required(matches, OUTPUT_PATH_FLAG)?; + + if !output_path.exists() { + fs::create_dir(&output_path) + .map_err(|e| format!("Failed to create {:?} directory: {:?}", output_path, e))?; + } else if !output_path.is_dir() { + return Err(format!("{:?} must be a directory", output_path)); + } + + let validators_path = output_path.join(VALIDATORS_FILENAME); + if validators_path.exists() { + return Err(format!( + "{:?} already exists, refusing to overwrite", + validators_path + )); + } + let deposits_path = output_path.join(DEPOSITS_FILENAME); + if deposits_path.exists() { + return Err(format!( + "{:?} already exists, refusing to overwrite", + deposits_path + )); + } + + let validators_and_deposits = build_validator_spec_from_cli(matches, spec)?; + + write_to_json_file(&validators_path, &validators_and_deposits.validators)?; + + if let Some(deposits) = validators_and_deposits.deposits { + write_to_json_file(&deposits_path, &validators_and_deposits.validators)?; + } + + Ok(()) +} + +fn write_to_json_file, S: Serialize>(path: P, contents: &S) -> Result<(), String> { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|e| format!("Failed to open {:?}: {:?}", path.as_ref(), e))?; + serde_json::to_writer(&mut file, contents) + .map_err(|e| format!("Failed to write JSON to {:?}: {:?}", path.as_ref(), e)) +} + +pub fn build_validator_spec_from_cli<'a>( + matches: &'a ArgMatches<'a>, + spec: &ChainSpec, +) -> Result { + let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)? + .unwrap_or(spec.max_effective_balance); + let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?; + let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?; + let mnemonic_path: Option = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?; + let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG); + let disable_deposits = matches.is_present(DISABLE_DEPOSITS_FLAG); + let specify_voting_keystore_password = + matches.is_present(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG); + let eth1_withdrawal_address: Option
= + clap_utils::parse_optional(matches, ETH1_WITHDRAWAL_ADDRESS_FLAG)?; + let ignore_duplicates = matches.is_present(IGNORE_DUPLICATES_FLAG); + let builder_proposals = matches.is_present(BUILDER_PROPOSALS_FLAG); + let fee_recipient: Option
= clap_utils::parse_optional(matches, FEE_RECIPIENT_FLAG)?; + let gas_limit: Option = clap_utils::parse_optional(matches, GAS_LIMIT_FLAG)?; + + let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?; + let voting_keystore_password = if specify_voting_keystore_password { + eprintln!("Please enter a voting keystore password when prompted."); + Some(read_password_from_user(stdin_inputs)?) + } else { + None + }; + + /* + * Generate a wallet to be used for HD key generation. + */ + + // A random password is always appropriate for the wallet since it is ephemeral. + let wallet_password = random_password_string(); + // A random password is always appropriate for the withdrawal keystore since we don't ever store + // it anywhere. + let withdrawal_keystore_password = random_password_string(); + let mut wallet = + WalletBuilder::from_mnemonic(&mnemonic, wallet_password.as_ref(), "".to_string()) + .map_err(|e| format!("Unable create seed from mnemonic: {:?}", e))? + .build() + .map_err(|e| format!("Unable to create wallet: {:?}", e))?; + + /* + * Start deriving individual validators. + */ + + let mut validators = Vec::with_capacity(count as usize); + let mut deposits = disable_deposits.then(|| vec![]); + + for (i, derivation_index) in (first_index..first_index + count).enumerate() { + // If the voting keystore password was not provided by the user then use a unique random + // string for each validator. + let voting_keystore_password = + voting_keystore_password.unwrap_or_else(|| random_password_string()); + + // Set the wallet to the appropriate derivation index. + wallet + .set_nextaccount(derivation_index) + .map_err(|e| format!("Failure to set validator derivation index: {:?}", e))?; + + // Derive the keystore from the HD wallet. + let keystores = wallet + .next_validator( + wallet_password.as_ref(), + voting_keystore_password.as_ref(), + withdrawal_keystore_password.as_ref(), + ) + .map_err(|e| format!("Failed to derive keystore {}: {:?}", i, e))?; + let voting_keystore = keystores.voting; + + if let Some(deposits) = &mut deposits { + // Decrypt the voting keystore so a deposit message can be signed. + let voting_keypair = voting_keystore + .decrypt_keypair(voting_keystore_password.as_ref()) + .map_err(|e| format!("Failed to decrypt voting keystore {}: {:?}", i, e))?; + + let withdrawal_credentials = if let Some(eth1_withdrawal_address) = + eth1_withdrawal_address + { + WithdrawalCredentials::eth1(eth1_withdrawal_address, &spec) + } else { + // Decrypt the withdrawal keystore so withdrawal credentials can be created. It's + // not strictly necessary to decrypt the keystore since we can read the pubkey + // directly from the keystore. However we decrypt the keystore to be more certain + // that we have access to the withdrawal keys. + let withdrawal_keypair = keystores + .withdrawal + .decrypt_keypair(withdrawal_keystore_password.as_ref()) + .map_err(|e| format!("Failed to decrypt withdrawal keystore {}: {:?}", i, e))?; + WithdrawalCredentials::bls(&withdrawal_keypair.pk, &spec) + }; + + // Create a JSON structure equivalent to the one generated by + // `ethereum/staking-deposit-cli`. + let json_deposit = StandardDepositDataJson::new( + &voting_keypair, + withdrawal_credentials.into(), + deposit_gwei, + &spec, + )?; + + deposits.push(json_deposit); + } + + let validator = ValidatorSpecification { + voting_keystore: KeystoreJsonStr(voting_keystore), + voting_keystore_password: voting_keystore_password.clone(), + fee_recipient, + gas_limit, + builder_proposals: Some(builder_proposals), + enabled: Some(true), + }; + validators.push(validator); + } + + Ok(ValidatorsAndDeposits { + validators, + deposits, + }) +} diff --git a/validator_manager/src/validator/import_validators.rs b/validator_manager/src/validator/import_validators.rs new file mode 100644 index 0000000000..4c82229174 --- /dev/null +++ b/validator_manager/src/validator/import_validators.rs @@ -0,0 +1,322 @@ +use super::common::*; +use account_utils::{ + random_password_string, read_mnemonic_from_cli, read_password_from_user, ZeroizeString, +}; +use clap::{App, Arg, ArgMatches}; +use environment::Environment; +use eth2::{ + lighthouse_vc::{ + http_client::ValidatorClientHttpClient, + std_types::{ImportKeystoresRequest, KeystoreJsonStr}, + types::UpdateFeeRecipientRequest, + }, + SensitiveUrl, +}; +use eth2_keystore::Keystore; +use eth2_wallet::{ + bip39::{Language, Mnemonic}, + WalletBuilder, +}; +use serde::Serialize; +use std::fs; +use std::path::PathBuf; +use types::*; + +pub const CMD: &str = "import"; +pub const VALIDATORS_FILE_FLAG: &str = "validators-file"; +pub const VALIDATOR_CLIENT_URL_FLAG: &str = "validator-client-url"; +pub const VALIDATOR_CLIENT_TOKEN_FLAG: &str = "validator-client-token"; +pub const IGNORE_DUPLICATES_FLAG: &str = "ignore-duplicates"; + +struct ValidatorKeystore { + voting_keystore: Keystore, + voting_keystore_password: ZeroizeString, + voting_pubkey_bytes: PublicKeyBytes, + fee_recipient: Option
, + gas_limit: Option, + builder_proposals: Option, + enabled: Option, +} + +struct ValidatorsAndDeposits { + validators: Vec, + deposits: Option>, +} + +pub fn cli_app<'a, 'b>() -> App<'a, 'b> { + App::new(CMD) + .about("Uploads validators to a validator client.") + .arg( + Arg::with_name(VALIDATORS_FILE_FLAG) + .long(VALIDATORS_FILE_FLAG) + .value_name("PATH_TO_JSON_FILE") + .help( + "The path to a JSON file containing a list of validators to be \ + imported to the validator client. This file is usually named \ + \"validators.json\".", + ) + .required(true) + .takes_value(true), + ) + .arg( + Arg::with_name(VALIDATOR_CLIENT_URL_FLAG) + .long(VALIDATOR_CLIENT_URL_FLAG) + .value_name("HTTP_ADDRESS") + .help( + "A HTTP(S) address of a validator client using the keymanager-API. \ + If this value is not supplied then a 'dry run' will be conducted where \ + no changes are made to the validator client.", + ) + .default_value("http://localhost:5062") + .requires(VALIDATOR_CLIENT_TOKEN_FLAG) + .takes_value(true), + ) + .arg( + Arg::with_name(VALIDATOR_CLIENT_TOKEN_FLAG) + .long(VALIDATOR_CLIENT_TOKEN_FLAG) + .value_name("PATH") + .help("The file containing a token required by the validator client.") + .takes_value(true), + ) + .arg( + Arg::with_name(IGNORE_DUPLICATES_FLAG) + .takes_value(false) + .long(IGNORE_DUPLICATES_FLAG) + .help( + "If present, ignore any validators which already exist on the VC. \ + Without this flag, the process will terminate without making any changes. \ + This flag should be used with caution, whilst it does not directly cause \ + slashable conditions, it might be an indicator that something is amiss. \ + Users should also be careful to avoid submitting duplicate deposits for \ + validators that already exist on the VC.", + ), + ) +} + +pub async fn cli_run<'a, T: EthSpec>( + matches: &'a ArgMatches<'a>, + mut env: Environment, +) -> Result<(), String> { + let spec = &env.core_context().eth2_config.spec; + + let create_spec = build_validator_spec_from_cli(matches, spec)?; + enact_spec(create_spec, spec).await +} + +pub async fn enact_spec<'a>(create_spec: CreateSpec, spec: &ChainSpec) -> Result<(), String> { + let CreateSpec { + mnemonic, + validator_client_url, + validator_client_token_path, + json_deposit_data_path, + ignore_duplicates, + validators, + } = create_spec; + + let count = validators.len(); + + let mnemonic = Mnemonic::from_phrase(&mnemonic, Language::English) + .map_err(|e| format!("Failed to parse mnemonic from create spec: {:?}", e))?; + + let http_client = match (validator_client_url, validator_client_token_path) { + (Some(vc_url), Some(vc_token_path)) => { + let token_bytes = fs::read(&vc_token_path) + .map_err(|e| format!("Failed to read {:?}: {:?}", vc_token_path, e))?; + let token_string = String::from_utf8(token_bytes) + .map_err(|e| format!("Failed to parse {:?} as utf8: {:?}", vc_token_path, e))?; + let http_client = ValidatorClientHttpClient::new(vc_url.clone(), token_string) + .map_err(|e| { + format!( + "Could not instantiate HTTP client from URL and secret: {:?}", + e + ) + })?; + + // Perform a request to check that the connection works + let remote_keystores = http_client + .get_keystores() + .await + .map_err(|e| format!("Failed to list keystores on VC: {:?}", e))?; + eprintln!( + "Validator client is reachable at {} and reports {} validators", + vc_url, + remote_keystores.data.len() + ); + + Some(http_client) + } + (None, None) => None, + _ => { + return Err(format!( + "Inconsistent use of {} and {}", + VALIDATOR_CLIENT_URL_FLAG, VALIDATOR_CLIENT_TOKEN_FLAG + )) + } + }; + + // A random password is always appropriate for the wallet since it is ephemeral. + let wallet_password = random_password_string(); + // A random password is always appropriate for the withdrawal keystore since we don't ever store + // it anywhere. + let withdrawal_keystore_password = random_password_string(); + + let mut wallet = + WalletBuilder::from_mnemonic(&mnemonic, wallet_password.as_ref(), "".to_string()) + .map_err(|e| format!("Unable create seed from mnemonic: {:?}", e))? + .build() + .map_err(|e| format!("Unable to create wallet: {:?}", e))?; + + let mut validator_keystores = Vec::with_capacity(count); + + eprintln!("Starting key generation. Each validator may take several seconds."); + + for (i, validator) in validators.into_iter().enumerate() { + let CreateValidatorSpec { + voting_keystore, + voting_keystore_password, + fee_recipient, + gas_limit, + builder_proposals, + enabled, + } = validator; + + let voting_keystore = voting_keystore.0; + + let voting_keypair = voting_keystore + .decrypt_keypair(voting_keystore_password.as_ref()) + .map_err(|e| format!("Failed to decrypt voting keystore {}: {:?}", i, e))?; + let voting_pubkey_bytes = voting_keypair.pk.clone().into(); + + // Check to see if this validator already exists in the VC. + if let Some(http_client) = &http_client { + let remote_keystores = http_client + .get_keystores() + .await + .map_err(|e| format!("Failed to list keystores on VC: {:?}", e))?; + + if remote_keystores + .data + .iter() + .find(|keystore| keystore.validating_pubkey == voting_pubkey_bytes) + .is_some() + { + if ignore_duplicates { + eprintln!( + "Validator {:?} already exists in the VC, be cautious of submitting \ + duplicate deposits", + IGNORE_DUPLICATES_FLAG + ); + } else { + return Err(format!( + "Duplicate validator {:?} detected, see --{} for more information", + voting_keypair.pk, IGNORE_DUPLICATES_FLAG + )); + } + } + } + + eprintln!( + "{}/{}: {:?}", + i.saturating_add(1), + count, + &voting_keypair.pk + ); + + validator_keystores.push(ValidatorKeystore { + voting_keystore, + voting_keystore_password, + voting_pubkey_bytes, + fee_recipient, + gas_limit, + builder_proposals, + enabled, + }); + } + + if let Some(http_client) = http_client { + eprintln!( + "Generated {} keystores. Starting to submit keystores to VC, \ + each keystore may take several seconds", + count + ); + + for (i, validator_keystore) in validator_keystores.into_iter().enumerate() { + let ValidatorKeystore { + voting_keystore, + voting_keystore_password, + voting_pubkey_bytes, + fee_recipient, + gas_limit, + builder_proposals, + enabled, + } = validator_keystore; + + let request = ImportKeystoresRequest { + keystores: vec![KeystoreJsonStr(voting_keystore)], + passwords: vec![voting_keystore_password], + // New validators have no slashing protection history. + slashing_protection: None, + }; + + if let Err(e) = http_client.post_keystores(&request).await { + eprintln!( + "Failed to upload batch {}. Some keys were imported whilst \ + others may not have been imported. A potential solution is to use the \ + --{} flag, however care should be taken to ensure that there are no \ + duplicate deposits submitted.", + i, IGNORE_DUPLICATES_FLAG + ); + // Return here *without* writing the deposit JSON file. This might help prevent + // users from submitting duplicate deposits or deposits for validators that weren't + // initialized on a VC. + // + // Next the the user runs with the --ignore-duplicates flag there should be a new, + // complete deposit JSON file created. + return Err(format!("Key upload failed: {:?}", e)); + } + + if let Some(fee_recipient) = fee_recipient { + http_client + .post_fee_recipient( + &voting_pubkey_bytes, + &UpdateFeeRecipientRequest { + ethaddress: fee_recipient, + }, + ) + .await + .map_err(|e| format!("Failed to update fee recipient on VC: {:?}", e))?; + } + + if gas_limit.is_some() || builder_proposals.is_some() || enabled.is_some() { + http_client + .patch_lighthouse_validators( + &voting_pubkey_bytes, + enabled, + gas_limit, + builder_proposals, + ) + .await + .map_err(|e| format!("Failed to update lighthouse validator on VC: {:?}", e))?; + } + + eprintln!("Uploaded keystore {} of {} to the VC", i + 1, count); + } + } + + // If configured, create a single JSON file which contains deposit data information for all + // validators. + if let Some(json_deposit_data_path) = json_deposit_data_path { + let json_deposits = json_deposits.ok_or("Internal error: JSON deposit data is None")?; + + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&json_deposit_data_path) + .map_err(|e| format!("Unable to create {:?}: {:?}", json_deposit_data_path, e))?; + + serde_json::to_writer(&mut file, &json_deposits) + .map_err(|e| format!("Unable write JSON to {:?}: {:?}", json_deposit_data_path, e))?; + } + + Ok(()) +} diff --git a/validator_manager/src/validator/mod.rs b/validator_manager/src/validator/mod.rs index 07c52b9b72..e3470f03f4 100644 --- a/validator_manager/src/validator/mod.rs +++ b/validator_manager/src/validator/mod.rs @@ -1,5 +1,7 @@ pub mod common; pub mod create; +pub mod create_validators; +pub mod import_validators; use clap::{App, ArgMatches}; use environment::Environment;