Start refactoring into separate commands

common/eth2/src/lighthouse_vc/std_types.rs

@@ -1,9 +1,10 @@
 use account_utils::ZeroizeString;
 use eth2_keystore::Keystore;
 use serde::{Deserialize, Serialize};
-use slashing_protection::interchange::Interchange;
 use types::{Address, PublicKeyBytes};
 
+pub use slashing_protection::interchange::Interchange;
+
 #[derive(Debug, Deserialize, Serialize, PartialEq)]
 pub struct GetFeeRecipientResponse {
     pub pubkey: PublicKeyBytes,

testing/ef_tests/src/cases/fork_choice.rs

@@ -153,7 +153,7 @@ impl<E: EthSpec> Case for ForkChoiceTest<E> {
         self.description.clone()
     }
 
-    fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {
+    fn result(&self, case_index: usize, fork_name: ForkName) -> Result<(), Error> {
         let tester = Tester::new(self, testing_spec::<E>(fork_name))?;
 
         for step in &self.steps {

validator_manager/src/validator/common/mod.rs

@@ -1,15 +1,15 @@
 use account_utils::ZeroizeString;
+use eth2::lighthouse_vc::std_types::KeystoreJsonStr;
 use eth2::SensitiveUrl;
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
+use tree_hash::TreeHash;
 use types::*;
 
 #[derive(Serialize, Deserialize)]
-pub struct CreateValidatorSpec {
-    pub derivation_index: u32,
-    pub voting_keystore_password: Option<ZeroizeString>,
-    pub deposit_gwei: u64,
-    pub eth1_withdrawal_address: Option<Address>,
+pub struct ValidatorSpecification {
+    pub voting_keystore: KeystoreJsonStr,
+    pub voting_keystore_password: ZeroizeString,
     pub fee_recipient: Option<Address>,
     pub gas_limit: Option<u64>,
     pub builder_proposals: Option<bool>,
@@ -23,5 +23,70 @@ pub struct CreateSpec {
     pub validator_client_token_path: Option<PathBuf>,
     pub json_deposit_data_path: Option<PathBuf>,
     pub ignore_duplicates: bool,
-    pub validators: Vec<CreateValidatorSpec>,
+    pub validators: Vec<ValidatorSpecification>,
+}
+
+/// The structure generated by the `staking-deposit-cli` which has become a quasi-standard for
+/// browser-based deposit submission tools (e.g., the Ethereum Launchpad and Lido).
+///
+/// We assume this code as the canonical definition:
+///
+/// https://github.com/ethereum/staking-deposit-cli/blob/76ed78224fdfe3daca788d12442b3d1a37978296/staking_deposit/credentials.py#L131-L144
+#[derive(Debug, PartialEq, Serialize)]
+pub struct StandardDepositDataJson {
+    pub pubkey: PublicKeyBytes,
+    pub withdrawal_credentials: Hash256,
+    #[serde(with = "eth2_serde_utils::quoted_u64")]
+    pub amount: u64,
+    pub signature: SignatureBytes,
+    #[serde(with = "eth2_serde_utils::bytes_4_hex")]
+    pub fork_version: [u8; 4],
+    pub eth2_network_name: String,
+    pub deposit_message_root: Hash256,
+    pub deposit_data_root: Hash256,
+}
+
+impl StandardDepositDataJson {
+    pub fn new(
+        keypair: &Keypair,
+        withdrawal_credentials: Hash256,
+        amount: u64,
+        spec: &ChainSpec,
+    ) -> Result<Self, String> {
+        let deposit_data = {
+            let mut deposit_data = DepositData {
+                pubkey: keypair.pk.clone().into(),
+                withdrawal_credentials,
+                amount,
+                signature: SignatureBytes::empty(),
+            };
+            deposit_data.signature = deposit_data.create_signature(&keypair.sk, spec);
+            deposit_data
+        };
+
+        let domain = spec.get_deposit_domain();
+        let deposit_message_root = deposit_data.as_deposit_message().signing_root(domain);
+        let deposit_data_root = deposit_data.tree_hash_root();
+
+        let DepositData {
+            pubkey,
+            withdrawal_credentials,
+            amount,
+            signature,
+        } = deposit_data;
+
+        Ok(Self {
+            pubkey,
+            withdrawal_credentials,
+            amount,
+            signature,
+            fork_version: spec.genesis_fork_version,
+            eth2_network_name: spec
+                .config_name
+                .clone()
+                .ok_or("The network specification does not have a CONFIG_NAME set")?,
+            deposit_message_root,
+            deposit_data_root,
+        })
+    }
 }

validator_manager/src/validator/create.rs

@@ -20,7 +20,6 @@ use eth2_wallet::{
 use serde::Serialize;
 use std::fs;
 use std::path::PathBuf;
-use tree_hash::TreeHash;
 use types::*;
 
 pub const CMD: &str = "create";
@@ -49,69 +48,9 @@ struct ValidatorKeystore {
     enabled: Option<bool>,
 }
 
-/// The structure generated by the `staking-deposit-cli` which has become a quasi-standard for
-/// browser-based deposit submission tools (e.g., the Ethereum Launchpad and Lido).
-///
-/// We assume this code as the canonical definition:
-///
-/// https://github.com/ethereum/staking-deposit-cli/blob/76ed78224fdfe3daca788d12442b3d1a37978296/staking_deposit/credentials.py#L131-L144
-#[derive(Debug, PartialEq, Serialize)]
-pub struct StandardDepositDataJson {
-    pub pubkey: PublicKeyBytes,
-    pub withdrawal_credentials: Hash256,
-    #[serde(with = "eth2_serde_utils::quoted_u64")]
-    pub amount: u64,
-    pub signature: SignatureBytes,
-    #[serde(with = "eth2_serde_utils::bytes_4_hex")]
-    pub fork_version: [u8; 4],
-    pub eth2_network_name: String,
-    pub deposit_message_root: Hash256,
-    pub deposit_data_root: Hash256,
-}
-
-impl StandardDepositDataJson {
-    fn new(
-        keypair: &Keypair,
-        withdrawal_credentials: Hash256,
-        amount: u64,
-        spec: &ChainSpec,
-    ) -> Result<Self, String> {
-        let deposit_data = {
-            let mut deposit_data = DepositData {
-                pubkey: keypair.pk.clone().into(),
-                withdrawal_credentials,
-                amount,
-                signature: SignatureBytes::empty(),
-            };
-            deposit_data.signature = deposit_data.create_signature(&keypair.sk, spec);
-            deposit_data
-        };
-
-        let domain = spec.get_deposit_domain();
-        let deposit_message_root = deposit_data.as_deposit_message().signing_root(domain);
-        let deposit_data_root = deposit_data.tree_hash_root();
-
-        let DepositData {
-            pubkey,
-            withdrawal_credentials,
-            amount,
-            signature,
-        } = deposit_data;
-
-        Ok(Self {
-            pubkey,
-            withdrawal_credentials,
-            amount,
-            signature,
-            fork_version: spec.genesis_fork_version,
-            eth2_network_name: spec
-                .config_name
-                .clone()
-                .ok_or("The network specification does not have a CONFIG_NAME set")?,
-            deposit_message_root,
-            deposit_data_root,
-        })
-    }
+struct ValidatorsAndDeposits {
+    validators: Vec<ValidatorSpecification>,
+    deposits: Option<Vec<StandardDepositDataJson>>,
 }
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
@@ -258,20 +197,21 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .required(false),
         )
 }
+
 pub async fn cli_run<'a, T: EthSpec>(
     matches: &'a ArgMatches<'a>,
     mut env: Environment<T>,
 ) -> Result<(), String> {
     let spec = &env.core_context().eth2_config.spec;
 
-    let create_spec = build_spec_from_cli(matches, spec)?;
+    let create_spec = build_validator_spec_from_cli(matches, spec)?;
     enact_spec(create_spec, spec).await
 }
 
-pub fn build_spec_from_cli<'a>(
+pub fn build_validator_spec_from_cli<'a>(
     matches: &'a ArgMatches<'a>,
     spec: &ChainSpec,
-) -> Result<CreateSpec, String> {
+) -> Result<ValidatorsAndDeposits, String> {
     let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
         .unwrap_or(spec.max_effective_balance);
     let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
@@ -301,13 +241,74 @@ pub fn build_spec_from_cli<'a>(
         None
     };
 
+    /*
+     * Generate a wallet to be used for HD key generation.
+     */
+
+    // A random password is always appropriate for the wallet since it is ephemeral.
+    let wallet_password = random_password_string();
+    // A random password is always appropriate for the withdrawal keystore since we don't ever store
+    // it anywhere.
+    let withdrawal_keystore_password = random_password_string();
+    let mut wallet =
+        WalletBuilder::from_mnemonic(&mnemonic, wallet_password.as_ref(), "".to_string())
+            .map_err(|e| format!("Unable create seed from mnemonic: {:?}", e))?
+            .build()
+            .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+    /*
+     * Start deriving individual validators.
+     */
+
     let mut validators = Vec::with_capacity(count as usize);
-    for derivation_index in first_index..first_index + count {
-        let validator = CreateValidatorSpec {
-            derivation_index,
+    let mut deposits = Some(vec![]).filter(|_| json_deposit_data_path.is_some());
+
+    for (i, derivation_index) in (first_index..first_index + count).enumerate() {
+        let voting_keystore_password =
+            voting_keystore_password.unwrap_or_else(|| random_password_string());
+
+        wallet
+            .set_nextaccount(derivation_index)
+            .map_err(|e| format!("Failure to set validator derivation index: {:?}", e))?;
+
+        let keystores = wallet
+            .next_validator(
+                wallet_password.as_ref(),
+                voting_keystore_password.as_ref(),
+                withdrawal_keystore_password.as_ref(),
+            )
+            .map_err(|e| format!("Failed to derive keystore {}: {:?}", i, e))?;
+        let voting_keystore = keystores.voting;
+        let voting_keypair = voting_keystore
+            .decrypt_keypair(voting_keystore_password.as_ref())
+            .map_err(|e| format!("Failed to decrypt voting keystore {}: {:?}", i, e))?;
+
+        if let Some(deposits) = &mut deposits {
+            let withdrawal_credentials = if let Some(eth1_withdrawal_address) =
+                eth1_withdrawal_address
+            {
+                WithdrawalCredentials::eth1(eth1_withdrawal_address, &spec)
+            } else {
+                let withdrawal_keypair = keystores
+                    .withdrawal
+                    .decrypt_keypair(withdrawal_keystore_password.as_ref())
+                    .map_err(|e| format!("Failed to decrypt withdrawal keystore {}: {:?}", i, e))?;
+                WithdrawalCredentials::bls(&withdrawal_keypair.pk, &spec)
+            };
+
+            let json_deposit = StandardDepositDataJson::new(
+                &voting_keypair,
+                withdrawal_credentials.into(),
+                deposit_gwei,
+                &spec,
+            )?;
+
+            deposits.push(json_deposit);
+        }
+
+        let validator = ValidatorSpecification {
+            voting_keystore: KeystoreJsonStr(voting_keystore),
             voting_keystore_password: voting_keystore_password.clone(),
-            deposit_gwei,
-            eth1_withdrawal_address,
             fee_recipient,
             gas_limit,
             builder_proposals: Some(builder_proposals),
@@ -316,13 +317,9 @@ pub fn build_spec_from_cli<'a>(
         validators.push(validator);
     }
 
-    Ok(CreateSpec {
-        mnemonic: mnemonic.to_string(),
-        validator_client_url: vc_url,
-        validator_client_token_path: vc_token_path,
-        json_deposit_data_path,
-        ignore_duplicates,
+    Ok(ValidatorsAndDeposits {
         validators,
+        deposits,
     })
 }
 
@@ -390,37 +387,21 @@ pub async fn enact_spec<'a>(create_spec: CreateSpec, spec: &ChainSpec) -> Result
             .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
 
     let mut validator_keystores = Vec::with_capacity(count);
-    let mut json_deposits = Some(vec![]).filter(|_| json_deposit_data_path.is_some());
 
     eprintln!("Starting key generation. Each validator may take several seconds.");
 
     for (i, validator) in validators.into_iter().enumerate() {
         let CreateValidatorSpec {
-            derivation_index,
+            voting_keystore,
             voting_keystore_password,
-            deposit_gwei,
-            eth1_withdrawal_address,
             fee_recipient,
             gas_limit,
             builder_proposals,
             enabled,
         } = validator;
 
-        let voting_keystore_password =
-            voting_keystore_password.unwrap_or_else(|| random_password_string());
+        let voting_keystore = voting_keystore.0;
 
-        wallet
-            .set_nextaccount(derivation_index)
-            .map_err(|e| format!("Failure to set validator derivation index: {:?}", e))?;
-
-        let keystores = wallet
-            .next_validator(
-                wallet_password.as_ref(),
-                voting_keystore_password.as_ref(),
-                withdrawal_keystore_password.as_ref(),
-            )
-            .map_err(|e| format!("Failed to derive keystore {}: {:?}", i, e))?;
-        let voting_keystore = keystores.voting;
         let voting_keypair = voting_keystore
             .decrypt_keypair(voting_keystore_password.as_ref())
             .map_err(|e| format!("Failed to decrypt voting keystore {}: {:?}", i, e))?;
@@ -454,28 +435,6 @@ pub async fn enact_spec<'a>(create_spec: CreateSpec, spec: &ChainSpec) -> Result
             }
         }
 
-        let withdrawal_credentials = if let Some(eth1_withdrawal_address) = eth1_withdrawal_address
-        {
-            WithdrawalCredentials::eth1(eth1_withdrawal_address, &spec)
-        } else {
-            let withdrawal_keypair = keystores
-                .withdrawal
-                .decrypt_keypair(withdrawal_keystore_password.as_ref())
-                .map_err(|e| format!("Failed to decrypt withdrawal keystore {}: {:?}", i, e))?;
-            WithdrawalCredentials::bls(&withdrawal_keypair.pk, &spec)
-        };
-
-        if let Some(json_deposits) = &mut json_deposits {
-            let json_deposit = StandardDepositDataJson::new(
-                &voting_keypair,
-                withdrawal_credentials.into(),
-                deposit_gwei,
-                &spec,
-            )?;
-
-            json_deposits.push(json_deposit);
-        }
-
         eprintln!(
             "{}/{}: {:?}",
             i.saturating_add(1),

validator_manager/src/validator/create_validators.rs

@@ -0,0 +1,344 @@
+use super::common::*;
+use account_utils::{
+    random_password_string, read_mnemonic_from_cli, read_password_from_user, ZeroizeString,
+};
+use clap::{App, Arg, ArgMatches};
+use environment::Environment;
+use eth2::lighthouse_vc::std_types::KeystoreJsonStr;
+use eth2_keystore::Keystore;
+use eth2_wallet::WalletBuilder;
+use serde::Serialize;
+use std::fs;
+use std::path::{Path, PathBuf};
+use types::*;
+
+pub const CMD: &str = "create";
+pub const OUTPUT_PATH_FLAG: &str = "output-path";
+pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
+pub const DISABLE_DEPOSITS_FLAG: &str = "disable-deposits";
+pub const COUNT_FLAG: &str = "count";
+pub const STDIN_INPUTS_FLAG: &str = "stdin-inputs";
+pub const FIRST_INDEX_FLAG: &str = "first-index";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+pub const SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG: &str = "specify-voting-keystore-password";
+pub const ETH1_WITHDRAWAL_ADDRESS_FLAG: &str = "eth1-withdrawal-address";
+pub const IGNORE_DUPLICATES_FLAG: &str = "ignore-duplicates";
+pub const GAS_LIMIT_FLAG: &str = "gas-limit";
+pub const FEE_RECIPIENT_FLAG: &str = "suggested-fee-recipient";
+pub const BUILDER_PROPOSALS_FLAG: &str = "builder-proposals";
+
+pub const VALIDATORS_FILENAME: &str = "validators.json";
+pub const DEPOSITS_FILENAME: &str = "deposits.json";
+
+struct ValidatorKeystore {
+    voting_keystore: Keystore,
+    voting_keystore_password: ZeroizeString,
+    voting_pubkey_bytes: PublicKeyBytes,
+    fee_recipient: Option<Address>,
+    gas_limit: Option<u64>,
+    builder_proposals: Option<bool>,
+    enabled: Option<bool>,
+}
+
+struct ValidatorsAndDeposits {
+    validators: Vec<ValidatorSpecification>,
+    deposits: Option<Vec<StandardDepositDataJson>>,
+}
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Creates new validators from BIP-39 mnemonic.")
+        .arg(
+            Arg::with_name(OUTPUT_PATH_FLAG)
+                .long(OUTPUT_PATH_FLAG)
+                .value_name("DIRECTORY")
+                .help(
+                    "The path to a directory where the validator and (optionally) deposits \
+                    files will be created. The directory will be created if it does not exist.",
+                )
+                .conflicts_with(DISABLE_DEPOSITS_FLAG)
+                .required(true)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(DEPOSIT_GWEI_FLAG)
+                .long(DEPOSIT_GWEI_FLAG)
+                .value_name("DEPOSIT_GWEI")
+                .help(
+                    "The GWEI value of the deposit amount. Defaults to the minimum amount \
+                    required for an active validator (MAX_EFFECTIVE_BALANCE)",
+                )
+                .conflicts_with(DISABLE_DEPOSITS_FLAG)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(FIRST_INDEX_FLAG)
+                .long(FIRST_INDEX_FLAG)
+                .value_name("FIRST_INDEX")
+                .help("The first of consecutive key indexes you wish to recover.")
+                .takes_value(true)
+                .required(false)
+                .default_value("0"),
+        )
+        .arg(
+            Arg::with_name(COUNT_FLAG)
+                .long(COUNT_FLAG)
+                .value_name("VALIDATOR_COUNT")
+                .help("The number of validators to create, regardless of how many already exist")
+                .conflicts_with("at-most")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help("If present, the mnemonic will be read in from this file.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .takes_value(false)
+                .hidden(cfg!(windows))
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+        .arg(
+            Arg::with_name(DISABLE_DEPOSITS_FLAG)
+                .long(DISABLE_DEPOSITS_FLAG)
+                .value_name("PATH")
+                .help(
+                    "When provided don't generate the deposits JSON file that is \
+                    commonly used for submitting validator deposits via a web UI. \
+                    Using this flag will save several seconds per validator if the \
+                    user has an alternate strategy for submitting deposits.",
+                ),
+        )
+        .arg(
+            Arg::with_name(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG)
+                .long(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG)
+                .value_name("STRING")
+                .takes_value(true)
+                .help(
+                    "If present, the user will be prompted to enter the voting keystore \
+                    password that will be used to encrypt the voting keystores. If this \
+                    flag is not provided, a random password will be used. It is not \
+                    necessary to keep backups of voting keystore passwords if the \
+                    mnemonic is safely backed up.",
+                ),
+        )
+        .arg(
+            Arg::with_name(ETH1_WITHDRAWAL_ADDRESS_FLAG)
+                .long(ETH1_WITHDRAWAL_ADDRESS_FLAG)
+                .value_name("ETH1_ADDRESS")
+                .help(
+                    "If this field is set, the given eth1 address will be used to create the \
+                    withdrawal credentials. Otherwise, it will generate withdrawal credentials \
+                    with the mnemonic-derived withdrawal public key in EIP-2334 format.",
+                )
+                .conflicts_with(DISABLE_DEPOSITS_FLAG)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(GAS_LIMIT_FLAG)
+                .long(GAS_LIMIT_FLAG)
+                .value_name("UINT64")
+                .help(
+                    "All created validators will use this gas limit. It is recommended \
+                    to leave this as the default value by not specifying this flag.",
+                )
+                .required(false)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(FEE_RECIPIENT_FLAG)
+                .long(FEE_RECIPIENT_FLAG)
+                .value_name("ETH1_ADDRESS")
+                .help(
+                    "All created validators will use this value for the suggested \
+                    fee recipient. Omit this flag to use the default value from the VC.",
+                )
+                .required(false)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(BUILDER_PROPOSALS_FLAG)
+                .long(BUILDER_PROPOSALS_FLAG)
+                .help(
+                    "When provided, all created validators will attempt to create \
+                    blocks via builder rather than the local EL.",
+                )
+                .required(false),
+        )
+}
+
+pub async fn cli_run<'a, T: EthSpec>(
+    matches: &'a ArgMatches<'a>,
+    mut env: Environment<T>,
+) -> Result<(), String> {
+    let spec = &env.core_context().eth2_config.spec;
+    let output_path: PathBuf = clap_utils::parse_required(matches, OUTPUT_PATH_FLAG)?;
+
+    if !output_path.exists() {
+        fs::create_dir(&output_path)
+            .map_err(|e| format!("Failed to create {:?} directory: {:?}", output_path, e))?;
+    } else if !output_path.is_dir() {
+        return Err(format!("{:?} must be a directory", output_path));
+    }
+
+    let validators_path = output_path.join(VALIDATORS_FILENAME);
+    if validators_path.exists() {
+        return Err(format!(
+            "{:?} already exists, refusing to overwrite",
+            validators_path
+        ));
+    }
+    let deposits_path = output_path.join(DEPOSITS_FILENAME);
+    if deposits_path.exists() {
+        return Err(format!(
+            "{:?} already exists, refusing to overwrite",
+            deposits_path
+        ));
+    }
+
+    let validators_and_deposits = build_validator_spec_from_cli(matches, spec)?;
+
+    write_to_json_file(&validators_path, &validators_and_deposits.validators)?;
+
+    if let Some(deposits) = validators_and_deposits.deposits {
+        write_to_json_file(&deposits_path, &validators_and_deposits.validators)?;
+    }
+
+    Ok(())
+}
+
+fn write_to_json_file<P: AsRef<Path>, S: Serialize>(path: P, contents: &S) -> Result<(), String> {
+    let mut file = fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(path)
+        .map_err(|e| format!("Failed to open {:?}: {:?}", path.as_ref(), e))?;
+    serde_json::to_writer(&mut file, contents)
+        .map_err(|e| format!("Failed to write JSON to {:?}: {:?}", path.as_ref(), e))
+}
+
+pub fn build_validator_spec_from_cli<'a>(
+    matches: &'a ArgMatches<'a>,
+    spec: &ChainSpec,
+) -> Result<ValidatorsAndDeposits, String> {
+    let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
+        .unwrap_or(spec.max_effective_balance);
+    let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
+    let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?;
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG);
+    let disable_deposits = matches.is_present(DISABLE_DEPOSITS_FLAG);
+    let specify_voting_keystore_password =
+        matches.is_present(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG);
+    let eth1_withdrawal_address: Option<Address> =
+        clap_utils::parse_optional(matches, ETH1_WITHDRAWAL_ADDRESS_FLAG)?;
+    let ignore_duplicates = matches.is_present(IGNORE_DUPLICATES_FLAG);
+    let builder_proposals = matches.is_present(BUILDER_PROPOSALS_FLAG);
+    let fee_recipient: Option<Address> = clap_utils::parse_optional(matches, FEE_RECIPIENT_FLAG)?;
+    let gas_limit: Option<u64> = clap_utils::parse_optional(matches, GAS_LIMIT_FLAG)?;
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+    let voting_keystore_password = if specify_voting_keystore_password {
+        eprintln!("Please enter a voting keystore password when prompted.");
+        Some(read_password_from_user(stdin_inputs)?)
+    } else {
+        None
+    };
+
+    /*
+     * Generate a wallet to be used for HD key generation.
+     */
+
+    // A random password is always appropriate for the wallet since it is ephemeral.
+    let wallet_password = random_password_string();
+    // A random password is always appropriate for the withdrawal keystore since we don't ever store
+    // it anywhere.
+    let withdrawal_keystore_password = random_password_string();
+    let mut wallet =
+        WalletBuilder::from_mnemonic(&mnemonic, wallet_password.as_ref(), "".to_string())
+            .map_err(|e| format!("Unable create seed from mnemonic: {:?}", e))?
+            .build()
+            .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+    /*
+     * Start deriving individual validators.
+     */
+
+    let mut validators = Vec::with_capacity(count as usize);
+    let mut deposits = disable_deposits.then(|| vec![]);
+
+    for (i, derivation_index) in (first_index..first_index + count).enumerate() {
+        // If the voting keystore password was not provided by the user then use a unique random
+        // string for each validator.
+        let voting_keystore_password =
+            voting_keystore_password.unwrap_or_else(|| random_password_string());
+
+        // Set the wallet to the appropriate derivation index.
+        wallet
+            .set_nextaccount(derivation_index)
+            .map_err(|e| format!("Failure to set validator derivation index: {:?}", e))?;
+
+        // Derive the keystore from the HD wallet.
+        let keystores = wallet
+            .next_validator(
+                wallet_password.as_ref(),
+                voting_keystore_password.as_ref(),
+                withdrawal_keystore_password.as_ref(),
+            )
+            .map_err(|e| format!("Failed to derive keystore {}: {:?}", i, e))?;
+        let voting_keystore = keystores.voting;
+
+        if let Some(deposits) = &mut deposits {
+            // Decrypt the voting keystore so a deposit message can be signed.
+            let voting_keypair = voting_keystore
+                .decrypt_keypair(voting_keystore_password.as_ref())
+                .map_err(|e| format!("Failed to decrypt voting keystore {}: {:?}", i, e))?;
+
+            let withdrawal_credentials = if let Some(eth1_withdrawal_address) =
+                eth1_withdrawal_address
+            {
+                WithdrawalCredentials::eth1(eth1_withdrawal_address, &spec)
+            } else {
+                // Decrypt the withdrawal keystore so withdrawal credentials can be created. It's
+                // not strictly necessary to decrypt the keystore since we can read the pubkey
+                // directly from the keystore. However we decrypt the keystore to be more certain
+                // that we have access to the withdrawal keys.
+                let withdrawal_keypair = keystores
+                    .withdrawal
+                    .decrypt_keypair(withdrawal_keystore_password.as_ref())
+                    .map_err(|e| format!("Failed to decrypt withdrawal keystore {}: {:?}", i, e))?;
+                WithdrawalCredentials::bls(&withdrawal_keypair.pk, &spec)
+            };
+
+            // Create a JSON structure equivalent to the one generated by
+            // `ethereum/staking-deposit-cli`.
+            let json_deposit = StandardDepositDataJson::new(
+                &voting_keypair,
+                withdrawal_credentials.into(),
+                deposit_gwei,
+                &spec,
+            )?;
+
+            deposits.push(json_deposit);
+        }
+
+        let validator = ValidatorSpecification {
+            voting_keystore: KeystoreJsonStr(voting_keystore),
+            voting_keystore_password: voting_keystore_password.clone(),
+            fee_recipient,
+            gas_limit,
+            builder_proposals: Some(builder_proposals),
+            enabled: Some(true),
+        };
+        validators.push(validator);
+    }
+
+    Ok(ValidatorsAndDeposits {
+        validators,
+        deposits,
+    })
+}

validator_manager/src/validator/import_validators.rs

@@ -0,0 +1,322 @@
+use super::common::*;
+use account_utils::{
+    random_password_string, read_mnemonic_from_cli, read_password_from_user, ZeroizeString,
+};
+use clap::{App, Arg, ArgMatches};
+use environment::Environment;
+use eth2::{
+    lighthouse_vc::{
+        http_client::ValidatorClientHttpClient,
+        std_types::{ImportKeystoresRequest, KeystoreJsonStr},
+        types::UpdateFeeRecipientRequest,
+    },
+    SensitiveUrl,
+};
+use eth2_keystore::Keystore;
+use eth2_wallet::{
+    bip39::{Language, Mnemonic},
+    WalletBuilder,
+};
+use serde::Serialize;
+use std::fs;
+use std::path::PathBuf;
+use types::*;
+
+pub const CMD: &str = "import";
+pub const VALIDATORS_FILE_FLAG: &str = "validators-file";
+pub const VALIDATOR_CLIENT_URL_FLAG: &str = "validator-client-url";
+pub const VALIDATOR_CLIENT_TOKEN_FLAG: &str = "validator-client-token";
+pub const IGNORE_DUPLICATES_FLAG: &str = "ignore-duplicates";
+
+struct ValidatorKeystore {
+    voting_keystore: Keystore,
+    voting_keystore_password: ZeroizeString,
+    voting_pubkey_bytes: PublicKeyBytes,
+    fee_recipient: Option<Address>,
+    gas_limit: Option<u64>,
+    builder_proposals: Option<bool>,
+    enabled: Option<bool>,
+}
+
+struct ValidatorsAndDeposits {
+    validators: Vec<ValidatorSpecification>,
+    deposits: Option<Vec<StandardDepositDataJson>>,
+}
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Uploads validators to a validator client.")
+        .arg(
+            Arg::with_name(VALIDATORS_FILE_FLAG)
+                .long(VALIDATORS_FILE_FLAG)
+                .value_name("PATH_TO_JSON_FILE")
+                .help(
+                    "The path to a JSON file containing a list of validators to be \
+                    imported to the validator client. This file is usually named \
+                    \"validators.json\".",
+                )
+                .required(true)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(VALIDATOR_CLIENT_URL_FLAG)
+                .long(VALIDATOR_CLIENT_URL_FLAG)
+                .value_name("HTTP_ADDRESS")
+                .help(
+                    "A HTTP(S) address of a validator client using the keymanager-API. \
+                    If this value is not supplied then a 'dry run' will be conducted where \
+                    no changes are made to the validator client.",
+                )
+                .default_value("http://localhost:5062")
+                .requires(VALIDATOR_CLIENT_TOKEN_FLAG)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(VALIDATOR_CLIENT_TOKEN_FLAG)
+                .long(VALIDATOR_CLIENT_TOKEN_FLAG)
+                .value_name("PATH")
+                .help("The file containing a token required by the validator client.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(IGNORE_DUPLICATES_FLAG)
+                .takes_value(false)
+                .long(IGNORE_DUPLICATES_FLAG)
+                .help(
+                    "If present, ignore any validators which already exist on the VC. \
+                    Without this flag, the process will terminate without making any changes. \
+                    This flag should be used with caution, whilst it does not directly cause \
+                    slashable conditions, it might be an indicator that something is amiss. \
+                    Users should also be careful to avoid submitting duplicate deposits for \
+                    validators that already exist on the VC.",
+                ),
+        )
+}
+
+pub async fn cli_run<'a, T: EthSpec>(
+    matches: &'a ArgMatches<'a>,
+    mut env: Environment<T>,
+) -> Result<(), String> {
+    let spec = &env.core_context().eth2_config.spec;
+
+    let create_spec = build_validator_spec_from_cli(matches, spec)?;
+    enact_spec(create_spec, spec).await
+}
+
+pub async fn enact_spec<'a>(create_spec: CreateSpec, spec: &ChainSpec) -> Result<(), String> {
+    let CreateSpec {
+        mnemonic,
+        validator_client_url,
+        validator_client_token_path,
+        json_deposit_data_path,
+        ignore_duplicates,
+        validators,
+    } = create_spec;
+
+    let count = validators.len();
+
+    let mnemonic = Mnemonic::from_phrase(&mnemonic, Language::English)
+        .map_err(|e| format!("Failed to parse mnemonic from create spec: {:?}", e))?;
+
+    let http_client = match (validator_client_url, validator_client_token_path) {
+        (Some(vc_url), Some(vc_token_path)) => {
+            let token_bytes = fs::read(&vc_token_path)
+                .map_err(|e| format!("Failed to read {:?}: {:?}", vc_token_path, e))?;
+            let token_string = String::from_utf8(token_bytes)
+                .map_err(|e| format!("Failed to parse {:?} as utf8: {:?}", vc_token_path, e))?;
+            let http_client = ValidatorClientHttpClient::new(vc_url.clone(), token_string)
+                .map_err(|e| {
+                    format!(
+                        "Could not instantiate HTTP client from URL and secret: {:?}",
+                        e
+                    )
+                })?;
+
+            // Perform a request to check that the connection works
+            let remote_keystores = http_client
+                .get_keystores()
+                .await
+                .map_err(|e| format!("Failed to list keystores on VC: {:?}", e))?;
+            eprintln!(
+                "Validator client is reachable at {} and reports {} validators",
+                vc_url,
+                remote_keystores.data.len()
+            );
+
+            Some(http_client)
+        }
+        (None, None) => None,
+        _ => {
+            return Err(format!(
+                "Inconsistent use of {} and {}",
+                VALIDATOR_CLIENT_URL_FLAG, VALIDATOR_CLIENT_TOKEN_FLAG
+            ))
+        }
+    };
+
+    // A random password is always appropriate for the wallet since it is ephemeral.
+    let wallet_password = random_password_string();
+    // A random password is always appropriate for the withdrawal keystore since we don't ever store
+    // it anywhere.
+    let withdrawal_keystore_password = random_password_string();
+
+    let mut wallet =
+        WalletBuilder::from_mnemonic(&mnemonic, wallet_password.as_ref(), "".to_string())
+            .map_err(|e| format!("Unable create seed from mnemonic: {:?}", e))?
+            .build()
+            .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+    let mut validator_keystores = Vec::with_capacity(count);
+
+    eprintln!("Starting key generation. Each validator may take several seconds.");
+
+    for (i, validator) in validators.into_iter().enumerate() {
+        let CreateValidatorSpec {
+            voting_keystore,
+            voting_keystore_password,
+            fee_recipient,
+            gas_limit,
+            builder_proposals,
+            enabled,
+        } = validator;
+
+        let voting_keystore = voting_keystore.0;
+
+        let voting_keypair = voting_keystore
+            .decrypt_keypair(voting_keystore_password.as_ref())
+            .map_err(|e| format!("Failed to decrypt voting keystore {}: {:?}", i, e))?;
+        let voting_pubkey_bytes = voting_keypair.pk.clone().into();
+
+        // Check to see if this validator already exists in the VC.
+        if let Some(http_client) = &http_client {
+            let remote_keystores = http_client
+                .get_keystores()
+                .await
+                .map_err(|e| format!("Failed to list keystores on VC: {:?}", e))?;
+
+            if remote_keystores
+                .data
+                .iter()
+                .find(|keystore| keystore.validating_pubkey == voting_pubkey_bytes)
+                .is_some()
+            {
+                if ignore_duplicates {
+                    eprintln!(
+                        "Validator {:?} already exists in the VC, be cautious of submitting \
+                        duplicate deposits",
+                        IGNORE_DUPLICATES_FLAG
+                    );
+                } else {
+                    return Err(format!(
+                        "Duplicate validator {:?} detected, see --{} for more information",
+                        voting_keypair.pk, IGNORE_DUPLICATES_FLAG
+                    ));
+                }
+            }
+        }
+
+        eprintln!(
+            "{}/{}: {:?}",
+            i.saturating_add(1),
+            count,
+            &voting_keypair.pk
+        );
+
+        validator_keystores.push(ValidatorKeystore {
+            voting_keystore,
+            voting_keystore_password,
+            voting_pubkey_bytes,
+            fee_recipient,
+            gas_limit,
+            builder_proposals,
+            enabled,
+        });
+    }
+
+    if let Some(http_client) = http_client {
+        eprintln!(
+            "Generated {} keystores. Starting to submit keystores to VC, \
+            each keystore may take several seconds",
+            count
+        );
+
+        for (i, validator_keystore) in validator_keystores.into_iter().enumerate() {
+            let ValidatorKeystore {
+                voting_keystore,
+                voting_keystore_password,
+                voting_pubkey_bytes,
+                fee_recipient,
+                gas_limit,
+                builder_proposals,
+                enabled,
+            } = validator_keystore;
+
+            let request = ImportKeystoresRequest {
+                keystores: vec![KeystoreJsonStr(voting_keystore)],
+                passwords: vec![voting_keystore_password],
+                // New validators have no slashing protection history.
+                slashing_protection: None,
+            };
+
+            if let Err(e) = http_client.post_keystores(&request).await {
+                eprintln!(
+                    "Failed to upload batch {}. Some keys were imported whilst \
+                    others may not have been imported. A potential solution is to use the \
+                    --{} flag, however care should be taken to ensure that there are no \
+                    duplicate deposits submitted.",
+                    i, IGNORE_DUPLICATES_FLAG
+                );
+                // Return here *without* writing the deposit JSON file. This might help prevent
+                // users from submitting duplicate deposits or deposits for validators that weren't
+                // initialized on a VC.
+                //
+                // Next the the user runs with the --ignore-duplicates flag there should be a new,
+                // complete deposit JSON file created.
+                return Err(format!("Key upload failed: {:?}", e));
+            }
+
+            if let Some(fee_recipient) = fee_recipient {
+                http_client
+                    .post_fee_recipient(
+                        &voting_pubkey_bytes,
+                        &UpdateFeeRecipientRequest {
+                            ethaddress: fee_recipient,
+                        },
+                    )
+                    .await
+                    .map_err(|e| format!("Failed to update fee recipient on VC: {:?}", e))?;
+            }
+
+            if gas_limit.is_some() || builder_proposals.is_some() || enabled.is_some() {
+                http_client
+                    .patch_lighthouse_validators(
+                        &voting_pubkey_bytes,
+                        enabled,
+                        gas_limit,
+                        builder_proposals,
+                    )
+                    .await
+                    .map_err(|e| format!("Failed to update lighthouse validator on VC: {:?}", e))?;
+            }
+
+            eprintln!("Uploaded keystore {} of {} to the VC", i + 1, count);
+        }
+    }
+
+    // If configured, create a single JSON file which contains deposit data information for all
+    // validators.
+    if let Some(json_deposit_data_path) = json_deposit_data_path {
+        let json_deposits = json_deposits.ok_or("Internal error: JSON deposit data is None")?;
+
+        let mut file = fs::OpenOptions::new()
+            .write(true)
+            .create_new(true)
+            .open(&json_deposit_data_path)
+            .map_err(|e| format!("Unable to create {:?}: {:?}", json_deposit_data_path, e))?;
+
+        serde_json::to_writer(&mut file, &json_deposits)
+            .map_err(|e| format!("Unable write JSON to {:?}: {:?}", json_deposit_data_path, e))?;
+    }
+
+    Ok(())
+}

validator_manager/src/validator/mod.rs

@@ -1,5 +1,7 @@
 pub mod common;
 pub mod create;
+pub mod create_validators;
+pub mod import_validators;
 
 use clap::{App, ArgMatches};
 use environment::Environment;