Add progress

validator_manager/src/validator/common/mod.rs

@@ -0,0 +1,27 @@
+use account_utils::ZeroizeString;
+use eth2::SensitiveUrl;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+use types::*;
+
+#[derive(Serialize, Deserialize)]
+pub struct CreateValidatorSpec {
+    pub derivation_index: u32,
+    pub voting_keystore_password: Option<ZeroizeString>,
+    pub deposit_gwei: u64,
+    pub eth1_withdrawal_address: Option<Address>,
+    pub fee_recipient: Option<Address>,
+    pub gas_limit: Option<u64>,
+    pub builder_proposals: Option<bool>,
+    pub enabled: Option<bool>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CreateSpec {
+    pub mnemonic: String,
+    pub validator_client_url: Option<SensitiveUrl>,
+    pub validator_client_token_path: Option<PathBuf>,
+    pub json_deposit_data_path: Option<PathBuf>,
+    pub ignore_duplicates: bool,
+    pub validators: Vec<CreateValidatorSpec>,
+}

validator_manager/src/validator/create.rs

@@ -1,14 +1,22 @@
-use account_utils::{random_password_string, read_mnemonic_from_cli, read_password_string};
+use super::common::*;
+use account_utils::{
+    random_password_string, read_mnemonic_from_cli, read_password_from_user, ZeroizeString,
+};
 use clap::{App, Arg, ArgMatches};
 use environment::Environment;
 use eth2::{
     lighthouse_vc::{
         http_client::ValidatorClientHttpClient,
         std_types::{ImportKeystoresRequest, KeystoreJsonStr},
+        types::UpdateFeeRecipientRequest,
     },
     SensitiveUrl,
 };
-use eth2_wallet::WalletBuilder;
+use eth2_keystore::Keystore;
+use eth2_wallet::{
+    bip39::{Language, Mnemonic},
+    WalletBuilder,
+};
 use serde::Serialize;
 use std::fs;
 use std::path::PathBuf;
@@ -22,13 +30,24 @@ pub const COUNT_FLAG: &str = "count";
 pub const STDIN_INPUTS_FLAG: &str = "stdin-inputs";
 pub const FIRST_INDEX_FLAG: &str = "first-index";
 pub const MNEMONIC_FLAG: &str = "mnemonic-path";
-pub const PASSWORD_FLAG: &str = "password-file";
+pub const SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG: &str = "specify-voting-keystore-password";
 pub const ETH1_WITHDRAWAL_ADDRESS_FLAG: &str = "eth1-withdrawal-address";
-pub const DRY_RUN_FLAG: &str = "dry-run";
 pub const VALIDATOR_CLIENT_URL_FLAG: &str = "validator-client-url";
 pub const VALIDATOR_CLIENT_TOKEN_FLAG: &str = "validator-client-token";
 pub const IGNORE_DUPLICATES_FLAG: &str = "ignore-duplicates";
-pub const KEYSTORE_UPLOAD_BATCH_SIZE: &str = "keystore-upload-batch-size";
+pub const GAS_LIMIT_FLAG: &str = "gas-limit";
+pub const FEE_RECIPIENT_FLAG: &str = "suggested-fee-recipient";
+pub const BUILDER_PROPOSALS_FLAG: &str = "builder-proposals";
+
+struct ValidatorKeystore {
+    voting_keystore: Keystore,
+    voting_keystore_password: ZeroizeString,
+    voting_pubkey_bytes: PublicKeyBytes,
+    fee_recipient: Option<Address>,
+    gas_limit: Option<u64>,
+    builder_proposals: Option<bool>,
+    enabled: Option<bool>,
+}
 
 /// The structure generated by the `staking-deposit-cli` which has become a quasi-standard for
 /// browser-based deposit submission tools (e.g., the Ethereum Launchpad and Lido).
@@ -151,16 +170,17 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name(PASSWORD_FLAG)
-                .long(PASSWORD_FLAG)
+            Arg::with_name(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG)
+                .long(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG)
                 .value_name("STRING")
+                .takes_value(true)
                 .help(
-                    "A path to a file containing the password which will unlock the wallet. \
-                    If the file does not exist, a random password will be generated and \
-                    saved at that path. To avoid confusion, if the file does not already \
-                    exist it must include a '.pass' suffix.",
-                )
-                .takes_value(true),
+                    "If present, the user will be prompted to enter the voting keystore \
+                    password that will be used to encrypt the voting keystores. If this \
+                    flag is not provided, a random password will be used. It is not \
+                    necessary to keep backups of voting keystore passwords if the \
+                    mnemonic is safely backed up.",
+                ),
         )
         .arg(
             Arg::with_name(ETH1_WITHDRAWAL_ADDRESS_FLAG)
@@ -173,20 +193,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 )
                 .takes_value(true),
         )
-        .arg(
-            Arg::with_name(DRY_RUN_FLAG)
-                .takes_value(false)
-                .long(DRY_RUN_FLAG)
-                .help(
-                    "If present, perform all actions without ever contacting the validator client.",
-                ),
-        )
         .arg(
             Arg::with_name(VALIDATOR_CLIENT_URL_FLAG)
                 .long(VALIDATOR_CLIENT_URL_FLAG)
                 .value_name("HTTP_ADDRESS")
-                .help("A HTTP(S) address of a validator client using the keymanager-API.")
-                .required_unless(DRY_RUN_FLAG)
+                .help(
+                    "A HTTP(S) address of a validator client using the keymanager-API. \
+                    If this value is not supplied then a 'dry run' will be conducted where \
+                    no changes are made to the validator client.",
+                )
+                .requires(VALIDATOR_CLIENT_TOKEN_FLAG)
                 .takes_value(true),
         )
         .arg(
@@ -194,7 +210,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .long(VALIDATOR_CLIENT_TOKEN_FLAG)
                 .value_name("PATH")
                 .help("The file containing a token required by the validator client.")
-                .required_unless(DRY_RUN_FLAG)
                 .takes_value(true),
         )
         .arg(
@@ -211,21 +226,52 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 ),
         )
         .arg(
-            Arg::with_name(KEYSTORE_UPLOAD_BATCH_SIZE)
-                .long(KEYSTORE_UPLOAD_BATCH_SIZE)
-                .value_name("INTEGER")
-                .help("The number of keystores to be submitted to the VC per request.")
-                .takes_value(true)
-                .default_value("4"),
+            Arg::with_name(GAS_LIMIT_FLAG)
+                .long(GAS_LIMIT_FLAG)
+                .value_name("UINT64")
+                .help(
+                    "All created validators will use this gas limit. It is recommended \
+                    to leave this as the default value by not specifying this flag.",
+                )
+                .required(false)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(FEE_RECIPIENT_FLAG)
+                .long(FEE_RECIPIENT_FLAG)
+                .value_name("ETH1_ADDRESS")
+                .help(
+                    "All created validators will use this value for the suggested \
+                    fee recipient. Omit this flag to use the default value from the VC.",
+                )
+                .required(false)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(BUILDER_PROPOSALS_FLAG)
+                .long(BUILDER_PROPOSALS_FLAG)
+                .help(
+                    "When provided, all created validators will attempt to create \
+                    blocks via builder rather than the local EL.",
+                )
+                .required(false)
+                .required(false),
         )
 }
-
 pub async fn cli_run<'a, T: EthSpec>(
     matches: &'a ArgMatches<'a>,
     mut env: Environment<T>,
 ) -> Result<(), String> {
-    let spec = env.core_context().eth2_config.spec;
+    let spec = &env.core_context().eth2_config.spec;
 
+    let create_spec = build_spec_from_cli(matches, spec)?;
+    enact_spec(create_spec, spec).await
+}
+
+pub fn build_spec_from_cli<'a>(
+    matches: &'a ArgMatches<'a>,
+    spec: &ChainSpec,
+) -> Result<CreateSpec, String> {
     let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
         .unwrap_or(spec.max_effective_balance);
     let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
@@ -234,24 +280,69 @@ pub async fn cli_run<'a, T: EthSpec>(
     let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG);
     let json_deposit_data_path: Option<PathBuf> =
         clap_utils::parse_optional(matches, JSON_DEPOSIT_DATA_PATH)?;
-    let wallet_password_path: Option<PathBuf> = clap_utils::parse_optional(matches, PASSWORD_FLAG)?;
+    let specify_voting_keystore_password =
+        matches.is_present(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG);
     let eth1_withdrawal_address: Option<Address> =
         clap_utils::parse_optional(matches, ETH1_WITHDRAWAL_ADDRESS_FLAG)?;
-    let dry_run = matches.is_present(DRY_RUN_FLAG);
     let vc_url: Option<SensitiveUrl> =
         clap_utils::parse_optional(matches, VALIDATOR_CLIENT_URL_FLAG)?;
     let vc_token_path: Option<PathBuf> =
         clap_utils::parse_optional(matches, VALIDATOR_CLIENT_TOKEN_FLAG)?;
     let ignore_duplicates = matches.is_present(IGNORE_DUPLICATES_FLAG);
-    let keystore_upload_batch_size: usize =
-        clap_utils::parse_required(matches, KEYSTORE_UPLOAD_BATCH_SIZE)?;
+    let builder_proposals = matches.is_present(BUILDER_PROPOSALS_FLAG);
+    let fee_recipient: Option<Address> = clap_utils::parse_optional(matches, FEE_RECIPIENT_FLAG)?;
+    let gas_limit: Option<u64> = clap_utils::parse_optional(matches, GAS_LIMIT_FLAG)?;
 
-    let num_batches = (count + (count - 1))
-        .checked_div(keystore_upload_batch_size as u32)
-        .ok_or_else(|| format!("--{} cannot be zero", KEYSTORE_UPLOAD_BATCH_SIZE))?;
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+    let voting_keystore_password = if specify_voting_keystore_password {
+        eprintln!("Please enter a voting keystore password when prompted.");
+        Some(read_password_from_user(stdin_inputs)?)
+    } else {
+        None
+    };
 
-    let http_client = match (dry_run, vc_url, vc_token_path) {
-        (false, Some(vc_url), Some(vc_token_path)) => {
+    let mut validators = Vec::with_capacity(count as usize);
+    for derivation_index in first_index..first_index + count {
+        let validator = CreateValidatorSpec {
+            derivation_index,
+            voting_keystore_password: voting_keystore_password.clone(),
+            deposit_gwei,
+            eth1_withdrawal_address,
+            fee_recipient,
+            gas_limit,
+            builder_proposals: Some(builder_proposals),
+            enabled: Some(true),
+        };
+        validators.push(validator);
+    }
+
+    Ok(CreateSpec {
+        mnemonic: mnemonic.to_string(),
+        validator_client_url: vc_url,
+        validator_client_token_path: vc_token_path,
+        json_deposit_data_path,
+        ignore_duplicates,
+        validators,
+    })
+}
+
+pub async fn enact_spec<'a>(create_spec: CreateSpec, spec: &ChainSpec) -> Result<(), String> {
+    let CreateSpec {
+        mnemonic,
+        validator_client_url,
+        validator_client_token_path,
+        json_deposit_data_path,
+        ignore_duplicates,
+        validators,
+    } = create_spec;
+
+    let count = validators.len();
+
+    let mnemonic = Mnemonic::from_phrase(&mnemonic, Language::English)
+        .map_err(|e| format!("Failed to parse mnemonic from create spec: {:?}", e))?;
+
+    let http_client = match (validator_client_url, validator_client_token_path) {
+        (Some(vc_url), Some(vc_token_path)) => {
             let token_bytes = fs::read(&vc_token_path)
                 .map_err(|e| format!("Failed to read {:?}: {:?}", vc_token_path, e))?;
             let token_string = String::from_utf8(token_bytes)
@@ -277,24 +368,17 @@ pub async fn cli_run<'a, T: EthSpec>(
 
             Some(http_client)
         }
-        (true, None, None) => None,
+        (None, None) => None,
         _ => {
             return Err(format!(
-                "Inconsistent use of {}, {} and {} flags",
-                DRY_RUN_FLAG, VALIDATOR_CLIENT_URL_FLAG, VALIDATOR_CLIENT_TOKEN_FLAG
+                "Inconsistent use of {} and {}",
+                VALIDATOR_CLIENT_URL_FLAG, VALIDATOR_CLIENT_TOKEN_FLAG
             ))
         }
     };
 
-    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
     // A random password is always appropriate for the wallet since it is ephemeral.
     let wallet_password = random_password_string();
-    let voting_keystore_password = if let Some(path) = wallet_password_path {
-        read_password_string(&path)
-            .map_err(|e| format!("Failed to read password from {:?}: {:?}", path, e))?
-    } else {
-        random_password_string()
-    };
     // A random password is always appropriate for the withdrawal keystore since we don't ever store
     // it anywhere.
     let withdrawal_keystore_password = random_password_string();
@@ -305,16 +389,30 @@ pub async fn cli_run<'a, T: EthSpec>(
             .build()
             .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
 
-    wallet
-        .set_nextaccount(first_index)
-        .map_err(|e| format!("Failure to set --{}: {:?}", FIRST_INDEX_FLAG, e))?;
-
-    let mut voting_keystores = Vec::with_capacity(count as usize);
+    let mut validator_keystores = Vec::with_capacity(count);
     let mut json_deposits = Some(vec![]).filter(|_| json_deposit_data_path.is_some());
 
     eprintln!("Starting key generation. Each validator may take several seconds.");
 
-    for i in 0..count {
+    for (i, validator) in validators.into_iter().enumerate() {
+        let CreateValidatorSpec {
+            derivation_index,
+            voting_keystore_password,
+            deposit_gwei,
+            eth1_withdrawal_address,
+            fee_recipient,
+            gas_limit,
+            builder_proposals,
+            enabled,
+        } = validator;
+
+        let voting_keystore_password =
+            voting_keystore_password.unwrap_or_else(|| random_password_string());
+
+        wallet
+            .set_nextaccount(derivation_index)
+            .map_err(|e| format!("Failure to set validator derivation index: {:?}", e))?;
+
         let keystores = wallet
             .next_validator(
                 wallet_password.as_ref(),
@@ -385,7 +483,15 @@ pub async fn cli_run<'a, T: EthSpec>(
             &voting_keypair.pk
         );
 
-        voting_keystores.push(voting_keystore);
+        validator_keystores.push(ValidatorKeystore {
+            voting_keystore,
+            voting_keystore_password,
+            voting_pubkey_bytes,
+            fee_recipient,
+            gas_limit,
+            builder_proposals,
+            enabled,
+        });
     }
 
     if let Some(http_client) = http_client {
@@ -395,22 +501,20 @@ pub async fn cli_run<'a, T: EthSpec>(
             count
         );
 
-        for (i, chunk) in voting_keystores
-            .chunks(keystore_upload_batch_size)
-            .enumerate()
-        {
-            let keystores = chunk
-                .iter()
-                .cloned()
-                .map(KeystoreJsonStr)
-                .collect::<Vec<_>>();
-            let passwords = keystores
-                .iter()
-                .map(|_| voting_keystore_password.clone())
-                .collect();
+        for (i, validator_keystore) in validator_keystores.into_iter().enumerate() {
+            let ValidatorKeystore {
+                voting_keystore,
+                voting_keystore_password,
+                voting_pubkey_bytes,
+                fee_recipient,
+                gas_limit,
+                builder_proposals,
+                enabled,
+            } = validator_keystore;
+
             let request = ImportKeystoresRequest {
-                keystores,
-                passwords,
+                keystores: vec![KeystoreJsonStr(voting_keystore)],
+                passwords: vec![voting_keystore_password],
                 // New validators have no slashing protection history.
                 slashing_protection: None,
             };
@@ -432,11 +536,31 @@ pub async fn cli_run<'a, T: EthSpec>(
                 return Err(format!("Key upload failed: {:?}", e));
             }
 
-            eprintln!(
-                "Uploaded keystore batch {} of {} to the VC",
-                i + 1,
-                num_batches
-            );
+            if let Some(fee_recipient) = fee_recipient {
+                http_client
+                    .post_fee_recipient(
+                        &voting_pubkey_bytes,
+                        &UpdateFeeRecipientRequest {
+                            ethaddress: fee_recipient,
+                        },
+                    )
+                    .await
+                    .map_err(|e| format!("Failed to update fee recipient on VC: {:?}", e))?;
+            }
+
+            if gas_limit.is_some() || builder_proposals.is_some() || enabled.is_some() {
+                http_client
+                    .patch_lighthouse_validators(
+                        &voting_pubkey_bytes,
+                        enabled,
+                        gas_limit,
+                        builder_proposals,
+                    )
+                    .await
+                    .map_err(|e| format!("Failed to update lighthouse validator on VC: {:?}", e))?;
+            }
+
+            eprintln!("Uploaded keystore {} of {} to the VC", i + 1, count);
         }
     }
 

validator_manager/src/validator/mod.rs

@@ -1,3 +1,4 @@
+pub mod common;
 pub mod create;
 
 use clap::{App, ArgMatches};