Validator manager commands for the Keymanager APIs (#6261)

* Validator manager commands for standard key-manager APIs

* Merge latest unstable

* Fix Some in lib.rs

* Replace Arg::with_name with Arg::new

* Update takes_value

* Remove clap::App

* Change App to Command

* Add command in use

* Remove generic in ArgMatches

* Fix matches.get_flag

* Fixes

* fix error handling

* SetTrue in import

* Fix

* Fix builder-proposal flag (will delete the flag later)

* Minor fix

* Fix prefer_builder_proposals

* Remove unwrap

* Error handling from Michael

* Add cli help text

* Use None in import to simplify

* Delete unwrap

* Revert flags option

* Simplify help command code

* Remove flag header in move

* Merge remote-tracking branch 'origin/unstable' into pahor/validator-manager-standard-keystore

* Add log in VC when keystore is deleted

* Delete duplicated log when validator does not exist

* Simplify log code

* Rename remove to delete

* cargo-fmt

* Try to remove a function

* make-cli

* Error handling

* Merge branch 'vm' of https://github.com/chong-he/lighthouse into vm

* Update CLI hel text

* make-cli

* Fix checks

* Merge branch 'vm' of https://github.com/chong-he/lighthouse into vm

* Try to fix check errors

* Fix test

* Remove changes

* Update flag name

* CLI display order

* Move builde_proposals flag

* Add doc

* mdlint

* Update validator_manager/src/list_validators.rs

Co-authored-by: Mac L <mjladson@pm.me>

* Delete empty line

* Fix list

* Simplify delete

* Add support to delete more validators

* Fix test

* Rename response

* Add (s)

* Add test to delete multiple validators

* Book and cli

* Make cli

* Only log when keystore is deleted

* Revise deletion log

* Add validator pubkey to error message

* Merge import

* Thank you Mac

* Test

* Add flags

* Error handling for password

* make cli

* Merge remote-tracking branch 'origin/unstable' into vm

* make cli

* Fix test

* Merge branch 'vm' of https://github.com/chong-he/lighthouse into vm

* Fix test

* vm test

* Debug trait thank you Michael

* Fix test

* Merge branch 'unstable' into vm

* test

* testing

* Combine import validator(s)

* make cli

* Add requires

* Update book

* mdlint

* Only show import log when import is successful

* delete testing

* Test for standard format

* Test standard format

* Test

* fix builder_proposals flag

* Fix test for standard format

* Add requires

* Fix vm test

* make cli

* Remove flag header

* Merge branch 'vm' of https://github.com/chong-he/lighthouse into vm

* make cli

* Delete space

* Merge branch 'vm' of https://github.com/chong-he/lighthouse into vm

* Merge branch 'unstable' into vm

* Rename delete_validator to delete_validators

* Rearrange

* Remove pub in run function

* Fix grammar

* Apply suggestions from code review

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Remove description

* Merge branch 'vm' of https://github.com/chong-he/lighthouse into vm

* Close bracket

* make cli

* Revise list code and test

* Revise import flag

* make cli

* Comment out test

* Update vm test

* Simplify

* Merge remote-tracking branch 'origin/unstable' into vm

* make cli

* Add test

* Add password as a requirement for keystore file

* Correct flags in docs

* typo

validator_manager/Cargo.toml

@@ -20,6 +20,7 @@ tree_hash = { workspace = true }
 eth2 = { workspace = true }
 hex = { workspace = true }
 tokio = { workspace = true }
+derivative = { workspace = true }
 
 [dev-dependencies]
 tempfile = { workspace = true }

validator_manager/src/create_validators.rs

@@ -45,15 +45,6 @@ pub fn cli_app() -> Command {
                 Another, optional JSON file is created which contains a list of validator \
                 deposits in the same format as the \"ethereum/staking-deposit-cli\" tool.",
         )
-        .arg(
-            Arg::new("help")
-                .long("help")
-                .short('h')
-                .help("Prints help information")
-                .action(ArgAction::HelpLong)
-                .display_order(0)
-                .help_heading(FLAG_HEADER),
-        )
         .arg(
             Arg::new(OUTPUT_PATH_FLAG)
                 .long(OUTPUT_PATH_FLAG)

validator_manager/src/delete_validators.rs

@@ -0,0 +1,293 @@
+use clap::{Arg, ArgAction, ArgMatches, Command};
+use eth2::{
+    lighthouse_vc::types::{DeleteKeystoreStatus, DeleteKeystoresRequest},
+    SensitiveUrl,
+};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+use types::PublicKeyBytes;
+
+use crate::{common::vc_http_client, DumpConfig};
+
+pub const CMD: &str = "delete";
+pub const VC_URL_FLAG: &str = "vc-url";
+pub const VC_TOKEN_FLAG: &str = "vc-token";
+pub const VALIDATOR_FLAG: &str = "validators";
+
+#[derive(Debug)]
+pub enum DeleteError {
+    InvalidPublicKey,
+    DeleteFailed(eth2::Error),
+}
+
+pub fn cli_app() -> Command {
+    Command::new(CMD)
+        .about("Deletes one or more validators from a validator client using the HTTP API.")
+        .arg(
+            Arg::new(VC_URL_FLAG)
+                .long(VC_URL_FLAG)
+                .value_name("HTTP_ADDRESS")
+                .help("A HTTP(S) address of a validator client using the keymanager-API.")
+                .default_value("http://localhost:5062")
+                .requires(VC_TOKEN_FLAG)
+                .action(ArgAction::Set)
+                .display_order(0),
+        )
+        .arg(
+            Arg::new(VC_TOKEN_FLAG)
+                .long(VC_TOKEN_FLAG)
+                .value_name("PATH")
+                .help("The file containing a token required by the validator client.")
+                .action(ArgAction::Set)
+                .display_order(0),
+        )
+        .arg(
+            Arg::new(VALIDATOR_FLAG)
+                .long(VALIDATOR_FLAG)
+                .value_name("STRING")
+                .help("Comma-separated list of validators (pubkey) that will be deleted.")
+                .action(ArgAction::Set)
+                .required(true)
+                .display_order(0),
+        )
+}
+
+#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+pub struct DeleteConfig {
+    pub vc_url: SensitiveUrl,
+    pub vc_token_path: PathBuf,
+    pub validators_to_delete: Vec<PublicKeyBytes>,
+}
+
+impl DeleteConfig {
+    fn from_cli(matches: &ArgMatches) -> Result<Self, String> {
+        let validators_to_delete_str =
+            clap_utils::parse_required::<String>(matches, VALIDATOR_FLAG)?;
+
+        let validators_to_delete = validators_to_delete_str
+            .split(',')
+            .map(|s| s.trim().parse())
+            .collect::<Result<Vec<PublicKeyBytes>, _>>()?;
+
+        Ok(Self {
+            vc_token_path: clap_utils::parse_required(matches, VC_TOKEN_FLAG)?,
+            validators_to_delete,
+            vc_url: clap_utils::parse_required(matches, VC_URL_FLAG)?,
+        })
+    }
+}
+
+pub async fn cli_run(matches: &ArgMatches, dump_config: DumpConfig) -> Result<(), String> {
+    let config = DeleteConfig::from_cli(matches)?;
+    if dump_config.should_exit_early(&config)? {
+        Ok(())
+    } else {
+        run(config).await
+    }
+}
+
+async fn run<'a>(config: DeleteConfig) -> Result<(), String> {
+    let DeleteConfig {
+        vc_url,
+        vc_token_path,
+        validators_to_delete,
+    } = config;
+
+    let (http_client, validators) = vc_http_client(vc_url.clone(), &vc_token_path).await?;
+
+    for validator_to_delete in &validators_to_delete {
+        if !validators
+            .iter()
+            .any(|validator| &validator.validating_pubkey == validator_to_delete)
+        {
+            return Err(format!("Validator {} doesn't exist", validator_to_delete));
+        }
+    }
+
+    let delete_request = DeleteKeystoresRequest {
+        pubkeys: validators_to_delete.clone(),
+    };
+
+    let responses = http_client
+        .delete_keystores(&delete_request)
+        .await
+        .map_err(|e| format!("Error deleting keystore {}", e))?
+        .data;
+
+    let mut error = false;
+    for (validator_to_delete, response) in validators_to_delete.iter().zip(responses.iter()) {
+        if response.status == DeleteKeystoreStatus::Error
+            || response.status == DeleteKeystoreStatus::NotFound
+            || response.status == DeleteKeystoreStatus::NotActive
+        {
+            error = true;
+            eprintln!(
+                "Problem with removing validator {:?}, status: {:?}",
+                validator_to_delete, response.status
+            );
+        }
+    }
+    if error {
+        return Err("Problem with removing one or more validators".to_string());
+    }
+
+    eprintln!("Validator(s) deleted");
+    Ok(())
+}
+
+#[cfg(not(debug_assertions))]
+#[cfg(test)]
+mod test {
+    use std::{
+        fs::{self, File},
+        io::Write,
+        str::FromStr,
+    };
+
+    use super::*;
+    use crate::{
+        common::ValidatorSpecification, import_validators::tests::TestBuilder as ImportTestBuilder,
+    };
+    use validator_client::http_api::{test_utils::ApiTester, Config as HttpConfig};
+
+    struct TestBuilder {
+        delete_config: Option<DeleteConfig>,
+        src_import_builder: Option<ImportTestBuilder>,
+        http_config: HttpConfig,
+        vc_token: Option<String>,
+        validators: Vec<ValidatorSpecification>,
+    }
+
+    impl TestBuilder {
+        async fn new() -> Self {
+            Self {
+                delete_config: None,
+                src_import_builder: None,
+                http_config: ApiTester::default_http_config(),
+                vc_token: None,
+                validators: vec![],
+            }
+        }
+
+        async fn with_validators(
+            mut self,
+            count: u32,
+            first_index: u32,
+            indices_of_validators_to_delete: Vec<usize>,
+        ) -> Self {
+            let builder = ImportTestBuilder::new_with_http_config(self.http_config.clone())
+                .await
+                .create_validators(count, first_index)
+                .await;
+
+            self.vc_token =
+                Some(fs::read_to_string(builder.get_import_config().vc_token_path).unwrap());
+
+            let local_validators: Vec<ValidatorSpecification> = {
+                let contents =
+                    fs::read_to_string(builder.get_import_config().validators_file_path.unwrap())
+                        .unwrap();
+                serde_json::from_str(&contents).unwrap()
+            };
+
+            let import_config = builder.get_import_config();
+
+            let validators_to_delete = indices_of_validators_to_delete
+                .iter()
+                .map(|&index| {
+                    PublicKeyBytes::from_str(
+                        format!("0x{}", local_validators[index].voting_keystore.pubkey()).as_str(),
+                    )
+                    .unwrap()
+                })
+                .collect();
+
+            self.delete_config = Some(DeleteConfig {
+                vc_url: import_config.vc_url,
+                vc_token_path: import_config.vc_token_path,
+                validators_to_delete,
+            });
+
+            self.validators = local_validators.clone();
+            self.src_import_builder = Some(builder);
+            self
+        }
+
+        pub async fn run_test(self) -> TestResult {
+            let import_builder = self.src_import_builder.unwrap();
+            let import_test_result = import_builder.run_test().await;
+            assert!(import_test_result.result.is_ok());
+
+            let path = self.delete_config.clone().unwrap().vc_token_path;
+            let url = self.delete_config.clone().unwrap().vc_url;
+            let parent = path.parent().unwrap();
+
+            fs::create_dir_all(parent).expect("Was not able to create parent directory");
+
+            File::options()
+                .write(true)
+                .read(true)
+                .create(true)
+                .truncate(true)
+                .open(path.clone())
+                .unwrap()
+                .write_all(self.vc_token.clone().unwrap().as_bytes())
+                .unwrap();
+
+            let result = run(self.delete_config.clone().unwrap()).await;
+
+            if result.is_ok() {
+                let (_, list_keystores_response) = vc_http_client(url, path.clone()).await.unwrap();
+
+                // The remaining number of active keystores (left) = Total validators - Deleted validators (right)
+                assert_eq!(
+                    list_keystores_response.len(),
+                    self.validators.len()
+                        - self
+                            .delete_config
+                            .clone()
+                            .unwrap()
+                            .validators_to_delete
+                            .len()
+                );
+
+                // Check the remaining validator keys are not in validators_to_delete
+                assert!(list_keystores_response.iter().all(|keystore| {
+                    !self
+                        .delete_config
+                        .clone()
+                        .unwrap()
+                        .validators_to_delete
+                        .contains(&keystore.validating_pubkey)
+                }));
+
+                return TestResult { result: Ok(()) };
+            }
+
+            TestResult {
+                result: Err(result.unwrap_err()),
+            }
+        }
+    }
+
+    #[must_use]
+    struct TestResult {
+        result: Result<(), String>,
+    }
+
+    impl TestResult {
+        fn assert_ok(self) {
+            assert_eq!(self.result, Ok(()))
+        }
+    }
+    #[tokio::test]
+    async fn delete_multiple_validators() {
+        TestBuilder::new()
+            .await
+            .with_validators(3, 0, vec![0, 1, 2])
+            .await
+            .run_test()
+            .await
+            .assert_ok();
+    }
+}

validator_manager/src/import_validators.rs

@@ -1,16 +1,28 @@
 use super::common::*;
 use crate::DumpConfig;
+use account_utils::{eth2_keystore::Keystore, ZeroizeString};
 use clap::{Arg, ArgAction, ArgMatches, Command};
 use clap_utils::FLAG_HEADER;
+use derivative::Derivative;
+use eth2::lighthouse_vc::types::KeystoreJsonStr;
 use eth2::{lighthouse_vc::std_types::ImportKeystoreStatus, SensitiveUrl};
 use serde::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
+use types::Address;
 
 pub const CMD: &str = "import";
 pub const VALIDATORS_FILE_FLAG: &str = "validators-file";
+pub const KEYSTORE_FILE_FLAG: &str = "keystore-file";
 pub const VC_URL_FLAG: &str = "vc-url";
 pub const VC_TOKEN_FLAG: &str = "vc-token";
+pub const PASSWORD: &str = "password";
+pub const FEE_RECIPIENT: &str = "suggested-fee-recipient";
+pub const GAS_LIMIT: &str = "gas-limit";
+pub const BUILDER_PROPOSALS: &str = "builder-proposals";
+pub const BUILDER_BOOST_FACTOR: &str = "builder-boost-factor";
+pub const PREFER_BUILDER_PROPOSALS: &str = "prefer-builder-proposals";
+pub const ENABLED: &str = "enabled";
 
 pub const DETECTED_DUPLICATE_MESSAGE: &str = "Duplicate validator detected!";
 
@@ -21,15 +33,6 @@ pub fn cli_app() -> Command {
                 are defined in a JSON file which can be generated using the \"create-validators\" \
                 command.",
         )
-        .arg(
-            Arg::new("help")
-                .long("help")
-                .short('h')
-                .help("Prints help information")
-                .action(ArgAction::HelpLong)
-                .display_order(0)
-                .help_heading(FLAG_HEADER),
-        )
         .arg(
             Arg::new(VALIDATORS_FILE_FLAG)
                 .long(VALIDATORS_FILE_FLAG)
@@ -39,19 +42,32 @@ pub fn cli_app() -> Command {
                     imported to the validator client. This file is usually named \
                     \"validators.json\".",
                 )
-                .required(true)
                 .action(ArgAction::Set)
-                .display_order(0),
+                .display_order(0)
+                .required_unless_present("keystore-file")
+                .conflicts_with("keystore-file"),
+        )
+        .arg(
+            Arg::new(KEYSTORE_FILE_FLAG)
+                .long(KEYSTORE_FILE_FLAG)
+                .value_name("PATH_TO_KEYSTORE_FILE")
+                .help(
+                    "The path to a keystore JSON file to be \
+                    imported to the validator client. This file is usually created \
+                    using staking-deposit-cli or ethstaker-deposit-cli",
+                )
+                .action(ArgAction::Set)
+                .display_order(0)
+                .conflicts_with("validators-file")
+                .required_unless_present("validators-file")
+                .requires(PASSWORD),
         )
         .arg(
             Arg::new(VC_URL_FLAG)
                 .long(VC_URL_FLAG)
                 .value_name("HTTP_ADDRESS")
                 .help(
-                    "A HTTP(S) address of a validator client using the keymanager-API. \
-                    If this value is not supplied then a 'dry run' will be conducted where \
-                    no changes are made to the validator client.",
-                )
+                    "A HTTP(S) address of a validator client using the keymanager-API.")
                 .default_value("http://localhost:5062")
                 .requires(VC_TOKEN_FLAG)
                 .action(ArgAction::Set)
@@ -80,29 +96,111 @@ pub fn cli_app() -> Command {
                 )
                 .display_order(0),
         )
+        .arg(
+            Arg::new(PASSWORD)
+                .long(PASSWORD)
+                .value_name("STRING")
+                .help("Password of the keystore file.")
+                .action(ArgAction::Set)
+                .display_order(0)
+                .requires(KEYSTORE_FILE_FLAG),
+        )
+        .arg(
+            Arg::new(FEE_RECIPIENT)
+                .long(FEE_RECIPIENT)
+                .value_name("ETH1_ADDRESS")
+                .help("When provided, the imported validator will use the suggested fee recipient. Omit this flag to use the default value from the VC.")
+                .action(ArgAction::Set)
+                .display_order(0)
+                .requires(KEYSTORE_FILE_FLAG),
+        )
+        .arg(
+            Arg::new(GAS_LIMIT)
+                .long(GAS_LIMIT)
+                .value_name("UINT64")
+                .help("When provided, the imported validator will use this gas limit. It is recommended \
+                to leave this as the default value by not specifying this flag.",)
+                .action(ArgAction::Set)
+                .display_order(0)
+                .requires(KEYSTORE_FILE_FLAG),
+        )
+        .arg(
+            Arg::new(BUILDER_PROPOSALS)
+                .long(BUILDER_PROPOSALS)
+                .help("When provided, the imported validator will attempt to create \
+                blocks via builder rather than the local EL.",)
+                .value_parser(["true","false"])
+                .action(ArgAction::Set)
+                .display_order(0)
+                .requires(KEYSTORE_FILE_FLAG),
+        )
+        .arg(
+            Arg::new(BUILDER_BOOST_FACTOR)
+                .long(BUILDER_BOOST_FACTOR)
+                .value_name("UINT64")
+                .help("When provided, the imported validator will use this \
+                percentage multiplier to apply to the builder's payload value \
+                when choosing between a builder payload header and payload from \
+                the local execution node.",)
+                .action(ArgAction::Set)
+                .display_order(0)
+                .requires(KEYSTORE_FILE_FLAG),
+        )
+        .arg(
+            Arg::new(PREFER_BUILDER_PROPOSALS)
+                .long(PREFER_BUILDER_PROPOSALS)
+                .help("When provided, the imported validator will always prefer blocks \
+                constructed by builders, regardless of payload value.",)
+                .value_parser(["true","false"])
+                .action(ArgAction::Set)
+                .display_order(0)
+                .requires(KEYSTORE_FILE_FLAG),
+        )
 }
 
-#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+#[derive(Clone, PartialEq, Serialize, Deserialize, Derivative)]
+#[derivative(Debug)]
 pub struct ImportConfig {
-    pub validators_file_path: PathBuf,
+    pub validators_file_path: Option<PathBuf>,
+    pub keystore_file_path: Option<PathBuf>,
     pub vc_url: SensitiveUrl,
     pub vc_token_path: PathBuf,
     pub ignore_duplicates: bool,
+    #[derivative(Debug = "ignore")]
+    pub password: Option<ZeroizeString>,
+    pub fee_recipient: Option<Address>,
+    pub gas_limit: Option<u64>,
+    pub builder_proposals: Option<bool>,
+    pub builder_boost_factor: Option<u64>,
+    pub prefer_builder_proposals: Option<bool>,
+    pub enabled: Option<bool>,
 }
 
 impl ImportConfig {
     fn from_cli(matches: &ArgMatches) -> Result<Self, String> {
         Ok(Self {
-            validators_file_path: clap_utils::parse_required(matches, VALIDATORS_FILE_FLAG)?,
+            validators_file_path: clap_utils::parse_optional(matches, VALIDATORS_FILE_FLAG)?,
+            keystore_file_path: clap_utils::parse_optional(matches, KEYSTORE_FILE_FLAG)?,
             vc_url: clap_utils::parse_required(matches, VC_URL_FLAG)?,
             vc_token_path: clap_utils::parse_required(matches, VC_TOKEN_FLAG)?,
             ignore_duplicates: matches.get_flag(IGNORE_DUPLICATES_FLAG),
+            password: clap_utils::parse_optional(matches, PASSWORD)?,
+            fee_recipient: clap_utils::parse_optional(matches, FEE_RECIPIENT)?,
+            gas_limit: clap_utils::parse_optional(matches, GAS_LIMIT)?,
+            builder_proposals: clap_utils::parse_optional(matches, BUILDER_PROPOSALS)?,
+            builder_boost_factor: clap_utils::parse_optional(matches, BUILDER_BOOST_FACTOR)?,
+            prefer_builder_proposals: clap_utils::parse_optional(
+                matches,
+                PREFER_BUILDER_PROPOSALS,
+            )?,
+            enabled: clap_utils::parse_optional(matches, ENABLED)?,
         })
     }
 }
 
 pub async fn cli_run(matches: &ArgMatches, dump_config: DumpConfig) -> Result<(), String> {
     let config = ImportConfig::from_cli(matches)?;
+
     if dump_config.should_exit_early(&config)? {
         Ok(())
     } else {
@@ -113,27 +211,61 @@ pub async fn cli_run(matches: &ArgMatches, dump_config: DumpConfig) -> Result<()
 async fn run<'a>(config: ImportConfig) -> Result<(), String> {
     let ImportConfig {
         validators_file_path,
+        keystore_file_path,
         vc_url,
         vc_token_path,
         ignore_duplicates,
+        password,
+        fee_recipient,
+        gas_limit,
+        builder_proposals,
+        builder_boost_factor,
+        prefer_builder_proposals,
+        enabled,
     } = config;
 
-    if !validators_file_path.exists() {
-        return Err(format!("Unable to find file at {:?}", validators_file_path));
-    }
+    let validators: Vec<ValidatorSpecification> =
+        if let Some(validators_format_path) = &validators_file_path {
+            if !validators_format_path.exists() {
+                return Err(format!(
+                    "Unable to find file at {:?}",
+                    validators_format_path
+                ));
+            }
 
-    let validators_file = fs::OpenOptions::new()
-        .read(true)
-        .create(false)
-        .open(&validators_file_path)
-        .map_err(|e| format!("Unable to open {:?}: {:?}", validators_file_path, e))?;
-    let validators: Vec<ValidatorSpecification> = serde_json::from_reader(&validators_file)
-        .map_err(|e| {
-            format!(
-                "Unable to parse JSON in {:?}: {:?}",
-                validators_file_path, e
-            )
-        })?;
+            let validators_file = fs::OpenOptions::new()
+                .read(true)
+                .create(false)
+                .open(validators_format_path)
+                .map_err(|e| format!("Unable to open {:?}: {:?}", validators_format_path, e))?;
+
+            serde_json::from_reader(&validators_file).map_err(|e| {
+                format!(
+                    "Unable to parse JSON in {:?}: {:?}",
+                    validators_format_path, e
+                )
+            })?
+        } else if let Some(keystore_format_path) = &keystore_file_path {
+            vec![ValidatorSpecification {
+                voting_keystore: KeystoreJsonStr(
+                    Keystore::from_json_file(keystore_format_path).map_err(|e| format!("{e:?}"))?,
+                ),
+                voting_keystore_password: password.ok_or_else(|| {
+                    "The --password flag is required to supply the keystore password".to_string()
+                })?,
+                slashing_protection: None,
+                fee_recipient,
+                gas_limit,
+                builder_proposals,
+                builder_boost_factor,
+                prefer_builder_proposals,
+                enabled,
+            }]
+        } else {
+            return Err(format!(
+                "One of the flag --{VALIDATORS_FILE_FLAG} or --{KEYSTORE_FILE_FLAG} is required."
+            ));
+        };
 
     let count = validators.len();
 
@@ -250,7 +382,10 @@ async fn run<'a>(config: ImportConfig) -> Result<(), String> {
 pub mod tests {
     use super::*;
     use crate::create_validators::tests::TestBuilder as CreateTestBuilder;
-    use std::fs;
+    use std::{
+        fs::{self, File},
+        str::FromStr,
+    };
     use tempfile::{tempdir, TempDir};
     use validator_client::http_api::{test_utils::ApiTester, Config as HttpConfig};
 
@@ -279,10 +414,18 @@ pub mod tests {
             Self {
                 import_config: ImportConfig {
                     // This field will be overwritten later on.
-                    validators_file_path: dir.path().into(),
+                    validators_file_path: Some(dir.path().into()),
+                    keystore_file_path: Some(dir.path().into()),
                     vc_url: vc.url.clone(),
                     vc_token_path,
                     ignore_duplicates: false,
+                    password: Some(ZeroizeString::from_str("password").unwrap()),
+                    fee_recipient: None,
+                    builder_boost_factor: None,
+                    gas_limit: None,
+                    builder_proposals: None,
+                    enabled: None,
+                    prefer_builder_proposals: None,
                 },
                 vc,
                 create_dir: None,
@@ -295,6 +438,10 @@ pub mod tests {
             self
         }
 
+        pub fn get_import_config(&self) -> ImportConfig {
+            self.import_config.clone()
+        }
+
         pub async fn create_validators(mut self, count: u32, first_index: u32) -> Self {
             let create_result = CreateTestBuilder::default()
                 .mutate_config(|config| {
@@ -307,7 +454,55 @@ pub mod tests {
                 create_result.result.is_ok(),
                 "precondition: validators are created"
             );
-            self.import_config.validators_file_path = create_result.validators_file_path();
+            self.import_config.validators_file_path = Some(create_result.validators_file_path());
+            self.create_dir = Some(create_result.output_dir);
+            self
+        }
+
+        // Keystore JSON requires a different format when creating valdiators
+        pub async fn create_validators_keystore_format(
+            mut self,
+            count: u32,
+            first_index: u32,
+        ) -> Self {
+            let create_result = CreateTestBuilder::default()
+                .mutate_config(|config| {
+                    config.count = count;
+                    config.first_index = first_index;
+                })
+                .run_test()
+                .await;
+            assert!(
+                create_result.result.is_ok(),
+                "precondition: validators are created"
+            );
+
+            let validators_file_path = create_result.validators_file_path();
+
+            let validators_file = fs::OpenOptions::new()
+                .read(true)
+                .create(false)
+                .open(&validators_file_path)
+                .map_err(|e| format!("Unable to open {:?}: {:?}", validators_file_path, e))
+                .unwrap();
+
+            let validators: Vec<ValidatorSpecification> = serde_json::from_reader(&validators_file)
+                .map_err(|e| {
+                    format!(
+                        "Unable to parse JSON in {:?}: {:?}",
+                        validators_file_path, e
+                    )
+                })
+                .unwrap();
+
+            let validator = &validators[0];
+            let validator_json = validator.voting_keystore.0.clone();
+
+            let keystore_file = File::create(&validators_file_path).unwrap();
+            let _ = validator_json.to_json_writer(keystore_file);
+
+            self.import_config.keystore_file_path = Some(create_result.validators_file_path());
+            self.import_config.password = Some(validator.voting_keystore_password.clone());
             self.create_dir = Some(create_result.output_dir);
             self
         }
@@ -327,7 +522,8 @@ pub mod tests {
 
                 let local_validators: Vec<ValidatorSpecification> = {
                     let contents =
-                        fs::read_to_string(&self.import_config.validators_file_path).unwrap();
+                        fs::read_to_string(&self.import_config.validators_file_path.unwrap())
+                            .unwrap();
                     serde_json::from_str(&contents).unwrap()
                 };
                 let list_keystores_response = self.vc.client.get_keystores().await.unwrap().data;
@@ -355,6 +551,39 @@ pub mod tests {
                 vc: self.vc,
             }
         }
+
+        pub async fn run_test_keystore_format(self) -> TestResult {
+            let result = run(self.import_config.clone()).await;
+
+            if result.is_ok() {
+                self.vc.ensure_key_cache_consistency().await;
+
+                let local_keystore: Keystore =
+                    Keystore::from_json_file(&self.import_config.keystore_file_path.unwrap())
+                        .unwrap();
+
+                let list_keystores_response = self.vc.client.get_keystores().await.unwrap().data;
+
+                assert_eq!(
+                    1,
+                    list_keystores_response.len(),
+                    "vc should have exactly the number of validators imported"
+                );
+
+                let local_pubkey = local_keystore.public_key().unwrap().into();
+                let remote_validator = list_keystores_response
+                    .iter()
+                    .find(|validator| validator.validating_pubkey == local_pubkey)
+                    .expect("validator must exist on VC");
+                assert_eq!(&remote_validator.derivation_path, &local_keystore.path());
+                assert_eq!(remote_validator.readonly, Some(false));
+            }
+
+            TestResult {
+                result,
+                vc: self.vc,
+            }
+        }
     }
 
     #[must_use] // Use the `assert_ok` or `assert_err` fns to "use" this value.
@@ -445,4 +674,66 @@ pub mod tests {
             .await
             .assert_ok();
     }
+
+    #[tokio::test]
+    async fn create_one_validator_keystore_format() {
+        TestBuilder::new()
+            .await
+            .mutate_import_config(|config| {
+                // Set validators_file_path to None so that keystore_file_path is used for tests with the keystore format
+                config.validators_file_path = None;
+            })
+            .create_validators_keystore_format(1, 0)
+            .await
+            .run_test_keystore_format()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn create_one_validator_with_offset_keystore_format() {
+        TestBuilder::new()
+            .await
+            .mutate_import_config(|config| {
+                config.validators_file_path = None;
+            })
+            .create_validators_keystore_format(1, 42)
+            .await
+            .run_test_keystore_format()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn import_duplicates_when_disallowed_keystore_format() {
+        TestBuilder::new()
+            .await
+            .mutate_import_config(|config| {
+                config.validators_file_path = None;
+            })
+            .create_validators_keystore_format(1, 0)
+            .await
+            .import_validators_without_checks()
+            .await
+            .run_test_keystore_format()
+            .await
+            .assert_err_contains("DuplicateValidator");
+    }
+
+    #[tokio::test]
+    async fn import_duplicates_when_allowed_keystore_format() {
+        TestBuilder::new()
+            .await
+            .mutate_import_config(|config| {
+                config.ignore_duplicates = true;
+                config.validators_file_path = None;
+            })
+            .create_validators_keystore_format(1, 0)
+            .await
+            .import_validators_without_checks()
+            .await
+            .run_test_keystore_format()
+            .await
+            .assert_ok();
+    }
 }

validator_manager/src/lib.rs

@@ -8,7 +8,9 @@ use types::EthSpec;
 
 pub mod common;
 pub mod create_validators;
+pub mod delete_validators;
 pub mod import_validators;
+pub mod list_validators;
 pub mod move_validators;
 
 pub const CMD: &str = "validator_manager";
@@ -51,11 +53,14 @@ pub fn cli_app() -> Command {
                 .help("Prints help information")
                 .action(ArgAction::HelpLong)
                 .display_order(0)
-                .help_heading(FLAG_HEADER),
+                .help_heading(FLAG_HEADER)
+                .global(true),
         )
         .subcommand(create_validators::cli_app())
         .subcommand(import_validators::cli_app())
         .subcommand(move_validators::cli_app())
+        .subcommand(list_validators::cli_app())
+        .subcommand(delete_validators::cli_app())
 }
 
 /// Run the account manager, returning an error if the operation did not succeed.
@@ -83,6 +88,13 @@ pub fn run<E: EthSpec>(matches: &ArgMatches, env: Environment<E>) -> Result<(),
                     Some((move_validators::CMD, matches)) => {
                         move_validators::cli_run(matches, dump_config).await
                     }
+                    Some((list_validators::CMD, matches)) => {
+                        list_validators::cli_run(matches, dump_config).await
+                    }
+                    Some((delete_validators::CMD, matches)) => {
+                        delete_validators::cli_run(matches, dump_config).await
+                    }
+                    Some(("", _)) => Err("No command supplied. See --help.".to_string()),
                     Some((unknown, _)) => Err(format!(
                         "{} is not a valid {} command. See --help.",
                         unknown, CMD

validator_manager/src/list_validators.rs

@@ -0,0 +1,201 @@
+use clap::{Arg, ArgAction, ArgMatches, Command};
+use eth2::lighthouse_vc::types::SingleKeystoreResponse;
+use eth2::SensitiveUrl;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+use crate::{common::vc_http_client, DumpConfig};
+
+pub const CMD: &str = "list";
+pub const VC_URL_FLAG: &str = "vc-url";
+pub const VC_TOKEN_FLAG: &str = "vc-token";
+
+pub fn cli_app() -> Command {
+    Command::new(CMD)
+        .about("Lists all validators in a validator client using the HTTP API.")
+        .arg(
+            Arg::new(VC_URL_FLAG)
+                .long(VC_URL_FLAG)
+                .value_name("HTTP_ADDRESS")
+                .help("A HTTP(S) address of a validator client using the keymanager-API.")
+                .default_value("http://localhost:5062")
+                .requires(VC_TOKEN_FLAG)
+                .action(ArgAction::Set)
+                .display_order(0),
+        )
+        .arg(
+            Arg::new(VC_TOKEN_FLAG)
+                .long(VC_TOKEN_FLAG)
+                .value_name("PATH")
+                .help("The file containing a token required by the validator client.")
+                .action(ArgAction::Set)
+                .display_order(0),
+        )
+}
+
+#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+pub struct ListConfig {
+    pub vc_url: SensitiveUrl,
+    pub vc_token_path: PathBuf,
+}
+
+impl ListConfig {
+    fn from_cli(matches: &ArgMatches) -> Result<Self, String> {
+        Ok(Self {
+            vc_token_path: clap_utils::parse_required(matches, VC_TOKEN_FLAG)?,
+            vc_url: clap_utils::parse_required(matches, VC_URL_FLAG)?,
+        })
+    }
+}
+
+pub async fn cli_run(matches: &ArgMatches, dump_config: DumpConfig) -> Result<(), String> {
+    let config = ListConfig::from_cli(matches)?;
+    if dump_config.should_exit_early(&config)? {
+        Ok(())
+    } else {
+        run(config).await?;
+        Ok(())
+    }
+}
+
+async fn run<'a>(config: ListConfig) -> Result<Vec<SingleKeystoreResponse>, String> {
+    let ListConfig {
+        vc_url,
+        vc_token_path,
+    } = config;
+
+    let (_, validators) = vc_http_client(vc_url.clone(), &vc_token_path).await?;
+
+    println!("List of validators ({}):", validators.len());
+
+    for validator in &validators {
+        println!("{}", validator.validating_pubkey);
+    }
+
+    Ok(validators)
+}
+
+#[cfg(not(debug_assertions))]
+#[cfg(test)]
+mod test {
+    use std::{
+        fs::{self, File},
+        io::Write,
+    };
+
+    use super::*;
+    use crate::{
+        common::ValidatorSpecification, import_validators::tests::TestBuilder as ImportTestBuilder,
+    };
+    use validator_client::http_api::{test_utils::ApiTester, Config as HttpConfig};
+
+    struct TestBuilder {
+        list_config: Option<ListConfig>,
+        src_import_builder: Option<ImportTestBuilder>,
+        http_config: HttpConfig,
+        vc_token: Option<String>,
+        validators: Vec<ValidatorSpecification>,
+    }
+
+    impl TestBuilder {
+        async fn new() -> Self {
+            Self {
+                list_config: None,
+                src_import_builder: None,
+                http_config: ApiTester::default_http_config(),
+                vc_token: None,
+                validators: vec![],
+            }
+        }
+
+        async fn with_validators(mut self, count: u32, first_index: u32) -> Self {
+            let builder = ImportTestBuilder::new_with_http_config(self.http_config.clone())
+                .await
+                .create_validators(count, first_index)
+                .await;
+            self.list_config = Some(ListConfig {
+                vc_url: builder.get_import_config().vc_url,
+                vc_token_path: builder.get_import_config().vc_token_path,
+            });
+
+            self.vc_token =
+                Some(fs::read_to_string(builder.get_import_config().vc_token_path).unwrap());
+
+            let local_validators: Vec<ValidatorSpecification> = {
+                let contents =
+                    fs::read_to_string(builder.get_import_config().validators_file_path.unwrap())
+                        .unwrap();
+                serde_json::from_str(&contents).unwrap()
+            };
+
+            self.validators = local_validators.clone();
+            self.src_import_builder = Some(builder);
+            self
+        }
+
+        pub async fn run_test(self) -> TestResult {
+            let import_test_result = self.src_import_builder.unwrap().run_test().await;
+            assert!(import_test_result.result.is_ok());
+
+            let path = self.list_config.clone().unwrap().vc_token_path;
+            let parent = path.parent().unwrap();
+
+            fs::create_dir_all(parent).expect("Was not able to create parent directory");
+
+            File::options()
+                .write(true)
+                .read(true)
+                .create(true)
+                .truncate(true)
+                .open(path)
+                .unwrap()
+                .write_all(self.vc_token.clone().unwrap().as_bytes())
+                .unwrap();
+
+            let result = run(self.list_config.clone().unwrap()).await;
+
+            if result.is_ok() {
+                let result_ref = result.as_ref().unwrap();
+
+                for local_validator in &self.validators {
+                    let local_keystore = &local_validator.voting_keystore.0;
+                    let local_pubkey = local_keystore.public_key().unwrap();
+                    assert!(
+                        result_ref
+                            .iter()
+                            .any(|validator| validator.validating_pubkey
+                                == local_pubkey.clone().into()),
+                        "local validator pubkey not found in result"
+                    );
+                }
+
+                return TestResult { result: Ok(()) };
+            }
+
+            TestResult {
+                result: Err(result.unwrap_err()),
+            }
+        }
+    }
+
+    #[must_use] // Use the `assert_ok` or `assert_err` fns to "use" this value.
+    struct TestResult {
+        result: Result<(), String>,
+    }
+
+    impl TestResult {
+        fn assert_ok(self) {
+            assert_eq!(self.result, Ok(()))
+        }
+    }
+    #[tokio::test]
+    async fn list_all_validators() {
+        TestBuilder::new()
+            .await
+            .with_validators(3, 0)
+            .await
+            .run_test()
+            .await
+            .assert_ok();
+    }
+}

validator_manager/src/move_validators.rs

@@ -2,7 +2,6 @@ use super::common::*;
 use crate::DumpConfig;
 use account_utils::{read_password_from_user, ZeroizeString};
 use clap::{Arg, ArgAction, ArgMatches, Command};
-use clap_utils::FLAG_HEADER;
 use eth2::{
     lighthouse_vc::{
         std_types::{
@@ -75,15 +74,6 @@ pub fn cli_app() -> Command {
                 command. This command only supports validators signing via a keystore on the local \
                 file system (i.e., not Web3Signer validators).",
         )
-        .arg(
-            Arg::new("help")
-                .long("help")
-                .short('h')
-                .help("Prints help information")
-                .action(ArgAction::HelpLong)
-                .display_order(0)
-                .help_heading(FLAG_HEADER),
-        )
         .arg(
             Arg::new(SRC_VC_URL_FLAG)
                 .long(SRC_VC_URL_FLAG)