Split the VC into crates making it more modular (#6453)

* Starting to modularize the VC

* Revert changes to eth2

* More progress

* More progress

* Compiles

* Merge latest unstable and make it compile

* Fix some lints

* Tests compile

* Merge latest unstable

* Remove unnecessary deps

* Merge latest unstable

* Correct release tests

* Merge latest unstable

* Merge remote-tracking branch 'origin/unstable' into modularize-vc

* Merge branch 'unstable' into modularize-vc

* Revert unnecessary cargo lock changes

* Update validator_client/beacon_node_fallback/Cargo.toml

* Update validator_client/http_metrics/Cargo.toml

* Update validator_client/http_metrics/src/lib.rs

* Update validator_client/initialized_validators/Cargo.toml

* Update validator_client/signing_method/Cargo.toml

* Update validator_client/validator_metrics/Cargo.toml

* Update validator_client/validator_services/Cargo.toml

* Update validator_client/validator_store/Cargo.toml

* Update validator_client/validator_store/src/lib.rs

* Merge remote-tracking branch 'origin/unstable' into modularize-vc

* Fix format string

* Rename doppelganger trait

* Don't drop the tempdir

* Cargo fmt

validator_client/http_api/src/api_secret.rs

@@ -0,0 +1,107 @@
+use filesystem::create_with_600_perms;
+use rand::distributions::Alphanumeric;
+use rand::{thread_rng, Rng};
+use std::fs;
+use std::path::{Path, PathBuf};
+use warp::Filter;
+
+/// The name of the file which stores the API token.
+pub const PK_FILENAME: &str = "api-token.txt";
+
+pub const PK_LEN: usize = 33;
+
+/// Contains a randomly generated string which is used for authorization of requests to the HTTP API.
+///
+/// Provides convenience functions to ultimately provide:
+///
+///  - Verification of proof-of-knowledge of the public key in `self` for incoming HTTP requests,
+///    via the `Authorization` header.
+///
+///  The aforementioned scheme was first defined here:
+///
+///  https://github.com/sigp/lighthouse/issues/1269#issuecomment-649879855
+///
+///  This scheme has since been tweaked to remove VC response signing and secp256k1 key generation.
+///  https://github.com/sigp/lighthouse/issues/5423
+pub struct ApiSecret {
+    pk: String,
+    pk_path: PathBuf,
+}
+
+impl ApiSecret {
+    /// If the public key is already on-disk, use it.
+    ///
+    /// The provided `dir` is a directory containing `PK_FILENAME`.
+    ///
+    /// If the public key file is missing on disk, create a new key and
+    /// write it to disk (over-writing any existing files).
+    pub fn create_or_open<P: AsRef<Path>>(dir: P) -> Result<Self, String> {
+        let pk_path = dir.as_ref().join(PK_FILENAME);
+
+        if !pk_path.exists() {
+            let length = PK_LEN;
+            let pk: String = thread_rng()
+                .sample_iter(&Alphanumeric)
+                .take(length)
+                .map(char::from)
+                .collect();
+
+            // Create and write the public key to file with appropriate permissions
+            create_with_600_perms(&pk_path, pk.to_string().as_bytes()).map_err(|e| {
+                format!(
+                    "Unable to create file with permissions for {:?}: {:?}",
+                    pk_path, e
+                )
+            })?;
+        }
+
+        let pk = fs::read(&pk_path)
+            .map_err(|e| format!("cannot read {}: {}", PK_FILENAME, e))?
+            .iter()
+            .map(|&c| char::from(c))
+            .collect();
+
+        Ok(Self { pk, pk_path })
+    }
+
+    /// Returns the API token.
+    pub fn api_token(&self) -> String {
+        self.pk.clone()
+    }
+
+    /// Returns the path for the API token file
+    pub fn api_token_path(&self) -> PathBuf {
+        self.pk_path.clone()
+    }
+
+    /// Returns the values of the `Authorization` header which indicate a valid incoming HTTP
+    /// request.
+    ///
+    /// For backwards-compatibility we accept the token in a basic authentication style, but this is
+    /// technically invalid according to RFC 7617 because the token is not a base64-encoded username
+    /// and password. As such, bearer authentication should be preferred.
+    fn auth_header_values(&self) -> Vec<String> {
+        vec![
+            format!("Basic {}", self.api_token()),
+            format!("Bearer {}", self.api_token()),
+        ]
+    }
+
+    /// Returns a `warp` header which filters out request that have a missing or inaccurate
+    /// `Authorization` header.
+    pub fn authorization_header_filter(&self) -> warp::filters::BoxedFilter<()> {
+        let expected = self.auth_header_values();
+        warp::any()
+            .map(move || expected.clone())
+            .and(warp::filters::header::header("Authorization"))
+            .and_then(move |expected: Vec<String>, header: String| async move {
+                if expected.contains(&header) {
+                    Ok(())
+                } else {
+                    Err(warp_utils::reject::invalid_auth(header))
+                }
+            })
+            .untuple_one()
+            .boxed()
+    }
+}

validator_client/http_api/src/create_signed_voluntary_exit.rs

@@ -0,0 +1,70 @@
+use bls::{PublicKey, PublicKeyBytes};
+use eth2::types::GenericResponse;
+use slog::{info, Logger};
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use types::{Epoch, EthSpec, SignedVoluntaryExit, VoluntaryExit};
+use validator_store::ValidatorStore;
+
+pub async fn create_signed_voluntary_exit<T: 'static + SlotClock + Clone, E: EthSpec>(
+    pubkey: PublicKey,
+    maybe_epoch: Option<Epoch>,
+    validator_store: Arc<ValidatorStore<T, E>>,
+    slot_clock: T,
+    log: Logger,
+) -> Result<GenericResponse<SignedVoluntaryExit>, warp::Rejection> {
+    let epoch = match maybe_epoch {
+        Some(epoch) => epoch,
+        None => get_current_epoch::<T, E>(slot_clock).ok_or_else(|| {
+            warp_utils::reject::custom_server_error("Unable to determine current epoch".to_string())
+        })?,
+    };
+
+    let pubkey_bytes = PublicKeyBytes::from(pubkey);
+    if !validator_store.has_validator(&pubkey_bytes) {
+        return Err(warp_utils::reject::custom_not_found(format!(
+            "{} is disabled or not managed by this validator client",
+            pubkey_bytes.as_hex_string()
+        )));
+    }
+
+    let validator_index = validator_store
+        .validator_index(&pubkey_bytes)
+        .ok_or_else(|| {
+            warp_utils::reject::custom_not_found(format!(
+                "The validator index for {} is not known. The validator client \
+                may still be initializing or the validator has not yet had a \
+                deposit processed.",
+                pubkey_bytes.as_hex_string()
+            ))
+        })?;
+
+    let voluntary_exit = VoluntaryExit {
+        epoch,
+        validator_index,
+    };
+
+    info!(
+        log,
+        "Signing voluntary exit";
+        "validator" => pubkey_bytes.as_hex_string(),
+        "epoch" => epoch
+    );
+
+    let signed_voluntary_exit = validator_store
+        .sign_voluntary_exit(pubkey_bytes, voluntary_exit)
+        .await
+        .map_err(|e| {
+            warp_utils::reject::custom_server_error(format!(
+                "Failed to sign voluntary exit: {:?}",
+                e
+            ))
+        })?;
+
+    Ok(GenericResponse::from(signed_voluntary_exit))
+}
+
+/// Calculates the current epoch from the genesis time and current time.
+fn get_current_epoch<T: 'static + SlotClock + Clone, E: EthSpec>(slot_clock: T) -> Option<Epoch> {
+    slot_clock.now().map(|s| s.epoch(E::slots_per_epoch()))
+}

validator_client/http_api/src/create_validator.rs

@@ -0,0 +1,218 @@
+use account_utils::validator_definitions::{PasswordStorage, ValidatorDefinition};
+use account_utils::{
+    eth2_keystore::Keystore,
+    eth2_wallet::{bip39::Mnemonic, WalletBuilder},
+    random_mnemonic, random_password, ZeroizeString,
+};
+use eth2::lighthouse_vc::types::{self as api_types};
+use slot_clock::SlotClock;
+use std::path::{Path, PathBuf};
+use types::ChainSpec;
+use types::EthSpec;
+use validator_dir::{keystore_password_path, Builder as ValidatorDirBuilder};
+use validator_store::ValidatorStore;
+
+/// Create some validator EIP-2335 keystores and store them on disk. Then, enroll the validators in
+/// this validator client.
+///
+/// Returns the list of created validators and the mnemonic used to derive them via EIP-2334.
+///
+/// ## Detail
+///
+/// If `mnemonic_opt` is not supplied it will be randomly generated and returned in the response.
+///
+/// If `key_derivation_path_offset` is supplied then the EIP-2334 validator index will start at
+/// this point.
+pub async fn create_validators_mnemonic<P: AsRef<Path>, T: 'static + SlotClock, E: EthSpec>(
+    mnemonic_opt: Option<Mnemonic>,
+    key_derivation_path_offset: Option<u32>,
+    validator_requests: &[api_types::ValidatorRequest],
+    validator_dir: P,
+    secrets_dir: Option<PathBuf>,
+    validator_store: &ValidatorStore<T, E>,
+    spec: &ChainSpec,
+) -> Result<(Vec<api_types::CreatedValidator>, Mnemonic), warp::Rejection> {
+    let mnemonic = mnemonic_opt.unwrap_or_else(random_mnemonic);
+
+    let wallet_password = random_password();
+    let mut wallet =
+        WalletBuilder::from_mnemonic(&mnemonic, wallet_password.as_bytes(), String::new())
+            .and_then(|builder| builder.build())
+            .map_err(|e| {
+                warp_utils::reject::custom_server_error(format!(
+                    "unable to create EIP-2386 wallet: {:?}",
+                    e
+                ))
+            })?;
+
+    if let Some(nextaccount) = key_derivation_path_offset {
+        wallet.set_nextaccount(nextaccount).map_err(|e| {
+            warp_utils::reject::custom_server_error(format!(
+                "unable to set wallet nextaccount: {:?}",
+                e
+            ))
+        })?;
+    }
+
+    let mut validators = Vec::with_capacity(validator_requests.len());
+
+    for request in validator_requests {
+        let voting_password = random_password();
+        let withdrawal_password = random_password();
+        let voting_password_string = ZeroizeString::from(
+            String::from_utf8(voting_password.as_bytes().to_vec()).map_err(|e| {
+                warp_utils::reject::custom_server_error(format!(
+                    "locally generated password is not utf8: {:?}",
+                    e
+                ))
+            })?,
+        );
+
+        let mut keystores = wallet
+            .next_validator(
+                wallet_password.as_bytes(),
+                voting_password.as_bytes(),
+                withdrawal_password.as_bytes(),
+            )
+            .map_err(|e| {
+                warp_utils::reject::custom_server_error(format!(
+                    "unable to create validator keys: {:?}",
+                    e
+                ))
+            })?;
+
+        keystores
+            .voting
+            .set_description(request.description.clone());
+        keystores
+            .withdrawal
+            .set_description(request.description.clone());
+
+        let voting_pubkey = format!("0x{}", keystores.voting.pubkey())
+            .parse()
+            .map_err(|e| {
+                warp_utils::reject::custom_server_error(format!(
+                    "created invalid public key: {:?}",
+                    e
+                ))
+            })?;
+
+        let voting_password_storage =
+            get_voting_password_storage(&secrets_dir, &keystores.voting, &voting_password_string)?;
+
+        let validator_dir = ValidatorDirBuilder::new(validator_dir.as_ref().into())
+            .password_dir_opt(secrets_dir.clone())
+            .voting_keystore(keystores.voting, voting_password.as_bytes())
+            .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
+            .create_eth1_tx_data(request.deposit_gwei, spec)
+            .store_withdrawal_keystore(false)
+            .build()
+            .map_err(|e| {
+                warp_utils::reject::custom_server_error(format!(
+                    "failed to build validator directory: {:?}",
+                    e
+                ))
+            })?;
+
+        let eth1_deposit_data = validator_dir
+            .eth1_deposit_data()
+            .map_err(|e| {
+                warp_utils::reject::custom_server_error(format!(
+                    "failed to read local deposit data: {:?}",
+                    e
+                ))
+            })?
+            .ok_or_else(|| {
+                warp_utils::reject::custom_server_error(
+                    "failed to create local deposit data: {:?}".to_string(),
+                )
+            })?;
+
+        if eth1_deposit_data.deposit_data.amount != request.deposit_gwei {
+            return Err(warp_utils::reject::custom_server_error(format!(
+                "invalid deposit_gwei {}, expected {}",
+                eth1_deposit_data.deposit_data.amount, request.deposit_gwei
+            )));
+        }
+
+        // Drop validator dir so that `add_validator_keystore` can re-lock the keystore.
+        let voting_keystore_path = validator_dir.voting_keystore_path();
+        drop(validator_dir);
+
+        validator_store
+            .add_validator_keystore(
+                voting_keystore_path,
+                voting_password_storage,
+                request.enable,
+                request.graffiti.clone(),
+                request.suggested_fee_recipient,
+                request.gas_limit,
+                request.builder_proposals,
+                request.builder_boost_factor,
+                request.prefer_builder_proposals,
+            )
+            .await
+            .map_err(|e| {
+                warp_utils::reject::custom_server_error(format!(
+                    "failed to initialize validator: {:?}",
+                    e
+                ))
+            })?;
+
+        validators.push(api_types::CreatedValidator {
+            enabled: request.enable,
+            description: request.description.clone(),
+            graffiti: request.graffiti.clone(),
+            suggested_fee_recipient: request.suggested_fee_recipient,
+            gas_limit: request.gas_limit,
+            builder_proposals: request.builder_proposals,
+            voting_pubkey,
+            eth1_deposit_tx_data: serde_utils::hex::encode(&eth1_deposit_data.rlp),
+            deposit_gwei: request.deposit_gwei,
+        });
+    }
+
+    Ok((validators, mnemonic))
+}
+
+pub async fn create_validators_web3signer<T: 'static + SlotClock, E: EthSpec>(
+    validators: Vec<ValidatorDefinition>,
+    validator_store: &ValidatorStore<T, E>,
+) -> Result<(), warp::Rejection> {
+    for validator in validators {
+        validator_store
+            .add_validator(validator)
+            .await
+            .map_err(|e| {
+                warp_utils::reject::custom_server_error(format!(
+                    "failed to initialize validator: {:?}",
+                    e
+                ))
+            })?;
+    }
+
+    Ok(())
+}
+
+/// Attempts to return a `PasswordStorage::File` if `secrets_dir` is defined.
+/// Otherwise, returns a `PasswordStorage::ValidatorDefinitions`.
+pub fn get_voting_password_storage(
+    secrets_dir: &Option<PathBuf>,
+    voting_keystore: &Keystore,
+    voting_password_string: &ZeroizeString,
+) -> Result<PasswordStorage, warp::Rejection> {
+    if let Some(secrets_dir) = &secrets_dir {
+        let password_path = keystore_password_path(secrets_dir, voting_keystore);
+        if password_path.exists() {
+            Err(warp_utils::reject::custom_server_error(
+                "Duplicate keystore password path".to_string(),
+            ))
+        } else {
+            Ok(PasswordStorage::File(password_path))
+        }
+    } else {
+        Ok(PasswordStorage::ValidatorDefinitions(
+            voting_password_string.clone(),
+        ))
+    }
+}

validator_client/http_api/src/graffiti.rs

@@ -0,0 +1,80 @@
+use bls::PublicKey;
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use types::{graffiti::GraffitiString, EthSpec, Graffiti};
+use validator_store::ValidatorStore;
+
+pub fn get_graffiti<T: 'static + SlotClock + Clone, E: EthSpec>(
+    validator_pubkey: PublicKey,
+    validator_store: Arc<ValidatorStore<T, E>>,
+    graffiti_flag: Option<Graffiti>,
+) -> Result<Graffiti, warp::Rejection> {
+    let initialized_validators_rw_lock = validator_store.initialized_validators();
+    let initialized_validators = initialized_validators_rw_lock.read();
+    match initialized_validators.validator(&validator_pubkey.compress()) {
+        None => Err(warp_utils::reject::custom_not_found(
+            "The key was not found on the server".to_string(),
+        )),
+        Some(_) => {
+            let Some(graffiti) = initialized_validators.graffiti(&validator_pubkey.into()) else {
+                return graffiti_flag.ok_or(warp_utils::reject::custom_server_error(
+                    "No graffiti found, unable to return the process-wide default".to_string(),
+                ));
+            };
+            Ok(graffiti)
+        }
+    }
+}
+
+pub fn set_graffiti<T: 'static + SlotClock + Clone, E: EthSpec>(
+    validator_pubkey: PublicKey,
+    graffiti: GraffitiString,
+    validator_store: Arc<ValidatorStore<T, E>>,
+) -> Result<(), warp::Rejection> {
+    let initialized_validators_rw_lock = validator_store.initialized_validators();
+    let mut initialized_validators = initialized_validators_rw_lock.write();
+    match initialized_validators.validator(&validator_pubkey.compress()) {
+        None => Err(warp_utils::reject::custom_not_found(
+            "The key was not found on the server, nothing to update".to_string(),
+        )),
+        Some(initialized_validator) => {
+            if initialized_validator.get_graffiti() == Some(graffiti.clone().into()) {
+                Ok(())
+            } else {
+                initialized_validators
+                    .set_graffiti(&validator_pubkey, graffiti)
+                    .map_err(|_| {
+                        warp_utils::reject::custom_server_error(
+                            "A graffiti was found, but failed to be updated.".to_string(),
+                        )
+                    })
+            }
+        }
+    }
+}
+
+pub fn delete_graffiti<T: 'static + SlotClock + Clone, E: EthSpec>(
+    validator_pubkey: PublicKey,
+    validator_store: Arc<ValidatorStore<T, E>>,
+) -> Result<(), warp::Rejection> {
+    let initialized_validators_rw_lock = validator_store.initialized_validators();
+    let mut initialized_validators = initialized_validators_rw_lock.write();
+    match initialized_validators.validator(&validator_pubkey.compress()) {
+        None => Err(warp_utils::reject::custom_not_found(
+            "The key was not found on the server, nothing to delete".to_string(),
+        )),
+        Some(initialized_validator) => {
+            if initialized_validator.get_graffiti().is_none() {
+                Ok(())
+            } else {
+                initialized_validators
+                    .delete_graffiti(&validator_pubkey)
+                    .map_err(|_| {
+                        warp_utils::reject::custom_server_error(
+                            "A graffiti was found, but failed to be removed.".to_string(),
+                        )
+                    })
+            }
+        }
+    }
+}

validator_client/http_api/src/keystores.rs

@@ -0,0 +1,379 @@
+//! Implementation of the standard keystore management API.
+use account_utils::{validator_definitions::PasswordStorage, ZeroizeString};
+use eth2::lighthouse_vc::{
+    std_types::{
+        DeleteKeystoreStatus, DeleteKeystoresRequest, DeleteKeystoresResponse,
+        ImportKeystoreStatus, ImportKeystoresRequest, ImportKeystoresResponse, InterchangeJsonStr,
+        KeystoreJsonStr, ListKeystoresResponse, SingleKeystoreResponse, Status,
+    },
+    types::{ExportKeystoresResponse, SingleExportKeystoresResponse},
+};
+use eth2_keystore::Keystore;
+use initialized_validators::{Error, InitializedValidators};
+use signing_method::SigningMethod;
+use slog::{info, warn, Logger};
+use slot_clock::SlotClock;
+use std::path::PathBuf;
+use std::sync::Arc;
+use task_executor::TaskExecutor;
+use tokio::runtime::Handle;
+use types::{EthSpec, PublicKeyBytes};
+use validator_dir::{keystore_password_path, Builder as ValidatorDirBuilder};
+use validator_store::ValidatorStore;
+use warp::Rejection;
+use warp_utils::reject::{custom_bad_request, custom_server_error};
+
+pub fn list<T: SlotClock + 'static, E: EthSpec>(
+    validator_store: Arc<ValidatorStore<T, E>>,
+) -> ListKeystoresResponse {
+    let initialized_validators_rwlock = validator_store.initialized_validators();
+    let initialized_validators = initialized_validators_rwlock.read();
+
+    let keystores = initialized_validators
+        .validator_definitions()
+        .iter()
+        .filter(|def| def.enabled)
+        .map(|def| {
+            let validating_pubkey = def.voting_public_key.compress();
+
+            let (derivation_path, readonly) = initialized_validators
+                .signing_method(&validating_pubkey)
+                .map_or((None, None), |signing_method| match *signing_method {
+                    SigningMethod::LocalKeystore {
+                        ref voting_keystore,
+                        ..
+                    } => (voting_keystore.path(), Some(false)),
+                    SigningMethod::Web3Signer { .. } => (None, Some(true)),
+                });
+
+            SingleKeystoreResponse {
+                validating_pubkey,
+                derivation_path,
+                readonly,
+            }
+        })
+        .collect::<Vec<_>>();
+
+    ListKeystoresResponse { data: keystores }
+}
+
+pub fn import<T: SlotClock + 'static, E: EthSpec>(
+    request: ImportKeystoresRequest,
+    validator_dir: PathBuf,
+    secrets_dir: Option<PathBuf>,
+    validator_store: Arc<ValidatorStore<T, E>>,
+    task_executor: TaskExecutor,
+    log: Logger,
+) -> Result<ImportKeystoresResponse, Rejection> {
+    // Check request validity. This is the only cases in which we should return a 4xx code.
+    if request.keystores.len() != request.passwords.len() {
+        return Err(custom_bad_request(format!(
+            "mismatched numbers of keystores ({}) and passwords ({})",
+            request.keystores.len(),
+            request.passwords.len(),
+        )));
+    }
+
+    // Import slashing protection data before keystores, so that new keystores don't start signing
+    // without it. Do not return early on failure, propagate the failure to each key.
+    let slashing_protection_status =
+        if let Some(InterchangeJsonStr(slashing_protection)) = request.slashing_protection {
+            // Warn for missing slashing protection.
+            for KeystoreJsonStr(ref keystore) in &request.keystores {
+                if let Some(public_key) = keystore.public_key() {
+                    let pubkey_bytes = public_key.compress();
+                    if !slashing_protection
+                        .data
+                        .iter()
+                        .any(|data| data.pubkey == pubkey_bytes)
+                    {
+                        warn!(
+                            log,
+                            "Slashing protection data not provided";
+                            "public_key" => ?public_key,
+                        );
+                    }
+                }
+            }
+
+            validator_store.import_slashing_protection(slashing_protection)
+        } else {
+            warn!(log, "No slashing protection data provided with keystores");
+            Ok(())
+        };
+
+    // Import each keystore. Some keystores may fail to be imported, so we record a status for each.
+    let mut statuses = Vec::with_capacity(request.keystores.len());
+
+    for (KeystoreJsonStr(keystore), password) in request
+        .keystores
+        .into_iter()
+        .zip(request.passwords.into_iter())
+    {
+        let pubkey_str = keystore.pubkey().to_string();
+
+        let status = if let Err(e) = &slashing_protection_status {
+            // Slashing protection import failed, do not attempt to import the key. Record an
+            // error status.
+            Status::error(
+                ImportKeystoreStatus::Error,
+                format!("slashing protection import failed: {:?}", e),
+            )
+        } else if let Some(handle) = task_executor.handle() {
+            // Import the keystore.
+            match import_single_keystore(
+                keystore,
+                password,
+                validator_dir.clone(),
+                secrets_dir.clone(),
+                &validator_store,
+                handle,
+            ) {
+                Ok(status) => Status::ok(status),
+                Err(e) => {
+                    warn!(
+                        log,
+                        "Error importing keystore, skipped";
+                        "pubkey" => pubkey_str,
+                        "error" => ?e,
+                    );
+                    Status::error(ImportKeystoreStatus::Error, e)
+                }
+            }
+        } else {
+            Status::error(
+                ImportKeystoreStatus::Error,
+                "validator client shutdown".into(),
+            )
+        };
+        statuses.push(status);
+    }
+
+    let successful_import = statuses
+        .iter()
+        .filter(|status| matches!(status.status, ImportKeystoreStatus::Imported))
+        .count();
+
+    if successful_import > 0 {
+        info!(
+            log,
+            "Imported keystores via standard HTTP API";
+            "count" => successful_import,
+        );
+    }
+
+    Ok(ImportKeystoresResponse { data: statuses })
+}
+
+fn import_single_keystore<T: SlotClock + 'static, E: EthSpec>(
+    keystore: Keystore,
+    password: ZeroizeString,
+    validator_dir_path: PathBuf,
+    secrets_dir: Option<PathBuf>,
+    validator_store: &ValidatorStore<T, E>,
+    handle: Handle,
+) -> Result<ImportKeystoreStatus, String> {
+    // Check if the validator key already exists, erroring if it is a remote signer validator.
+    let pubkey = keystore
+        .public_key()
+        .ok_or_else(|| format!("invalid pubkey: {}", keystore.pubkey()))?;
+    if let Some(def) = validator_store
+        .initialized_validators()
+        .read()
+        .validator_definitions()
+        .iter()
+        .find(|def| def.voting_public_key == pubkey)
+    {
+        if !def.signing_definition.is_local_keystore() {
+            return Err("cannot import duplicate of existing remote signer validator".into());
+        } else if def.enabled {
+            return Ok(ImportKeystoreStatus::Duplicate);
+        }
+    }
+
+    let password_storage = if let Some(secrets_dir) = &secrets_dir {
+        let password_path = keystore_password_path(secrets_dir, &keystore);
+        if password_path.exists() {
+            return Ok(ImportKeystoreStatus::Duplicate);
+        }
+        PasswordStorage::File(password_path)
+    } else {
+        PasswordStorage::ValidatorDefinitions(password.clone())
+    };
+
+    // Check that the password is correct.
+    // In future we should re-structure to avoid the double decryption here. It's not as simple
+    // as removing this check because `add_validator_keystore` will break if provided with an
+    // invalid validator definition (`update_validators` will get stuck trying to decrypt with the
+    // wrong password indefinitely).
+    keystore
+        .decrypt_keypair(password.as_ref())
+        .map_err(|e| format!("incorrect password: {:?}", e))?;
+
+    let validator_dir = ValidatorDirBuilder::new(validator_dir_path)
+        .password_dir_opt(secrets_dir)
+        .voting_keystore(keystore, password.as_ref())
+        .store_withdrawal_keystore(false)
+        .build()
+        .map_err(|e| format!("failed to build validator directory: {:?}", e))?;
+
+    // Drop validator dir so that `add_validator_keystore` can re-lock the keystore.
+    let voting_keystore_path = validator_dir.voting_keystore_path();
+    drop(validator_dir);
+
+    handle
+        .block_on(validator_store.add_validator_keystore(
+            voting_keystore_path,
+            password_storage,
+            true,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+        ))
+        .map_err(|e| format!("failed to initialize validator: {:?}", e))?;
+
+    Ok(ImportKeystoreStatus::Imported)
+}
+
+pub fn delete<T: SlotClock + 'static, E: EthSpec>(
+    request: DeleteKeystoresRequest,
+    validator_store: Arc<ValidatorStore<T, E>>,
+    task_executor: TaskExecutor,
+    log: Logger,
+) -> Result<DeleteKeystoresResponse, Rejection> {
+    let export_response = export(request, validator_store, task_executor, log.clone())?;
+
+    // Check the status is Deleted to confirm deletion is successful, then only display the log
+    let successful_deletion = export_response
+        .data
+        .iter()
+        .filter(|response| matches!(response.status.status, DeleteKeystoreStatus::Deleted))
+        .count();
+
+    if successful_deletion > 0 {
+        info!(
+            log,
+            "Deleted keystore via standard HTTP API";
+            "count" => successful_deletion,
+        );
+    }
+
+    Ok(DeleteKeystoresResponse {
+        data: export_response
+            .data
+            .into_iter()
+            .map(|response| response.status)
+            .collect(),
+        slashing_protection: export_response.slashing_protection,
+    })
+}
+
+pub fn export<T: SlotClock + 'static, E: EthSpec>(
+    request: DeleteKeystoresRequest,
+    validator_store: Arc<ValidatorStore<T, E>>,
+    task_executor: TaskExecutor,
+    log: Logger,
+) -> Result<ExportKeystoresResponse, Rejection> {
+    // Remove from initialized validators.
+    let initialized_validators_rwlock = validator_store.initialized_validators();
+    let mut initialized_validators = initialized_validators_rwlock.write();
+
+    let mut responses = request
+        .pubkeys
+        .iter()
+        .map(|pubkey_bytes| {
+            match delete_single_keystore(
+                pubkey_bytes,
+                &mut initialized_validators,
+                task_executor.clone(),
+            ) {
+                Ok(status) => status,
+                Err(error) => {
+                    warn!(
+                        log,
+                        "Error deleting keystore";
+                        "pubkey" => ?pubkey_bytes,
+                        "error" => ?error,
+                    );
+                    SingleExportKeystoresResponse {
+                        status: Status::error(DeleteKeystoreStatus::Error, error),
+                        validating_keystore: None,
+                        validating_keystore_password: None,
+                    }
+                }
+            }
+        })
+        .collect::<Vec<_>>();
+
+    // Use `update_validators` to update the key cache. It is safe to let the key cache get a bit out
+    // of date as it resets when it can't be decrypted. We update it just a single time to avoid
+    // continually resetting it after each key deletion.
+    if let Some(handle) = task_executor.handle() {
+        handle
+            .block_on(initialized_validators.update_validators())
+            .map_err(|e| custom_server_error(format!("unable to update key cache: {:?}", e)))?;
+    }
+
+    // Export the slashing protection data.
+    let slashing_protection = validator_store
+        .export_slashing_protection_for_keys(&request.pubkeys)
+        .map_err(|e| {
+            custom_server_error(format!("error exporting slashing protection: {:?}", e))
+        })?;
+
+    // Update stasuses based on availability of slashing protection data.
+    for (pubkey, response) in request.pubkeys.iter().zip(responses.iter_mut()) {
+        if response.status.status == DeleteKeystoreStatus::NotFound
+            && slashing_protection
+                .data
+                .iter()
+                .any(|interchange_data| interchange_data.pubkey == *pubkey)
+        {
+            response.status.status = DeleteKeystoreStatus::NotActive;
+        }
+    }
+
+    Ok(ExportKeystoresResponse {
+        data: responses,
+        slashing_protection,
+    })
+}
+
+fn delete_single_keystore(
+    pubkey_bytes: &PublicKeyBytes,
+    initialized_validators: &mut InitializedValidators,
+    task_executor: TaskExecutor,
+) -> Result<SingleExportKeystoresResponse, String> {
+    if let Some(handle) = task_executor.handle() {
+        let pubkey = pubkey_bytes
+            .decompress()
+            .map_err(|e| format!("invalid pubkey, {:?}: {:?}", pubkey_bytes, e))?;
+
+        match handle.block_on(initialized_validators.delete_definition_and_keystore(&pubkey, true))
+        {
+            Ok(Some(keystore_and_password)) => Ok(SingleExportKeystoresResponse {
+                status: Status::ok(DeleteKeystoreStatus::Deleted),
+                validating_keystore: Some(KeystoreJsonStr(keystore_and_password.keystore)),
+                validating_keystore_password: keystore_and_password.password,
+            }),
+            Ok(None) => Ok(SingleExportKeystoresResponse {
+                status: Status::ok(DeleteKeystoreStatus::Deleted),
+                validating_keystore: None,
+                validating_keystore_password: None,
+            }),
+            Err(e) => match e {
+                Error::ValidatorNotInitialized(_) => Ok(SingleExportKeystoresResponse {
+                    status: Status::ok(DeleteKeystoreStatus::NotFound),
+                    validating_keystore: None,
+                    validating_keystore_password: None,
+                }),
+                _ => Err(format!("unable to disable and delete: {:?}", e)),
+            },
+        }
+    } else {
+        Err("validator client shutdown".into())
+    }
+}

validator_client/http_api/src/remotekeys.rs

@@ -0,0 +1,218 @@
+//! Implementation of the standard remotekey management API.
+use account_utils::validator_definitions::{
+    SigningDefinition, ValidatorDefinition, Web3SignerDefinition,
+};
+use eth2::lighthouse_vc::std_types::{
+    DeleteRemotekeyStatus, DeleteRemotekeysRequest, DeleteRemotekeysResponse,
+    ImportRemotekeyStatus, ImportRemotekeysRequest, ImportRemotekeysResponse,
+    ListRemotekeysResponse, SingleListRemotekeysResponse, Status,
+};
+use initialized_validators::{Error, InitializedValidators};
+use slog::{info, warn, Logger};
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use task_executor::TaskExecutor;
+use tokio::runtime::Handle;
+use types::{EthSpec, PublicKeyBytes};
+use url::Url;
+use validator_store::ValidatorStore;
+use warp::Rejection;
+use warp_utils::reject::custom_server_error;
+
+pub fn list<T: SlotClock + 'static, E: EthSpec>(
+    validator_store: Arc<ValidatorStore<T, E>>,
+) -> ListRemotekeysResponse {
+    let initialized_validators_rwlock = validator_store.initialized_validators();
+    let initialized_validators = initialized_validators_rwlock.read();
+
+    let keystores = initialized_validators
+        .validator_definitions()
+        .iter()
+        .filter(|def| def.enabled)
+        .filter_map(|def| {
+            let validating_pubkey = def.voting_public_key.compress();
+
+            match &def.signing_definition {
+                SigningDefinition::LocalKeystore { .. } => None,
+                SigningDefinition::Web3Signer(Web3SignerDefinition { url, .. }) => {
+                    Some(SingleListRemotekeysResponse {
+                        pubkey: validating_pubkey,
+                        url: url.clone(),
+                        readonly: false,
+                    })
+                }
+            }
+        })
+        .collect::<Vec<_>>();
+
+    ListRemotekeysResponse { data: keystores }
+}
+
+pub fn import<T: SlotClock + 'static, E: EthSpec>(
+    request: ImportRemotekeysRequest,
+    validator_store: Arc<ValidatorStore<T, E>>,
+    task_executor: TaskExecutor,
+    log: Logger,
+) -> Result<ImportRemotekeysResponse, Rejection> {
+    info!(
+        log,
+        "Importing remotekeys via standard HTTP API";
+        "count" => request.remote_keys.len(),
+    );
+    // Import each remotekey. Some remotekeys may fail to be imported, so we record a status for each.
+    let mut statuses = Vec::with_capacity(request.remote_keys.len());
+
+    for remotekey in request.remote_keys {
+        let status = if let Some(handle) = task_executor.handle() {
+            // Import the keystore.
+            match import_single_remotekey(remotekey.pubkey, remotekey.url, &validator_store, handle)
+            {
+                Ok(status) => Status::ok(status),
+                Err(e) => {
+                    warn!(
+                        log,
+                        "Error importing keystore, skipped";
+                        "pubkey" => remotekey.pubkey.to_string(),
+                        "error" => ?e,
+                    );
+                    Status::error(ImportRemotekeyStatus::Error, e)
+                }
+            }
+        } else {
+            Status::error(
+                ImportRemotekeyStatus::Error,
+                "validator client shutdown".into(),
+            )
+        };
+        statuses.push(status);
+    }
+    Ok(ImportRemotekeysResponse { data: statuses })
+}
+
+fn import_single_remotekey<T: SlotClock + 'static, E: EthSpec>(
+    pubkey: PublicKeyBytes,
+    url: String,
+    validator_store: &ValidatorStore<T, E>,
+    handle: Handle,
+) -> Result<ImportRemotekeyStatus, String> {
+    if let Err(url_err) = Url::parse(&url) {
+        return Err(format!("failed to parse remotekey URL: {}", url_err));
+    }
+
+    let pubkey = pubkey
+        .decompress()
+        .map_err(|_| format!("invalid pubkey: {}", pubkey))?;
+
+    if let Some(def) = validator_store
+        .initialized_validators()
+        .read()
+        .validator_definitions()
+        .iter()
+        .find(|def| def.voting_public_key == pubkey)
+    {
+        if def.signing_definition.is_local_keystore() {
+            return Err("Pubkey already present in local keystore.".into());
+        } else if def.enabled {
+            return Ok(ImportRemotekeyStatus::Duplicate);
+        }
+    }
+
+    // Remotekeys are stored as web3signers.
+    // The remotekey API provides less confgiuration option than the web3signer API.
+    let web3signer_validator = ValidatorDefinition {
+        enabled: true,
+        voting_public_key: pubkey,
+        graffiti: None,
+        suggested_fee_recipient: None,
+        gas_limit: None,
+        builder_proposals: None,
+        builder_boost_factor: None,
+        prefer_builder_proposals: None,
+        description: String::from("Added by remotekey API"),
+        signing_definition: SigningDefinition::Web3Signer(Web3SignerDefinition {
+            url,
+            root_certificate_path: None,
+            request_timeout_ms: None,
+            client_identity_path: None,
+            client_identity_password: None,
+        }),
+    };
+    handle
+        .block_on(validator_store.add_validator(web3signer_validator))
+        .map_err(|e| format!("failed to initialize validator: {:?}", e))?;
+
+    Ok(ImportRemotekeyStatus::Imported)
+}
+
+pub fn delete<T: SlotClock + 'static, E: EthSpec>(
+    request: DeleteRemotekeysRequest,
+    validator_store: Arc<ValidatorStore<T, E>>,
+    task_executor: TaskExecutor,
+    log: Logger,
+) -> Result<DeleteRemotekeysResponse, Rejection> {
+    info!(
+        log,
+        "Deleting remotekeys via standard HTTP API";
+        "count" => request.pubkeys.len(),
+    );
+    // Remove from initialized validators.
+    let initialized_validators_rwlock = validator_store.initialized_validators();
+    let mut initialized_validators = initialized_validators_rwlock.write();
+
+    let statuses = request
+        .pubkeys
+        .iter()
+        .map(|pubkey_bytes| {
+            match delete_single_remotekey(
+                pubkey_bytes,
+                &mut initialized_validators,
+                task_executor.clone(),
+            ) {
+                Ok(status) => Status::ok(status),
+                Err(error) => {
+                    warn!(
+                        log,
+                        "Error deleting keystore";
+                        "pubkey" => ?pubkey_bytes,
+                        "error" => ?error,
+                    );
+                    Status::error(DeleteRemotekeyStatus::Error, error)
+                }
+            }
+        })
+        .collect::<Vec<_>>();
+
+    // Use `update_validators` to update the key cache. It is safe to let the key cache get a bit out
+    // of date as it resets when it can't be decrypted. We update it just a single time to avoid
+    // continually resetting it after each key deletion.
+    if let Some(handle) = task_executor.handle() {
+        handle
+            .block_on(initialized_validators.update_validators())
+            .map_err(|e| custom_server_error(format!("unable to update key cache: {:?}", e)))?;
+    }
+
+    Ok(DeleteRemotekeysResponse { data: statuses })
+}
+
+fn delete_single_remotekey(
+    pubkey_bytes: &PublicKeyBytes,
+    initialized_validators: &mut InitializedValidators,
+    task_executor: TaskExecutor,
+) -> Result<DeleteRemotekeyStatus, String> {
+    if let Some(handle) = task_executor.handle() {
+        let pubkey = pubkey_bytes
+            .decompress()
+            .map_err(|e| format!("invalid pubkey, {:?}: {:?}", pubkey_bytes, e))?;
+
+        match handle.block_on(initialized_validators.delete_definition_and_keystore(&pubkey, false))
+        {
+            Ok(_) => Ok(DeleteRemotekeyStatus::Deleted),
+            Err(e) => match e {
+                Error::ValidatorNotInitialized(_) => Ok(DeleteRemotekeyStatus::NotFound),
+                _ => Err(format!("unable to disable and delete: {:?}", e)),
+            },
+        }
+    } else {
+        Err("validator client shutdown".into())
+    }
+}

validator_client/http_api/src/test_utils.rs

@@ -0,0 +1,650 @@
+use crate::{ApiSecret, Config as HttpConfig, Context};
+use account_utils::validator_definitions::ValidatorDefinitions;
+use account_utils::{
+    eth2_wallet::WalletBuilder, mnemonic_from_phrase, random_mnemonic, random_password,
+    ZeroizeString,
+};
+use deposit_contract::decode_eth1_tx_data;
+use doppelganger_service::DoppelgangerService;
+use eth2::{
+    lighthouse_vc::{http_client::ValidatorClientHttpClient, types::*},
+    types::ErrorMessage as ApiErrorMessage,
+    Error as ApiError,
+};
+use eth2_keystore::KeystoreBuilder;
+use initialized_validators::key_cache::{KeyCache, CACHE_FILENAME};
+use initialized_validators::{InitializedValidators, OnDecryptFailure};
+use logging::test_logger;
+use parking_lot::RwLock;
+use sensitive_url::SensitiveUrl;
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
+use slot_clock::{SlotClock, TestingSlotClock};
+use std::future::Future;
+use std::marker::PhantomData;
+use std::net::{IpAddr, Ipv4Addr};
+use std::sync::Arc;
+use std::time::Duration;
+use task_executor::test_utils::TestRuntime;
+use tempfile::{tempdir, TempDir};
+use tokio::sync::oneshot;
+use validator_store::{Config as ValidatorStoreConfig, ValidatorStore};
+
+pub const PASSWORD_BYTES: &[u8] = &[42, 50, 37];
+pub const TEST_DEFAULT_FEE_RECIPIENT: Address = Address::repeat_byte(42);
+
+type E = MainnetEthSpec;
+
+pub struct HdValidatorScenario {
+    pub count: usize,
+    pub specify_mnemonic: bool,
+    pub key_derivation_path_offset: u32,
+    pub disabled: Vec<usize>,
+}
+
+pub struct KeystoreValidatorScenario {
+    pub enabled: bool,
+    pub correct_password: bool,
+}
+
+pub struct Web3SignerValidatorScenario {
+    pub count: usize,
+    pub enabled: bool,
+}
+
+pub struct ApiTester {
+    pub client: ValidatorClientHttpClient,
+    pub initialized_validators: Arc<RwLock<InitializedValidators>>,
+    pub validator_store: Arc<ValidatorStore<TestingSlotClock, E>>,
+    pub url: SensitiveUrl,
+    pub api_token: String,
+    pub test_runtime: TestRuntime,
+    pub _server_shutdown: oneshot::Sender<()>,
+    pub validator_dir: TempDir,
+    pub secrets_dir: TempDir,
+}
+
+impl ApiTester {
+    pub async fn new() -> Self {
+        Self::new_with_http_config(Self::default_http_config()).await
+    }
+
+    pub async fn new_with_http_config(http_config: HttpConfig) -> Self {
+        let log = test_logger();
+
+        let validator_dir = tempdir().unwrap();
+        let secrets_dir = tempdir().unwrap();
+
+        let validator_defs = ValidatorDefinitions::open_or_create(validator_dir.path()).unwrap();
+
+        let initialized_validators = InitializedValidators::from_definitions(
+            validator_defs,
+            validator_dir.path().into(),
+            Default::default(),
+            log.clone(),
+        )
+        .await
+        .unwrap();
+
+        let api_secret = ApiSecret::create_or_open(validator_dir.path()).unwrap();
+        let api_pubkey = api_secret.api_token();
+
+        let config = ValidatorStoreConfig {
+            fee_recipient: Some(TEST_DEFAULT_FEE_RECIPIENT),
+            ..Default::default()
+        };
+
+        let spec = Arc::new(E::default_spec());
+
+        let slashing_db_path = validator_dir.path().join(SLASHING_PROTECTION_FILENAME);
+        let slashing_protection = SlashingDatabase::open_or_create(&slashing_db_path).unwrap();
+
+        let slot_clock =
+            TestingSlotClock::new(Slot::new(0), Duration::from_secs(0), Duration::from_secs(1));
+
+        let test_runtime = TestRuntime::default();
+
+        let validator_store = Arc::new(ValidatorStore::<_, E>::new(
+            initialized_validators,
+            slashing_protection,
+            Hash256::repeat_byte(42),
+            spec.clone(),
+            Some(Arc::new(DoppelgangerService::new(log.clone()))),
+            slot_clock.clone(),
+            &config,
+            test_runtime.task_executor.clone(),
+            log.clone(),
+        ));
+
+        validator_store
+            .register_all_in_doppelganger_protection_if_enabled()
+            .expect("Should attach doppelganger service");
+
+        let initialized_validators = validator_store.initialized_validators();
+
+        let context = Arc::new(Context {
+            task_executor: test_runtime.task_executor.clone(),
+            api_secret,
+            block_service: None,
+            validator_dir: Some(validator_dir.path().into()),
+            secrets_dir: Some(secrets_dir.path().into()),
+            validator_store: Some(validator_store.clone()),
+            graffiti_file: None,
+            graffiti_flag: Some(Graffiti::default()),
+            spec,
+            config: http_config,
+            log,
+            sse_logging_components: None,
+            slot_clock,
+            _phantom: PhantomData,
+        });
+        let ctx = context;
+        let (shutdown_tx, shutdown_rx) = oneshot::channel();
+        let server_shutdown = async {
+            // It's not really interesting why this triggered, just that it happened.
+            let _ = shutdown_rx.await;
+        };
+        let (listening_socket, server) = super::serve(ctx, server_shutdown).unwrap();
+
+        tokio::spawn(server);
+
+        let url = SensitiveUrl::parse(&format!(
+            "http://{}:{}",
+            listening_socket.ip(),
+            listening_socket.port()
+        ))
+        .unwrap();
+
+        let client = ValidatorClientHttpClient::new(url.clone(), api_pubkey.clone()).unwrap();
+
+        Self {
+            client,
+            initialized_validators,
+            validator_store,
+            url,
+            api_token: api_pubkey,
+            test_runtime,
+            _server_shutdown: shutdown_tx,
+            validator_dir,
+            secrets_dir,
+        }
+    }
+
+    pub fn default_http_config() -> HttpConfig {
+        HttpConfig {
+            enabled: true,
+            listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+            listen_port: 0,
+            allow_origin: None,
+            allow_keystore_export: true,
+            store_passwords_in_secrets_dir: false,
+        }
+    }
+
+    /// Checks that the key cache exists and can be decrypted with the current
+    /// set of known validators.
+    #[allow(clippy::await_holding_lock)] // This is a test, so it should be fine.
+    pub async fn ensure_key_cache_consistency(&self) {
+        assert!(
+            self.validator_dir.as_ref().join(CACHE_FILENAME).exists(),
+            "the key cache should exist"
+        );
+        let key_cache =
+            KeyCache::open_or_create(self.validator_dir.as_ref()).expect("should open a key cache");
+
+        self.initialized_validators
+            .read()
+            .decrypt_key_cache(key_cache, &mut <_>::default(), OnDecryptFailure::Error)
+            .await
+            .expect("key cache should decypt");
+    }
+
+    pub fn invalid_token_client(&self) -> ValidatorClientHttpClient {
+        let tmp = tempdir().unwrap();
+        let api_secret = ApiSecret::create_or_open(tmp.path()).unwrap();
+        let invalid_pubkey = api_secret.api_token();
+        ValidatorClientHttpClient::new(self.url.clone(), invalid_pubkey).unwrap()
+    }
+
+    pub async fn test_with_invalid_auth<F, A, T>(self, func: F) -> Self
+    where
+        F: Fn(ValidatorClientHttpClient) -> A,
+        A: Future<Output = Result<T, ApiError>>,
+    {
+        /*
+         * Test with an invalid Authorization header.
+         */
+        match func(self.invalid_token_client()).await {
+            Err(ApiError::ServerMessage(ApiErrorMessage { code: 403, .. })) => (),
+            Err(other) => panic!("expected authorized error, got {:?}", other),
+            Ok(_) => panic!("expected authorized error, got Ok"),
+        }
+
+        /*
+         * Test with a missing Authorization header.
+         */
+        let mut missing_token_client = self.client.clone();
+        missing_token_client.send_authorization_header(false);
+        match func(missing_token_client).await {
+            Err(ApiError::ServerMessage(ApiErrorMessage {
+                code: 401, message, ..
+            })) if message.contains("missing Authorization header") => (),
+            Err(other) => panic!("expected missing header error, got {:?}", other),
+            Ok(_) => panic!("expected missing header error, got Ok"),
+        }
+
+        self
+    }
+
+    pub fn invalidate_api_token(mut self) -> Self {
+        self.client = self.invalid_token_client();
+        self
+    }
+
+    pub async fn test_get_lighthouse_version_invalid(self) -> Self {
+        self.client.get_lighthouse_version().await.unwrap_err();
+        self
+    }
+
+    pub async fn test_get_lighthouse_spec(self) -> Self {
+        let result = self
+            .client
+            .get_lighthouse_spec::<ConfigAndPresetElectra>()
+            .await
+            .map(|res| ConfigAndPreset::Electra(res.data))
+            .unwrap();
+        let expected = ConfigAndPreset::from_chain_spec::<E>(&E::default_spec(), None);
+
+        assert_eq!(result, expected);
+
+        self
+    }
+
+    pub async fn test_get_lighthouse_version(self) -> Self {
+        let result = self.client.get_lighthouse_version().await.unwrap().data;
+
+        let expected = VersionData {
+            version: lighthouse_version::version_with_platform(),
+        };
+
+        assert_eq!(result, expected);
+
+        self
+    }
+
+    #[cfg(target_os = "linux")]
+    pub async fn test_get_lighthouse_health(self) -> Self {
+        self.client.get_lighthouse_health().await.unwrap();
+
+        self
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    pub async fn test_get_lighthouse_health(self) -> Self {
+        self.client.get_lighthouse_health().await.unwrap_err();
+
+        self
+    }
+    pub fn vals_total(&self) -> usize {
+        self.initialized_validators.read().num_total()
+    }
+
+    pub fn vals_enabled(&self) -> usize {
+        self.initialized_validators.read().num_enabled()
+    }
+
+    pub fn assert_enabled_validators_count(self, count: usize) -> Self {
+        assert_eq!(self.vals_enabled(), count);
+        self
+    }
+
+    pub fn assert_validators_count(self, count: usize) -> Self {
+        assert_eq!(self.vals_total(), count);
+        self
+    }
+
+    pub async fn create_hd_validators(self, s: HdValidatorScenario) -> Self {
+        let initial_vals = self.vals_total();
+        let initial_enabled_vals = self.vals_enabled();
+
+        let validators = (0..s.count)
+            .map(|i| ValidatorRequest {
+                enable: !s.disabled.contains(&i),
+                description: format!("boi #{}", i),
+                graffiti: None,
+                suggested_fee_recipient: None,
+                gas_limit: None,
+                builder_proposals: None,
+                builder_boost_factor: None,
+                prefer_builder_proposals: None,
+                deposit_gwei: E::default_spec().max_effective_balance,
+            })
+            .collect::<Vec<_>>();
+
+        let (response, mnemonic) = if s.specify_mnemonic {
+            let mnemonic = ZeroizeString::from(random_mnemonic().phrase().to_string());
+            let request = CreateValidatorsMnemonicRequest {
+                mnemonic: mnemonic.clone(),
+                key_derivation_path_offset: s.key_derivation_path_offset,
+                validators: validators.clone(),
+            };
+            let response = self
+                .client
+                .post_lighthouse_validators_mnemonic(&request)
+                .await
+                .unwrap()
+                .data;
+
+            (response, mnemonic)
+        } else {
+            assert_eq!(
+                s.key_derivation_path_offset, 0,
+                "cannot use a derivation offset without specifying a mnemonic"
+            );
+            let response = self
+                .client
+                .post_lighthouse_validators(validators.clone())
+                .await
+                .unwrap()
+                .data;
+            (response.validators.clone(), response.mnemonic)
+        };
+
+        assert_eq!(response.len(), s.count);
+        assert_eq!(self.vals_total(), initial_vals + s.count);
+        assert_eq!(
+            self.vals_enabled(),
+            initial_enabled_vals + s.count - s.disabled.len()
+        );
+
+        let server_vals = self.client.get_lighthouse_validators().await.unwrap().data;
+
+        assert_eq!(server_vals.len(), self.vals_total());
+
+        // Ensure the server lists all of these newly created validators.
+        for validator in &response {
+            assert!(server_vals
+                .iter()
+                .any(|server_val| server_val.voting_pubkey == validator.voting_pubkey));
+        }
+
+        /*
+         * Verify that we can regenerate all the keys from the mnemonic.
+         */
+
+        let mnemonic = mnemonic_from_phrase(mnemonic.as_str()).unwrap();
+        let mut wallet = WalletBuilder::from_mnemonic(&mnemonic, PASSWORD_BYTES, "".to_string())
+            .unwrap()
+            .build()
+            .unwrap();
+
+        wallet
+            .set_nextaccount(s.key_derivation_path_offset)
+            .unwrap();
+
+        for item in response.iter().take(s.count) {
+            let keypairs = wallet
+                .next_validator(PASSWORD_BYTES, PASSWORD_BYTES, PASSWORD_BYTES)
+                .unwrap();
+            let voting_keypair = keypairs.voting.decrypt_keypair(PASSWORD_BYTES).unwrap();
+
+            assert_eq!(
+                item.voting_pubkey,
+                voting_keypair.pk.clone().into(),
+                "the locally generated voting pk should match the server response"
+            );
+
+            let withdrawal_keypair = keypairs.withdrawal.decrypt_keypair(PASSWORD_BYTES).unwrap();
+
+            let deposit_bytes = serde_utils::hex::decode(&item.eth1_deposit_tx_data).unwrap();
+
+            let (deposit_data, _) =
+                decode_eth1_tx_data(&deposit_bytes, E::default_spec().max_effective_balance)
+                    .unwrap();
+
+            assert_eq!(
+                deposit_data.pubkey,
+                voting_keypair.pk.clone().into(),
+                "the locally generated voting pk should match the deposit data"
+            );
+
+            assert_eq!(
+                deposit_data.withdrawal_credentials,
+                Hash256::from_slice(&bls::get_withdrawal_credentials(
+                    &withdrawal_keypair.pk,
+                    E::default_spec().bls_withdrawal_prefix_byte
+                )),
+                "the locally generated withdrawal creds should match the deposit data"
+            );
+
+            assert_eq!(
+                deposit_data.signature,
+                deposit_data.create_signature(&voting_keypair.sk, &E::default_spec()),
+                "the locally-generated deposit sig should create the same deposit sig"
+            );
+        }
+
+        self
+    }
+
+    pub async fn create_keystore_validators(self, s: KeystoreValidatorScenario) -> Self {
+        let initial_vals = self.vals_total();
+        let initial_enabled_vals = self.vals_enabled();
+
+        let password = random_password();
+        let keypair = Keypair::random();
+        let keystore = KeystoreBuilder::new(&keypair, password.as_bytes(), String::new())
+            .unwrap()
+            .build()
+            .unwrap();
+
+        if !s.correct_password {
+            let request = KeystoreValidatorsPostRequest {
+                enable: s.enabled,
+                password: String::from_utf8(random_password().as_ref().to_vec())
+                    .unwrap()
+                    .into(),
+                keystore,
+                graffiti: None,
+                suggested_fee_recipient: None,
+                gas_limit: None,
+                builder_proposals: None,
+                builder_boost_factor: None,
+                prefer_builder_proposals: None,
+            };
+
+            self.client
+                .post_lighthouse_validators_keystore(&request)
+                .await
+                .unwrap_err();
+
+            return self;
+        }
+
+        let request = KeystoreValidatorsPostRequest {
+            enable: s.enabled,
+            password: String::from_utf8(password.as_ref().to_vec())
+                .unwrap()
+                .into(),
+            keystore,
+            graffiti: None,
+            suggested_fee_recipient: None,
+            gas_limit: None,
+            builder_proposals: None,
+            builder_boost_factor: None,
+            prefer_builder_proposals: None,
+        };
+
+        let response = self
+            .client
+            .post_lighthouse_validators_keystore(&request)
+            .await
+            .unwrap()
+            .data;
+
+        let num_enabled = s.enabled as usize;
+
+        assert_eq!(self.vals_total(), initial_vals + 1);
+        assert_eq!(self.vals_enabled(), initial_enabled_vals + num_enabled);
+
+        let server_vals = self.client.get_lighthouse_validators().await.unwrap().data;
+
+        assert_eq!(server_vals.len(), self.vals_total());
+
+        assert_eq!(response.voting_pubkey, keypair.pk.into());
+        assert_eq!(response.enabled, s.enabled);
+
+        self
+    }
+
+    pub async fn create_web3signer_validators(self, s: Web3SignerValidatorScenario) -> Self {
+        let initial_vals = self.vals_total();
+        let initial_enabled_vals = self.vals_enabled();
+
+        let request: Vec<_> = (0..s.count)
+            .map(|i| {
+                let kp = Keypair::random();
+                Web3SignerValidatorRequest {
+                    enable: s.enabled,
+                    description: format!("{}", i),
+                    graffiti: None,
+                    suggested_fee_recipient: None,
+                    gas_limit: None,
+                    builder_proposals: None,
+                    voting_public_key: kp.pk,
+                    url: format!("http://signer_{}.com/", i),
+                    root_certificate_path: None,
+                    request_timeout_ms: None,
+                    client_identity_path: None,
+                    client_identity_password: None,
+                    builder_boost_factor: None,
+                    prefer_builder_proposals: None,
+                }
+            })
+            .collect();
+
+        self.client
+            .post_lighthouse_validators_web3signer(&request)
+            .await
+            .unwrap();
+
+        assert_eq!(self.vals_total(), initial_vals + s.count);
+        if s.enabled {
+            assert_eq!(self.vals_enabled(), initial_enabled_vals + s.count);
+        } else {
+            assert_eq!(self.vals_enabled(), initial_enabled_vals);
+        };
+
+        self
+    }
+
+    pub async fn set_validator_enabled(self, index: usize, enabled: bool) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        self.client
+            .patch_lighthouse_validators(
+                &validator.voting_pubkey,
+                Some(enabled),
+                None,
+                None,
+                None,
+                None,
+                None,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(
+            self.initialized_validators
+                .read()
+                .is_enabled(&validator.voting_pubkey.decompress().unwrap())
+                .unwrap(),
+            enabled
+        );
+
+        assert!(self
+            .client
+            .get_lighthouse_validators()
+            .await
+            .unwrap()
+            .data
+            .into_iter()
+            .find(|v| v.voting_pubkey == validator.voting_pubkey)
+            .map(|v| v.enabled == enabled)
+            .unwrap());
+
+        // Check the server via an individual request.
+        assert_eq!(
+            self.client
+                .get_lighthouse_validators_pubkey(&validator.voting_pubkey)
+                .await
+                .unwrap()
+                .unwrap()
+                .data
+                .enabled,
+            enabled
+        );
+
+        self
+    }
+
+    pub async fn set_gas_limit(self, index: usize, gas_limit: u64) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        self.client
+            .patch_lighthouse_validators(
+                &validator.voting_pubkey,
+                None,
+                Some(gas_limit),
+                None,
+                None,
+                None,
+                None,
+            )
+            .await
+            .unwrap();
+
+        self
+    }
+
+    pub async fn assert_gas_limit(self, index: usize, gas_limit: u64) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        assert_eq!(
+            self.validator_store.get_gas_limit(&validator.voting_pubkey),
+            gas_limit
+        );
+
+        self
+    }
+
+    pub async fn set_builder_proposals(self, index: usize, builder_proposals: bool) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        self.client
+            .patch_lighthouse_validators(
+                &validator.voting_pubkey,
+                None,
+                None,
+                Some(builder_proposals),
+                None,
+                None,
+                None,
+            )
+            .await
+            .unwrap();
+
+        self
+    }
+
+    pub async fn assert_builder_proposals(self, index: usize, builder_proposals: bool) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        assert_eq!(
+            self.validator_store
+                .get_builder_proposals(&validator.voting_pubkey),
+            builder_proposals
+        );
+
+        self
+    }
+}