Add validator-manager (#3502)

## Issue Addressed

Addresses #2557

## Proposed Changes

Adds the `lighthouse validator-manager` command, which provides:

- `lighthouse validator-manager create`
    - Creates a `validators.json` file and a `deposits.json` (same format as https://github.com/ethereum/staking-deposit-cli)
- `lighthouse validator-manager import`
    - Imports validators from a `validators.json` file to the VC via the HTTP API.
- `lighthouse validator-manager move`
    - Moves validators from one VC to the other, utilizing only the VC API.

## Additional Info

In 98bcb947c I've reduced some VC `ERRO` and `CRIT` warnings to `WARN` or `DEBG` for the case where a pubkey is missing from the validator store. These were being triggered when we removed a validator but still had it in caches. It seems to me that `UnknownPubkey` will only happen in the case where we've removed a validator, so downgrading the logs is prudent. All the logs are `DEBG` apart from attestations and blocks which are `WARN`. I thought having *some* logging about this condition might help us down the track.

In 856cd7e37d I've made the VC delete the corresponding password file when it's deleting a keystore. This seemed like nice hygiene. Notably, it'll only delete that password file after it scans the validator definitions and finds that no other validator is also using that password file.

validator_client/src/http_api/mod.rs

@@ -5,6 +5,8 @@ mod keystores;
 mod remotekeys;
 mod tests;
 
+pub mod test_utils;
+
 use crate::http_api::create_signed_voluntary_exit::create_signed_voluntary_exit;
 use crate::{determine_graffiti, GraffitiFile, ValidatorStore};
 use account_utils::{
@@ -12,7 +14,9 @@ use account_utils::{
     validator_definitions::{SigningDefinition, ValidatorDefinition, Web3SignerDefinition},
 };
 pub use api_secret::ApiSecret;
-use create_validator::{create_validators_mnemonic, create_validators_web3signer};
+use create_validator::{
+    create_validators_mnemonic, create_validators_web3signer, get_voting_password_storage,
+};
 use eth2::lighthouse_vc::{
     std_types::{AuthResponse, GetFeeRecipientResponse, GetGasLimitResponse},
     types::{self as api_types, GenericResponse, Graffiti, PublicKey, PublicKeyBytes},
@@ -71,6 +75,7 @@ pub struct Context<T: SlotClock, E: EthSpec> {
     pub api_secret: ApiSecret,
     pub validator_store: Option<Arc<ValidatorStore<T, E>>>,
     pub validator_dir: Option<PathBuf>,
+    pub secrets_dir: Option<PathBuf>,
     pub graffiti_file: Option<GraffitiFile>,
     pub graffiti_flag: Option<Graffiti>,
     pub spec: ChainSpec,
@@ -88,6 +93,8 @@ pub struct Config {
     pub listen_addr: IpAddr,
     pub listen_port: u16,
     pub allow_origin: Option<String>,
+    pub allow_keystore_export: bool,
+    pub store_passwords_in_secrets_dir: bool,
 }
 
 impl Default for Config {
@@ -97,6 +104,8 @@ impl Default for Config {
             listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
             listen_port: 5062,
             allow_origin: None,
+            allow_keystore_export: false,
+            store_passwords_in_secrets_dir: false,
         }
     }
 }
@@ -121,6 +130,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
     shutdown: impl Future<Output = ()> + Send + Sync + 'static,
 ) -> Result<(SocketAddr, impl Future<Output = ()>), Error> {
     let config = &ctx.config;
+    let allow_keystore_export = config.allow_keystore_export;
+    let store_passwords_in_secrets_dir = config.store_passwords_in_secrets_dir;
     let log = ctx.log.clone();
 
     // Configure CORS.
@@ -187,6 +198,17 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
             })
         });
 
+    let inner_secrets_dir = ctx.secrets_dir.clone();
+    let secrets_dir_filter = warp::any().map(move || inner_secrets_dir.clone()).and_then(
+        |secrets_dir: Option<_>| async move {
+            secrets_dir.ok_or_else(|| {
+                warp_utils::reject::custom_not_found(
+                    "secrets_dir directory is not initialized.".to_string(),
+                )
+            })
+        },
+    );
+
     let inner_graffiti_file = ctx.graffiti_file.clone();
     let graffiti_file_filter = warp::any().map(move || inner_graffiti_file.clone());
 
@@ -394,18 +416,21 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::path::end())
         .and(warp::body::json())
         .and(validator_dir_filter.clone())
+        .and(secrets_dir_filter.clone())
         .and(validator_store_filter.clone())
         .and(spec_filter.clone())
         .and(signer.clone())
         .and(task_executor_filter.clone())
         .and_then(
-            |body: Vec<api_types::ValidatorRequest>,
-             validator_dir: PathBuf,
-             validator_store: Arc<ValidatorStore<T, E>>,
-             spec: Arc<ChainSpec>,
-             signer,
-             task_executor: TaskExecutor| {
+            move |body: Vec<api_types::ValidatorRequest>,
+                  validator_dir: PathBuf,
+                  secrets_dir: PathBuf,
+                  validator_store: Arc<ValidatorStore<T, E>>,
+                  spec: Arc<ChainSpec>,
+                  signer,
+                  task_executor: TaskExecutor| {
                 blocking_signed_json_task(signer, move || {
+                    let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
                     if let Some(handle) = task_executor.handle() {
                         let (validators, mnemonic) =
                             handle.block_on(create_validators_mnemonic(
@@ -413,6 +438,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                                 None,
                                 &body,
                                 &validator_dir,
+                                secrets_dir,
                                 &validator_store,
                                 &spec,
                             ))?;
@@ -437,18 +463,21 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::path::end())
         .and(warp::body::json())
         .and(validator_dir_filter.clone())
+        .and(secrets_dir_filter.clone())
         .and(validator_store_filter.clone())
         .and(spec_filter)
         .and(signer.clone())
         .and(task_executor_filter.clone())
         .and_then(
-            |body: api_types::CreateValidatorsMnemonicRequest,
-             validator_dir: PathBuf,
-             validator_store: Arc<ValidatorStore<T, E>>,
-             spec: Arc<ChainSpec>,
-             signer,
-             task_executor: TaskExecutor| {
+            move |body: api_types::CreateValidatorsMnemonicRequest,
+                  validator_dir: PathBuf,
+                  secrets_dir: PathBuf,
+                  validator_store: Arc<ValidatorStore<T, E>>,
+                  spec: Arc<ChainSpec>,
+                  signer,
+                  task_executor: TaskExecutor| {
                 blocking_signed_json_task(signer, move || {
+                    let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
                     if let Some(handle) = task_executor.handle() {
                         let mnemonic =
                             mnemonic_from_phrase(body.mnemonic.as_str()).map_err(|e| {
@@ -463,6 +492,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                                 Some(body.key_derivation_path_offset),
                                 &body.validators,
                                 &validator_dir,
+                                secrets_dir,
                                 &validator_store,
                                 &spec,
                             ))?;
@@ -483,15 +513,17 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::path::end())
         .and(warp::body::json())
         .and(validator_dir_filter.clone())
+        .and(secrets_dir_filter.clone())
         .and(validator_store_filter.clone())
         .and(signer.clone())
         .and(task_executor_filter.clone())
         .and_then(
-            |body: api_types::KeystoreValidatorsPostRequest,
-             validator_dir: PathBuf,
-             validator_store: Arc<ValidatorStore<T, E>>,
-             signer,
-             task_executor: TaskExecutor| {
+            move |body: api_types::KeystoreValidatorsPostRequest,
+                  validator_dir: PathBuf,
+                  secrets_dir: PathBuf,
+                  validator_store: Arc<ValidatorStore<T, E>>,
+                  signer,
+                  task_executor: TaskExecutor| {
                 blocking_signed_json_task(signer, move || {
                     // Check to ensure the password is correct.
                     let keypair = body
@@ -504,7 +536,12 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                             ))
                         })?;
 
+                    let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
+                    let password_storage =
+                        get_voting_password_storage(&secrets_dir, &body.keystore, &body.password)?;
+
                     let validator_dir = ValidatorDirBuilder::new(validator_dir.clone())
+                        .password_dir_opt(secrets_dir)
                         .voting_keystore(body.keystore.clone(), body.password.as_ref())
                         .store_withdrawal_keystore(false)
                         .build()
@@ -518,7 +555,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                     // Drop validator dir so that `add_validator_keystore` can re-lock the keystore.
                     let voting_keystore_path = validator_dir.voting_keystore_path();
                     drop(validator_dir);
-                    let voting_password = body.password.clone();
                     let graffiti = body.graffiti.clone();
                     let suggested_fee_recipient = body.suggested_fee_recipient;
                     let gas_limit = body.gas_limit;
@@ -529,7 +565,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                             handle
                                 .block_on(validator_store.add_validator_keystore(
                                     voting_keystore_path,
-                                    voting_password,
+                                    password_storage,
                                     body.enable,
                                     graffiti,
                                     suggested_fee_recipient,
@@ -698,6 +734,29 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
             })
         });
 
+    // DELETE /lighthouse/keystores
+    let delete_lighthouse_keystores = warp::path("lighthouse")
+        .and(warp::path("keystores"))
+        .and(warp::path::end())
+        .and(warp::body::json())
+        .and(signer.clone())
+        .and(validator_store_filter.clone())
+        .and(task_executor_filter.clone())
+        .and(log_filter.clone())
+        .and_then(
+            move |request, signer, validator_store, task_executor, log| {
+                blocking_signed_json_task(signer, move || {
+                    if allow_keystore_export {
+                        keystores::export(request, validator_store, task_executor, log)
+                    } else {
+                        Err(warp_utils::reject::custom_bad_request(
+                            "keystore export is disabled".to_string(),
+                        ))
+                    }
+                })
+            },
+        );
+
     // Standard key-manager endpoints.
     let eth_v1 = warp::path("eth").and(warp::path("v1"));
     let std_keystores = eth_v1.and(warp::path("keystores")).and(warp::path::end());
@@ -982,13 +1041,28 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::body::json())
         .and(signer.clone())
         .and(validator_dir_filter)
+        .and(secrets_dir_filter)
         .and(validator_store_filter.clone())
         .and(task_executor_filter.clone())
         .and(log_filter.clone())
         .and_then(
-            |request, signer, validator_dir, validator_store, task_executor, log| {
+            move |request,
+                  signer,
+                  validator_dir,
+                  secrets_dir,
+                  validator_store,
+                  task_executor,
+                  log| {
+                let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
                 blocking_signed_json_task(signer, move || {
-                    keystores::import(request, validator_dir, validator_store, task_executor, log)
+                    keystores::import(
+                        request,
+                        validator_dir,
+                        secrets_dir,
+                        validator_store,
+                        task_executor,
+                        log,
+                    )
                 })
             },
         );
@@ -1117,7 +1191,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                 ))
                 .or(warp::patch().and(patch_validators))
                 .or(warp::delete().and(
-                    delete_fee_recipient
+                    delete_lighthouse_keystores
+                        .or(delete_fee_recipient)
                         .or(delete_gas_limit)
                         .or(delete_std_keystores)
                         .or(delete_std_remotekeys),