From 05bafee5a8bd070a3a958283f19c314e243976c8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 25 Aug 2022 09:40:01 +1000 Subject: [PATCH] Retry rather than saving file to disk --- validator_manager/Cargo.toml | 2 +- .../src/validators/move_validators.rs | 131 ++++++++++-------- 2 files changed, 77 insertions(+), 56 deletions(-) diff --git a/validator_manager/Cargo.toml b/validator_manager/Cargo.toml index 6e56d52310..e32f35bb7b 100644 --- a/validator_manager/Cargo.toml +++ b/validator_manager/Cargo.toml @@ -21,10 +21,10 @@ eth2_serde_utils = "0.1.1" tree_hash = "0.4.1" eth2 = { path = "../common/eth2", features = ["lighthouse"]} hex = "0.4.2" +tokio = { version = "1.14.0", features = ["time", "rt-multi-thread", "macros"] } [dev-dependencies] tempfile = "3.1.0" -tokio = { version = "1.14.0", features = ["time", "rt-multi-thread", "macros"] } regex = "1.6.0" eth2_network_config = { path = "../common/eth2_network_config" } validator_client = { path = "../validator_client" } diff --git a/validator_manager/src/validators/move_validators.rs b/validator_manager/src/validators/move_validators.rs index 02223df587..055dd417a7 100644 --- a/validator_manager/src/validators/move_validators.rs +++ b/validator_manager/src/validators/move_validators.rs @@ -13,6 +13,8 @@ use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; use std::str::FromStr; +use std::time::Duration; +use tokio::time::sleep; use types::{Address, PublicKeyBytes}; pub const MOVE_DIR_NAME: &str = "lighthouse-validator-move"; @@ -31,6 +33,8 @@ pub const BUILDER_PROPOSALS_FLAG: &str = "builder-proposals"; const NO_VALIDATORS_MSG: &str = "No validators present on source validator client"; +const UPLOAD_RETRY_WAIT: Duration = Duration::from_secs(5); + pub fn cli_app<'a, 'b>() -> App<'a, 'b> { App::new(CMD) .about( @@ -410,6 +414,8 @@ async fn run<'a>(config: MoveConfig) -> Result<(), String> { } }; + let keystore_derivation_path = voting_keystore.0.path(); + let validator_specification = ValidatorSpecification { voting_keystore, voting_keystore_password, @@ -424,70 +430,85 @@ async fn run<'a>(config: MoveConfig) -> Result<(), String> { // there doesn't appear to be much harm just adding them again. let ignore_duplicates = true; - match validator_specification - .clone() - .upload(&dest_http_client, ignore_duplicates) - .await - { - Ok(()) => eprintln!( + loop { + match validator_specification + .clone() + .upload(&dest_http_client, ignore_duplicates) + .await + { + Ok(()) => break, + e @ Err(UploadError::InvalidPublicKey) => { + eprintln!("Validator {} has an invalid public key", i); + return Err(format!("{:?}", e)); + } + Err(UploadError::DuplicateValidator(_)) => { + return Err( + "Duplicate validator detected when duplicates are ignored".to_string() + ); + } + Err(UploadError::FailedToListKeys(e)) => { + eprintln!( + "Failed to list keystores. Some keys may have been moved whilst \ + others may not. Error was {:?}", + e + ); + // Retry uploading this validator. + sleep_with_retry_message(&pubkey_to_move, keystore_derivation_path.as_deref()) + .await; + } + Err(UploadError::KeyUploadFailed(e)) => { + eprintln!( + "Failed to upload keystore. Some keys may have been moved whilst \ + others may not. Error was {:?}", + e + ); + // Retry uploading this validator. + sleep_with_retry_message(&pubkey_to_move, keystore_derivation_path.as_deref()) + .await; + } + Err(UploadError::FeeRecipientUpdateFailed(e)) => { + eprintln!( + "Failed to set fee recipient for validator {}. This value may need \ + to be set manually. Continuing with other validators. Error was {:?}", + i, e + ); + // Continue onto the next validator. + break; + } + Err(UploadError::PatchValidatorFailed(e)) => { + eprintln!( + "Failed to set some values on validator {} (e.g., builder, enabled or gas limit. \ + These values value may need to be set manually. Continuing with other validators. \ + Error was {:?}", + i, e + ); + // Continue onto the next validator. + break; + } + } + eprintln!( "Uploaded keystore {} of {} to the destination VC", i + 1, count - ), - e @ Err(UploadError::InvalidPublicKey) => { - eprintln!("Validator {} has an invalid public key", i); - return Err(format!("{:?}", e)); - } - Err(UploadError::DuplicateValidator(_)) => { - return Err("Duplicate validator detected when duplicates are ignored".to_string()); - } - Err(UploadError::FailedToListKeys(e)) => { - eprintln!( - "Failed to list keystores. Some keys may have been moved whilst \ - others may not.", - ); - backup_validator( - &validator_specification, - &working_directory_path, - &dest_vc_url, - &dest_vc_token_path, - ); - return Err(format!("{:?}", e)); - } - Err(UploadError::KeyUploadFailed(e)) => { - eprintln!( - "Failed to upload keystore. Some keys may have been moved whilst \ - others may not.", - ); - backup_validator( - &validator_specification, - &working_directory_path, - &dest_vc_url, - &dest_vc_token_path, - ); - return Err(format!("{:?}", e)); - } - Err(UploadError::FeeRecipientUpdateFailed(e)) => { - eprintln!( - "Failed to set fee recipient for validator {}. This value may need \ - to be set manually. Continuing with other validators. Error was {:?}", - i, e - ); - } - Err(UploadError::PatchValidatorFailed(e)) => { - eprintln!( - "Failed to set some values on validator {} (e.g., builder, enabled or gas limit. \ - These values value may need to be set manually. Continuing with other validators. \ - Error was {:?}", - i, e - ); - } + ); } } Ok(()) } +async fn sleep_with_retry_message(pubkey: &PublicKeyBytes, path: Option<&str>) { + let path = path.unwrap_or(""); + eprintln!( + "Sleeping for {:?} before retrying. Exiting the application before it completes \ + may result in the loss of a validator keystore. The keystore would need to be \ + restored from a backup or mnemonic. The keystore which may be lost has a public \ + key of {:?} and a derivation path of {}", + UPLOAD_RETRY_WAIT, pubkey, path + ); + sleep(UPLOAD_RETRY_WAIT).await +} + pub fn backup_validator>( validator_specification: &ValidatorSpecification, working_directory_path: P,