Retry rather than saving file to disk

validator_manager/Cargo.toml

@@ -21,10 +21,10 @@ eth2_serde_utils = "0.1.1"
 tree_hash = "0.4.1"
 eth2 = { path = "../common/eth2", features = ["lighthouse"]}
 hex = "0.4.2"
+tokio = { version = "1.14.0", features = ["time", "rt-multi-thread", "macros"] }
 
 [dev-dependencies]
 tempfile = "3.1.0"
-tokio = { version = "1.14.0", features = ["time", "rt-multi-thread", "macros"] }
 regex = "1.6.0"
 eth2_network_config = { path = "../common/eth2_network_config" }
 validator_client = { path = "../validator_client" }

validator_manager/src/validators/move_validators.rs

@@ -13,6 +13,8 @@ use std::collections::{HashMap, HashSet};
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
+use std::time::Duration;
+use tokio::time::sleep;
 use types::{Address, PublicKeyBytes};
 
 pub const MOVE_DIR_NAME: &str = "lighthouse-validator-move";
@@ -31,6 +33,8 @@ pub const BUILDER_PROPOSALS_FLAG: &str = "builder-proposals";
 
 const NO_VALIDATORS_MSG: &str = "No validators present on source validator client";
 
+const UPLOAD_RETRY_WAIT: Duration = Duration::from_secs(5);
+
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about(
@@ -410,6 +414,8 @@ async fn run<'a>(config: MoveConfig) -> Result<(), String> {
             }
         };
 
+        let keystore_derivation_path = voting_keystore.0.path();
+
         let validator_specification = ValidatorSpecification {
             voting_keystore,
             voting_keystore_password,
@@ -424,48 +430,41 @@ async fn run<'a>(config: MoveConfig) -> Result<(), String> {
         // there doesn't appear to be much harm just adding them again.
         let ignore_duplicates = true;
 
+        loop {
             match validator_specification
                 .clone()
                 .upload(&dest_http_client, ignore_duplicates)
                 .await
             {
-            Ok(()) => eprintln!(
+                Ok(()) => break,
-                "Uploaded keystore {} of {} to the destination VC",
-                i + 1,
-                count
-            ),
                 e @ Err(UploadError::InvalidPublicKey) => {
                     eprintln!("Validator {} has an invalid public key", i);
                     return Err(format!("{:?}", e));
                 }
                 Err(UploadError::DuplicateValidator(_)) => {
-                return Err("Duplicate validator detected when duplicates are ignored".to_string());
+                    return Err(
+                        "Duplicate validator detected when duplicates are ignored".to_string()
+                    );
                 }
                 Err(UploadError::FailedToListKeys(e)) => {
                     eprintln!(
                         "Failed to list keystores. Some keys may have been moved whilst \
-                    others may not.",
+                        others may not. Error was {:?}",
+                        e
                     );
-                backup_validator(
+                    // Retry uploading this validator.
-                    &validator_specification,
+                    sleep_with_retry_message(&pubkey_to_move, keystore_derivation_path.as_deref())
-                    &working_directory_path,
+                        .await;
-                    &dest_vc_url,
-                    &dest_vc_token_path,
-                );
-                return Err(format!("{:?}", e));
                 }
                 Err(UploadError::KeyUploadFailed(e)) => {
                     eprintln!(
                         "Failed to upload keystore. Some keys may have been moved whilst \
-                    others may not.",
+                        others may not. Error was {:?}",
+                        e
                     );
-                backup_validator(
+                    // Retry uploading this validator.
-                    &validator_specification,
+                    sleep_with_retry_message(&pubkey_to_move, keystore_derivation_path.as_deref())
-                    &working_directory_path,
+                        .await;
-                    &dest_vc_url,
-                    &dest_vc_token_path,
-                );
-                return Err(format!("{:?}", e));
                 }
                 Err(UploadError::FeeRecipientUpdateFailed(e)) => {
                     eprintln!(
@@ -473,6 +472,8 @@ async fn run<'a>(config: MoveConfig) -> Result<(), String> {
                         to be set manually. Continuing with other validators. Error was {:?}",
                         i, e
                     );
+                    // Continue onto the next validator.
+                    break;
                 }
                 Err(UploadError::PatchValidatorFailed(e)) => {
                     eprintln!(
@@ -481,13 +482,33 @@ async fn run<'a>(config: MoveConfig) -> Result<(), String> {
                         Error was {:?}",
                         i, e
                     );
+                    // Continue onto the next validator.
+                    break;
                 }
             }
+            eprintln!(
+                "Uploaded keystore {} of {} to the destination VC",
+                i + 1,
+                count
+            );
+        }
     }
 
     Ok(())
 }
 
+async fn sleep_with_retry_message(pubkey: &PublicKeyBytes, path: Option<&str>) {
+    let path = path.unwrap_or("<unspecified>");
+    eprintln!(
+        "Sleeping for {:?} before retrying. Exiting the application before it completes \
+        may result in the loss of a validator keystore. The keystore would need to be \
+        restored from a backup or mnemonic. The keystore which may be lost has a public \
+        key of {:?} and a derivation path of {}",
+        UPLOAD_RETRY_WAIT, pubkey, path
+    );
+    sleep(UPLOAD_RETRY_WAIT).await
+}
+
 pub fn backup_validator<P: AsRef<Path>>(
     validator_specification: &ValidatorSpecification,
     working_directory_path: P,