Optimizations, disable val client sync check & additional lcli tools (#834)

* Start adding interop genesis state to lcli

* Use more efficient method to generate genesis state

* Remove duplicate int_to_bytes32

* Add lcli command to change state genesis time

* Add option to allow VC to start with unsynced BN

* Set VC to do parallel key loading

* Don't default to dummy eth1 backend

* Add endpoint to dump operation pool

* Add metrics for op pool

* Remove state clone for slot notifier

* Add mem size approximation for tree hash cache

* Avoid cloning tree hash when getting head

* Fix failing API tests

* Address Michael's comments

* Add HashMap::from_par_iter

validator_client/src/cli.rs

@@ -14,6 +14,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .default_value(&DEFAULT_HTTP_SERVER)
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name("allow-unsynced")
+                .long("allow-unsynced")
+                .help("If present, the validator client will still poll for duties if the beacon
+                      node is not synced.")
+        )
         /*
          * The "testnet" sub-command.
          *

validator_client/src/config.rs

@@ -32,6 +32,9 @@ pub struct Config {
     ///
     /// Should be similar to `http://localhost:8080`
     pub http_server: String,
+    /// If true, the validator client will still poll for duties and produce blocks even if the
+    /// beacon node is not synced at startup.
+    pub allow_unsynced_beacon_node: bool,
 }
 
 impl Default for Config {
@@ -44,6 +47,7 @@ impl Default for Config {
             data_dir,
             key_source: <_>::default(),
             http_server: DEFAULT_HTTP_SERVER.to_string(),
+            allow_unsynced_beacon_node: false,
         }
     }
 }
@@ -71,7 +75,7 @@ impl Config {
             config.http_server = server.to_string();
         }
 
-        let config = match cli_args.subcommand() {
+        let mut config = match cli_args.subcommand() {
             ("testnet", Some(sub_cli_args)) => {
                 if cli_args.is_present("eth2-config") && sub_cli_args.is_present("bootstrap") {
                     return Err(
@@ -88,6 +92,8 @@ impl Config {
             }
         };
 
+        config.allow_unsynced_beacon_node = cli_args.is_present("allow-unsynced");
+
         Ok(config)
     }
 }

validator_client/src/duties_service.rs

@@ -190,6 +190,7 @@ pub struct DutiesServiceBuilder<T, E: EthSpec> {
     slot_clock: Option<T>,
     beacon_node: Option<RemoteBeaconNode<E>>,
     context: Option<RuntimeContext<E>>,
+    allow_unsynced_beacon_node: bool,
 }
 
 impl<T: SlotClock + 'static, E: EthSpec> DutiesServiceBuilder<T, E> {
@@ -199,6 +200,7 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesServiceBuilder<T, E> {
             slot_clock: None,
             beacon_node: None,
             context: None,
+            allow_unsynced_beacon_node: false,
         }
     }
 
@@ -222,6 +224,12 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesServiceBuilder<T, E> {
         self
     }
 
+    /// Set to `true` to allow polling for duties when the beacon node is not synced.
+    pub fn allow_unsynced_beacon_node(mut self, allow_unsynced_beacon_node: bool) -> Self {
+        self.allow_unsynced_beacon_node = allow_unsynced_beacon_node;
+        self
+    }
+
     pub fn build(self) -> Result<DutiesService<T, E>, String> {
         Ok(DutiesService {
             inner: Arc::new(Inner {
@@ -238,6 +246,7 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesServiceBuilder<T, E> {
                 context: self
                     .context
                     .ok_or_else(|| "Cannot build DutiesService without runtime_context")?,
+                allow_unsynced_beacon_node: self.allow_unsynced_beacon_node,
             }),
         })
     }
@@ -250,6 +259,9 @@ pub struct Inner<T, E: EthSpec> {
     pub(crate) slot_clock: T,
     beacon_node: RemoteBeaconNode<E>,
     context: RuntimeContext<E>,
+    /// If true, the duties service will poll for duties from the beacon node even if it is not
+    /// synced.
+    allow_unsynced_beacon_node: bool,
 }
 
 /// Maintains a store of the duties for all voting validators in the `validator_store`.
@@ -404,36 +416,38 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesService<T, E> {
             .and_then(move |(current_epoch, beacon_head_epoch)| {
                 let log = service_3.context.log.clone();
 
-                let future: Box<dyn Future<Item = (), Error = ()> + Send> =
-                    if beacon_head_epoch + 1 < current_epoch {
-                        error!(
-                            log,
-                            "Beacon node is not synced";
-                            "node_head_epoch" => format!("{}", beacon_head_epoch),
-                            "current_epoch" => format!("{}", current_epoch),
-                        );
+                let future: Box<dyn Future<Item = (), Error = ()> + Send> = if beacon_head_epoch + 1
+                    < current_epoch
+                    && !service_3.allow_unsynced_beacon_node
+                {
+                    error!(
+                        log,
+                        "Beacon node is not synced";
+                        "node_head_epoch" => format!("{}", beacon_head_epoch),
+                        "current_epoch" => format!("{}", current_epoch),
+                    );
 
-                        Box::new(future::ok(()))
-                    } else {
-                        Box::new(service_3.update_epoch(current_epoch).then(move |result| {
-                            if let Err(e) = result {
-                                error!(
-                                    log,
-                                    "Failed to get current epoch duties";
-                                    "http_error" => format!("{:?}", e)
-                                );
-                            }
+                    Box::new(future::ok(()))
+                } else {
+                    Box::new(service_3.update_epoch(current_epoch).then(move |result| {
+                        if let Err(e) = result {
+                            error!(
+                                log,
+                                "Failed to get current epoch duties";
+                                "http_error" => format!("{:?}", e)
+                            );
+                        }
 
-                            let log = service_4.context.log.clone();
-                            service_4.update_epoch(current_epoch + 1).map_err(move |e| {
-                                error!(
-                                    log,
-                                    "Failed to get next epoch duties";
-                                    "http_error" => format!("{:?}", e)
-                                );
-                            })
-                        }))
-                    };
+                        let log = service_4.context.log.clone();
+                        service_4.update_epoch(current_epoch + 1).map_err(move |e| {
+                            error!(
+                                log,
+                                "Failed to get next epoch duties";
+                                "http_error" => format!("{:?}", e)
+                            );
+                        })
+                    }))
+                };
 
                 future
             })

validator_client/src/lib.rs

@@ -210,6 +210,7 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
                     .validator_store(validator_store.clone())
                     .beacon_node(beacon_node.clone())
                     .runtime_context(context.service_context("duties".into()))
+                    .allow_unsynced_beacon_node(config.allow_unsynced_beacon_node)
                     .build()?;
 
                 let block_service = BlockServiceBuilder::new()

validator_client/src/validator_store.rs

@@ -33,8 +33,10 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
         fork_service: ForkService<T, E>,
         log: Logger,
     ) -> Result<Self, String> {
-        let validator_iter = read_dir(&base_dir)
+        let validator_key_values = read_dir(&base_dir)
             .map_err(|e| format!("Failed to read base directory {:?}: {:?}", base_dir, e))?
+            .collect::<Vec<_>>()
+            .into_par_iter()
             .filter_map(|validator_dir| {
                 let path = validator_dir.ok()?.path();
 
@@ -63,7 +65,7 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
             });
 
         Ok(Self {
-            validators: Arc::new(RwLock::new(HashMap::from_iter(validator_iter))),
+            validators: Arc::new(RwLock::new(HashMap::from_par_iter(validator_key_values))),
             spec: Arc::new(spec),
             log,
             temp_dir: None,