Validator on-boarding docs (#656)

* Add first draft of validator onboarding

* Update docs

* Add documentation link to main README

* Continue docs development

* Update book readme

* Update docs

* Allow vc to run without testnet subcommand

* Small change to onboarding docs

* Tidy CLI help messages

* Update docs

* Add check to val client see if beacon node is synced

* Add notifier service to validator client

* Re-order onboarding steps

* Update deposit contract address

* Update testnet dir

* Add note about public eth1 node

* Set default eth1 endpoint to sigp

* Fix broken test

* Try fix eth1 cache locking

* Be more specific about eth1 endpoint

* Increase gas limit for deposit

* Fix default deposit amount

validator_client/src/cli.rs

@@ -4,7 +4,8 @@ use clap::{App, Arg, SubCommand};
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new("validator_client")
         .visible_aliases(&["v", "vc", "validator"])
-        .about("Ethereum 2.0 Validator Client")
+        .about("When connected to a beacon node, performs the duties of a staked \
+                validator (e.g., proposing blocks and attestations).")
         .arg(
             Arg::with_name("server")
                 .long("server")

validator_client/src/config.rs

@@ -80,10 +80,13 @@ impl Config {
                             .into(),
                     );
                 }
-                process_testnet_subcommand(sub_cli_args, config)
+                process_testnet_subcommand(sub_cli_args, config)?
             }
-            _ => return Err("You must use the testnet command. See '--help'.".into()),
-        }?;
+            _ => {
+                config.key_source = KeySource::Disk;
+                config
+            }
+        };
 
         Ok(config)
     }

validator_client/src/duties_service.rs

@@ -1,10 +1,10 @@
 use crate::validator_store::ValidatorStore;
 use environment::RuntimeContext;
 use exit_future::Signal;
-use futures::{Future, IntoFuture, Stream};
+use futures::{future, Future, IntoFuture, Stream};
 use parking_lot::RwLock;
 use remote_beacon_node::RemoteBeaconNode;
-use slog::{crit, error, info, trace, warn};
+use slog::{crit, debug, error, info, trace, warn};
 use slot_clock::SlotClock;
 use std::collections::HashMap;
 use std::convert::TryInto;
@@ -74,6 +74,34 @@ pub struct DutiesStore {
 }
 
 impl DutiesStore {
+    /// Returns the total number of validators that should propose in the given epoch.
+    fn proposer_count(&self, epoch: Epoch) -> usize {
+        self.store
+            .read()
+            .iter()
+            .filter(|(_validator_pubkey, validator_map)| {
+                validator_map
+                    .get(&epoch)
+                    .map(|duties| !duties.block_proposal_slots.is_empty())
+                    .unwrap_or_else(|| false)
+            })
+            .count()
+    }
+
+    /// Returns the total number of validators that should attest in the given epoch.
+    fn attester_count(&self, epoch: Epoch) -> usize {
+        self.store
+            .read()
+            .iter()
+            .filter(|(_validator_pubkey, validator_map)| {
+                validator_map
+                    .get(&epoch)
+                    .map(|duties| duties.attestation_slot.is_some())
+                    .unwrap_or_else(|| false)
+            })
+            .count()
+    }
+
     fn block_producers(&self, slot: Slot, slots_per_epoch: u64) -> Vec<PublicKey> {
         self.store
             .read()
@@ -219,7 +247,7 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesServiceBuilder<T, E> {
 pub struct Inner<T, E: EthSpec> {
     store: Arc<DutiesStore>,
     validator_store: ValidatorStore<T, E>,
-    slot_clock: T,
+    pub(crate) slot_clock: T,
     beacon_node: RemoteBeaconNode<E>,
     context: RuntimeContext<E>,
 }
@@ -249,6 +277,21 @@ impl<T, E: EthSpec> Deref for DutiesService<T, E> {
 }
 
 impl<T: SlotClock + 'static, E: EthSpec> DutiesService<T, E> {
+    /// Returns the total number of validators known to the duties service.
+    pub fn total_validator_count(&self) -> usize {
+        self.validator_store.num_voting_validators()
+    }
+
+    /// Returns the total number of validators that should propose in the given epoch.
+    pub fn proposer_count(&self, epoch: Epoch) -> usize {
+        self.store.proposer_count(epoch)
+    }
+
+    /// Returns the total number of validators that should attest in the given epoch.
+    pub fn attester_count(&self, epoch: Epoch) -> usize {
+        self.store.attester_count(epoch)
+    }
+
     /// Returns the pubkeys of the validators which are assigned to propose in the given slot.
     ///
     /// In normal cases, there should be 0 or 1 validators returned. In extreme cases (i.e., deep forking)
@@ -313,6 +356,7 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesService<T, E> {
         let service_1 = self.clone();
         let service_2 = self.clone();
         let service_3 = self.clone();
+        let service_4 = self.clone();
         let log_1 = self.context.log.clone();
         let log_2 = self.context.log.clone();
 
@@ -342,24 +386,56 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesService<T, E> {
             })
             .and_then(move |epoch| {
                 let log = service_2.context.log.clone();
-                service_2.update_epoch(epoch).then(move |result| {
-                    if let Err(e) = result {
-                        error!(
-                            log,
-                            "Failed to get current epoch duties";
-                            "http_error" => format!("{:?}", e)
-                        );
-                    }
 
-                    let log = service_3.context.log.clone();
-                    service_3.update_epoch(epoch + 1).map_err(move |e| {
+                service_2
+                    .beacon_node
+                    .http
+                    .beacon()
+                    .get_head()
+                    .map(move |head| (epoch, head.slot.epoch(E::slots_per_epoch())))
+                    .map_err(move |e| {
+                        error!(
+                                log,
+                                "Failed to contact beacon node";
+                                "error" => format!("{:?}", e)
+                        )
+                    })
+            })
+            .and_then(move |(current_epoch, beacon_head_epoch)| {
+                let log = service_3.context.log.clone();
+
+                let future: Box<dyn Future<Item = (), Error = ()> + Send> =
+                    if beacon_head_epoch + 1 < current_epoch {
                         error!(
                             log,
-                            "Failed to get next epoch duties";
-                            "http_error" => format!("{:?}", e)
+                            "Beacon node is not synced";
+                            "node_head_epoch" => format!("{}", beacon_head_epoch),
+                            "current_epoch" => format!("{}", current_epoch),
                         );
-                    })
-                })
+
+                        Box::new(future::ok(()))
+                    } else {
+                        Box::new(service_3.update_epoch(current_epoch).then(move |result| {
+                            if let Err(e) = result {
+                                error!(
+                                    log,
+                                    "Failed to get current epoch duties";
+                                    "http_error" => format!("{:?}", e)
+                                );
+                            }
+
+                            let log = service_4.context.log.clone();
+                            service_4.update_epoch(current_epoch + 1).map_err(move |e| {
+                                error!(
+                                    log,
+                                    "Failed to get next epoch duties";
+                                    "http_error" => format!("{:?}", e)
+                                );
+                            })
+                        }))
+                    };
+
+                future
             })
             .map(|_| ())
     }
@@ -394,7 +470,7 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesService<T, E> {
                         .insert(epoch, duties.clone(), E::slots_per_epoch())
                     {
                         InsertOutcome::NewValidator => {
-                            info!(
+                            debug!(
                                 log,
                                 "First duty assignment for validator";
                                 "proposal_slots" => format!("{:?}", &duties.block_proposal_slots),

validator_client/src/lib.rs

@@ -4,6 +4,7 @@ mod cli;
 mod config;
 mod duties_service;
 mod fork_service;
+mod notifier;
 mod validator_store;
 
 pub mod validator_directory;
@@ -22,6 +23,7 @@ use futures::{
     future::{self, loop_fn, Loop},
     Future, IntoFuture,
 };
+use notifier::spawn_notifier;
 use remote_beacon_node::RemoteBeaconNode;
 use slog::{error, info, Logger};
 use slot_clock::SlotClock;
@@ -258,7 +260,16 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
             .start_update_service(&self.context.eth2_config.spec)
             .map_err(|e| format!("Unable to start attestation service: {}", e))?;
 
-        self.exit_signals = vec![duties_exit, fork_exit, block_exit, attestation_exit];
+        let notifier_exit =
+            spawn_notifier(self).map_err(|e| format!("Failed to start notifier: {}", e))?;
+
+        self.exit_signals = vec![
+            duties_exit,
+            fork_exit,
+            block_exit,
+            attestation_exit,
+            notifier_exit,
+        ];
 
         Ok(())
     }

validator_client/src/notifier.rs

@@ -0,0 +1,91 @@
+use crate::ProductionValidatorClient;
+use exit_future::Signal;
+use futures::{Future, Stream};
+use slog::{error, info};
+use slot_clock::SlotClock;
+use std::time::{Duration, Instant};
+use tokio::timer::Interval;
+use types::EthSpec;
+
+/// Spawns a notifier service which periodically logs information about the node.
+pub fn spawn_notifier<T: EthSpec>(client: &ProductionValidatorClient<T>) -> Result<Signal, String> {
+    let context = client.context.service_context("notifier".into());
+
+    let slot_duration = Duration::from_millis(context.eth2_config.spec.milliseconds_per_slot);
+    let duration_to_next_slot = client
+        .duties_service
+        .slot_clock
+        .duration_to_next_slot()
+        .ok_or_else(|| "slot_notifier unable to determine time to next slot")?;
+
+    // Run this half way through each slot.
+    let start_instant = Instant::now() + duration_to_next_slot + (slot_duration / 2);
+
+    // Run this each slot.
+    let interval_duration = slot_duration;
+
+    let duties_service = client.duties_service.clone();
+    let log_1 = context.log.clone();
+    let log_2 = context.log.clone();
+
+    let interval_future = Interval::new(start_instant, interval_duration)
+        .map_err(
+            move |e| error!(log_1, "Slot notifier timer failed"; "error" => format!("{:?}", e)),
+        )
+        .for_each(move |_| {
+            let log = log_2.clone();
+
+            if let Some(slot) = duties_service.slot_clock.now() {
+                let epoch = slot.epoch(T::slots_per_epoch());
+
+                let total_validators = duties_service.total_validator_count();
+                let proposing_validators = duties_service.proposer_count(epoch);
+                let attesting_validators = duties_service.attester_count(epoch);
+
+                if total_validators == 0 {
+                    error!(log, "No validators present")
+                } else if total_validators == attesting_validators {
+                    info!(
+                        log_2,
+                        "All validators active";
+                        "proposers" => proposing_validators,
+                        "active_validators" => attesting_validators,
+                        "total_validators" => total_validators,
+                        "epoch" => format!("{}", epoch),
+                        "slot" => format!("{}", slot),
+                    );
+                } else if attesting_validators > 0 {
+                    info!(
+                        log_2,
+                        "Some validators active";
+                        "proposers" => proposing_validators,
+                        "active_validators" => attesting_validators,
+                        "total_validators" => total_validators,
+                        "epoch" => format!("{}", epoch),
+                        "slot" => format!("{}", slot),
+                    );
+                } else {
+                    info!(
+                        log_2,
+                        "Awaiting activation";
+                        "validators" => total_validators,
+                        "epoch" => format!("{}", epoch),
+                        "slot" => format!("{}", slot),
+                    );
+                }
+            } else {
+                error!(log, "Unable to read slot clock");
+            }
+
+            Ok(())
+        });
+
+    let (exit_signal, exit) = exit_future::signal();
+    let log = context.log.clone();
+    client.context.executor.spawn(
+        exit.until(interval_future)
+            .map(move |_| info!(log, "Shutdown complete")),
+    );
+
+    Ok(exit_signal)
+}