Validator client refactor (#618)

* Update to spec v0.9.0

* Update to v0.9.1

* Bump spec tags for v0.9.1

* Formatting, fix CI failures

* Resolve accidental KeyPair merge conflict

* Document new BeaconState functions

* Add `validator` changes from `validator-to-rest`

* Add initial (failing) REST api tests

* Fix signature parsing

* Add more tests

* Refactor http router

* Add working tests for publish beacon block

* Add validator duties tests

* Move account_manager under `lighthouse` binary

* Unify logfile handling in `environment` crate.

* Fix incorrect cache drops in `advance_caches`

* Update fork choice for v0.9.1

* Add `deposit_contract` crate

* Add progress on validator onboarding

* Add unfinished attesation code

* Update account manager CLI

* Write eth1 data file as hex string

* Integrate ValidatorDirectory with validator_client

* Move ValidatorDirectory into validator_client

* Clean up some FIXMEs

* Add beacon_chain_sim

* Fix a few docs/logs

* Expand `beacon_chain_sim`

* Fix spec for `beacon_chain_sim

* More testing for api

* Start work on attestation endpoint

* Reject empty attestations

* Allow attestations to genesis block

* Add working tests for `rest_api` validator endpoint

* Remove grpc from beacon_node

* Start heavy refactor of validator client

- Block production is working

* Prune old validator client files

* Start works on attestation service

* Add attestation service to validator client

* Use full pubkey for validator directories

* Add validator duties post endpoint

* Use par_iter for keypair generation

* Use bulk duties request in validator client

* Add version http endpoint tests

* Add interop keys and startup wait

* Ensure a prompt exit

* Add duties pruning

* Fix compile error in beacon node tests

* Add github workflow

* Modify rust.yaml

* Modify gitlab actions

* Add to CI file

* Add sudo to CI npm install

* Move cargo fmt to own job in tests

* Fix cargo fmt in CI

* Add rustup update before cargo fmt

* Change name of CI job

* Make other CI jobs require cargo fmt

* Add CI badge

* Remove gitlab and travis files

* Add different http timeout for debug

* Update docker file, use makefile in CI

* Use make in the dockerfile, skip the test

* Use the makefile for debug GI test

* Update book

* Tidy grpc and misc things

* Apply discv5 fixes

* Address other minor issues

* Fix warnings

* Attempt fix for addr parsing

* Tidy validator config, CLIs

* Tidy comments

* Tidy signing, reduce ForkService duplication

* Fail if skipping too many slots

* Set default recent genesis time to 0

* Add custom http timeout to validator

* Fix compile bug in node_test_rig

* Remove old bootstrap flag from val CLI

* Update docs

* Tidy val client

* Change val client log levels

* Add comments, more validity checks

* Fix compile error, add comments

* Undo changes to eth2-libp2p/src

* Reduce duplication of keypair generation

* Add more logging for validator duties

* Fix beacon_chain_sim, nitpicks

* Fix compile error, minor nits

* Address Michael's comments

lighthouse/environment/src/lib.rs

@@ -193,7 +193,7 @@ impl<E: EthSpec> Environment<E> {
     }
 
     /// Returns a `Context` where the `service_name` is added to the logger output.
-    pub fn service_context(&mut self, service_name: &'static str) -> RuntimeContext<E> {
+    pub fn service_context(&mut self, service_name: String) -> RuntimeContext<E> {
         RuntimeContext {
             executor: self.runtime.executor(),
             log: self.log.new(o!("service" => service_name)),

lighthouse/src/main.rs

@@ -29,11 +29,11 @@ fn main() {
                 .short("s")
                 .long("spec")
                 .value_name("TITLE")
-                .help("Specifies the default eth2 spec type. Only effective when creating a new datadir.")
+                .help("Specifies the default eth2 spec type.")
                 .takes_value(true)
                 .possible_values(&["mainnet", "minimal", "interop"])
                 .global(true)
-                .default_value("minimal")
+                .default_value("minimal"),
         )
         .arg(
             Arg::with_name("logfile")
@@ -51,6 +51,15 @@ fn main() {
                 .possible_values(&["info", "debug", "trace", "warn", "error", "crit"])
                 .default_value("trace"),
         )
+        .arg(
+            Arg::with_name("datadir")
+                .long("datadir")
+                .short("d")
+                .value_name("DIR")
+                .global(true)
+                .help("Data directory for keys and databases.")
+                .takes_value(true),
+        )
         .subcommand(beacon_node::cli_app())
         .subcommand(validator_client::cli_app())
         .subcommand(account_manager::cli_app())
@@ -123,17 +132,17 @@ fn run<E: EthSpec>(
     //
     // Creating a command which can run both might be useful future works.
 
-    if let Some(sub_matches) = matches.subcommand_matches("Account Manager") {
+    if let Some(sub_matches) = matches.subcommand_matches("account_manager") {
         let runtime_context = environment.core_context();
 
         account_manager::run(sub_matches, runtime_context);
 
-        // Exit early if the account manager was run. It does not used the tokio executor, so no
-        // need to wait for it to shutdown.
+        // Exit early if the account manager was run. It does not use the tokio executor, no need
+        // to wait for it to shutdown.
         return Ok(());
     }
 
-    let beacon_node = if let Some(sub_matches) = matches.subcommand_matches("Beacon Node") {
+    let beacon_node = if let Some(sub_matches) = matches.subcommand_matches("beacon_node") {
         let runtime_context = environment.core_context();
 
         let beacon = environment
@@ -149,11 +158,16 @@ fn run<E: EthSpec>(
         None
     };
 
-    let validator_client = if let Some(sub_matches) = matches.subcommand_matches("Validator Client")
+    let validator_client = if let Some(sub_matches) = matches.subcommand_matches("validator_client")
     {
         let runtime_context = environment.core_context();
 
-        let validator = ProductionValidatorClient::new_from_cli(runtime_context, sub_matches)
+        let mut validator = environment
+            .runtime()
+            .block_on(ProductionValidatorClient::new_from_cli(
+                runtime_context,
+                sub_matches,
+            ))
             .map_err(|e| format!("Failed to init validator client: {}", e))?;
 
         validator