Optimize attestation processing (#841)

* Start updating types

* WIP

* Signature hacking

* Existing EF tests passing with fake_crypto

* Updates

* Delete outdated API spec

* The refactor continues

* It compiles

* WIP test fixes

* All release tests passing bar genesis state parsing

* Update and test YamlConfig

* Update to spec v0.10 compatible BLS

* Updates to BLS EF tests

* Add EF test for AggregateVerify

And delete unused hash2curve tests for uncompressed points

* Update EF tests to v0.10.1

* Use optional block root correctly in block proc

* Use genesis fork in deposit domain. All tests pass

* Cargo fmt

* Fast aggregate verify test

* Update REST API docs

* Cargo fmt

* Fix unused import

* Bump spec tags to v0.10.1

* Add `seconds_per_eth1_block` to chainspec

* Update to timestamp based eth1 voting scheme

* Return None from `get_votes_to_consider` if block cache is empty

* Handle overflows in `is_candidate_block`

* Revert to failing tests

* Fix eth1 data sets test

* Choose default vote according to spec

* Fix collect_valid_votes tests

* Fix `get_votes_to_consider` to choose all eligible blocks

* Uncomment winning_vote tests

* Add comments; remove unused code

* Reduce seconds_per_eth1_block for simulation

* Addressed review comments

* Add test for default vote case

* Fix logs

* Remove unused functions

* Meter default eth1 votes

* Fix comments

* Address review comments; remove unused dependency

* Add first attempt at attestation proc. re-write

* Add version 2 of attestation processing

* Minor fixes

* Add validator pubkey cache

* Make get_indexed_attestation take a committee

* Link signature processing into new attn verification

* First working version

* Ensure pubkey cache is updated

* Add more metrics, slight optimizations

* Clone committee cache during attestation processing

* Update shuffling cache during block processing

* Remove old commented-out code

* Fix shuffling cache insert bug

* Used indexed attestation in fork choice

* Restructure attn processing, add metrics

* Add more detailed metrics

* Tidy, fix failing tests

* Fix failing tests, tidy

* Disable/delete two outdated tests

* Tidy

* Add pubkey cache persistence file

* Add more comments

* Integrate persistence file into builder

* Add pubkey cache tests

* Add data_dir to beacon chain builder

* Remove Option in pubkey cache persistence file

* Ensure consistency between datadir/data_dir

* Fix failing network test

* Tidy

* Fix todos

* Add attestation processing tests

* Add another test

* Only run attestation tests in release

* Make attestation tests MainnetEthSpec

* Address Michael's comments

* Remove redundant check

* Fix warning

* Fix failing test

Co-authored-by: Michael Sproul <micsproul@gmail.com>
Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>

beacon_node/client/src/builder.rs

@@ -117,13 +117,14 @@ where
     pub fn beacon_chain_builder(
         mut self,
         client_genesis: ClientGenesis,
-        config: Eth1Config,
+        config: ClientConfig,
     ) -> impl Future<Item = Self, Error = String> {
         let store = self.store.clone();
         let store_migrator = self.store_migrator.take();
         let chain_spec = self.chain_spec.clone();
         let runtime_context = self.runtime_context.clone();
         let eth_spec_instance = self.eth_spec_instance.clone();
+        let data_dir = config.data_dir.clone();
 
         future::ok(())
             .and_then(move |()| {
@@ -144,6 +145,7 @@ where
                     .logger(context.log.clone())
                     .store(store)
                     .store_migrator(store_migrator)
+                    .data_dir(data_dir)
                     .custom_spec(spec.clone());
 
                 Ok((builder, spec, context))
@@ -197,11 +199,11 @@ where
                             info!(
                                 context.log,
                                 "Waiting for eth2 genesis from eth1";
-                                "eth1_node" => &config.endpoint
+                                "eth1_node" => &config.eth1.endpoint
                             );
 
                             let genesis_service =
-                                Eth1GenesisService::new(config, context.log.clone());
+                                Eth1GenesisService::new(config.eth1, context.log.clone());
 
                             let future = genesis_service
                                 .wait_for_genesis_state(
@@ -233,7 +235,7 @@ where
                         }
                         ClientGenesis::Resume => {
                             let future = builder
-                                .resume_from_db(config)
+                                .resume_from_db(config.eth1)
                                 .into_future()
                                 .map(|v| (v, None));
 

beacon_node/client/src/config.rs

@@ -3,6 +3,8 @@ use serde_derive::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
 
+pub const DEFAULT_DATADIR: &str = ".lighthouse";
+
 /// The number initial validators when starting the `Minimal`.
 const TESTNET_SPEC_CONSTANTS: &str = "minimal";
 
@@ -70,7 +72,7 @@ pub struct Config {
 impl Default for Config {
     fn default() -> Self {
         Self {
-            data_dir: PathBuf::from(".lighthouse"),
+            data_dir: PathBuf::from(DEFAULT_DATADIR),
             db_name: "chain_db".to_string(),
             freezer_db_path: None,
             testnet_dir: None,

beacon_node/client/src/lib.rs

@@ -1,6 +1,6 @@
 extern crate slog;
 
-mod config;
+pub mod config;
 mod notifier;
 
 pub mod builder;