Optimize attestation processing (#841)

* Start updating types

* WIP

* Signature hacking

* Existing EF tests passing with fake_crypto

* Updates

* Delete outdated API spec

* The refactor continues

* It compiles

* WIP test fixes

* All release tests passing bar genesis state parsing

* Update and test YamlConfig

* Update to spec v0.10 compatible BLS

* Updates to BLS EF tests

* Add EF test for AggregateVerify

And delete unused hash2curve tests for uncompressed points

* Update EF tests to v0.10.1

* Use optional block root correctly in block proc

* Use genesis fork in deposit domain. All tests pass

* Cargo fmt

* Fast aggregate verify test

* Update REST API docs

* Cargo fmt

* Fix unused import

* Bump spec tags to v0.10.1

* Add `seconds_per_eth1_block` to chainspec

* Update to timestamp based eth1 voting scheme

* Return None from `get_votes_to_consider` if block cache is empty

* Handle overflows in `is_candidate_block`

* Revert to failing tests

* Fix eth1 data sets test

* Choose default vote according to spec

* Fix collect_valid_votes tests

* Fix `get_votes_to_consider` to choose all eligible blocks

* Uncomment winning_vote tests

* Add comments; remove unused code

* Reduce seconds_per_eth1_block for simulation

* Addressed review comments

* Add test for default vote case

* Fix logs

* Remove unused functions

* Meter default eth1 votes

* Fix comments

* Address review comments; remove unused dependency

* Add first attempt at attestation proc. re-write

* Add version 2 of attestation processing

* Minor fixes

* Add validator pubkey cache

* Make get_indexed_attestation take a committee

* Link signature processing into new attn verification

* First working version

* Ensure pubkey cache is updated

* Add more metrics, slight optimizations

* Clone committee cache during attestation processing

* Update shuffling cache during block processing

* Remove old commented-out code

* Fix shuffling cache insert bug

* Used indexed attestation in fork choice

* Restructure attn processing, add metrics

* Add more detailed metrics

* Tidy, fix failing tests

* Fix failing tests, tidy

* Disable/delete two outdated tests

* Tidy

* Add pubkey cache persistence file

* Add more comments

* Integrate persistence file into builder

* Add pubkey cache tests

* Add data_dir to beacon chain builder

* Remove Option in pubkey cache persistence file

* Ensure consistency between datadir/data_dir

* Fix failing network test

* Tidy

* Fix todos

* Add attestation processing tests

* Add another test

* Only run attestation tests in release

* Make attestation tests MainnetEthSpec

* Address Michael's comments

* Remove redundant check

* Fix warning

* Fix failing test

Co-authored-by: Michael Sproul <micsproul@gmail.com>
Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>

beacon_node/beacon_chain/src/test_utils.rs

@@ -18,6 +18,7 @@ use store::{
     migrate::{BlockingMigrator, NullMigrator},
     DiskStore, MemoryStore, Migrate, Store,
 };
+use tempfile::{tempdir, TempDir};
 use types::{
     AggregateSignature, Attestation, BeaconState, ChainSpec, Domain, EthSpec, Hash256, Keypair,
     SecretKey, Signature, SignedBeaconBlock, SignedRoot, Slot,
@@ -76,11 +77,13 @@ pub struct BeaconChainHarness<T: BeaconChainTypes> {
     pub chain: BeaconChain<T>,
     pub keypairs: Vec<Keypair>,
     pub spec: ChainSpec,
+    pub data_dir: TempDir,
 }
 
 impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
     /// Instantiate a new harness with `validator_count` initial validators.
     pub fn new(eth_spec_instance: E, keypairs: Vec<Keypair>) -> Self {
+        let data_dir = tempdir().expect("should create temporary data_dir");
         let spec = E::default_spec();
 
         let log = NullLoggerBuilder.build().expect("logger should build");
@@ -90,6 +93,7 @@ impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
             .custom_spec(spec.clone())
             .store(Arc::new(MemoryStore::open()))
             .store_migrator(NullMigrator)
+            .data_dir(data_dir.path().to_path_buf())
             .genesis_state(
                 interop_genesis_state::<E>(&keypairs, HARNESS_GENESIS_TIME, &spec)
                     .expect("should generate interop state"),
@@ -109,6 +113,7 @@ impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
             spec: chain.spec.clone(),
             chain,
             keypairs,
+            data_dir,
         }
     }
 }
@@ -120,6 +125,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
         store: Arc<DiskStore<E>>,
         keypairs: Vec<Keypair>,
     ) -> Self {
+        let data_dir = tempdir().expect("should create temporary data_dir");
         let spec = E::default_spec();
 
         let log = NullLoggerBuilder.build().expect("logger should build");
@@ -129,6 +135,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
             .custom_spec(spec.clone())
             .store(store.clone())
             .store_migrator(<BlockingMigrator<_> as Migrate<_, E>>::new(store))
+            .data_dir(data_dir.path().to_path_buf())
             .genesis_state(
                 interop_genesis_state::<E>(&keypairs, HARNESS_GENESIS_TIME, &spec)
                     .expect("should generate interop state"),
@@ -148,6 +155,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
             spec: chain.spec.clone(),
             chain,
             keypairs,
+            data_dir,
         }
     }
 
@@ -156,6 +164,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
         eth_spec_instance: E,
         store: Arc<DiskStore<E>>,
         keypairs: Vec<Keypair>,
+        data_dir: TempDir,
     ) -> Self {
         let spec = E::default_spec();
 
@@ -166,6 +175,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
             .custom_spec(spec)
             .store(store.clone())
             .store_migrator(<BlockingMigrator<_> as Migrate<_, E>>::new(store))
+            .data_dir(data_dir.path().to_path_buf())
             .resume_from_db(Eth1Config::default())
             .expect("should resume beacon chain from db")
             .dummy_eth1_backend()
@@ -182,6 +192,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
             spec: chain.spec.clone(),
             chain,
             keypairs,
+            data_dir,
         }
     }
 }
@@ -400,6 +411,7 @@ where
                                 let message = attestation.data.signing_root(domain);
 
                                 let mut agg_sig = AggregateSignature::new();
+
                                 agg_sig.add(&Signature::new(
                                     message.as_bytes(),
                                     self.get_sk(*validator_index),