Optimize attestation processing (#841)

* Start updating types

* WIP

* Signature hacking

* Existing EF tests passing with fake_crypto

* Updates

* Delete outdated API spec

* The refactor continues

* It compiles

* WIP test fixes

* All release tests passing bar genesis state parsing

* Update and test YamlConfig

* Update to spec v0.10 compatible BLS

* Updates to BLS EF tests

* Add EF test for AggregateVerify

And delete unused hash2curve tests for uncompressed points

* Update EF tests to v0.10.1

* Use optional block root correctly in block proc

* Use genesis fork in deposit domain. All tests pass

* Cargo fmt

* Fast aggregate verify test

* Update REST API docs

* Cargo fmt

* Fix unused import

* Bump spec tags to v0.10.1

* Add `seconds_per_eth1_block` to chainspec

* Update to timestamp based eth1 voting scheme

* Return None from `get_votes_to_consider` if block cache is empty

* Handle overflows in `is_candidate_block`

* Revert to failing tests

* Fix eth1 data sets test

* Choose default vote according to spec

* Fix collect_valid_votes tests

* Fix `get_votes_to_consider` to choose all eligible blocks

* Uncomment winning_vote tests

* Add comments; remove unused code

* Reduce seconds_per_eth1_block for simulation

* Addressed review comments

* Add test for default vote case

* Fix logs

* Remove unused functions

* Meter default eth1 votes

* Fix comments

* Address review comments; remove unused dependency

* Add first attempt at attestation proc. re-write

* Add version 2 of attestation processing

* Minor fixes

* Add validator pubkey cache

* Make get_indexed_attestation take a committee

* Link signature processing into new attn verification

* First working version

* Ensure pubkey cache is updated

* Add more metrics, slight optimizations

* Clone committee cache during attestation processing

* Update shuffling cache during block processing

* Remove old commented-out code

* Fix shuffling cache insert bug

* Used indexed attestation in fork choice

* Restructure attn processing, add metrics

* Add more detailed metrics

* Tidy, fix failing tests

* Fix failing tests, tidy

* Disable/delete two outdated tests

* Tidy

* Add pubkey cache persistence file

* Add more comments

* Integrate persistence file into builder

* Add pubkey cache tests

* Add data_dir to beacon chain builder

* Remove Option in pubkey cache persistence file

* Ensure consistency between datadir/data_dir

* Fix failing network test

* Tidy

* Fix todos

* Add attestation processing tests

* Add another test

* Only run attestation tests in release

* Make attestation tests MainnetEthSpec

* Address Michael's comments

* Remove redundant check

* Fix warning

* Fix failing test

Co-authored-by: Michael Sproul <micsproul@gmail.com>
Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>

beacon_node/beacon_chain/src/builder.rs

@@ -2,7 +2,9 @@ use crate::eth1_chain::CachingEth1Backend;
 use crate::events::NullEventHandler;
 use crate::head_tracker::HeadTracker;
 use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY};
+use crate::shuffling_cache::ShufflingCache;
 use crate::timeout_rw_lock::TimeoutRwLock;
+use crate::validator_pubkey_cache::ValidatorPubkeyCache;
 use crate::{
     BeaconChain, BeaconChainTypes, CheckPoint, Eth1Chain, Eth1ChainBackend, EventHandler,
     ForkChoice,
@@ -13,6 +15,7 @@ use proto_array_fork_choice::ProtoArrayForkChoice;
 use slog::{info, Logger};
 use slot_clock::{SlotClock, TestingSlotClock};
 use std::marker::PhantomData;
+use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Duration;
 use store::Store;
@@ -20,6 +23,8 @@ use types::{
     BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Signature, SignedBeaconBlock, Slot,
 };
 
+pub const PUBKEY_CACHE_FILENAME: &str = "pubkey_cache.ssz";
+
 /// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing
 /// functionality and only exists to satisfy the type system.
 pub struct Witness<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>(
@@ -73,6 +78,9 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     slot_clock: Option<T::SlotClock>,
     persisted_beacon_chain: Option<PersistedBeaconChain<T>>,
     head_tracker: Option<HeadTracker>,
+    data_dir: Option<PathBuf>,
+    pubkey_cache_path: Option<PathBuf>,
+    validator_pubkey_cache: Option<ValidatorPubkeyCache>,
     spec: ChainSpec,
     log: Option<Logger>,
 }
@@ -106,6 +114,9 @@ where
             slot_clock: None,
             persisted_beacon_chain: None,
             head_tracker: None,
+            pubkey_cache_path: None,
+            data_dir: None,
+            validator_pubkey_cache: None,
             spec: TEthSpec::default_spec(),
             log: None,
         }
@@ -142,6 +153,15 @@ where
         self
     }
 
+    /// Sets the location to the pubkey cache file.
+    ///
+    /// Should generally be called early in the build chain.
+    pub fn data_dir(mut self, path: PathBuf) -> Self {
+        self.pubkey_cache_path = Some(path.join(PUBKEY_CACHE_FILENAME));
+        self.data_dir = Some(path);
+        self
+    }
+
     /// Attempt to load an existing chain from the builder's `Store`.
     ///
     /// May initialize several components; including the op_pool and finalized checkpoints.
@@ -151,6 +171,11 @@ where
             .as_ref()
             .ok_or_else(|| "resume_from_db requires a log".to_string())?;
 
+        let pubkey_cache_path = self
+            .pubkey_cache_path
+            .as_ref()
+            .ok_or_else(|| "resume_from_db requires a data_dir".to_string())?;
+
         info!(
             log,
             "Starting beacon chain";
@@ -194,6 +219,11 @@ where
         };
         self.persisted_beacon_chain = Some(p);
 
+        let pubkey_cache = ValidatorPubkeyCache::load_from_file(pubkey_cache_path)
+            .map_err(|e| format!("Unable to open persisted pubkey cache: {:?}", e))?;
+
+        self.validator_pubkey_cache = Some(pubkey_cache);
+
         Ok(self)
     }
 
@@ -308,6 +338,18 @@ where
             return Err("beacon_block.state_root != beacon_state".to_string());
         }
 
+        let pubkey_cache_path = self
+            .pubkey_cache_path
+            .ok_or_else(|| "Cannot build without a pubkey cache path".to_string())?;
+
+        let validator_pubkey_cache = self
+            .validator_pubkey_cache
+            .map(|cache| Ok(cache))
+            .unwrap_or_else(|| {
+                ValidatorPubkeyCache::new(&canonical_head.beacon_state, pubkey_cache_path)
+                    .map_err(|e| format!("Unable to init validator pubkey cache: {:?}", e))
+            })?;
+
         let beacon_chain = BeaconChain {
             spec: self.spec,
             store: self
@@ -334,6 +376,8 @@ where
                 .event_handler
                 .ok_or_else(|| "Cannot build without an event handler".to_string())?,
             head_tracker: self.head_tracker.unwrap_or_default(),
+            shuffling_cache: TimeoutRwLock::new(ShufflingCache::new()),
+            validator_pubkey_cache: TimeoutRwLock::new(validator_pubkey_cache),
             log: log.clone(),
         };
 
@@ -384,6 +428,7 @@ where
 
             let backend = ProtoArrayForkChoice::new(
                 finalized_checkpoint.beacon_block.message.slot,
+                finalized_checkpoint.beacon_block.message.state_root,
                 // Note: here we set the `justified_epoch` to be the same as the epoch of the
                 // finalized checkpoint. Whilst this finalized checkpoint may actually point to
                 // a _later_ justified checkpoint, that checkpoint won't yet exist in the fork
@@ -539,6 +584,7 @@ mod test {
     use ssz::Encode;
     use std::time::Duration;
     use store::{migrate::NullMigrator, MemoryStore};
+    use tempfile::tempdir;
     use types::{EthSpec, MinimalEthSpec, Slot};
 
     type TestEthSpec = MinimalEthSpec;
@@ -556,6 +602,7 @@ mod test {
         let log = get_logger();
         let store = Arc::new(MemoryStore::open());
         let spec = MinimalEthSpec::default_spec();
+        let data_dir = tempdir().expect("should create temporary data_dir");
 
         let genesis_state = interop_genesis_state(
             &generate_deterministic_keypairs(validator_count),
@@ -568,6 +615,7 @@ mod test {
             .logger(log.clone())
             .store(store)
             .store_migrator(NullMigrator)
+            .data_dir(data_dir.path().to_path_buf())
             .genesis_state(genesis_state)
             .expect("should build state using recent genesis")
             .dummy_eth1_backend()