Optimize attestation processing (#841)

* Start updating types

* WIP

* Signature hacking

* Existing EF tests passing with fake_crypto

* Updates

* Delete outdated API spec

* The refactor continues

* It compiles

* WIP test fixes

* All release tests passing bar genesis state parsing

* Update and test YamlConfig

* Update to spec v0.10 compatible BLS

* Updates to BLS EF tests

* Add EF test for AggregateVerify

And delete unused hash2curve tests for uncompressed points

* Update EF tests to v0.10.1

* Use optional block root correctly in block proc

* Use genesis fork in deposit domain. All tests pass

* Cargo fmt

* Fast aggregate verify test

* Update REST API docs

* Cargo fmt

* Fix unused import

* Bump spec tags to v0.10.1

* Add `seconds_per_eth1_block` to chainspec

* Update to timestamp based eth1 voting scheme

* Return None from `get_votes_to_consider` if block cache is empty

* Handle overflows in `is_candidate_block`

* Revert to failing tests

* Fix eth1 data sets test

* Choose default vote according to spec

* Fix collect_valid_votes tests

* Fix `get_votes_to_consider` to choose all eligible blocks

* Uncomment winning_vote tests

* Add comments; remove unused code

* Reduce seconds_per_eth1_block for simulation

* Addressed review comments

* Add test for default vote case

* Fix logs

* Remove unused functions

* Meter default eth1 votes

* Fix comments

* Address review comments; remove unused dependency

* Add first attempt at attestation proc. re-write

* Add version 2 of attestation processing

* Minor fixes

* Add validator pubkey cache

* Make get_indexed_attestation take a committee

* Link signature processing into new attn verification

* First working version

* Ensure pubkey cache is updated

* Add more metrics, slight optimizations

* Clone committee cache during attestation processing

* Update shuffling cache during block processing

* Remove old commented-out code

* Fix shuffling cache insert bug

* Used indexed attestation in fork choice

* Restructure attn processing, add metrics

* Add more detailed metrics

* Tidy, fix failing tests

* Fix failing tests, tidy

* Disable/delete two outdated tests

* Tidy

* Add pubkey cache persistence file

* Add more comments

* Integrate persistence file into builder

* Add pubkey cache tests

* Add data_dir to beacon chain builder

* Remove Option in pubkey cache persistence file

* Ensure consistency between datadir/data_dir

* Fix failing network test

* Tidy

* Fix todos

* Add attestation processing tests

* Add another test

* Only run attestation tests in release

* Make attestation tests MainnetEthSpec

* Address Michael's comments

* Remove redundant check

* Fix warning

* Fix failing test

Co-authored-by: Michael Sproul <micsproul@gmail.com>
Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>

eth2/operation_pool/src/attestation.rs

@@ -18,7 +18,10 @@ impl<'a, T: EthSpec> AttMaxCover<'a, T> {
         spec: &ChainSpec,
     ) -> Option<Self> {
         let fresh_validators = earliest_attestation_validators(att, state);
-        let indices = get_attesting_indices(state, &att.data, &fresh_validators).ok()?;
+        let committee = state
+            .get_beacon_committee(att.data.slot, att.data.index)
+            .ok()?;
+        let indices = get_attesting_indices::<T>(committee.committee, &fresh_validators).ok()?;
         let fresh_validators_rewards: HashMap<u64, u64> = indices
             .iter()
             .map(|i| *i as u64)

eth2/operation_pool/src/attestation_id.rs

@@ -2,7 +2,7 @@ use int_to_bytes::int_to_bytes8;
 use serde_derive::{Deserialize, Serialize};
 use ssz::ssz_encode;
 use ssz_derive::{Decode, Encode};
-use types::{AttestationData, BeaconState, ChainSpec, Domain, Epoch, EthSpec};
+use types::{AttestationData, ChainSpec, Domain, Epoch, Fork};
 
 /// Serialized `AttestationData` augmented with a domain to encode the fork info.
 #[derive(
@@ -16,23 +16,15 @@ pub struct AttestationId {
 const DOMAIN_BYTES_LEN: usize = 8;
 
 impl AttestationId {
-    pub fn from_data<T: EthSpec>(
-        attestation: &AttestationData,
-        state: &BeaconState<T>,
-        spec: &ChainSpec,
-    ) -> Self {
+    pub fn from_data(attestation: &AttestationData, fork: &Fork, spec: &ChainSpec) -> Self {
         let mut bytes = ssz_encode(attestation);
         let epoch = attestation.target.epoch;
-        bytes.extend_from_slice(&AttestationId::compute_domain_bytes(epoch, state, spec));
+        bytes.extend_from_slice(&AttestationId::compute_domain_bytes(epoch, fork, spec));
         AttestationId { v: bytes }
     }
 
-    pub fn compute_domain_bytes<T: EthSpec>(
-        epoch: Epoch,
-        state: &BeaconState<T>,
-        spec: &ChainSpec,
-    ) -> Vec<u8> {
-        int_to_bytes8(spec.get_domain(epoch, Domain::BeaconAttester, &state.fork))
+    pub fn compute_domain_bytes(epoch: Epoch, fork: &Fork, spec: &ChainSpec) -> Vec<u8> {
+        int_to_bytes8(spec.get_domain(epoch, Domain::BeaconAttester, fork))
     }
 
     pub fn domain_bytes_match(&self, domain_bytes: &[u8]) -> bool {

eth2/operation_pool/src/lib.rs

@@ -22,7 +22,7 @@ use std::collections::{hash_map, HashMap, HashSet};
 use std::marker::PhantomData;
 use types::{
     typenum::Unsigned, Attestation, AttesterSlashing, BeaconState, BeaconStateError, ChainSpec,
-    EthSpec, ProposerSlashing, RelativeEpoch, SignedVoluntaryExit, Validator,
+    EthSpec, Fork, ProposerSlashing, RelativeEpoch, SignedVoluntaryExit, Validator,
 };
 
 #[derive(Default, Debug)]
@@ -57,10 +57,10 @@ impl<T: EthSpec> OperationPool<T> {
     pub fn insert_attestation(
         &self,
         attestation: Attestation<T>,
-        state: &BeaconState<T>,
+        fork: &Fork,
         spec: &ChainSpec,
     ) -> Result<(), AttestationValidationError> {
-        let id = AttestationId::from_data(&attestation.data, state, spec);
+        let id = AttestationId::from_data(&attestation.data, fork, spec);
 
         // Take a write lock on the attestations map.
         let mut attestations = self.attestations.write();
@@ -106,8 +106,9 @@ impl<T: EthSpec> OperationPool<T> {
         // Attestations for the current fork, which may be from the current or previous epoch.
         let prev_epoch = state.previous_epoch();
         let current_epoch = state.current_epoch();
-        let prev_domain_bytes = AttestationId::compute_domain_bytes(prev_epoch, state, spec);
-        let curr_domain_bytes = AttestationId::compute_domain_bytes(current_epoch, state, spec);
+        let prev_domain_bytes = AttestationId::compute_domain_bytes(prev_epoch, &state.fork, spec);
+        let curr_domain_bytes =
+            AttestationId::compute_domain_bytes(current_epoch, &state.fork, spec);
         let reader = self.attestations.read();
         let active_indices = state
             .get_cached_active_validator_indices(RelativeEpoch::Current)
@@ -180,8 +181,8 @@ impl<T: EthSpec> OperationPool<T> {
         spec: &ChainSpec,
     ) -> (AttestationId, AttestationId) {
         (
-            AttestationId::from_data(&slashing.attestation_1.data, state, spec),
-            AttestationId::from_data(&slashing.attestation_2.data, state, spec),
+            AttestationId::from_data(&slashing.attestation_1.data, &state.fork, spec),
+            AttestationId::from_data(&slashing.attestation_2.data, &state.fork, spec),
         )
     }
 
@@ -547,7 +548,7 @@ mod release_tests {
                     spec,
                     None,
                 );
-                op_pool.insert_attestation(att, state, spec).unwrap();
+                op_pool.insert_attestation(att, &state.fork, spec).unwrap();
             }
         }
 
@@ -616,9 +617,9 @@ mod release_tests {
                 None,
             );
             op_pool
-                .insert_attestation(att.clone(), state, spec)
+                .insert_attestation(att.clone(), &state.fork, spec)
                 .unwrap();
-            op_pool.insert_attestation(att, state, spec).unwrap();
+            op_pool.insert_attestation(att, &state.fork, spec).unwrap();
         }
 
         assert_eq!(op_pool.num_attestations(), committees.len());
@@ -655,7 +656,7 @@ mod release_tests {
                     spec,
                     None,
                 );
-                op_pool.insert_attestation(att, state, spec).unwrap();
+                op_pool.insert_attestation(att, &state.fork, spec).unwrap();
             }
         }
 
@@ -703,7 +704,7 @@ mod release_tests {
                     spec,
                     if i == 0 { None } else { Some(0) },
                 );
-                op_pool.insert_attestation(att, state, spec).unwrap();
+                op_pool.insert_attestation(att, &state.fork, spec).unwrap();
             }
         };
 
@@ -776,7 +777,7 @@ mod release_tests {
                     spec,
                     if i == 0 { None } else { Some(0) },
                 );
-                op_pool.insert_attestation(att, state, spec).unwrap();
+                op_pool.insert_attestation(att, &state.fork, spec).unwrap();
             }
         };
 
@@ -816,8 +817,14 @@ mod release_tests {
 
         for att in &best_attestations {
             let fresh_validators_bitlist = earliest_attestation_validators(att, state);
-            let att_indices =
-                get_attesting_indices(state, &att.data, &fresh_validators_bitlist).unwrap();
+            let committee = state
+                .get_beacon_committee(att.data.slot, att.data.index)
+                .expect("should get beacon committee");
+            let att_indices = get_attesting_indices::<MainnetEthSpec>(
+                committee.committee,
+                &fresh_validators_bitlist,
+            )
+            .unwrap();
             let fresh_indices = &att_indices - &seen_indices;
 
             let rewards = fresh_indices