Attestation processing (#497)

* Renamed fork_choice::process_attestation_from_block

* Processing attestation in fork choice

* Retrieving state from store and checking signature

* Looser check on beacon state validity.

* Cleaned up get_attestation_state

* Expanded fork choice api to provide latest validator message.

* Checking if the an attestation contains a latest message

* Correct process_attestation error handling.

* Copy paste error in comment fixed.

* Tidy ancestor iterators

* Getting attestation slot via helper method

* Refactored attestation creation in test utils

* Revert "Refactored attestation creation in test utils"

This reverts commit 4d277fe4239a7194758b18fb5c00dfe0b8231306.

* Integration tests for free attestation processing

* Implicit conflicts resolved.

* formatting

* Do first pass on Grants code

* Add another attestation processing test

* Tidy attestation processing

* Remove old code fragment

* Add non-compiling half finished changes

* Simplify, fix bugs, add tests for chain iters

* Remove attestation processing from op pool

* Fix bug with fork choice, tidy

* Fix overly restrictive check in fork choice.

* Ensure committee cache is build during attn proc

* Ignore unknown blocks at fork choice

* Various minor fixes

* Make fork choice write lock in to read lock

* Remove unused method

* Tidy comments

* Fix attestation prod. target roots change

* Fix compile error in store iters

* Reject any attestation prior to finalization

* Fix minor PR comments

* Remove duplicated attestation finalization check

* Remove awkward `let` statement

eth2/lmd_ghost/src/lib.rs

@@ -43,4 +43,7 @@ pub trait LmdGhost<S: Store, E: EthSpec>: Send + Sync {
         finalized_block: &BeaconBlock<E>,
         finalized_block_root: Hash256,
     ) -> Result<()>;
+
+    /// Returns the latest message for a given validator index.
+    fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Slot)>;
 }

eth2/lmd_ghost/src/reduced_tree.rs

@@ -109,6 +109,10 @@ where
             .update_root(new_block.slot, new_root)
             .map_err(|e| format!("update_finalized_root failed: {:?}", e))
     }
+
+    fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Slot)> {
+        self.core.read().latest_message(validator_index)
+    }
 }
 
 struct ReducedTree<T, E> {
@@ -254,6 +258,13 @@ where
         Ok(head_node.block_hash)
     }
 
+    pub fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Slot)> {
+        match self.latest_votes.get_ref(validator_index) {
+            Some(Some(v)) => Some((v.hash.clone(), v.slot.clone())),
+            _ => None,
+        }
+    }
+
     fn find_head_from<'a>(&'a self, start_node: &'a Node) -> Result<&'a Node> {
         if start_node.does_not_have_children() {
             Ok(start_node)
@@ -600,11 +611,7 @@ where
         let block = self.get_block(child)?;
         let state = self.get_state(block.state_root)?;
 
-        Ok(BlockRootsIterator::owned(
-            self.store.clone(),
-            state,
-            block.slot - 1,
-        ))
+        Ok(BlockRootsIterator::owned(self.store.clone(), state))
     }
 
     /// Verify the integrity of `self`. Returns `Ok(())` if the tree has integrity, otherwise returns `Err(description)`.
@@ -769,6 +776,10 @@ where
         &self.0[i]
     }
 
+    pub fn get_ref(&self, i: usize) -> Option<&T> {
+        self.0.get(i)
+    }
+
     pub fn insert(&mut self, i: usize, element: T) {
         self.ensure(i);
         self.0[i] = element;

eth2/operation_pool/src/lib.rs

@@ -15,9 +15,10 @@ use state_processing::per_block_processing::errors::{
     ExitValidationError, ProposerSlashingValidationError, TransferValidationError,
 };
 use state_processing::per_block_processing::{
-    get_slashable_indices_modular, verify_attestation, verify_attestation_time_independent_only,
+    get_slashable_indices_modular, verify_attestation_for_block_inclusion,
     verify_attester_slashing, verify_exit, verify_exit_time_independent_only,
     verify_proposer_slashing, verify_transfer, verify_transfer_time_independent_only,
+    VerifySignatures,
 };
 use std::collections::{btree_map::Entry, hash_map, BTreeMap, HashMap, HashSet};
 use std::marker::PhantomData;
@@ -64,15 +65,16 @@ impl<T: EthSpec> OperationPool<T> {
     }
 
     /// Insert an attestation into the pool, aggregating it with existing attestations if possible.
+    ///
+    /// ## Note
+    ///
+    /// This function assumes the given `attestation` is valid.
     pub fn insert_attestation(
         &self,
         attestation: Attestation<T>,
         state: &BeaconState<T>,
         spec: &ChainSpec,
     ) -> Result<(), AttestationValidationError> {
-        // Check that attestation signatures are valid.
-        verify_attestation_time_independent_only(state, &attestation, spec)?;
-
         let id = AttestationId::from_data(&attestation.data, state, spec);
 
         // Take a write lock on the attestations map.
@@ -128,7 +130,15 @@ impl<T: EthSpec> OperationPool<T> {
             })
             .flat_map(|(_, attestations)| attestations)
             // That are valid...
-            .filter(|attestation| verify_attestation(state, attestation, spec).is_ok())
+            .filter(|attestation| {
+                verify_attestation_for_block_inclusion(
+                    state,
+                    attestation,
+                    spec,
+                    VerifySignatures::True,
+                )
+                .is_ok()
+            })
             .map(|att| AttMaxCover::new(att, earliest_attestation_validators(att, state)));
 
         maximum_cover(valid_attestations, T::MaxAttestations::to_usize())

eth2/state_processing/src/per_block_processing.rs

@@ -15,8 +15,7 @@ pub use is_valid_indexed_attestation::{
     is_valid_indexed_attestation, is_valid_indexed_attestation_without_signature,
 };
 pub use verify_attestation::{
-    verify_attestation, verify_attestation_time_independent_only,
-    verify_attestation_without_signature,
+    verify_attestation_for_block_inclusion, verify_attestation_for_state,
 };
 pub use verify_deposit::{
     get_existing_validator_index, verify_deposit_merkle_proof, verify_deposit_signature,
@@ -37,6 +36,12 @@ mod verify_exit;
 mod verify_proposer_slashing;
 mod verify_transfer;
 
+#[derive(PartialEq)]
+pub enum VerifySignatures {
+    True,
+    False,
+}
+
 /// Updates the state for a new block, whilst validating that the block is valid.
 ///
 /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise
@@ -312,7 +317,8 @@ pub fn process_attestations<T: EthSpec>(
         .par_iter()
         .enumerate()
         .try_for_each(|(i, attestation)| {
-            verify_attestation(state, attestation, spec).map_err(|e| e.into_with_index(i))
+            verify_attestation_for_block_inclusion(state, attestation, spec, VerifySignatures::True)
+                .map_err(|e| e.into_with_index(i))
         })?;
 
     // Update the state in series.

eth2/state_processing/src/per_block_processing/verify_attestation.rs

@@ -1,4 +1,5 @@
 use super::errors::{AttestationInvalid as Invalid, AttestationValidationError as Error};
+use super::VerifySignatures;
 use crate::common::get_indexed_attestation;
 use crate::per_block_processing::{
     is_valid_indexed_attestation, is_valid_indexed_attestation_without_signature,
@@ -6,67 +7,25 @@ use crate::per_block_processing::{
 use tree_hash::TreeHash;
 use types::*;
 
-/// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the
-/// given state.
+/// Returns `Ok(())` if the given `attestation` is valid to be included in a block that is applied
+/// to `state`. Otherwise, returns a descriptive `Err`.
 ///
-/// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity.
+/// Optionally verifies the aggregate signature, depending on `verify_signatures`.
 ///
 /// Spec v0.8.0
-pub fn verify_attestation<T: EthSpec>(
+pub fn verify_attestation_for_block_inclusion<T: EthSpec>(
     state: &BeaconState<T>,
     attestation: &Attestation<T>,
     spec: &ChainSpec,
-) -> Result<(), Error> {
-    verify_attestation_parametric(state, attestation, spec, true, false)
-}
-
-/// Like `verify_attestation` but doesn't run checks which may become true in future states.
-pub fn verify_attestation_time_independent_only<T: EthSpec>(
-    state: &BeaconState<T>,
-    attestation: &Attestation<T>,
-    spec: &ChainSpec,
-) -> Result<(), Error> {
-    verify_attestation_parametric(state, attestation, spec, true, true)
-}
-
-/// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the
-/// given state, without validating the aggregate signature.
-///
-/// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity.
-///
-/// Spec v0.8.0
-pub fn verify_attestation_without_signature<T: EthSpec>(
-    state: &BeaconState<T>,
-    attestation: &Attestation<T>,
-    spec: &ChainSpec,
-) -> Result<(), Error> {
-    verify_attestation_parametric(state, attestation, spec, false, false)
-}
-
-/// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the
-/// given state, optionally validating the aggregate signature.
-///
-///
-/// Spec v0.8.0
-fn verify_attestation_parametric<T: EthSpec>(
-    state: &BeaconState<T>,
-    attestation: &Attestation<T>,
-    spec: &ChainSpec,
-    verify_signature: bool,
-    time_independent_only: bool,
+    verify_signatures: VerifySignatures,
 ) -> Result<(), Error> {
     let data = &attestation.data;
-    verify!(
-        data.crosslink.shard < T::ShardCount::to_u64(),
-        Invalid::BadShard
-    );
 
     // Check attestation slot.
     let attestation_slot = state.get_attestation_data_slot(&data)?;
 
     verify!(
-        time_independent_only
-            || attestation_slot + spec.min_attestation_inclusion_delay <= state.slot,
+        attestation_slot + spec.min_attestation_inclusion_delay <= state.slot,
         Invalid::IncludedTooEarly {
             state: state.slot,
             delay: spec.min_attestation_inclusion_delay,
@@ -81,27 +40,47 @@ fn verify_attestation_parametric<T: EthSpec>(
         }
     );
 
-    // Verify the Casper FFG vote and crosslink data.
-    if !time_independent_only {
-        let parent_crosslink = verify_casper_ffg_vote(attestation, state)?;
+    verify_attestation_for_state(state, attestation, spec, verify_signatures)
+}
 
-        verify!(
-            data.crosslink.parent_root == Hash256::from_slice(&parent_crosslink.tree_hash_root()),
-            Invalid::BadParentCrosslinkHash
-        );
-        verify!(
-            data.crosslink.start_epoch == parent_crosslink.end_epoch,
-            Invalid::BadParentCrosslinkStartEpoch
-        );
-        verify!(
-            data.crosslink.end_epoch
-                == std::cmp::min(
-                    data.target.epoch,
-                    parent_crosslink.end_epoch + spec.max_epochs_per_crosslink
-                ),
-            Invalid::BadParentCrosslinkEndEpoch
-        );
-    }
+/// Returns `Ok(())` if `attestation` is a valid attestation to the chain that precedes the given
+/// `state`.
+///
+/// Returns a descriptive `Err` if the attestation is malformed or does not accurately reflect the
+/// prior blocks in `state`.
+///
+/// Spec v0.8.0
+pub fn verify_attestation_for_state<T: EthSpec>(
+    state: &BeaconState<T>,
+    attestation: &Attestation<T>,
+    spec: &ChainSpec,
+    verify_signature: VerifySignatures,
+) -> Result<(), Error> {
+    let data = &attestation.data;
+    verify!(
+        data.crosslink.shard < T::ShardCount::to_u64(),
+        Invalid::BadShard
+    );
+
+    // Verify the Casper FFG vote and crosslink data.
+    let parent_crosslink = verify_casper_ffg_vote(attestation, state)?;
+
+    verify!(
+        data.crosslink.parent_root == Hash256::from_slice(&parent_crosslink.tree_hash_root()),
+        Invalid::BadParentCrosslinkHash
+    );
+    verify!(
+        data.crosslink.start_epoch == parent_crosslink.end_epoch,
+        Invalid::BadParentCrosslinkStartEpoch
+    );
+    verify!(
+        data.crosslink.end_epoch
+            == std::cmp::min(
+                data.target.epoch,
+                parent_crosslink.end_epoch + spec.max_epochs_per_crosslink
+            ),
+        Invalid::BadParentCrosslinkEndEpoch
+    );
 
     // Crosslink data root is zero (to be removed in phase 1).
     verify!(
@@ -111,7 +90,7 @@ fn verify_attestation_parametric<T: EthSpec>(
 
     // Check signature and bitfields
     let indexed_attestation = get_indexed_attestation(state, attestation)?;
-    if verify_signature {
+    if verify_signature == VerifySignatures::True {
         is_valid_indexed_attestation(state, &indexed_attestation, spec)?;
     } else {
         is_valid_indexed_attestation_without_signature(state, &indexed_attestation, spec)?;

eth2/types/src/beacon_block.rs

@@ -61,6 +61,11 @@ impl<T: EthSpec> BeaconBlock<T> {
         }
     }
 
+    /// Returns the epoch corresponding to `self.slot`.
+    pub fn epoch(&self) -> Epoch {
+        self.slot.epoch(T::slots_per_epoch())
+    }
+
     /// Returns the `signed_root` of the block.
     ///
     /// Spec v0.8.1