Attestation processing (#497)

* Renamed fork_choice::process_attestation_from_block

* Processing attestation in fork choice

* Retrieving state from store and checking signature

* Looser check on beacon state validity.

* Cleaned up get_attestation_state

* Expanded fork choice api to provide latest validator message.

* Checking if the an attestation contains a latest message

* Correct process_attestation error handling.

* Copy paste error in comment fixed.

* Tidy ancestor iterators

* Getting attestation slot via helper method

* Refactored attestation creation in test utils

* Revert "Refactored attestation creation in test utils"

This reverts commit 4d277fe4239a7194758b18fb5c00dfe0b8231306.

* Integration tests for free attestation processing

* Implicit conflicts resolved.

* formatting

* Do first pass on Grants code

* Add another attestation processing test

* Tidy attestation processing

* Remove old code fragment

* Add non-compiling half finished changes

* Simplify, fix bugs, add tests for chain iters

* Remove attestation processing from op pool

* Fix bug with fork choice, tidy

* Fix overly restrictive check in fork choice.

* Ensure committee cache is build during attn proc

* Ignore unknown blocks at fork choice

* Various minor fixes

* Make fork choice write lock in to read lock

* Remove unused method

* Tidy comments

* Fix attestation prod. target roots change

* Fix compile error in store iters

* Reject any attestation prior to finalization

* Fix minor PR comments

* Remove duplicated attestation finalization check

* Remove awkward `let` statement

beacon_node/beacon_chain/tests/tests.rs

@@ -1,29 +1,104 @@
 #![cfg(not(debug_assertions))]
 
+#[macro_use]
+extern crate lazy_static;
+
 use beacon_chain::test_utils::{
     AttestationStrategy, BeaconChainHarness, BlockStrategy, CommonTypes, PersistedBeaconChain,
     BEACON_CHAIN_DB_KEY,
 };
+use beacon_chain::AttestationProcessingOutcome;
 use lmd_ghost::ThreadSafeReducedTree;
 use rand::Rng;
 use store::{MemoryStore, Store};
 use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
-use types::{Deposit, EthSpec, Hash256, MinimalEthSpec, Slot};
+use types::{Deposit, EthSpec, Hash256, Keypair, MinimalEthSpec, RelativeEpoch, Slot};
 
 // Should ideally be divisible by 3.
 pub const VALIDATOR_COUNT: usize = 24;
 
+lazy_static! {
+    /// A cached set of keys.
+    static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
+}
+
 type TestForkChoice = ThreadSafeReducedTree<MemoryStore, MinimalEthSpec>;
 
 fn get_harness(validator_count: usize) -> BeaconChainHarness<TestForkChoice, MinimalEthSpec> {
-    let harness = BeaconChainHarness::new(validator_count);
+    let harness = BeaconChainHarness::from_keypairs(KEYPAIRS[0..validator_count].to_vec());
 
-    // Move past the zero slot.
     harness.advance_slot();
 
     harness
 }
 
+#[test]
+fn iterators() {
+    let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1;
+
+    let harness = get_harness(VALIDATOR_COUNT);
+
+    harness.extend_chain(
+        num_blocks_produced as usize,
+        BlockStrategy::OnCanonicalHead,
+        // No need to produce attestations for this test.
+        AttestationStrategy::SomeValidators(vec![]),
+    );
+
+    let block_roots: Vec<(Hash256, Slot)> = harness.chain.rev_iter_block_roots().collect();
+    let state_roots: Vec<(Hash256, Slot)> = harness.chain.rev_iter_state_roots().collect();
+
+    assert_eq!(
+        block_roots.len(),
+        state_roots.len(),
+        "should be an equal amount of block and state roots"
+    );
+
+    assert!(
+        block_roots.iter().any(|(_root, slot)| *slot == 0),
+        "should contain genesis block root"
+    );
+    assert!(
+        state_roots.iter().any(|(_root, slot)| *slot == 0),
+        "should contain genesis state root"
+    );
+
+    assert_eq!(
+        block_roots.len(),
+        num_blocks_produced as usize + 1,
+        "should contain all produced blocks, plus the genesis block"
+    );
+
+    block_roots.windows(2).for_each(|x| {
+        assert_eq!(
+            x[1].1,
+            x[0].1 - 1,
+            "block root slots should be decreasing by one"
+        )
+    });
+    state_roots.windows(2).for_each(|x| {
+        assert_eq!(
+            x[1].1,
+            x[0].1 - 1,
+            "state root slots should be decreasing by one"
+        )
+    });
+
+    let head = &harness.chain.head();
+
+    assert_eq!(
+        *block_roots.first().expect("should have some block roots"),
+        (head.beacon_block_root, head.beacon_block.slot),
+        "first block root and slot should be for the head block"
+    );
+
+    assert_eq!(
+        *state_roots.first().expect("should have some state roots"),
+        (head.beacon_state_root, head.beacon_state.slot),
+        "first state root and slot should be for the head state"
+    );
+}
+
 #[test]
 fn chooses_fork() {
     let harness = get_harness(VALIDATOR_COUNT);
@@ -251,3 +326,136 @@ fn roundtrip_operation_pool() {
 
     assert_eq!(harness.chain.op_pool, restored_op_pool);
 }
+
+#[test]
+fn free_attestations_added_to_fork_choice_some_none() {
+    let num_blocks_produced = MinimalEthSpec::slots_per_epoch() / 2;
+
+    let harness = get_harness(VALIDATOR_COUNT);
+
+    harness.extend_chain(
+        num_blocks_produced as usize,
+        BlockStrategy::OnCanonicalHead,
+        AttestationStrategy::AllValidators,
+    );
+
+    let state = &harness.chain.head().beacon_state;
+    let fork_choice = &harness.chain.fork_choice;
+
+    let validator_slots: Vec<(usize, Slot)> = (0..VALIDATOR_COUNT)
+        .into_iter()
+        .map(|validator_index| {
+            let slot = state
+                .get_attestation_duties(validator_index, RelativeEpoch::Current)
+                .expect("should get attester duties")
+                .unwrap()
+                .slot;
+
+            (validator_index, slot)
+        })
+        .collect();
+
+    for (validator, slot) in validator_slots.clone() {
+        let latest_message = fork_choice.latest_message(validator);
+
+        if slot <= num_blocks_produced && slot != 0 {
+            assert_eq!(
+                latest_message.unwrap().1,
+                slot,
+                "Latest message slot for {} should be equal to slot {}.",
+                validator,
+                slot
+            )
+        } else {
+            assert!(
+                latest_message.is_none(),
+                "Latest message slot should be None."
+            )
+        }
+    }
+}
+
+#[test]
+fn attestations_with_increasing_slots() {
+    let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
+
+    let harness = get_harness(VALIDATOR_COUNT);
+
+    let mut attestations = vec![];
+
+    for _ in 0..num_blocks_produced {
+        harness.extend_chain(
+            2,
+            BlockStrategy::OnCanonicalHead,
+            // Don't produce & include any attestations (we'll collect them later).
+            AttestationStrategy::SomeValidators(vec![]),
+        );
+
+        attestations.append(&mut harness.get_free_attestations(
+            &AttestationStrategy::AllValidators,
+            &harness.chain.head().beacon_state,
+            harness.chain.head().beacon_block_root,
+            harness.chain.head().beacon_block.slot,
+        ));
+
+        harness.advance_slot();
+    }
+
+    for attestation in attestations {
+        assert_eq!(
+            harness.chain.process_attestation(attestation),
+            Ok(AttestationProcessingOutcome::Processed)
+        )
+    }
+}
+
+#[test]
+fn free_attestations_added_to_fork_choice_all_updated() {
+    let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1;
+
+    let harness = get_harness(VALIDATOR_COUNT);
+
+    harness.extend_chain(
+        num_blocks_produced as usize,
+        BlockStrategy::OnCanonicalHead,
+        AttestationStrategy::AllValidators,
+    );
+
+    let state = &harness.chain.head().beacon_state;
+    let fork_choice = &harness.chain.fork_choice;
+
+    let validators: Vec<usize> = (0..VALIDATOR_COUNT).collect();
+    let slots: Vec<Slot> = validators
+        .iter()
+        .map(|&v| {
+            state
+                .get_attestation_duties(v, RelativeEpoch::Current)
+                .expect("should get attester duties")
+                .unwrap()
+                .slot
+        })
+        .collect();
+    let validator_slots: Vec<(&usize, Slot)> = validators.iter().zip(slots).collect();
+
+    for (validator, slot) in validator_slots {
+        let latest_message = fork_choice.latest_message(*validator);
+
+        assert_eq!(
+            latest_message.unwrap().1,
+            slot,
+            "Latest message slot should be equal to attester duty."
+        );
+
+        if slot != num_blocks_produced {
+            let block_root = state
+                .get_block_root(slot)
+                .expect("Should get block root at slot");
+
+            assert_eq!(
+                latest_message.unwrap().0,
+                *block_root,
+                "Latest message block root should be equal to block at slot."
+            );
+        }
+    }
+}