v0.12 fork choice update (#1229)

* Incomplete scraps

* Add progress on new fork choice impl

* Further progress

* First complete compiling version

* Remove chain reference

* Add new lmd_ghost crate

* Start integrating into beacon chain

* Update `milagro_bls` to new release (#1183)

* Update milagro_bls to new release

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Tidy up fake cryptos

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* move SecretHash to bls and put plaintext back

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Update state processing for v0.12

* Fix EF test runners for v0.12

* Fix some tests

* Fix broken attestation verification test

* More test fixes

* Rough beacon chain impl working

* Remove fork_choice_2

* Remove checkpoint manager

* Half finished ssz impl

* Add missed file

* Add persistence

* Tidy, fix some compile errors

* Remove RwLock from ProtoArrayForkChoice

* Fix store-based compile errors

* Add comments, tidy

* Move function out of ForkChoice struct

* Start testing

* More testing

* Fix compile error

* Tidy beacon_chain::fork_choice

* Queue attestations from the current slot

* Allow fork choice to handle prior-to-genesis start

* Improve error granularity

* Test attestation dequeuing

* Process attestations during block

* Store target root in fork choice

* Move fork choice verification into new crate

* Update tests

* Consensus updates for v0.12 (#1228)

* Update state processing for v0.12

* Fix EF test runners for v0.12

* Fix some tests

* Fix broken attestation verification test

* More test fixes

* Fix typo found in review

* Add `Block` struct to ProtoArray

* Start fixing get_ancestor

* Add rough progress on testing

* Get fork choice tests working

* Progress with testing

* Fix partialeq impl

* Move slot clock from fc_store

* Improve testing

* Add testing for best justified

* Add clone back to SystemTimeSlotClock

* Add balances test

* Start adding balances cache again

* Wire-in balances cache

* Improve tests

* Remove commented-out tests

* Remove beacon_chain::ForkChoice

* Rename crates

* Update wider codebase to new fork_choice layout

* Move advance_slot in test harness

* Tidy ForkChoice::update_time

* Fix verification tests

* Fix compile error with iter::once

* Fix fork choice tests

* Ensure block attestations are processed

* Fix failing beacon_chain tests

* Add first invalid block check

* Add finalized block check

* Progress with testing, new store builder

* Add fixes to get_ancestor

* Fix old genesis justification test

* Fix remaining fork choice tests

* Change root iteration method

* Move on_verified_block

* Remove unused method

* Start adding attestation verification tests

* Add invalid ffg target test

* Add target epoch test

* Add queued attestation test

* Remove old fork choice verification tests

* Tidy, add test

* Move fork choice lock drop

* Rename BeaconForkChoiceStore

* Add comments, tidy BeaconForkChoiceStore

* Update metrics, rename fork_choice_store.rs

* Remove genesis_block_root from ForkChoice

* Tidy

* Update fork_choice comments

* Tidy, add comments

* Tidy, simplify ForkChoice, fix compile issue

* Tidy, removed dead file

* Increase http request timeout

* Fix failing rest_api test

* Set HTTP timeout back to 5s

* Apply fix to get_ancestor

* Address Michael's comments

* Fix typo

* Revert "Fix broken attestation verification test"

This reverts commit 722cdc903b.

Co-authored-by: Kirk Baird <baird.k@outlook.com>
Co-authored-by: Michael Sproul <michael@sigmaprime.io>

beacon_node/beacon_chain/src/test_utils.rs

@@ -122,8 +122,6 @@ impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
             .null_event_handler()
             .testing_slot_clock(HARNESS_SLOT_TIME)
             .expect("should configure testing slot clock")
-            .reduced_tree_fork_choice()
-            .expect("should add fork choice to builder")
             .build()
             .expect("should build");
 
@@ -164,8 +162,6 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
             .null_event_handler()
             .testing_slot_clock(HARNESS_SLOT_TIME)
             .expect("should configure testing slot clock")
-            .reduced_tree_fork_choice()
-            .expect("should add fork choice to builder")
             .build()
             .expect("should build");
 
@@ -201,8 +197,6 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
             .null_event_handler()
             .testing_slot_clock(Duration::from_secs(1))
             .expect("should configure testing slot clock")
-            .reduced_tree_fork_choice()
-            .expect("should add fork choice to builder")
             .build()
             .expect("should build");
 
@@ -243,6 +237,35 @@ where
         block_strategy: BlockStrategy,
         attestation_strategy: AttestationStrategy,
     ) -> Hash256 {
+        let mut i = 0;
+        self.extend_chain_while(
+            |_, _| {
+                i += 1;
+                i <= num_blocks
+            },
+            block_strategy,
+            attestation_strategy,
+        )
+    }
+
+    /// Extend the `BeaconChain` with some blocks and attestations. Returns the root of the
+    /// last-produced block (the head of the chain).
+    ///
+    /// Chain will be extended while `predidcate` returns `true`.
+    ///
+    /// The `block_strategy` dictates where the new blocks will be placed.
+    ///
+    /// The `attestation_strategy` dictates which validators will attest to the newly created
+    /// blocks.
+    pub fn extend_chain_while<F>(
+        &self,
+        mut predicate: F,
+        block_strategy: BlockStrategy,
+        attestation_strategy: AttestationStrategy,
+    ) -> Hash256
+    where
+        F: FnMut(&SignedBeaconBlock<E>, &BeaconState<E>) -> bool,
+    {
         let mut state = {
             // Determine the slot for the first block (or skipped block).
             let state_slot = match block_strategy {
@@ -265,13 +288,17 @@ where
 
         let mut head_block_root = None;
 
-        for _ in 0..num_blocks {
+        loop {
+            let (block, new_state) = self.build_block(state.clone(), slot, block_strategy);
+
+            if !predicate(&block, &new_state) {
+                break;
+            }
+
             while self.chain.slot().expect("should have a slot") < slot {
                 self.advance_slot();
             }
 
-            let (block, new_state) = self.build_block(state.clone(), slot, block_strategy);
-
             let block_root = self
                 .chain
                 .process_block(block)
@@ -289,6 +316,39 @@ where
         head_block_root.expect("did not produce any blocks")
     }
 
+    /// A simple method to produce a block at the current slot without applying it to the chain.
+    ///
+    /// Always uses `BlockStrategy::OnCanonicalHead`.
+    pub fn get_block(&self) -> (SignedBeaconBlock<E>, BeaconState<E>) {
+        let state = self
+            .chain
+            .state_at_slot(
+                self.chain.slot().unwrap() - 1,
+                StateSkipConfig::WithStateRoots,
+            )
+            .unwrap();
+
+        let slot = self.chain.slot().unwrap();
+
+        self.build_block(state, slot, BlockStrategy::OnCanonicalHead)
+    }
+
+    /// A simple method to produce and process all attestation at the current slot. Always uses
+    /// `AttestationStrategy::AllValidators`.
+    pub fn generate_all_attestations(&self) {
+        let slot = self.chain.slot().unwrap();
+        let (state, block_root) = {
+            let head = self.chain.head().unwrap();
+            (head.beacon_state.clone(), head.beacon_block_root)
+        };
+        self.add_attestations_for_slot(
+            &AttestationStrategy::AllValidators,
+            &state,
+            block_root,
+            slot,
+        );
+    }
+
     /// Returns current canonical head slot
     pub fn get_chain_slot(&self) -> Slot {
         self.chain.slot().unwrap()
@@ -626,14 +686,16 @@ where
                         spec,
                     );
 
-                    self.chain
+                    let attn = self.chain
                         .verify_aggregated_attestation_for_gossip(signed_aggregate)
-                        .expect("should not error during attestation processing")
-                        .add_to_pool(&self.chain)
-                        .expect("should add attestation to naive aggregation pool")
-                        .add_to_fork_choice(&self.chain)
+                        .expect("should not error during attestation processing");
+
+                    self.chain.apply_attestation_to_fork_choice(&attn)
                         .expect("should add attestation to fork choice");
-                    }
+
+                    self.chain.add_to_block_inclusion_pool(attn)
+                        .expect("should add attestation to op pool");
+                }
             });
     }