Add proto_array fork choice (#804)

* Start implementing proto_array

* Add progress

* Add unfinished progress

* Add further progress

* Add progress

* Add tree filtering

* Add half-finished modifications

* Add refactored version

* Tidy, add incomplete LmdGhost impl

* Move impls in LmdGhost trait def

* Remove old reduced_tree fork choice

* Combine two functions in to `compute_deltas`

* Start testing

* Add more compute_deltas tests

* Add fork choice testing

* Add more fork choice testing

* Add more fork choice tests

* Add more testing to proto-array

* Remove old tests

* Modify tests

* Add more tests

* Add more testing

* Add comments and fixes

* Re-organise crate

* Tidy, finish pruning tests

* Add ssz encoding, other pub fns

* Rename lmd_ghost > proto_array_fork_choice

* Integrate proto_array into lighthouse

* Add first pass at fixing filter

* Clean out old comments

* Add more comments

* Attempt to fix prune error

* Adjust TODO

* Fix test compile errors

* Add extra justification change check

* Update cargo.lock

* Fix fork choice test compile errors

* Most remove ffg_update_required

* Fix bug with epoch of attestation votes

* Start adding new test format

* Make fork choice tests declarative

* Create test def concept

* Move test defs into crate

* Add binary, re-org crate

* Shuffle files

* Start adding ffg tests

* Add more fork choice tests

* Add fork choice JSON dumping

* Add more detail to best node error

* Ensure fin+just checkpoints from from same block

* Rename JustificationManager

* Move checkpoint manager into own file

* Tidy

* Add targetted logging for sneaky sync bug

* Fix justified balances bug

* Add cache metrics

* Add metrics for log levels

* Fix bug in checkpoint manager

* Fix compile error in fork choice tests

* Ignore duplicate blocks in fork choice

* Add block to fock choice before db

* Rename on_new_block fn

* Fix spec inconsistency in `CheckpointManager`

* Remove BlockRootTree

* Remove old reduced_tree code fragment

* Add API endpoint for fork choice

* Add more ffg tests

* Remove block_root_tree reminents

* Ensure effective balances are used

* Remove old debugging code, fix API fault

* Add check to ensure parent block is in fork choice

* Update readme dates

* Fix readme

* Tidy checkpoint manager

* Remove fork choice yaml files from repo

* Remove fork choice yaml from repo

* General tidy

* Address majority of Michael's comments

* Tidy bin/lib business

* Remove dangling file

* Undo changes for rpc/handler from master

* Revert "Undo changes for rpc/handler from master"

This reverts commit 876edff0e4.

Co-authored-by: Age Manning <Age@AgeManning.com>

beacon_node/beacon_chain/src/fork_choice.rs

@@ -1,13 +1,14 @@
+mod checkpoint_manager;
+
 use crate::{errors::BeaconChainError, metrics, BeaconChain, BeaconChainTypes};
-use lmd_ghost::LmdGhost;
-use parking_lot::RwLock;
+use checkpoint_manager::{get_effective_balances, CheckpointManager, CheckpointWithBalances};
+use parking_lot::{RwLock, RwLockReadGuard};
+use proto_array_fork_choice::{core::ProtoArray, ProtoArrayForkChoice};
 use ssz_derive::{Decode, Encode};
-use state_processing::{common::get_attesting_indices, per_slot_processing};
-use std::sync::Arc;
-use store::{BlockRootTree, Error as StoreError, Store};
-use types::{
-    Attestation, BeaconBlock, BeaconState, BeaconStateError, Checkpoint, EthSpec, Hash256, Slot,
-};
+use state_processing::common::get_attesting_indices;
+use std::marker::PhantomData;
+use store::Error as StoreError;
+use types::{Attestation, BeaconBlock, BeaconState, BeaconStateError, Epoch, Hash256};
 
 type Result<T> = std::result::Result<T, Error>;
 
@@ -19,27 +20,29 @@ pub enum Error {
     BeaconStateError(BeaconStateError),
     StoreError(StoreError),
     BeaconChainError(Box<BeaconChainError>),
+    UnknownBlockSlot(Hash256),
+    UnknownJustifiedBlock(Hash256),
+    UnknownJustifiedState(Hash256),
+    UnableToJsonEncode(String),
 }
 
 pub struct ForkChoice<T: BeaconChainTypes> {
-    backend: T::LmdGhost,
+    backend: ProtoArrayForkChoice,
     /// Used for resolving the `0x00..00` alias back to genesis.
     ///
     /// Does not necessarily need to be the _actual_ genesis, it suffices to be the finalized root
     /// whenever the struct was instantiated.
     genesis_block_root: Hash256,
-    /// The fork choice rule's current view of the justified checkpoint.
-    justified_checkpoint: RwLock<Checkpoint>,
-    /// The best justified checkpoint we've seen, which may be ahead of `justified_checkpoint`.
-    best_justified_checkpoint: RwLock<Checkpoint>,
+    checkpoint_manager: RwLock<CheckpointManager>,
+    _phantom: PhantomData<T>,
 }
 
 impl<T: BeaconChainTypes> PartialEq for ForkChoice<T> {
+    /// This implementation ignores the `store`.
     fn eq(&self, other: &Self) -> bool {
         self.backend == other.backend
             && self.genesis_block_root == other.genesis_block_root
-            && *self.justified_checkpoint.read() == *other.justified_checkpoint.read()
-            && *self.best_justified_checkpoint.read() == *other.best_justified_checkpoint.read()
+            && *self.checkpoint_manager.read() == *other.checkpoint_manager.read()
     }
 }
 
@@ -48,122 +51,48 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
     ///
     /// "Genesis" does not necessarily need to be the absolute genesis, it can be some finalized
     /// block.
-    pub fn new(backend: T::LmdGhost, genesis_block_root: Hash256, genesis_slot: Slot) -> Self {
-        let justified_checkpoint = Checkpoint {
-            epoch: genesis_slot.epoch(T::EthSpec::slots_per_epoch()),
+    pub fn new(
+        backend: ProtoArrayForkChoice,
+        genesis_block_root: Hash256,
+        genesis_state: &BeaconState<T::EthSpec>,
+    ) -> Self {
+        let genesis_checkpoint = CheckpointWithBalances {
+            epoch: genesis_state.current_epoch(),
             root: genesis_block_root,
+            balances: get_effective_balances(genesis_state),
         };
+
         Self {
             backend,
             genesis_block_root,
-            justified_checkpoint: RwLock::new(justified_checkpoint.clone()),
-            best_justified_checkpoint: RwLock::new(justified_checkpoint),
+            checkpoint_manager: RwLock::new(CheckpointManager::new(genesis_checkpoint)),
+            _phantom: PhantomData,
         }
     }
 
-    /// Determine whether the fork choice's view of the justified checkpoint should be updated.
-    ///
-    /// To prevent the bouncing attack, an update is allowed only in these conditions:
-    ///
-    /// * We're in the first SAFE_SLOTS_TO_UPDATE_JUSTIFIED slots of the epoch, or
-    /// * The new justified checkpoint is a descendant of the current justified checkpoint
-    fn should_update_justified_checkpoint(
-        &self,
-        chain: &BeaconChain<T>,
-        new_justified_checkpoint: &Checkpoint,
-    ) -> Result<bool> {
-        if Self::compute_slots_since_epoch_start(chain.slot()?)
-            < chain.spec.safe_slots_to_update_justified
-        {
-            return Ok(true);
-        }
-
-        let justified_checkpoint = self.justified_checkpoint.read().clone();
-
-        let current_justified_block = chain
-            .get_block(&justified_checkpoint.root)?
-            .ok_or_else(|| Error::MissingBlock(justified_checkpoint.root))?;
-
-        let new_justified_block = chain
-            .get_block(&new_justified_checkpoint.root)?
-            .ok_or_else(|| Error::MissingBlock(new_justified_checkpoint.root))?;
-
-        let slots_per_epoch = T::EthSpec::slots_per_epoch();
-
-        Ok(
-            new_justified_block.slot > justified_checkpoint.epoch.start_slot(slots_per_epoch)
-                && chain.get_ancestor_block_root(
-                    new_justified_checkpoint.root,
-                    current_justified_block.slot,
-                )? == Some(justified_checkpoint.root),
-        )
-    }
-
-    /// Calculate how far `slot` lies from the start of its epoch.
-    fn compute_slots_since_epoch_start(slot: Slot) -> u64 {
-        let slots_per_epoch = T::EthSpec::slots_per_epoch();
-        (slot - slot.epoch(slots_per_epoch).start_slot(slots_per_epoch)).as_u64()
-    }
-
     /// Run the fork choice rule to determine the head.
     pub fn find_head(&self, chain: &BeaconChain<T>) -> Result<Hash256> {
         let timer = metrics::start_timer(&metrics::FORK_CHOICE_FIND_HEAD_TIMES);
 
-        let (start_state, start_block_root, start_block_slot) = {
-            // Check if we should update our view of the justified checkpoint.
-            // Doing this check here should be quasi-equivalent to the update in the `on_tick`
-            // function of the spec, so long as `find_head` is called at least once during the first
-            // SAFE_SLOTS_TO_UPDATE_JUSTIFIED slots.
-            let best_justified_checkpoint = self.best_justified_checkpoint.read();
-            if self.should_update_justified_checkpoint(chain, &best_justified_checkpoint)? {
-                *self.justified_checkpoint.write() = best_justified_checkpoint.clone();
-            }
-
-            let current_justified_checkpoint = self.justified_checkpoint.read().clone();
-
-            let (block_root, block_justified_slot) = (
-                current_justified_checkpoint.root,
-                current_justified_checkpoint
-                    .epoch
-                    .start_slot(T::EthSpec::slots_per_epoch()),
-            );
-
-            let block = chain
-                .store
-                .get::<BeaconBlock<T::EthSpec>>(&block_root)?
-                .ok_or_else(|| Error::MissingBlock(block_root))?;
-
-            // Resolve the `0x00.. 00` alias back to genesis
-            let block_root = if block_root == Hash256::zero() {
+        let remove_alias = |root| {
+            if root == Hash256::zero() {
                 self.genesis_block_root
             } else {
-                block_root
-            };
-
-            let mut state: BeaconState<T::EthSpec> = chain
-                .get_state_caching_only_with_committee_caches(&block.state_root, Some(block.slot))?
-                .ok_or_else(|| Error::MissingState(block.state_root))?;
-
-            // Fast-forward the state to the start slot of the epoch where it was justified.
-            for _ in block.slot.as_u64()..block_justified_slot.as_u64() {
-                per_slot_processing(&mut state, None, &chain.spec)
-                    .map_err(BeaconChainError::SlotProcessingError)?
+                root
             }
-
-            (state, block_root, block_justified_slot)
         };
 
-        // A function that returns the weight for some validator index.
-        let weight = |validator_index: usize| -> Option<u64> {
-            start_state
-                .validators
-                .get(validator_index)
-                .map(|v| v.effective_balance)
-        };
+        let mut manager = self.checkpoint_manager.write();
+        manager.maybe_update(chain.slot()?, chain)?;
 
         let result = self
             .backend
-            .find_head(start_block_slot, start_block_root, weight)
+            .find_head(
+                manager.current.justified.epoch,
+                remove_alias(manager.current.justified.root),
+                manager.current.finalized.epoch,
+                &manager.current.justified.balances,
+            )
             .map_err(Into::into);
 
         metrics::stop_timer(timer);
@@ -171,6 +100,11 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
         result
     }
 
+    /// Returns true if the given block is known to fork choice.
+    pub fn contains_block(&self, block_root: &Hash256) -> bool {
+        self.backend.contains_block(block_root)
+    }
+
     /// Process all attestations in the given `block`.
     ///
     /// Assumes the block (and therefore its attestations) are valid. It is a logic error to
@@ -183,36 +117,35 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
         block_root: Hash256,
     ) -> Result<()> {
         let timer = metrics::start_timer(&metrics::FORK_CHOICE_PROCESS_BLOCK_TIMES);
-        // Note: we never count the block as a latest message, only attestations.
-        //
-        // I (Paul H) do not have an explicit reference to this, but I derive it from this
-        // document:
-        //
-        // https://github.com/ethereum/eth2.0-specs/blob/v0.7.0/specs/core/0_fork-choice.md
-        for attestation in &block.body.attestations {
-            // If the `data.beacon_block_root` block is not known to us, simply ignore the latest
-            // vote.
-            if let Some(block) = chain.get_block_caching(&attestation.data.beacon_block_root)? {
-                self.process_attestation(state, attestation, &block)?;
-            }
-        }
 
-        // Check if we should update our view of the justified checkpoint
-        if state.current_justified_checkpoint.epoch > self.justified_checkpoint.read().epoch {
-            *self.best_justified_checkpoint.write() = state.current_justified_checkpoint.clone();
+        self.checkpoint_manager
+            .write()
+            .process_state(block_root, state, chain, &self.backend)?;
+        self.checkpoint_manager
+            .write()
+            .maybe_update(chain.slot()?, chain)?;
+
+        // Note: we never count the block as a latest message, only attestations.
+        for attestation in &block.body.attestations {
+            // If the `data.beacon_block_root` block is not known to the fork choice, simply ignore
+            // the vote.
             if self
-                .should_update_justified_checkpoint(chain, &state.current_justified_checkpoint)?
+                .backend
+                .contains_block(&attestation.data.beacon_block_root)
             {
-                *self.justified_checkpoint.write() = state.current_justified_checkpoint.clone();
+                self.process_attestation(state, attestation)?;
             }
         }
 
         // This does not apply a vote to the block, it just makes fork choice aware of the block so
         // it can still be identified as the head even if it doesn't have any votes.
-        //
-        // A case where a block without any votes can be the head is where it is the only child of
-        // a block that has the majority of votes applied to it.
-        self.backend.process_block(block, block_root)?;
+        self.backend.process_block(
+            block.slot,
+            block_root,
+            block.parent_root,
+            state.current_justified_checkpoint.epoch,
+            state.finalized_checkpoint.epoch,
+        )?;
 
         metrics::stop_timer(timer);
 
@@ -226,7 +159,6 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
         &self,
         state: &BeaconState<T::EthSpec>,
         attestation: &Attestation<T::EthSpec>,
-        block: &BeaconBlock<T::EthSpec>,
     ) -> Result<()> {
         let timer = metrics::start_timer(&metrics::FORK_CHOICE_PROCESS_ATTESTATION_TIMES);
 
@@ -252,8 +184,11 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
                 get_attesting_indices(state, &attestation.data, &attestation.aggregation_bits)?;
 
             for validator_index in validator_indices {
-                self.backend
-                    .process_attestation(validator_index, block_hash, block.slot)?;
+                self.backend.process_attestation(
+                    validator_index,
+                    block_hash,
+                    attestation.data.target.epoch,
+                )?;
             }
         }
 
@@ -265,38 +200,29 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
     /// Returns the latest message for a given validator, if any.
     ///
     /// Returns `(block_root, block_slot)`.
-    pub fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Slot)> {
+    pub fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Epoch)> {
         self.backend.latest_message(validator_index)
     }
 
-    /// Runs an integrity verification function on the underlying fork choice algorithm.
-    ///
-    /// Returns `Ok(())` if the underlying fork choice has maintained it's integrity,
-    /// `Err(description)` otherwise.
-    pub fn verify_integrity(&self) -> core::result::Result<(), String> {
-        self.backend.verify_integrity()
+    /// Trigger a prune on the underlying fork choice backend.
+    pub fn prune(&self) -> Result<()> {
+        let finalized_root = self.checkpoint_manager.read().current.finalized.root;
+
+        self.backend.maybe_prune(finalized_root).map_err(Into::into)
     }
 
-    /// Inform the fork choice that the given block (and corresponding root) have been finalized so
-    /// it may prune it's storage.
+    /// Returns a read-lock to the core `ProtoArray` struct.
     ///
-    /// `finalized_block_root` must be the root of `finalized_block`.
-    pub fn process_finalization(
-        &self,
-        finalized_block: &BeaconBlock<T::EthSpec>,
-        finalized_block_root: Hash256,
-    ) -> Result<()> {
-        self.backend
-            .update_finalized_root(finalized_block, finalized_block_root)
-            .map_err(Into::into)
+    /// Should only be used when encoding/decoding during troubleshooting.
+    pub fn core_proto_array(&self) -> RwLockReadGuard<ProtoArray> {
+        self.backend.core_proto_array()
     }
 
     /// Returns a `SszForkChoice` which contains the current state of `Self`.
     pub fn as_ssz_container(&self) -> SszForkChoice {
         SszForkChoice {
-            genesis_block_root: self.genesis_block_root,
-            justified_checkpoint: self.justified_checkpoint.read().clone(),
-            best_justified_checkpoint: self.best_justified_checkpoint.read().clone(),
+            genesis_block_root: self.genesis_block_root.clone(),
+            checkpoint_manager: self.checkpoint_manager.read().clone(),
             backend_bytes: self.backend.as_bytes(),
         }
     }
@@ -304,18 +230,14 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
     /// Instantiates `Self` from a prior `SszForkChoice`.
     ///
     /// The created `Self` will have the same state as the `Self` that created the `SszForkChoice`.
-    pub fn from_ssz_container(
-        ssz_container: SszForkChoice,
-        store: Arc<T::Store>,
-        block_root_tree: Arc<BlockRootTree>,
-    ) -> Result<Self> {
-        let backend = LmdGhost::from_bytes(&ssz_container.backend_bytes, store, block_root_tree)?;
+    pub fn from_ssz_container(ssz_container: SszForkChoice) -> Result<Self> {
+        let backend = ProtoArrayForkChoice::from_bytes(&ssz_container.backend_bytes)?;
 
         Ok(Self {
             backend,
             genesis_block_root: ssz_container.genesis_block_root,
-            justified_checkpoint: RwLock::new(ssz_container.justified_checkpoint),
-            best_justified_checkpoint: RwLock::new(ssz_container.best_justified_checkpoint),
+            checkpoint_manager: RwLock::new(ssz_container.checkpoint_manager),
+            _phantom: PhantomData,
         })
     }
 }
@@ -326,8 +248,7 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
 #[derive(Encode, Decode, Clone)]
 pub struct SszForkChoice {
     genesis_block_root: Hash256,
-    justified_checkpoint: Checkpoint,
-    best_justified_checkpoint: Checkpoint,
+    checkpoint_manager: CheckpointManager,
     backend_bytes: Vec<u8>,
 }