Enable proposer boost re-orging (#2860)

## Proposed Changes

With proposer boosting implemented (#2822) we have an opportunity to re-org out late blocks.

This PR adds three flags to the BN to control this behaviour:

* `--disable-proposer-reorgs`: turn aggressive re-orging off (it's on by default).
* `--proposer-reorg-threshold N`: attempt to orphan blocks with less than N% of the committee vote. If this parameter isn't set then N defaults to 20% when the feature is enabled.
* `--proposer-reorg-epochs-since-finalization N`: only attempt to re-org late blocks when the number of epochs since finalization is less than or equal to N. The default is 2 epochs, meaning re-orgs will only be attempted when the chain is finalizing optimally.

For safety Lighthouse will only attempt a re-org under very specific conditions:

1. The block being proposed is 1 slot after the canonical head, and the canonical head is 1 slot after its parent. i.e. at slot `n + 1` rather than building on the block from slot `n` we build on the block from slot `n - 1`.
2. The current canonical head received less than N% of the committee vote. N should be set depending on the proposer boost fraction itself, the fraction of the network that is believed to be applying it, and the size of the largest entity that could be hoarding votes.
3. The current canonical head arrived after the attestation deadline from our perspective. This condition was only added to support suppression of forkchoiceUpdated messages, but makes intuitive sense.
4. The block is being proposed in the first 2 seconds of the slot. This gives it time to propagate and receive the proposer boost.


## Additional Info

For the initial idea and background, see: https://github.com/ethereum/consensus-specs/pull/2353#issuecomment-950238004

There is also a specification for this feature here: https://github.com/ethereum/consensus-specs/pull/3034

Co-authored-by: Michael Sproul <micsproul@gmail.com>
Co-authored-by: pawan <pawandhananjay@gmail.com>

consensus/proto_array/src/proto_array_fork_choice.rs

@@ -1,9 +1,12 @@
-use crate::error::Error;
-use crate::proto_array::CountUnrealizedFull;
-use crate::proto_array::{
-    calculate_proposer_boost, InvalidationOperation, Iter, ProposerBoost, ProtoArray, ProtoNode,
+use crate::{
+    error::Error,
+    proto_array::{
+        calculate_committee_fraction, CountUnrealizedFull, InvalidationOperation, Iter,
+        ProposerBoost, ProtoArray, ProtoNode,
+    },
+    ssz_container::SszContainer,
+    JustifiedBalances,
 };
-use crate::ssz_container::SszContainer;
 use serde_derive::{Deserialize, Serialize};
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
@@ -170,11 +173,128 @@ where
     }
 }
 
+/// Information about the proposer head used for opportunistic re-orgs.
+#[derive(Clone)]
+pub struct ProposerHeadInfo {
+    /// Information about the *current* head block, which may be re-orged.
+    pub head_node: ProtoNode,
+    /// Information about the parent of the current head, which should be selected as the parent
+    /// for a new proposal *if* a re-org is decided on.
+    pub parent_node: ProtoNode,
+    /// The computed fraction of the active committee balance below which we can re-org.
+    pub re_org_weight_threshold: u64,
+    /// The current slot from fork choice's point of view, may lead the wall-clock slot by upto
+    /// 500ms.
+    pub current_slot: Slot,
+}
+
+/// Error type to enable short-circuiting checks in `get_proposer_head`.
+///
+/// This type intentionally does not implement `Debug` so that callers are forced to handle the
+/// enum.
+#[derive(Clone, PartialEq)]
+pub enum ProposerHeadError<E> {
+    DoNotReOrg(DoNotReOrg),
+    Error(E),
+}
+
+impl<E> From<DoNotReOrg> for ProposerHeadError<E> {
+    fn from(e: DoNotReOrg) -> ProposerHeadError<E> {
+        Self::DoNotReOrg(e)
+    }
+}
+
+impl From<Error> for ProposerHeadError<Error> {
+    fn from(e: Error) -> Self {
+        Self::Error(e)
+    }
+}
+
+impl<E1> ProposerHeadError<E1> {
+    pub fn convert_inner_error<E2>(self) -> ProposerHeadError<E2>
+    where
+        E2: From<E1>,
+    {
+        self.map_inner_error(E2::from)
+    }
+
+    pub fn map_inner_error<E2>(self, f: impl FnOnce(E1) -> E2) -> ProposerHeadError<E2> {
+        match self {
+            ProposerHeadError::DoNotReOrg(reason) => ProposerHeadError::DoNotReOrg(reason),
+            ProposerHeadError::Error(error) => ProposerHeadError::Error(f(error)),
+        }
+    }
+}
+
+/// Reasons why a re-org should not be attempted.
+///
+/// This type intentionally does not implement `Debug` so that the `Display` impl must be used.
+#[derive(Clone, PartialEq)]
+pub enum DoNotReOrg {
+    MissingHeadOrParentNode,
+    MissingHeadFinalizedCheckpoint,
+    ParentDistance,
+    HeadDistance,
+    ShufflingUnstable,
+    JustificationAndFinalizationNotCompetitive,
+    ChainNotFinalizing {
+        epochs_since_finalization: u64,
+    },
+    HeadNotWeak {
+        head_weight: u64,
+        re_org_weight_threshold: u64,
+    },
+    HeadNotLate,
+    NotProposing,
+    ReOrgsDisabled,
+}
+
+impl std::fmt::Display for DoNotReOrg {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            Self::MissingHeadOrParentNode => write!(f, "unknown head or parent"),
+            Self::MissingHeadFinalizedCheckpoint => write!(f, "finalized checkpoint missing"),
+            Self::ParentDistance => write!(f, "parent too far from head"),
+            Self::HeadDistance => write!(f, "head too far from current slot"),
+            Self::ShufflingUnstable => write!(f, "shuffling unstable at epoch boundary"),
+            Self::JustificationAndFinalizationNotCompetitive => {
+                write!(f, "justification or finalization not competitive")
+            }
+            Self::ChainNotFinalizing {
+                epochs_since_finalization,
+            } => write!(
+                f,
+                "chain not finalizing ({epochs_since_finalization} epochs since finalization)"
+            ),
+            Self::HeadNotWeak {
+                head_weight,
+                re_org_weight_threshold,
+            } => {
+                write!(f, "head not weak ({head_weight}/{re_org_weight_threshold})")
+            }
+            Self::HeadNotLate => {
+                write!(f, "head arrived on time")
+            }
+            Self::NotProposing => {
+                write!(f, "not proposing at next slot")
+            }
+            Self::ReOrgsDisabled => {
+                write!(f, "re-orgs disabled in config")
+            }
+        }
+    }
+}
+
+/// New-type for the re-org threshold percentage.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct ReOrgThreshold(pub u64);
+
 #[derive(PartialEq)]
 pub struct ProtoArrayForkChoice {
     pub(crate) proto_array: ProtoArray,
     pub(crate) votes: ElasticList<VoteTracker>,
-    pub(crate) balances: Vec<u64>,
+    pub(crate) balances: JustifiedBalances,
 }
 
 impl ProtoArrayForkChoice {
@@ -223,7 +343,7 @@ impl ProtoArrayForkChoice {
         Ok(Self {
             proto_array,
             votes: ElasticList::default(),
-            balances: vec![],
+            balances: JustifiedBalances::default(),
         })
     }
 
@@ -282,21 +402,20 @@ impl ProtoArrayForkChoice {
         &mut self,
         justified_checkpoint: Checkpoint,
         finalized_checkpoint: Checkpoint,
-        justified_state_balances: &[u64],
+        justified_state_balances: &JustifiedBalances,
         proposer_boost_root: Hash256,
         equivocating_indices: &BTreeSet<u64>,
         current_slot: Slot,
         spec: &ChainSpec,
     ) -> Result<Hash256, String> {
         let old_balances = &mut self.balances;
-
         let new_balances = justified_state_balances;
 
         let deltas = compute_deltas(
             &self.proto_array.indices,
             &mut self.votes,
-            old_balances,
-            new_balances,
+            &old_balances.effective_balances,
+            &new_balances.effective_balances,
             equivocating_indices,
         )
         .map_err(|e| format!("find_head compute_deltas failed: {:?}", e))?;
@@ -313,13 +432,129 @@ impl ProtoArrayForkChoice {
             )
             .map_err(|e| format!("find_head apply_score_changes failed: {:?}", e))?;
 
-        *old_balances = new_balances.to_vec();
+        *old_balances = new_balances.clone();
 
         self.proto_array
             .find_head::<E>(&justified_checkpoint.root, current_slot)
             .map_err(|e| format!("find_head failed: {:?}", e))
     }
 
+    /// Get the block to propose on during `current_slot`.
+    ///
+    /// This function returns a *definitive* result which should be acted on.
+    pub fn get_proposer_head<E: EthSpec>(
+        &self,
+        current_slot: Slot,
+        canonical_head: Hash256,
+        justified_balances: &JustifiedBalances,
+        re_org_threshold: ReOrgThreshold,
+        max_epochs_since_finalization: Epoch,
+    ) -> Result<ProposerHeadInfo, ProposerHeadError<Error>> {
+        let info = self.get_proposer_head_info::<E>(
+            current_slot,
+            canonical_head,
+            justified_balances,
+            re_org_threshold,
+            max_epochs_since_finalization,
+        )?;
+
+        // Only re-org a single slot. This prevents cascading failures during asynchrony.
+        let head_slot_ok = info.head_node.slot + 1 == current_slot;
+        if !head_slot_ok {
+            return Err(DoNotReOrg::HeadDistance.into());
+        }
+
+        // Only re-org if the head's weight is less than the configured committee fraction.
+        let head_weight = info.head_node.weight;
+        let re_org_weight_threshold = info.re_org_weight_threshold;
+        let weak_head = head_weight < re_org_weight_threshold;
+        if !weak_head {
+            return Err(DoNotReOrg::HeadNotWeak {
+                head_weight,
+                re_org_weight_threshold,
+            }
+            .into());
+        }
+
+        // All checks have passed, build upon the parent to re-org the head.
+        Ok(info)
+    }
+
+    /// Get information about the block to propose on during `current_slot`.
+    ///
+    /// This function returns a *partial* result which must be processed further.
+    pub fn get_proposer_head_info<E: EthSpec>(
+        &self,
+        current_slot: Slot,
+        canonical_head: Hash256,
+        justified_balances: &JustifiedBalances,
+        re_org_threshold: ReOrgThreshold,
+        max_epochs_since_finalization: Epoch,
+    ) -> Result<ProposerHeadInfo, ProposerHeadError<Error>> {
+        let mut nodes = self
+            .proto_array
+            .iter_nodes(&canonical_head)
+            .take(2)
+            .cloned()
+            .collect::<Vec<_>>();
+
+        let parent_node = nodes.pop().ok_or(DoNotReOrg::MissingHeadOrParentNode)?;
+        let head_node = nodes.pop().ok_or(DoNotReOrg::MissingHeadOrParentNode)?;
+
+        let parent_slot = parent_node.slot;
+        let head_slot = head_node.slot;
+        let re_org_block_slot = head_slot + 1;
+
+        // Check finalization distance.
+        let proposal_epoch = re_org_block_slot.epoch(E::slots_per_epoch());
+        let finalized_epoch = head_node
+            .unrealized_finalized_checkpoint
+            .ok_or(DoNotReOrg::MissingHeadFinalizedCheckpoint)?
+            .epoch;
+        let epochs_since_finalization = proposal_epoch.saturating_sub(finalized_epoch).as_u64();
+        if epochs_since_finalization > max_epochs_since_finalization.as_u64() {
+            return Err(DoNotReOrg::ChainNotFinalizing {
+                epochs_since_finalization,
+            }
+            .into());
+        }
+
+        // Check parent distance from head.
+        // Do not check head distance from current slot, as that condition needs to be
+        // late-evaluated and is elided when `current_slot == head_slot`.
+        let parent_slot_ok = parent_slot + 1 == head_slot;
+        if !parent_slot_ok {
+            return Err(DoNotReOrg::ParentDistance.into());
+        }
+
+        // Check shuffling stability.
+        let shuffling_stable = re_org_block_slot % E::slots_per_epoch() != 0;
+        if !shuffling_stable {
+            return Err(DoNotReOrg::ShufflingUnstable.into());
+        }
+
+        // Check FFG.
+        let ffg_competitive = parent_node.unrealized_justified_checkpoint
+            == head_node.unrealized_justified_checkpoint
+            && parent_node.unrealized_finalized_checkpoint
+                == head_node.unrealized_finalized_checkpoint;
+        if !ffg_competitive {
+            return Err(DoNotReOrg::JustificationAndFinalizationNotCompetitive.into());
+        }
+
+        // Compute re-org weight threshold.
+        let re_org_weight_threshold =
+            calculate_committee_fraction::<E>(justified_balances, re_org_threshold.0)
+                .ok_or(Error::ReOrgThresholdOverflow)?;
+
+        Ok(ProposerHeadInfo {
+            head_node,
+            parent_node,
+            re_org_weight_threshold,
+            current_slot,
+        })
+    }
+
     /// Returns `true` if there are any blocks in `self` with an `INVALID` execution payload status.
     ///
     /// This will operate on *all* blocks, even those that do not descend from the finalized
@@ -368,7 +603,7 @@ impl ProtoArrayForkChoice {
                             if vote.current_root == node.root {
                                 // Any voting validator that does not have a balance should be
                                 // ignored. This is consistent with `compute_deltas`.
-                                self.balances.get(validator_index)
+                                self.balances.effective_balances.get(validator_index)
                             } else {
                                 None
                             }
@@ -382,9 +617,11 @@ impl ProtoArrayForkChoice {
                             // Compute the score based upon the current balances. We can't rely on
                             // the `previous_proposr_boost.score` since it is set to zero with an
                             // invalid node.
-                            let proposer_score =
-                                calculate_proposer_boost::<E>(&self.balances, proposer_score_boost)
-                                    .ok_or("Failed to compute proposer boost")?;
+                            let proposer_score = calculate_committee_fraction::<E>(
+                                &self.balances,
+                                proposer_score_boost,
+                            )
+                            .ok_or("Failed to compute proposer boost")?;
                             // Store the score we've applied here so it can be removed in
                             // a later call to `apply_score_changes`.
                             self.proto_array.previous_proposer_boost.score = proposer_score;
@@ -538,10 +775,11 @@ impl ProtoArrayForkChoice {
         bytes: &[u8],
         count_unrealized_full: CountUnrealizedFull,
     ) -> Result<Self, String> {
-        SszContainer::from_ssz_bytes(bytes)
-            .map(|container| (container, count_unrealized_full))
-            .map(Into::into)
-            .map_err(|e| format!("Failed to decode ProtoArrayForkChoice: {:?}", e))
+        let container = SszContainer::from_ssz_bytes(bytes)
+            .map_err(|e| format!("Failed to decode ProtoArrayForkChoice: {:?}", e))?;
+        (container, count_unrealized_full)
+            .try_into()
+            .map_err(|e| format!("Failed to initialize ProtoArrayForkChoice: {e:?}"))
     }
 
     /// Returns a read-lock to core `ProtoArray` struct.