Fork choice modifications and cleanup (#3962)

## Issue Addressed

NA

## Proposed Changes

- Implements https://github.com/ethereum/consensus-specs/pull/3290/
- Bumps `ef-tests` to [v1.3.0-rc.4](https://github.com/ethereum/consensus-spec-tests/releases/tag/v1.3.0-rc.4).

The `CountRealizedFull` concept has been removed and the `--count-unrealized-full` and `--count-unrealized` BN flags now do nothing but log a `WARN` when used.

## Database Migration Debt

This PR removes the `best_justified_checkpoint` from fork choice. This field is persisted on-disk and the correct way to go about this would be to make a DB migration to remove the field. However, in this PR I've simply stubbed out the value with a junk value. I've taken this approach because if we're going to do a DB migration I'd love to remove the `Option`s around the justified and finalized checkpoints on `ProtoNode` whilst we're at it. Those options were added in #2822 which was included in Lighthouse v2.1.0. The options were only put there to handle the migration and they've been set to `Some` ever since v2.1.0. There's no reason to keep them as options anymore.

I started adding the DB migration to this branch but I started to feel like I was bloating this rather critical PR with nice-to-haves. I've kept the partially-complete migration [over in my repo](https://github.com/paulhauner/lighthouse/tree/fc-pr-18-migration) so we can pick it up after this PR is merged.

consensus/proto_array/src/fork_choice_test_definition.rs

@@ -3,7 +3,6 @@ mod ffg_updates;
 mod no_votes;
 mod votes;
 
-use crate::proto_array::CountUnrealizedFull;
 use crate::proto_array_fork_choice::{Block, ExecutionStatus, ProtoArrayForkChoice};
 use crate::{InvalidationOperation, JustifiedBalances};
 use serde_derive::{Deserialize, Serialize};
@@ -88,7 +87,6 @@ impl ForkChoiceTestDefinition {
             junk_shuffling_id.clone(),
             junk_shuffling_id,
             ExecutionStatus::Optimistic(ExecutionBlockHash::zero()),
-            CountUnrealizedFull::default(),
         )
         .expect("should create fork choice struct");
         let equivocating_indices = BTreeSet::new();
@@ -307,8 +305,8 @@ fn get_checkpoint(i: u64) -> Checkpoint {
 
 fn check_bytes_round_trip(original: &ProtoArrayForkChoice) {
     let bytes = original.as_bytes();
-    let decoded = ProtoArrayForkChoice::from_bytes(&bytes, CountUnrealizedFull::default())
-        .expect("fork choice should decode from bytes");
+    let decoded =
+        ProtoArrayForkChoice::from_bytes(&bytes).expect("fork choice should decode from bytes");
     assert!(
         *original == decoded,
         "fork choice should encode and decode without change"

consensus/proto_array/src/justified_balances.rs

@@ -24,7 +24,7 @@ impl JustifiedBalances {
             .validators()
             .iter()
             .map(|validator| {
-                if validator.is_active_at(current_epoch) {
+                if !validator.slashed && validator.is_active_at(current_epoch) {
                     total_effective_balance.safe_add_assign(validator.effective_balance)?;
                     num_active_validators.safe_add_assign(1)?;
 

consensus/proto_array/src/lib.rs

@@ -6,9 +6,7 @@ mod proto_array_fork_choice;
 mod ssz_container;
 
 pub use crate::justified_balances::JustifiedBalances;
-pub use crate::proto_array::{
-    calculate_committee_fraction, CountUnrealizedFull, InvalidationOperation,
-};
+pub use crate::proto_array::{calculate_committee_fraction, InvalidationOperation};
 pub use crate::proto_array_fork_choice::{
     Block, DoNotReOrg, ExecutionStatus, ProposerHeadError, ProposerHeadInfo, ProtoArrayForkChoice,
     ReOrgThreshold,

consensus/proto_array/src/proto_array.rs

@@ -118,24 +118,6 @@ impl Default for ProposerBoost {
     }
 }
 
-/// Indicate whether we should strictly count unrealized justification/finalization votes.
-#[derive(Default, PartialEq, Eq, Debug, Serialize, Deserialize, Copy, Clone)]
-pub enum CountUnrealizedFull {
-    True,
-    #[default]
-    False,
-}
-
-impl From<bool> for CountUnrealizedFull {
-    fn from(b: bool) -> Self {
-        if b {
-            CountUnrealizedFull::True
-        } else {
-            CountUnrealizedFull::False
-        }
-    }
-}
-
 #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)]
 pub struct ProtoArray {
     /// Do not attempt to prune the tree unless it has at least this many nodes. Small prunes
@@ -146,7 +128,6 @@ pub struct ProtoArray {
     pub nodes: Vec<ProtoNode>,
     pub indices: HashMap<Hash256, usize>,
     pub previous_proposer_boost: ProposerBoost,
-    pub count_unrealized_full: CountUnrealizedFull,
 }
 
 impl ProtoArray {
@@ -900,55 +881,44 @@ impl ProtoArray {
         }
 
         let genesis_epoch = Epoch::new(0);
-
-        let checkpoint_match_predicate =
-            |node_justified_checkpoint: Checkpoint, node_finalized_checkpoint: Checkpoint| {
-                let correct_justified = node_justified_checkpoint == self.justified_checkpoint
-                    || self.justified_checkpoint.epoch == genesis_epoch;
-                let correct_finalized = node_finalized_checkpoint == self.finalized_checkpoint
-                    || self.finalized_checkpoint.epoch == genesis_epoch;
-                correct_justified && correct_finalized
+        let current_epoch = current_slot.epoch(E::slots_per_epoch());
+        let node_epoch = node.slot.epoch(E::slots_per_epoch());
+        let node_justified_checkpoint =
+            if let Some(justified_checkpoint) = node.justified_checkpoint {
+                justified_checkpoint
+            } else {
+                // The node does not have any information about the justified
+                // checkpoint. This indicates an inconsistent proto-array.
+                return false;
             };
 
-        if let (
-            Some(unrealized_justified_checkpoint),
-            Some(unrealized_finalized_checkpoint),
-            Some(justified_checkpoint),
-            Some(finalized_checkpoint),
-        ) = (
-            node.unrealized_justified_checkpoint,
-            node.unrealized_finalized_checkpoint,
-            node.justified_checkpoint,
-            node.finalized_checkpoint,
-        ) {
-            let current_epoch = current_slot.epoch(E::slots_per_epoch());
-
-            // If previous epoch is justified, pull up all tips to at least the previous epoch
-            if CountUnrealizedFull::True == self.count_unrealized_full
-                && (current_epoch > genesis_epoch
-                    && self.justified_checkpoint.epoch + 1 == current_epoch)
-            {
-                unrealized_justified_checkpoint.epoch + 1 >= current_epoch
-            // If previous epoch is not justified, pull up only tips from past epochs up to the current epoch
-            } else {
-                // If block is from a previous epoch, filter using unrealized justification & finalization information
-                if node.slot.epoch(E::slots_per_epoch()) < current_epoch {
-                    checkpoint_match_predicate(
-                        unrealized_justified_checkpoint,
-                        unrealized_finalized_checkpoint,
-                    )
-                // If block is from the current epoch, filter using the head state's justification & finalization information
-                } else {
-                    checkpoint_match_predicate(justified_checkpoint, finalized_checkpoint)
-                }
-            }
-        } else if let (Some(justified_checkpoint), Some(finalized_checkpoint)) =
-            (node.justified_checkpoint, node.finalized_checkpoint)
-        {
-            checkpoint_match_predicate(justified_checkpoint, finalized_checkpoint)
+        let voting_source = if current_epoch > node_epoch {
+            // The block is from a prior epoch, the voting source will be pulled-up.
+            node.unrealized_justified_checkpoint
+                // Sometimes we don't track the unrealized justification. In
+                // that case, just use the fully-realized justified checkpoint.
+                .unwrap_or(node_justified_checkpoint)
         } else {
-            false
+            // The block is not from a prior epoch, therefore the voting source
+            // is not pulled up.
+            node_justified_checkpoint
+        };
+
+        let mut correct_justified = self.justified_checkpoint.epoch == genesis_epoch
+            || voting_source.epoch == self.justified_checkpoint.epoch;
+
+        if let Some(node_unrealized_justified_checkpoint) = node.unrealized_justified_checkpoint {
+            if !correct_justified && self.justified_checkpoint.epoch + 1 == current_epoch {
+                correct_justified = node_unrealized_justified_checkpoint.epoch
+                    >= self.justified_checkpoint.epoch
+                    && voting_source.epoch + 2 >= current_epoch;
+            }
         }
+
+        let correct_finalized = self.finalized_checkpoint.epoch == genesis_epoch
+            || self.is_finalized_checkpoint_or_descendant::<E>(node.root);
+
+        correct_justified && correct_finalized
     }
 
     /// Return a reverse iterator over the nodes which comprise the chain ending at `block_root`.

consensus/proto_array/src/proto_array_fork_choice.rs

@@ -1,8 +1,8 @@
 use crate::{
     error::Error,
     proto_array::{
-        calculate_committee_fraction, CountUnrealizedFull, InvalidationOperation, Iter,
-        ProposerBoost, ProtoArray, ProtoNode,
+        calculate_committee_fraction, InvalidationOperation, Iter, ProposerBoost, ProtoArray,
+        ProtoNode,
     },
     ssz_container::SszContainer,
     JustifiedBalances,
@@ -307,7 +307,6 @@ impl ProtoArrayForkChoice {
         current_epoch_shuffling_id: AttestationShufflingId,
         next_epoch_shuffling_id: AttestationShufflingId,
         execution_status: ExecutionStatus,
-        count_unrealized_full: CountUnrealizedFull,
     ) -> Result<Self, String> {
         let mut proto_array = ProtoArray {
             prune_threshold: DEFAULT_PRUNE_THRESHOLD,
@@ -316,7 +315,6 @@ impl ProtoArrayForkChoice {
             nodes: Vec::with_capacity(1),
             indices: HashMap::with_capacity(1),
             previous_proposer_boost: ProposerBoost::default(),
-            count_unrealized_full,
         };
 
         let block = Block {
@@ -780,13 +778,10 @@ impl ProtoArrayForkChoice {
         SszContainer::from(self).as_ssz_bytes()
     }
 
-    pub fn from_bytes(
-        bytes: &[u8],
-        count_unrealized_full: CountUnrealizedFull,
-    ) -> Result<Self, String> {
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
         let container = SszContainer::from_ssz_bytes(bytes)
             .map_err(|e| format!("Failed to decode ProtoArrayForkChoice: {:?}", e))?;
-        (container, count_unrealized_full)
+        container
             .try_into()
             .map_err(|e| format!("Failed to initialize ProtoArrayForkChoice: {e:?}"))
     }
@@ -950,7 +945,6 @@ mod test_compute_deltas {
             junk_shuffling_id.clone(),
             junk_shuffling_id.clone(),
             execution_status,
-            CountUnrealizedFull::default(),
         )
         .unwrap();
 
@@ -1076,7 +1070,6 @@ mod test_compute_deltas {
             junk_shuffling_id.clone(),
             junk_shuffling_id.clone(),
             execution_status,
-            CountUnrealizedFull::default(),
         )
         .unwrap();
 

consensus/proto_array/src/ssz_container.rs

@@ -1,6 +1,6 @@
 use crate::proto_array::ProposerBoost;
 use crate::{
-    proto_array::{CountUnrealizedFull, ProtoArray, ProtoNode},
+    proto_array::{ProtoArray, ProtoNode},
     proto_array_fork_choice::{ElasticList, ProtoArrayForkChoice, VoteTracker},
     Error, JustifiedBalances,
 };
@@ -43,12 +43,10 @@ impl From<&ProtoArrayForkChoice> for SszContainer {
     }
 }
 
-impl TryFrom<(SszContainer, CountUnrealizedFull)> for ProtoArrayForkChoice {
+impl TryFrom<SszContainer> for ProtoArrayForkChoice {
     type Error = Error;
 
-    fn try_from(
-        (from, count_unrealized_full): (SszContainer, CountUnrealizedFull),
-    ) -> Result<Self, Error> {
+    fn try_from(from: SszContainer) -> Result<Self, Error> {
         let proto_array = ProtoArray {
             prune_threshold: from.prune_threshold,
             justified_checkpoint: from.justified_checkpoint,
@@ -56,7 +54,6 @@ impl TryFrom<(SszContainer, CountUnrealizedFull)> for ProtoArrayForkChoice {
             nodes: from.nodes,
             indices: from.indices.into_iter().collect::<HashMap<_, _>>(),
             previous_proposer_boost: from.previous_proposer_boost,
-            count_unrealized_full,
         };
 
         Ok(Self {