DB migration for fork choice cleanup (#4265)

## Issue Addressed

#4233

## Proposed Changes

Remove the `best_justified_checkpoint` from the `PersistedForkChoiceStore` type as it is now unused.
Additionally, remove the `Option`'s wrapping the `justified_checkpoint` and `finalized_checkpoint` fields on `ProtoNode` which were only present to facilitate a previous migration.

Include the necessary code to facilitate the migration to a new DB schema.

consensus/proto_array/src/error.rs

@@ -14,6 +14,8 @@ pub enum Error {
     InvalidBestDescendant(usize),
     InvalidParentDelta(usize),
     InvalidNodeDelta(usize),
+    MissingJustifiedCheckpoint,
+    MissingFinalizedCheckpoint,
     DeltaOverflow(usize),
     ProposerBoostOverflow(usize),
     ReOrgThresholdOverflow,
@@ -67,6 +69,6 @@ pub struct InvalidBestNodeInfo {
     pub justified_checkpoint: Checkpoint,
     pub finalized_checkpoint: Checkpoint,
     pub head_root: Hash256,
-    pub head_justified_checkpoint: Option<Checkpoint>,
-    pub head_finalized_checkpoint: Option<Checkpoint>,
+    pub head_justified_checkpoint: Checkpoint,
+    pub head_finalized_checkpoint: Checkpoint,
 }

consensus/proto_array/src/lib.rs

@@ -16,5 +16,5 @@ pub use error::Error;
 pub mod core {
     pub use super::proto_array::{ProposerBoost, ProtoArray, ProtoNode};
     pub use super::proto_array_fork_choice::VoteTracker;
-    pub use super::ssz_container::SszContainer;
+    pub use super::ssz_container::{SszContainer, SszContainerV16, SszContainerV17};
 }

consensus/proto_array/src/proto_array.rs

@@ -5,6 +5,7 @@ use ssz::four_byte_option_impl;
 use ssz::Encode;
 use ssz_derive::{Decode, Encode};
 use std::collections::{HashMap, HashSet};
+use superstruct::superstruct;
 use types::{
     AttestationShufflingId, ChainSpec, Checkpoint, Epoch, EthSpec, ExecutionBlockHash, Hash256,
     Slot,
@@ -66,7 +67,13 @@ impl InvalidationOperation {
     }
 }
 
-#[derive(Clone, PartialEq, Debug, Encode, Decode, Serialize, Deserialize)]
+pub type ProtoNode = ProtoNodeV17;
+
+#[superstruct(
+    variants(V16, V17),
+    variant_attributes(derive(Clone, PartialEq, Debug, Encode, Decode, Serialize, Deserialize)),
+    no_enum
+)]
 pub struct ProtoNode {
     /// The `slot` is not necessary for `ProtoArray`, it just exists so external components can
     /// easily query the block slot. This is useful for upstream fork choice logic.
@@ -85,10 +92,16 @@ pub struct ProtoNode {
     pub root: Hash256,
     #[ssz(with = "four_byte_option_usize")]
     pub parent: Option<usize>,
+    #[superstruct(only(V16))]
     #[ssz(with = "four_byte_option_checkpoint")]
     pub justified_checkpoint: Option<Checkpoint>,
+    #[superstruct(only(V16))]
     #[ssz(with = "four_byte_option_checkpoint")]
     pub finalized_checkpoint: Option<Checkpoint>,
+    #[superstruct(only(V17))]
+    pub justified_checkpoint: Checkpoint,
+    #[superstruct(only(V17))]
+    pub finalized_checkpoint: Checkpoint,
     pub weight: u64,
     #[ssz(with = "four_byte_option_usize")]
     pub best_child: Option<usize>,
@@ -103,6 +116,57 @@ pub struct ProtoNode {
     pub unrealized_finalized_checkpoint: Option<Checkpoint>,
 }
 
+impl TryInto<ProtoNode> for ProtoNodeV16 {
+    type Error = Error;
+
+    fn try_into(self) -> Result<ProtoNode, Error> {
+        let result = ProtoNode {
+            slot: self.slot,
+            state_root: self.state_root,
+            target_root: self.target_root,
+            current_epoch_shuffling_id: self.current_epoch_shuffling_id,
+            next_epoch_shuffling_id: self.next_epoch_shuffling_id,
+            root: self.root,
+            parent: self.parent,
+            justified_checkpoint: self
+                .justified_checkpoint
+                .ok_or(Error::MissingJustifiedCheckpoint)?,
+            finalized_checkpoint: self
+                .finalized_checkpoint
+                .ok_or(Error::MissingFinalizedCheckpoint)?,
+            weight: self.weight,
+            best_child: self.best_child,
+            best_descendant: self.best_descendant,
+            execution_status: self.execution_status,
+            unrealized_justified_checkpoint: self.unrealized_justified_checkpoint,
+            unrealized_finalized_checkpoint: self.unrealized_finalized_checkpoint,
+        };
+        Ok(result)
+    }
+}
+
+impl Into<ProtoNodeV16> for ProtoNode {
+    fn into(self) -> ProtoNodeV16 {
+        ProtoNodeV16 {
+            slot: self.slot,
+            state_root: self.state_root,
+            target_root: self.target_root,
+            current_epoch_shuffling_id: self.current_epoch_shuffling_id,
+            next_epoch_shuffling_id: self.next_epoch_shuffling_id,
+            root: self.root,
+            parent: self.parent,
+            justified_checkpoint: Some(self.justified_checkpoint),
+            finalized_checkpoint: Some(self.finalized_checkpoint),
+            weight: self.weight,
+            best_child: self.best_child,
+            best_descendant: self.best_descendant,
+            execution_status: self.execution_status,
+            unrealized_justified_checkpoint: self.unrealized_justified_checkpoint,
+            unrealized_finalized_checkpoint: self.unrealized_finalized_checkpoint,
+        }
+    }
+}
+
 #[derive(PartialEq, Debug, Encode, Decode, Serialize, Deserialize, Copy, Clone)]
 pub struct ProposerBoost {
     pub root: Hash256,
@@ -320,8 +384,8 @@ impl ProtoArray {
             parent: block
                 .parent_root
                 .and_then(|parent| self.indices.get(&parent).copied()),
-            justified_checkpoint: Some(block.justified_checkpoint),
-            finalized_checkpoint: Some(block.finalized_checkpoint),
+            justified_checkpoint: block.justified_checkpoint,
+            finalized_checkpoint: block.finalized_checkpoint,
             weight: 0,
             best_child: None,
             best_descendant: None,
@@ -883,14 +947,7 @@ impl ProtoArray {
         let genesis_epoch = Epoch::new(0);
         let current_epoch = current_slot.epoch(E::slots_per_epoch());
         let node_epoch = node.slot.epoch(E::slots_per_epoch());
-        let node_justified_checkpoint =
-            if let Some(justified_checkpoint) = node.justified_checkpoint {
-                justified_checkpoint
-            } else {
-                // The node does not have any information about the justified
-                // checkpoint. This indicates an inconsistent proto-array.
-                return false;
-            };
+        let node_justified_checkpoint = node.justified_checkpoint;
 
         let voting_source = if current_epoch > node_epoch {
             // The block is from a prior epoch, the voting source will be pulled-up.
@@ -998,9 +1055,13 @@ impl ProtoArray {
         // Run this check once, outside of the loop rather than inside the loop.
         // If the conditions don't match for this node then they're unlikely to
         // start matching for its ancestors.
+        for checkpoint in &[node.finalized_checkpoint, node.justified_checkpoint] {
+            if checkpoint == &self.finalized_checkpoint {
+                return true;
+            }
+        }
+
         for checkpoint in &[
-            node.finalized_checkpoint,
-            node.justified_checkpoint,
             node.unrealized_finalized_checkpoint,
             node.unrealized_justified_checkpoint,
         ] {

consensus/proto_array/src/proto_array_fork_choice.rs

@@ -754,29 +754,20 @@ impl ProtoArrayForkChoice {
             .and_then(|i| self.proto_array.nodes.get(i))
             .map(|parent| parent.root);
 
-        // If a node does not have a `finalized_checkpoint` or `justified_checkpoint` populated,
-        // it means it is not a descendant of the finalized checkpoint, so it is valid to return
-        // `None` here.
-        if let (Some(justified_checkpoint), Some(finalized_checkpoint)) =
-            (block.justified_checkpoint, block.finalized_checkpoint)
-        {
-            Some(Block {
-                slot: block.slot,
-                root: block.root,
-                parent_root,
-                state_root: block.state_root,
-                target_root: block.target_root,
-                current_epoch_shuffling_id: block.current_epoch_shuffling_id.clone(),
-                next_epoch_shuffling_id: block.next_epoch_shuffling_id.clone(),
-                justified_checkpoint,
-                finalized_checkpoint,
-                execution_status: block.execution_status,
-                unrealized_justified_checkpoint: block.unrealized_justified_checkpoint,
-                unrealized_finalized_checkpoint: block.unrealized_finalized_checkpoint,
-            })
-        } else {
-            None
-        }
+        Some(Block {
+            slot: block.slot,
+            root: block.root,
+            parent_root,
+            state_root: block.state_root,
+            target_root: block.target_root,
+            current_epoch_shuffling_id: block.current_epoch_shuffling_id.clone(),
+            next_epoch_shuffling_id: block.next_epoch_shuffling_id.clone(),
+            justified_checkpoint: block.justified_checkpoint,
+            finalized_checkpoint: block.finalized_checkpoint,
+            execution_status: block.execution_status,
+            unrealized_justified_checkpoint: block.unrealized_justified_checkpoint,
+            unrealized_finalized_checkpoint: block.unrealized_finalized_checkpoint,
+        })
     }
 
     /// Returns the `block.execution_status` field, if the block is present.

consensus/proto_array/src/ssz_container.rs

@@ -1,6 +1,6 @@
 use crate::proto_array::ProposerBoost;
 use crate::{
-    proto_array::{ProtoArray, ProtoNode},
+    proto_array::{ProtoArray, ProtoNodeV16, ProtoNodeV17},
     proto_array_fork_choice::{ElasticList, ProtoArrayForkChoice, VoteTracker},
     Error, JustifiedBalances,
 };
@@ -8,24 +8,71 @@ use ssz::{four_byte_option_impl, Encode};
 use ssz_derive::{Decode, Encode};
 use std::collections::HashMap;
 use std::convert::TryFrom;
+use superstruct::superstruct;
 use types::{Checkpoint, Hash256};
 
 // Define a "legacy" implementation of `Option<usize>` which uses four bytes for encoding the union
 // selector.
 four_byte_option_impl!(four_byte_option_checkpoint, Checkpoint);
 
-#[derive(Encode, Decode)]
+pub type SszContainer = SszContainerV17;
+
+#[superstruct(
+    variants(V16, V17),
+    variant_attributes(derive(Encode, Decode)),
+    no_enum
+)]
 pub struct SszContainer {
     pub votes: Vec<VoteTracker>,
     pub balances: Vec<u64>,
     pub prune_threshold: usize,
     pub justified_checkpoint: Checkpoint,
     pub finalized_checkpoint: Checkpoint,
-    pub nodes: Vec<ProtoNode>,
+    #[superstruct(only(V16))]
+    pub nodes: Vec<ProtoNodeV16>,
+    #[superstruct(only(V17))]
+    pub nodes: Vec<ProtoNodeV17>,
     pub indices: Vec<(Hash256, usize)>,
     pub previous_proposer_boost: ProposerBoost,
 }
 
+impl TryInto<SszContainer> for SszContainerV16 {
+    type Error = Error;
+
+    fn try_into(self) -> Result<SszContainer, Error> {
+        let nodes: Result<Vec<ProtoNodeV17>, Error> =
+            self.nodes.into_iter().map(TryInto::try_into).collect();
+
+        Ok(SszContainer {
+            votes: self.votes,
+            balances: self.balances,
+            prune_threshold: self.prune_threshold,
+            justified_checkpoint: self.justified_checkpoint,
+            finalized_checkpoint: self.finalized_checkpoint,
+            nodes: nodes?,
+            indices: self.indices,
+            previous_proposer_boost: self.previous_proposer_boost,
+        })
+    }
+}
+
+impl Into<SszContainerV16> for SszContainer {
+    fn into(self) -> SszContainerV16 {
+        let nodes = self.nodes.into_iter().map(Into::into).collect();
+
+        SszContainerV16 {
+            votes: self.votes,
+            balances: self.balances,
+            prune_threshold: self.prune_threshold,
+            justified_checkpoint: self.justified_checkpoint,
+            finalized_checkpoint: self.finalized_checkpoint,
+            nodes,
+            indices: self.indices,
+            previous_proposer_boost: self.previous_proposer_boost,
+        }
+    }
+}
+
 impl From<&ProtoArrayForkChoice> for SszContainer {
     fn from(from: &ProtoArrayForkChoice) -> Self {
         let proto_array = &from.proto_array;