Deposit Cache Finalization & Fast WS Sync (#2915)

## Summary

The deposit cache now has the ability to finalize deposits. This will cause it to drop unneeded deposit logs and hashes in the deposit Merkle tree that are no longer required to construct deposit proofs. The cache is finalized whenever the latest finalized checkpoint has a new `Eth1Data` with all deposits imported.

This has three benefits:

1. Improves the speed of constructing Merkle proofs for deposits as we can just replay deposits since the last finalized checkpoint instead of all historical deposits when re-constructing the Merkle tree.
2. Significantly faster weak subjectivity sync as the deposit cache can be transferred to the newly syncing node in compressed form. The Merkle tree that stores `N` finalized deposits requires a maximum of `log2(N)` hashes. The newly syncing node then only needs to download deposits since the last finalized checkpoint to have a full tree.
3. Future proofing in preparation for [EIP-4444](https://eips.ethereum.org/EIPS/eip-4444) as execution nodes will no longer be required to store logs permanently so we won't always have all historical logs available to us.

## More Details

Image to illustrate how the deposit contract merkle tree evolves and finalizes along with the resulting `DepositTreeSnapshot`
![image](https://user-images.githubusercontent.com/37123614/151465302-5fc56284-8a69-4998-b20e-45db3934ac70.png)

## Other Considerations

I've changed the structure of the `SszDepositCache` so once you load & save your database from this version of lighthouse, you will no longer be able to load it from older versions.

Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com>

consensus/merkle_proof/src/lib.rs

@@ -19,6 +19,8 @@ lazy_static! {
 /// indices are populated by non-zero leaves (perfect for the deposit contract tree).
 #[derive(Debug, PartialEq)]
 pub enum MerkleTree {
+    /// Finalized Node
+    Finalized(H256),
     /// Leaf node with the hash of its content.
     Leaf(H256),
     /// Internal node with hash, left subtree and right subtree.
@@ -41,6 +43,24 @@ pub enum MerkleTreeError {
     DepthTooSmall,
     // Overflow occurred
     ArithError,
+    // Can't finalize a zero node
+    ZeroNodeFinalized,
+    // Can't push to finalized node
+    FinalizedNodePushed,
+    // Invalid Snapshot
+    InvalidSnapshot(InvalidSnapshot),
+    // Can't proof a finalized node
+    ProofEncounteredFinalizedNode,
+    // This should never happen
+    PleaseNotifyTheDevs,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum InvalidSnapshot {
+    // Branch hashes are empty but deposits are not
+    EmptyBranchWithNonZeroDeposits(usize),
+    // End of tree reached but deposits != 1
+    EndOfTree,
 }
 
 impl MerkleTree {
@@ -97,9 +117,11 @@ impl MerkleTree {
                 let right: &mut MerkleTree = &mut *right;
                 match (&*left, &*right) {
                     // Tree is full
-                    (Leaf(_), Leaf(_)) => return Err(MerkleTreeError::MerkleTreeFull),
+                    (Leaf(_), Leaf(_)) | (Finalized(_), Leaf(_)) => {
+                        return Err(MerkleTreeError::MerkleTreeFull)
+                    }
                     // There is a right node so insert in right node
-                    (Node(_, _, _), Node(_, _, _)) => {
+                    (Node(_, _, _), Node(_, _, _)) | (Finalized(_), Node(_, _, _)) => {
                         right.push_leaf(elem, depth - 1)?;
                     }
                     // Both branches are zero, insert in left one
@@ -107,7 +129,7 @@ impl MerkleTree {
                         *left = MerkleTree::create(&[elem], depth - 1);
                     }
                     // Leaf on left branch and zero on right branch, insert on right side
-                    (Leaf(_), Zero(_)) => {
+                    (Leaf(_), Zero(_)) | (Finalized(_), Zero(_)) => {
                         *right = MerkleTree::create(&[elem], depth - 1);
                     }
                     // Try inserting on the left node -> if it fails because it is full, insert in right side.
@@ -129,6 +151,7 @@ impl MerkleTree {
                     right.hash().as_bytes(),
                 ));
             }
+            Finalized(_) => return Err(MerkleTreeError::FinalizedNodePushed),
         }
 
         Ok(())
@@ -137,6 +160,7 @@ impl MerkleTree {
     /// Retrieve the root hash of this Merkle tree.
     pub fn hash(&self) -> H256 {
         match *self {
+            MerkleTree::Finalized(h) => h,
             MerkleTree::Leaf(h) => h,
             MerkleTree::Node(h, _, _) => h,
             MerkleTree::Zero(depth) => H256::from_slice(&ZERO_HASHES[depth]),
@@ -146,7 +170,7 @@ impl MerkleTree {
     /// Get a reference to the left and right subtrees if they exist.
     pub fn left_and_right_branches(&self) -> Option<(&Self, &Self)> {
         match *self {
-            MerkleTree::Leaf(_) | MerkleTree::Zero(0) => None,
+            MerkleTree::Finalized(_) | MerkleTree::Leaf(_) | MerkleTree::Zero(0) => None,
             MerkleTree::Node(_, ref l, ref r) => Some((l, r)),
             MerkleTree::Zero(depth) => Some((&ZERO_NODES[depth - 1], &ZERO_NODES[depth - 1])),
         }
@@ -157,16 +181,125 @@ impl MerkleTree {
         matches!(self, MerkleTree::Leaf(_))
     }
 
+    /// Finalize deposits up to deposit with count = deposits_to_finalize
+    pub fn finalize_deposits(
+        &mut self,
+        deposits_to_finalize: usize,
+        level: usize,
+    ) -> Result<(), MerkleTreeError> {
+        match self {
+            MerkleTree::Finalized(_) => Ok(()),
+            MerkleTree::Zero(_) => Err(MerkleTreeError::ZeroNodeFinalized),
+            MerkleTree::Leaf(hash) => {
+                if level != 0 {
+                    // This shouldn't happen but this is a sanity check
+                    return Err(MerkleTreeError::PleaseNotifyTheDevs);
+                }
+                *self = MerkleTree::Finalized(*hash);
+                Ok(())
+            }
+            MerkleTree::Node(hash, left, right) => {
+                if level == 0 {
+                    // this shouldn't happen but we'll put it here for safety
+                    return Err(MerkleTreeError::PleaseNotifyTheDevs);
+                }
+                let deposits = 0x1 << level;
+                if deposits <= deposits_to_finalize {
+                    *self = MerkleTree::Finalized(*hash);
+                    return Ok(());
+                }
+                left.finalize_deposits(deposits_to_finalize, level - 1)?;
+                if deposits_to_finalize > deposits / 2 {
+                    let remaining = deposits_to_finalize - deposits / 2;
+                    right.finalize_deposits(remaining, level - 1)?;
+                }
+                Ok(())
+            }
+        }
+    }
+
+    fn append_finalized_hashes(&self, result: &mut Vec<H256>) {
+        match self {
+            MerkleTree::Zero(_) | MerkleTree::Leaf(_) => {}
+            MerkleTree::Finalized(h) => result.push(*h),
+            MerkleTree::Node(_, left, right) => {
+                left.append_finalized_hashes(result);
+                right.append_finalized_hashes(result);
+            }
+        }
+    }
+
+    pub fn get_finalized_hashes(&self) -> Vec<H256> {
+        let mut result = vec![];
+        self.append_finalized_hashes(&mut result);
+        result
+    }
+
+    pub fn from_finalized_snapshot(
+        finalized_branch: &[H256],
+        deposit_count: usize,
+        level: usize,
+    ) -> Result<Self, MerkleTreeError> {
+        if finalized_branch.is_empty() {
+            return if deposit_count == 0 {
+                Ok(MerkleTree::Zero(level))
+            } else {
+                Err(InvalidSnapshot::EmptyBranchWithNonZeroDeposits(deposit_count).into())
+            };
+        }
+        if deposit_count == (0x1 << level) {
+            return Ok(MerkleTree::Finalized(
+                *finalized_branch
+                    .get(0)
+                    .ok_or(MerkleTreeError::PleaseNotifyTheDevs)?,
+            ));
+        }
+        if level == 0 {
+            return Err(InvalidSnapshot::EndOfTree.into());
+        }
+
+        let (left, right) = match deposit_count.checked_sub(0x1 << (level - 1)) {
+            // left tree is fully finalized
+            Some(right_deposits) => {
+                let (left_hash, right_branch) = finalized_branch
+                    .split_first()
+                    .ok_or(MerkleTreeError::PleaseNotifyTheDevs)?;
+                (
+                    MerkleTree::Finalized(*left_hash),
+                    MerkleTree::from_finalized_snapshot(right_branch, right_deposits, level - 1)?,
+                )
+            }
+            // left tree is not fully finalized -> right tree is zero
+            None => (
+                MerkleTree::from_finalized_snapshot(finalized_branch, deposit_count, level - 1)?,
+                MerkleTree::Zero(level - 1),
+            ),
+        };
+
+        let hash = H256::from_slice(&hash32_concat(
+            left.hash().as_bytes(),
+            right.hash().as_bytes(),
+        ));
+        Ok(MerkleTree::Node(hash, Box::new(left), Box::new(right)))
+    }
+
     /// Return the leaf at `index` and a Merkle proof of its inclusion.
     ///
     /// The Merkle proof is in "bottom-up" order, starting with a leaf node
     /// and moving up the tree. Its length will be exactly equal to `depth`.
-    pub fn generate_proof(&self, index: usize, depth: usize) -> (H256, Vec<H256>) {
+    pub fn generate_proof(
+        &self,
+        index: usize,
+        depth: usize,
+    ) -> Result<(H256, Vec<H256>), MerkleTreeError> {
         let mut proof = vec![];
         let mut current_node = self;
         let mut current_depth = depth;
         while current_depth > 0 {
             let ith_bit = (index >> (current_depth - 1)) & 0x01;
+            if let &MerkleTree::Finalized(_) = current_node {
+                return Err(MerkleTreeError::ProofEncounteredFinalizedNode);
+            }
             // Note: unwrap is safe because leaves are only ever constructed at depth == 0.
             let (left, right) = current_node.left_and_right_branches().unwrap();
 
@@ -187,7 +320,33 @@ impl MerkleTree {
         // Put proof in bottom-up order.
         proof.reverse();
 
-        (current_node.hash(), proof)
+        Ok((current_node.hash(), proof))
+    }
+
+    /// useful for debugging
+    pub fn print_node(&self, mut space: u32) {
+        const SPACES: u32 = 10;
+        space += SPACES;
+        let (pair, text) = match self {
+            MerkleTree::Node(hash, left, right) => (Some((left, right)), format!("Node({})", hash)),
+            MerkleTree::Leaf(hash) => (None, format!("Leaf({})", hash)),
+            MerkleTree::Zero(depth) => (
+                None,
+                format!("Z[{}]({})", depth, H256::from_slice(&ZERO_HASHES[*depth])),
+            ),
+            MerkleTree::Finalized(hash) => (None, format!("Finl({})", hash)),
+        };
+        if let Some((_, right)) = pair {
+            right.print_node(space);
+        }
+        println!();
+        for _i in SPACES..space {
+            print!(" ");
+        }
+        println!("{}", text);
+        if let Some((left, _)) = pair {
+            left.print_node(space);
+        }
     }
 }
 
@@ -235,6 +394,12 @@ impl From<ArithError> for MerkleTreeError {
     }
 }
 
+impl From<InvalidSnapshot> for MerkleTreeError {
+    fn from(e: InvalidSnapshot) -> Self {
+        MerkleTreeError::InvalidSnapshot(e)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -255,7 +420,9 @@ mod tests {
         let merkle_root = merkle_tree.hash();
 
         let proofs_ok = (0..leaves.len()).all(|i| {
-            let (leaf, branch) = merkle_tree.generate_proof(i, depth);
+            let (leaf, branch) = merkle_tree
+                .generate_proof(i, depth)
+                .expect("should generate proof");
             leaf == leaves[i] && verify_merkle_proof(leaf, &branch, depth, i, merkle_root)
         });
 
@@ -274,7 +441,9 @@ mod tests {
 
         let proofs_ok = leaves_iter.enumerate().all(|(i, leaf)| {
             assert_eq!(merkle_tree.push_leaf(leaf, depth), Ok(()));
-            let (stored_leaf, branch) = merkle_tree.generate_proof(i, depth);
+            let (stored_leaf, branch) = merkle_tree
+                .generate_proof(i, depth)
+                .expect("should generate proof");
             stored_leaf == leaf && verify_merkle_proof(leaf, &branch, depth, i, merkle_tree.hash())
         });
 

consensus/ssz/src/decode/impls.rs

@@ -246,6 +246,20 @@ impl Decode for NonZeroUsize {
     }
 }
 
+impl<T: Decode> Decode for Option<T> {
+    fn is_ssz_fixed_len() -> bool {
+        false
+    }
+    fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
+        let (selector, body) = split_union_bytes(bytes)?;
+        match selector.into() {
+            0u8 => Ok(None),
+            1u8 => <T as Decode>::from_ssz_bytes(body).map(Option::Some),
+            other => Err(DecodeError::UnionSelectorInvalid(other)),
+        }
+    }
+}
+
 impl<T: Decode> Decode for Arc<T> {
     fn is_ssz_fixed_len() -> bool {
         T::is_ssz_fixed_len()

consensus/ssz/src/encode/impls.rs

@@ -203,6 +203,34 @@ impl_encode_for_tuples! {
     }
 }
 
+impl<T: Encode> Encode for Option<T> {
+    fn is_ssz_fixed_len() -> bool {
+        false
+    }
+    fn ssz_append(&self, buf: &mut Vec<u8>) {
+        match self {
+            Option::None => {
+                let union_selector: u8 = 0u8;
+                buf.push(union_selector);
+            }
+            Option::Some(ref inner) => {
+                let union_selector: u8 = 1u8;
+                buf.push(union_selector);
+                inner.ssz_append(buf);
+            }
+        }
+    }
+    fn ssz_bytes_len(&self) -> usize {
+        match self {
+            Option::None => 1usize,
+            Option::Some(ref inner) => inner
+                .ssz_bytes_len()
+                .checked_add(1)
+                .expect("encoded length must be less than usize::max_value"),
+        }
+    }
+}
+
 impl<T: Encode> Encode for Arc<T> {
     fn is_ssz_fixed_len() -> bool {
         T::is_ssz_fixed_len()
@@ -561,6 +589,14 @@ mod tests {
         );
     }
 
+    #[test]
+    fn ssz_encode_option_u8() {
+        let opt: Option<u8> = None;
+        assert_eq!(opt.as_ssz_bytes(), vec![0]);
+        let opt: Option<u8> = Some(2);
+        assert_eq!(opt.as_ssz_bytes(), vec![1, 2]);
+    }
+
     #[test]
     fn ssz_encode_bool() {
         assert_eq!(true.as_ssz_bytes(), vec![1]);

consensus/ssz/tests/tests.rs

@@ -22,6 +22,13 @@ mod round_trip {
         round_trip(items);
     }
 
+    #[test]
+    fn option_u16() {
+        let items: Vec<Option<u16>> = vec![None, Some(2u16)];
+
+        round_trip(items);
+    }
+
     #[test]
     fn u8_array_4() {
         let items: Vec<[u8; 4]> = vec![[0, 0, 0, 0], [1, 0, 0, 0], [1, 2, 3, 4], [1, 2, 0, 4]];
@@ -46,6 +53,17 @@ mod round_trip {
         round_trip(items);
     }
 
+    #[test]
+    fn option_vec_h256() {
+        let items: Vec<Option<Vec<H256>>> = vec![
+            None,
+            Some(vec![]),
+            Some(vec![H256::zero(), H256::from([1; 32]), H256::random()]),
+        ];
+
+        round_trip(items);
+    }
+
     #[test]
     fn vec_u16() {
         let items: Vec<Vec<u16>> = vec![

consensus/state_processing/src/common/deposit_data_tree.rs

@@ -2,12 +2,14 @@ use eth2_hashing::hash;
 use int_to_bytes::int_to_bytes32;
 use merkle_proof::{MerkleTree, MerkleTreeError};
 use safe_arith::SafeArith;
-use types::Hash256;
+use types::{DepositTreeSnapshot, FinalizedExecutionBlock, Hash256};
 
 /// Emulates the eth1 deposit contract merkle tree.
+#[derive(PartialEq)]
 pub struct DepositDataTree {
     tree: MerkleTree,
     mix_in_length: usize,
+    finalized_execution_block: Option<FinalizedExecutionBlock>,
     depth: usize,
 }
 
@@ -17,6 +19,7 @@ impl DepositDataTree {
         Self {
             tree: MerkleTree::create(leaves, depth),
             mix_in_length,
+            finalized_execution_block: None,
             depth,
         }
     }
@@ -38,10 +41,10 @@ impl DepositDataTree {
     ///
     /// The Merkle proof is in "bottom-up" order, starting with a leaf node
     /// and moving up the tree. Its length will be exactly equal to `depth + 1`.
-    pub fn generate_proof(&self, index: usize) -> (Hash256, Vec<Hash256>) {
-        let (root, mut proof) = self.tree.generate_proof(index, self.depth);
+    pub fn generate_proof(&self, index: usize) -> Result<(Hash256, Vec<Hash256>), MerkleTreeError> {
+        let (root, mut proof) = self.tree.generate_proof(index, self.depth)?;
         proof.push(Hash256::from_slice(&self.length_bytes()));
-        (root, proof)
+        Ok((root, proof))
     }
 
     /// Add a deposit to the merkle tree.
@@ -50,4 +53,50 @@ impl DepositDataTree {
         self.mix_in_length.safe_add_assign(1)?;
         Ok(())
     }
+
+    /// Finalize deposits up to `finalized_execution_block.deposit_count`
+    pub fn finalize(
+        &mut self,
+        finalized_execution_block: FinalizedExecutionBlock,
+    ) -> Result<(), MerkleTreeError> {
+        self.tree
+            .finalize_deposits(finalized_execution_block.deposit_count as usize, self.depth)?;
+        self.finalized_execution_block = Some(finalized_execution_block);
+        Ok(())
+    }
+
+    /// Get snapshot of finalized deposit tree (if tree is finalized)
+    pub fn get_snapshot(&self) -> Option<DepositTreeSnapshot> {
+        let finalized_execution_block = self.finalized_execution_block.as_ref()?;
+        Some(DepositTreeSnapshot {
+            finalized: self.tree.get_finalized_hashes(),
+            deposit_root: finalized_execution_block.deposit_root,
+            deposit_count: finalized_execution_block.deposit_count,
+            execution_block_hash: finalized_execution_block.block_hash,
+            execution_block_height: finalized_execution_block.block_height,
+        })
+    }
+
+    /// Create a new Merkle tree from a snapshot
+    pub fn from_snapshot(
+        snapshot: &DepositTreeSnapshot,
+        depth: usize,
+    ) -> Result<Self, MerkleTreeError> {
+        Ok(Self {
+            tree: MerkleTree::from_finalized_snapshot(
+                &snapshot.finalized,
+                snapshot.deposit_count as usize,
+                depth,
+            )?,
+            mix_in_length: snapshot.deposit_count as usize,
+            finalized_execution_block: Some(snapshot.into()),
+            depth,
+        })
+    }
+
+    #[allow(dead_code)]
+    pub fn print_tree(&self) {
+        self.tree.print_node(0);
+        println!("========================================================");
+    }
 }

consensus/types/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "types"
-version = "0.2.0"
+version = "0.2.1"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com>"]
 edition = "2021"
 

consensus/types/src/deposit_tree_snapshot.rs

@@ -0,0 +1,83 @@
+use crate::*;
+use eth2_hashing::{hash32_concat, ZERO_HASHES};
+use int_to_bytes::int_to_bytes32;
+use serde_derive::{Deserialize, Serialize};
+use ssz_derive::{Decode, Encode};
+use test_random_derive::TestRandom;
+use test_utils::TestRandom;
+use DEPOSIT_TREE_DEPTH;
+
+#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq, TestRandom)]
+pub struct FinalizedExecutionBlock {
+    pub deposit_root: Hash256,
+    pub deposit_count: u64,
+    pub block_hash: Hash256,
+    pub block_height: u64,
+}
+
+impl From<&DepositTreeSnapshot> for FinalizedExecutionBlock {
+    fn from(snapshot: &DepositTreeSnapshot) -> Self {
+        Self {
+            deposit_root: snapshot.deposit_root,
+            deposit_count: snapshot.deposit_count,
+            block_hash: snapshot.execution_block_hash,
+            block_height: snapshot.execution_block_height,
+        }
+    }
+}
+
+#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq, TestRandom)]
+pub struct DepositTreeSnapshot {
+    pub finalized: Vec<Hash256>,
+    pub deposit_root: Hash256,
+    pub deposit_count: u64,
+    pub execution_block_hash: Hash256,
+    pub execution_block_height: u64,
+}
+
+impl Default for DepositTreeSnapshot {
+    fn default() -> Self {
+        let mut result = Self {
+            finalized: vec![],
+            deposit_root: Hash256::default(),
+            deposit_count: 0,
+            execution_block_hash: Hash256::zero(),
+            execution_block_height: 0,
+        };
+        // properly set the empty deposit root
+        result.deposit_root = result.calculate_root().unwrap();
+        result
+    }
+}
+
+impl DepositTreeSnapshot {
+    // Calculates the deposit tree root from the hashes in the snapshot
+    pub fn calculate_root(&self) -> Option<Hash256> {
+        let mut size = self.deposit_count;
+        let mut index = self.finalized.len();
+        let mut deposit_root = [0; 32];
+        for height in 0..DEPOSIT_TREE_DEPTH {
+            deposit_root = if (size & 1) == 1 {
+                index = index.checked_sub(1)?;
+                hash32_concat(self.finalized.get(index)?.as_bytes(), &deposit_root)
+            } else {
+                hash32_concat(&deposit_root, ZERO_HASHES.get(height)?)
+            };
+            size /= 2;
+        }
+        // add mix-in-length
+        deposit_root = hash32_concat(&deposit_root, &int_to_bytes32(self.deposit_count));
+
+        Some(Hash256::from_slice(&deposit_root))
+    }
+    pub fn is_valid(&self) -> bool {
+        self.calculate_root()
+            .map_or(false, |calculated| self.deposit_root == calculated)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    ssz_tests!(DepositTreeSnapshot);
+}

consensus/types/src/lib.rs

@@ -36,6 +36,7 @@ pub mod contribution_and_proof;
 pub mod deposit;
 pub mod deposit_data;
 pub mod deposit_message;
+pub mod deposit_tree_snapshot;
 pub mod enr_fork_id;
 pub mod eth1_data;
 pub mod eth_spec;
@@ -120,6 +121,7 @@ pub use crate::contribution_and_proof::ContributionAndProof;
 pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH};
 pub use crate::deposit_data::DepositData;
 pub use crate::deposit_message::DepositMessage;
+pub use crate::deposit_tree_snapshot::{DepositTreeSnapshot, FinalizedExecutionBlock};
 pub use crate::enr_fork_id::EnrForkId;
 pub use crate::eth1_data::Eth1Data;
 pub use crate::eth_spec::EthSpecId;