Optimizations, disable val client sync check & additional lcli tools (#834)

* Start adding interop genesis state to lcli

* Use more efficient method to generate genesis state

* Remove duplicate int_to_bytes32

* Add lcli command to change state genesis time

* Add option to allow VC to start with unsynced BN

* Set VC to do parallel key loading

* Don't default to dummy eth1 backend

* Add endpoint to dump operation pool

* Add metrics for op pool

* Remove state clone for slot notifier

* Add mem size approximation for tree hash cache

* Avoid cloning tree hash when getting head

* Fix failing API tests

* Address Michael's comments

* Add HashMap::from_par_iter

eth2/state_processing/src/common/deposit_data_tree.rs

@@ -0,0 +1,52 @@
+use eth2_hashing::hash;
+use int_to_bytes::int_to_bytes32;
+use merkle_proof::{MerkleTree, MerkleTreeError};
+use types::Hash256;
+
+/// Emulates the eth1 deposit contract merkle tree.
+pub struct DepositDataTree {
+    tree: MerkleTree,
+    mix_in_length: usize,
+    depth: usize,
+}
+
+impl DepositDataTree {
+    /// Create a new Merkle tree from a list of leaves (`DepositData::tree_hash_root`) and a fixed depth.
+    pub fn create(leaves: &[Hash256], mix_in_length: usize, depth: usize) -> Self {
+        Self {
+            tree: MerkleTree::create(leaves, depth),
+            mix_in_length,
+            depth,
+        }
+    }
+
+    /// Returns 32 bytes representing the "mix in length" for the merkle root of this tree.
+    fn length_bytes(&self) -> Vec<u8> {
+        int_to_bytes32(self.mix_in_length as u64)
+    }
+
+    /// Retrieve the root hash of this Merkle tree with the length mixed in.
+    pub fn root(&self) -> Hash256 {
+        let mut preimage = [0; 64];
+        preimage[0..32].copy_from_slice(&self.tree.hash()[..]);
+        preimage[32..64].copy_from_slice(&self.length_bytes());
+        Hash256::from_slice(&hash(&preimage))
+    }
+
+    /// Return the leaf at `index` and a Merkle proof of its inclusion.
+    ///
+    /// The Merkle proof is in "bottom-up" order, starting with a leaf node
+    /// and moving up the tree. Its length will be exactly equal to `depth + 1`.
+    pub fn generate_proof(&self, index: usize) -> (Hash256, Vec<Hash256>) {
+        let (root, mut proof) = self.tree.generate_proof(index, self.depth);
+        proof.push(Hash256::from_slice(&self.length_bytes()));
+        (root, proof)
+    }
+
+    /// Add a deposit to the merkle tree.
+    pub fn push_leaf(&mut self, leaf: Hash256) -> Result<(), MerkleTreeError> {
+        self.tree.push_leaf(leaf, self.depth)?;
+        self.mix_in_length += 1;
+        Ok(())
+    }
+}

eth2/state_processing/src/common/mod.rs

@@ -1,9 +1,11 @@
+mod deposit_data_tree;
 mod get_attesting_indices;
 mod get_base_reward;
 mod get_indexed_attestation;
 mod initiate_validator_exit;
 mod slash_validator;
 
+pub use deposit_data_tree::DepositDataTree;
 pub use get_attesting_indices::get_attesting_indices;
 pub use get_base_reward::get_base_reward;
 pub use get_indexed_attestation::get_indexed_attestation;

eth2/state_processing/src/genesis.rs

@@ -1,6 +1,7 @@
 use super::per_block_processing::{errors::BlockProcessingError, process_deposit};
+use crate::common::DepositDataTree;
 use tree_hash::TreeHash;
-use types::typenum::U4294967296;
+use types::DEPOSIT_TREE_DEPTH;
 use types::*;
 
 /// Initialize a `BeaconState` from genesis data.
@@ -26,14 +27,13 @@ pub fn initialize_beacon_state_from_eth1<T: EthSpec>(
     // Seed RANDAO with Eth1 entropy
     state.fill_randao_mixes_with(eth1_block_hash);
 
-    // Process deposits
-    let leaves: Vec<_> = deposits
-        .iter()
-        .map(|deposit| deposit.data.clone())
-        .collect();
-    for (index, deposit) in deposits.into_iter().enumerate() {
-        let deposit_data_list = VariableList::<_, U4294967296>::from(leaves[..=index].to_vec());
-        state.eth1_data.deposit_root = Hash256::from_slice(&deposit_data_list.tree_hash_root());
+    let mut deposit_tree = DepositDataTree::create(&[], 0, DEPOSIT_TREE_DEPTH);
+
+    for deposit in deposits.iter() {
+        deposit_tree
+            .push_leaf(Hash256::from_slice(&deposit.data.tree_hash_root()))
+            .map_err(BlockProcessingError::MerkleTreeError)?;
+        state.eth1_data.deposit_root = deposit_tree.root();
         process_deposit(&mut state, &deposit, spec, true)?;
     }
 

eth2/state_processing/src/per_block_processing/errors.rs

@@ -1,4 +1,5 @@
 use super::signature_sets::Error as SignatureSetError;
+use merkle_proof::MerkleTreeError;
 use types::*;
 
 /// The error returned from the `per_block_processing` function. Indicates that a block is either
@@ -46,6 +47,7 @@ pub enum BlockProcessingError {
     BeaconStateError(BeaconStateError),
     SignatureSetError(SignatureSetError),
     SszTypesError(ssz_types::Error),
+    MerkleTreeError(MerkleTreeError),
 }
 
 impl From<BeaconStateError> for BlockProcessingError {