Optimizations, disable val client sync check & additional lcli tools (#834)

* Start adding interop genesis state to lcli

* Use more efficient method to generate genesis state

* Remove duplicate int_to_bytes32

* Add lcli command to change state genesis time

* Add option to allow VC to start with unsynced BN

* Set VC to do parallel key loading

* Don't default to dummy eth1 backend

* Add endpoint to dump operation pool

* Add metrics for op pool

* Remove state clone for slot notifier

* Add mem size approximation for tree hash cache

* Avoid cloning tree hash when getting head

* Fix failing API tests

* Address Michael's comments

* Add HashMap::from_par_iter

eth2/state_processing/src/common/deposit_data_tree.rs

@@ -0,0 +1,52 @@
+use eth2_hashing::hash;
+use int_to_bytes::int_to_bytes32;
+use merkle_proof::{MerkleTree, MerkleTreeError};
+use types::Hash256;
+
+/// Emulates the eth1 deposit contract merkle tree.
+pub struct DepositDataTree {
+    tree: MerkleTree,
+    mix_in_length: usize,
+    depth: usize,
+}
+
+impl DepositDataTree {
+    /// Create a new Merkle tree from a list of leaves (`DepositData::tree_hash_root`) and a fixed depth.
+    pub fn create(leaves: &[Hash256], mix_in_length: usize, depth: usize) -> Self {
+        Self {
+            tree: MerkleTree::create(leaves, depth),
+            mix_in_length,
+            depth,
+        }
+    }
+
+    /// Returns 32 bytes representing the "mix in length" for the merkle root of this tree.
+    fn length_bytes(&self) -> Vec<u8> {
+        int_to_bytes32(self.mix_in_length as u64)
+    }
+
+    /// Retrieve the root hash of this Merkle tree with the length mixed in.
+    pub fn root(&self) -> Hash256 {
+        let mut preimage = [0; 64];
+        preimage[0..32].copy_from_slice(&self.tree.hash()[..]);
+        preimage[32..64].copy_from_slice(&self.length_bytes());
+        Hash256::from_slice(&hash(&preimage))
+    }
+
+    /// Return the leaf at `index` and a Merkle proof of its inclusion.
+    ///
+    /// The Merkle proof is in "bottom-up" order, starting with a leaf node
+    /// and moving up the tree. Its length will be exactly equal to `depth + 1`.
+    pub fn generate_proof(&self, index: usize) -> (Hash256, Vec<Hash256>) {
+        let (root, mut proof) = self.tree.generate_proof(index, self.depth);
+        proof.push(Hash256::from_slice(&self.length_bytes()));
+        (root, proof)
+    }
+
+    /// Add a deposit to the merkle tree.
+    pub fn push_leaf(&mut self, leaf: Hash256) -> Result<(), MerkleTreeError> {
+        self.tree.push_leaf(leaf, self.depth)?;
+        self.mix_in_length += 1;
+        Ok(())
+    }
+}

eth2/state_processing/src/common/mod.rs

@@ -1,9 +1,11 @@
+mod deposit_data_tree;
 mod get_attesting_indices;
 mod get_base_reward;
 mod get_indexed_attestation;
 mod initiate_validator_exit;
 mod slash_validator;
 
+pub use deposit_data_tree::DepositDataTree;
 pub use get_attesting_indices::get_attesting_indices;
 pub use get_base_reward::get_base_reward;
 pub use get_indexed_attestation::get_indexed_attestation;