Optimizations, disable val client sync check & additional lcli tools (#834)

* Start adding interop genesis state to lcli

* Use more efficient method to generate genesis state

* Remove duplicate int_to_bytes32

* Add lcli command to change state genesis time

* Add option to allow VC to start with unsynced BN

* Set VC to do parallel key loading

* Don't default to dummy eth1 backend

* Add endpoint to dump operation pool

* Add metrics for op pool

* Remove state clone for slot notifier

* Add mem size approximation for tree hash cache

* Avoid cloning tree hash when getting head

* Fix failing API tests

* Address Michael's comments

* Add HashMap::from_par_iter

eth2/operation_pool/Cargo.toml

@@ -11,6 +11,8 @@ types = { path = "../types" }
 state_processing = { path = "../state_processing" }
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
+serde = "1.0.102"
+serde_derive = "1.0.102"
 
 [dev-dependencies]
 rand = "0.7.2"

eth2/operation_pool/src/attestation_id.rs

@@ -1,10 +1,13 @@
 use int_to_bytes::int_to_bytes8;
+use serde_derive::{Deserialize, Serialize};
 use ssz::ssz_encode;
 use ssz_derive::{Decode, Encode};
 use types::{AttestationData, BeaconState, ChainSpec, Domain, Epoch, EthSpec};
 
 /// Serialized `AttestationData` augmented with a domain to encode the fork info.
-#[derive(PartialEq, Eq, Clone, Hash, Debug, PartialOrd, Ord, Encode, Decode)]
+#[derive(
+    PartialEq, Eq, Clone, Hash, Debug, PartialOrd, Ord, Encode, Decode, Serialize, Deserialize,
+)]
 pub struct AttestationId {
     v: Vec<u8>,
 }

eth2/operation_pool/src/lib.rs

@@ -284,6 +284,16 @@ impl<T: EthSpec> OperationPool<T> {
         });
     }
 
+    /// Total number of attester slashings in the pool.
+    pub fn num_attester_slashings(&self) -> usize {
+        self.attester_slashings.read().len()
+    }
+
+    /// Total number of proposer slashings in the pool.
+    pub fn num_proposer_slashings(&self) -> usize {
+        self.proposer_slashings.read().len()
+    }
+
     /// Insert a voluntary exit, validating it almost-entirely (future exits are permitted).
     pub fn insert_voluntary_exit(
         &self,
@@ -327,6 +337,11 @@ impl<T: EthSpec> OperationPool<T> {
         self.prune_attester_slashings(finalized_state, spec);
         self.prune_voluntary_exits(finalized_state);
     }
+
+    /// Total number of voluntary exits in the pool.
+    pub fn num_voluntary_exits(&self) -> usize {
+        self.voluntary_exits.read().len()
+    }
 }
 
 /// Filter up to a maximum number of operations out of an iterator.

eth2/operation_pool/src/persistence.rs

@@ -1,6 +1,7 @@
 use crate::attestation_id::AttestationId;
 use crate::OperationPool;
 use parking_lot::RwLock;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use types::*;
 
@@ -8,7 +9,8 @@ use types::*;
 ///
 /// Operations are stored in arbitrary order, so it's not a good idea to compare instances
 /// of this type (or its encoded form) for equality. Convert back to an `OperationPool` first.
-#[derive(Clone, Encode, Decode)]
+#[derive(Clone, PartialEq, Debug, Encode, Decode, Serialize, Deserialize)]
+#[serde(bound = "T: EthSpec")]
 pub struct PersistedOperationPool<T: EthSpec> {
     /// Mapping from attestation ID to attestation mappings.
     // We could save space by not storing the attestation ID, but it might

eth2/state_processing/Cargo.toml

@@ -31,6 +31,8 @@ tree_hash = "0.1.0"
 tree_hash_derive = "0.2"
 types = { path = "../types" }
 rayon = "1.2.0"
+eth2_hashing = { path = "../utils/eth2_hashing" }
+int_to_bytes = { path = "../utils/int_to_bytes" }
 
 [features]
 fake_crypto = ["bls/fake_crypto"]

eth2/state_processing/src/common/deposit_data_tree.rs

@@ -0,0 +1,52 @@
+use eth2_hashing::hash;
+use int_to_bytes::int_to_bytes32;
+use merkle_proof::{MerkleTree, MerkleTreeError};
+use types::Hash256;
+
+/// Emulates the eth1 deposit contract merkle tree.
+pub struct DepositDataTree {
+    tree: MerkleTree,
+    mix_in_length: usize,
+    depth: usize,
+}
+
+impl DepositDataTree {
+    /// Create a new Merkle tree from a list of leaves (`DepositData::tree_hash_root`) and a fixed depth.
+    pub fn create(leaves: &[Hash256], mix_in_length: usize, depth: usize) -> Self {
+        Self {
+            tree: MerkleTree::create(leaves, depth),
+            mix_in_length,
+            depth,
+        }
+    }
+
+    /// Returns 32 bytes representing the "mix in length" for the merkle root of this tree.
+    fn length_bytes(&self) -> Vec<u8> {
+        int_to_bytes32(self.mix_in_length as u64)
+    }
+
+    /// Retrieve the root hash of this Merkle tree with the length mixed in.
+    pub fn root(&self) -> Hash256 {
+        let mut preimage = [0; 64];
+        preimage[0..32].copy_from_slice(&self.tree.hash()[..]);
+        preimage[32..64].copy_from_slice(&self.length_bytes());
+        Hash256::from_slice(&hash(&preimage))
+    }
+
+    /// Return the leaf at `index` and a Merkle proof of its inclusion.
+    ///
+    /// The Merkle proof is in "bottom-up" order, starting with a leaf node
+    /// and moving up the tree. Its length will be exactly equal to `depth + 1`.
+    pub fn generate_proof(&self, index: usize) -> (Hash256, Vec<Hash256>) {
+        let (root, mut proof) = self.tree.generate_proof(index, self.depth);
+        proof.push(Hash256::from_slice(&self.length_bytes()));
+        (root, proof)
+    }
+
+    /// Add a deposit to the merkle tree.
+    pub fn push_leaf(&mut self, leaf: Hash256) -> Result<(), MerkleTreeError> {
+        self.tree.push_leaf(leaf, self.depth)?;
+        self.mix_in_length += 1;
+        Ok(())
+    }
+}

eth2/state_processing/src/common/mod.rs

@@ -1,9 +1,11 @@
+mod deposit_data_tree;
 mod get_attesting_indices;
 mod get_base_reward;
 mod get_indexed_attestation;
 mod initiate_validator_exit;
 mod slash_validator;
 
+pub use deposit_data_tree::DepositDataTree;
 pub use get_attesting_indices::get_attesting_indices;
 pub use get_base_reward::get_base_reward;
 pub use get_indexed_attestation::get_indexed_attestation;

eth2/state_processing/src/genesis.rs

@@ -1,6 +1,7 @@
 use super::per_block_processing::{errors::BlockProcessingError, process_deposit};
+use crate::common::DepositDataTree;
 use tree_hash::TreeHash;
-use types::typenum::U4294967296;
+use types::DEPOSIT_TREE_DEPTH;
 use types::*;
 
 /// Initialize a `BeaconState` from genesis data.
@@ -26,14 +27,13 @@ pub fn initialize_beacon_state_from_eth1<T: EthSpec>(
     // Seed RANDAO with Eth1 entropy
     state.fill_randao_mixes_with(eth1_block_hash);
 
-    // Process deposits
-    let leaves: Vec<_> = deposits
-        .iter()
-        .map(|deposit| deposit.data.clone())
-        .collect();
-    for (index, deposit) in deposits.into_iter().enumerate() {
-        let deposit_data_list = VariableList::<_, U4294967296>::from(leaves[..=index].to_vec());
-        state.eth1_data.deposit_root = Hash256::from_slice(&deposit_data_list.tree_hash_root());
+    let mut deposit_tree = DepositDataTree::create(&[], 0, DEPOSIT_TREE_DEPTH);
+
+    for deposit in deposits.iter() {
+        deposit_tree
+            .push_leaf(Hash256::from_slice(&deposit.data.tree_hash_root()))
+            .map_err(BlockProcessingError::MerkleTreeError)?;
+        state.eth1_data.deposit_root = deposit_tree.root();
         process_deposit(&mut state, &deposit, spec, true)?;
     }
 

eth2/state_processing/src/per_block_processing/errors.rs

@@ -1,4 +1,5 @@
 use super::signature_sets::Error as SignatureSetError;
+use merkle_proof::MerkleTreeError;
 use types::*;
 
 /// The error returned from the `per_block_processing` function. Indicates that a block is either
@@ -46,6 +47,7 @@ pub enum BlockProcessingError {
     BeaconStateError(BeaconStateError),
     SignatureSetError(SignatureSetError),
     SszTypesError(ssz_types::Error),
+    MerkleTreeError(MerkleTreeError),
 }
 
 impl From<BeaconStateError> for BlockProcessingError {

eth2/types/src/beacon_state.rs

@@ -97,6 +97,21 @@ impl BeaconTreeHashCache {
     pub fn is_initialized(&self) -> bool {
         self.initialized
     }
+
+    /// Returns the approximate size of the cache in bytes.
+    ///
+    /// The size is approximate because we ignore some stack-allocated `u64` and `Vec` pointers.
+    /// We focus instead on the lists of hashes, which should massively outweigh the items that we
+    /// ignore.
+    pub fn approx_mem_size(&self) -> usize {
+        self.block_roots.approx_mem_size()
+            + self.state_roots.approx_mem_size()
+            + self.historical_roots.approx_mem_size()
+            + self.validators.approx_mem_size()
+            + self.balances.approx_mem_size()
+            + self.randao_mixes.approx_mem_size()
+            + self.slashings.approx_mem_size()
+    }
 }
 
 /// The state of the `BeaconChain` at some slot.
@@ -996,6 +1011,12 @@ impl<T: EthSpec> BeaconState<T> {
             tree_hash_cache: BeaconTreeHashCache::default(),
         }
     }
+
+    pub fn clone_with_only_committee_caches(&self) -> Self {
+        let mut state = self.clone_without_caches();
+        state.committee_caches = self.committee_caches.clone();
+        state
+    }
 }
 
 impl From<RelativeEpochError> for Error {

eth2/utils/cached_tree_hash/src/cache.rs

@@ -127,6 +127,15 @@ impl TreeHashCache {
     pub fn leaves(&mut self) -> &mut Vec<Hash256> {
         &mut self.layers[self.depth]
     }
+
+    /// Returns the approximate size of the cache in bytes.
+    ///
+    /// The size is approximate because we ignore some stack-allocated `u64` and `Vec` pointers.
+    /// We focus instead on the lists of hashes, which should massively outweigh the items that we
+    /// ignore.
+    pub fn approx_mem_size(&self) -> usize {
+        self.layers.iter().map(|layer| layer.len() * 32).sum()
+    }
 }
 
 /// Compute the dirty indices for one layer up.

eth2/utils/cached_tree_hash/src/multi_cache.rs

@@ -16,6 +16,22 @@ pub struct MultiTreeHashCache {
     value_caches: Vec<TreeHashCache>,
 }
 
+impl MultiTreeHashCache {
+    /// Returns the approximate size of the cache in bytes.
+    ///
+    /// The size is approximate because we ignore some stack-allocated `u64` and `Vec` pointers.
+    /// We focus instead on the lists of hashes, which should massively outweigh the items that we
+    /// ignore.
+    pub fn approx_mem_size(&self) -> usize {
+        self.list_cache.approx_mem_size()
+            + self
+                .value_caches
+                .iter()
+                .map(TreeHashCache::approx_mem_size)
+                .sum::<usize>()
+    }
+}
+
 impl<T, N> CachedTreeHash<MultiTreeHashCache> for VariableList<T, N>
 where
     T: CachedTreeHash<TreeHashCache>,

eth2/utils/merkle_proof/src/lib.rs

@@ -28,7 +28,7 @@ pub enum MerkleTree {
     Zero(usize),
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum MerkleTreeError {
     // Trying to push in a leaf
     LeafReached,

eth2/utils/remote_beacon_node/Cargo.toml

@@ -18,3 +18,4 @@ eth2_ssz = { path = "../../../eth2/utils/ssz" }
 serde_json = "^1.0"
 eth2_config = { path = "../../../eth2/utils/eth2_config" }
 proto_array_fork_choice = { path = "../../../eth2/proto_array_fork_choice" }
+operation_pool = { path = "../../../eth2/operation_pool" }

eth2/utils/remote_beacon_node/src/lib.rs

@@ -5,7 +5,6 @@
 
 use eth2_config::Eth2Config;
 use futures::{future, Future, IntoFuture};
-use proto_array_fork_choice::core::ProtoArray;
 use reqwest::{
     r#async::{Client, ClientBuilder, Response},
     StatusCode,
@@ -20,6 +19,8 @@ use types::{
 };
 use url::Url;
 
+pub use operation_pool::PersistedOperationPool;
+pub use proto_array_fork_choice::core::ProtoArray;
 pub use rest_api::{
     CanonicalHeadResponse, Committee, HeadBeaconBlock, ValidatorDutiesRequest, ValidatorDuty,
     ValidatorRequest, ValidatorResponse,
@@ -560,6 +561,16 @@ impl<E: EthSpec> Advanced<E> {
             .into_future()
             .and_then(move |url| client.json_get(url, vec![]))
     }
+
+    /// Gets the core `PersistedOperationPool` struct from the node.
+    pub fn get_operation_pool(
+        &self,
+    ) -> impl Future<Item = PersistedOperationPool<E>, Error = Error> {
+        let client = self.0.clone();
+        self.url("operation_pool")
+            .into_future()
+            .and_then(move |url| client.json_get(url, vec![]))
+    }
 }
 
 #[derive(Deserialize)]