Optimizations, disable val client sync check & additional lcli tools (#834)

* Start adding interop genesis state to lcli

* Use more efficient method to generate genesis state

* Remove duplicate int_to_bytes32

* Add lcli command to change state genesis time

* Add option to allow VC to start with unsynced BN

* Set VC to do parallel key loading

* Don't default to dummy eth1 backend

* Add endpoint to dump operation pool

* Add metrics for op pool

* Remove state clone for slot notifier

* Add mem size approximation for tree hash cache

* Avoid cloning tree hash when getting head

* Fix failing API tests

* Address Michael's comments

* Add HashMap::from_par_iter

eth2/utils/cached_tree_hash/src/cache.rs

@@ -127,6 +127,15 @@ impl TreeHashCache {
     pub fn leaves(&mut self) -> &mut Vec<Hash256> {
         &mut self.layers[self.depth]
     }
+
+    /// Returns the approximate size of the cache in bytes.
+    ///
+    /// The size is approximate because we ignore some stack-allocated `u64` and `Vec` pointers.
+    /// We focus instead on the lists of hashes, which should massively outweigh the items that we
+    /// ignore.
+    pub fn approx_mem_size(&self) -> usize {
+        self.layers.iter().map(|layer| layer.len() * 32).sum()
+    }
 }
 
 /// Compute the dirty indices for one layer up.

eth2/utils/cached_tree_hash/src/multi_cache.rs

@@ -16,6 +16,22 @@ pub struct MultiTreeHashCache {
     value_caches: Vec<TreeHashCache>,
 }
 
+impl MultiTreeHashCache {
+    /// Returns the approximate size of the cache in bytes.
+    ///
+    /// The size is approximate because we ignore some stack-allocated `u64` and `Vec` pointers.
+    /// We focus instead on the lists of hashes, which should massively outweigh the items that we
+    /// ignore.
+    pub fn approx_mem_size(&self) -> usize {
+        self.list_cache.approx_mem_size()
+            + self
+                .value_caches
+                .iter()
+                .map(TreeHashCache::approx_mem_size)
+                .sum::<usize>()
+    }
+}
+
 impl<T, N> CachedTreeHash<MultiTreeHashCache> for VariableList<T, N>
 where
     T: CachedTreeHash<TreeHashCache>,

eth2/utils/merkle_proof/src/lib.rs

@@ -28,7 +28,7 @@ pub enum MerkleTree {
     Zero(usize),
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum MerkleTreeError {
     // Trying to push in a leaf
     LeafReached,

eth2/utils/remote_beacon_node/Cargo.toml

@@ -18,3 +18,4 @@ eth2_ssz = { path = "../../../eth2/utils/ssz" }
 serde_json = "^1.0"
 eth2_config = { path = "../../../eth2/utils/eth2_config" }
 proto_array_fork_choice = { path = "../../../eth2/proto_array_fork_choice" }
+operation_pool = { path = "../../../eth2/operation_pool" }

eth2/utils/remote_beacon_node/src/lib.rs

@@ -5,7 +5,6 @@
 
 use eth2_config::Eth2Config;
 use futures::{future, Future, IntoFuture};
-use proto_array_fork_choice::core::ProtoArray;
 use reqwest::{
     r#async::{Client, ClientBuilder, Response},
     StatusCode,
@@ -20,6 +19,8 @@ use types::{
 };
 use url::Url;
 
+pub use operation_pool::PersistedOperationPool;
+pub use proto_array_fork_choice::core::ProtoArray;
 pub use rest_api::{
     CanonicalHeadResponse, Committee, HeadBeaconBlock, ValidatorDutiesRequest, ValidatorDuty,
     ValidatorRequest, ValidatorResponse,
@@ -560,6 +561,16 @@ impl<E: EthSpec> Advanced<E> {
             .into_future()
             .and_then(move |url| client.json_get(url, vec![]))
     }
+
+    /// Gets the core `PersistedOperationPool` struct from the node.
+    pub fn get_operation_pool(
+        &self,
+    ) -> impl Future<Item = PersistedOperationPool<E>, Error = Error> {
+        let client = self.0.clone();
+        self.url("operation_pool")
+            .into_future()
+            .and_then(move |url| client.json_get(url, vec![]))
+    }
 }
 
 #[derive(Deserialize)]