Optimizations, disable val client sync check & additional lcli tools (#834)

* Start adding interop genesis state to lcli

* Use more efficient method to generate genesis state

* Remove duplicate int_to_bytes32

* Add lcli command to change state genesis time

* Add option to allow VC to start with unsynced BN

* Set VC to do parallel key loading

* Don't default to dummy eth1 backend

* Add endpoint to dump operation pool

* Add metrics for op pool

* Remove state clone for slot notifier

* Add mem size approximation for tree hash cache

* Avoid cloning tree hash when getting head

* Fix failing API tests

* Address Michael's comments

* Add HashMap::from_par_iter

lcli/src/change_genesis_time.rs

@@ -0,0 +1,40 @@
+use clap::ArgMatches;
+use ssz::{Decode, Encode};
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::PathBuf;
+use types::{BeaconState, EthSpec};
+
+pub fn run<T: EthSpec>(matches: &ArgMatches) -> Result<(), String> {
+    let path = matches
+        .value_of("ssz-state")
+        .ok_or_else(|| "ssz-state not specified")?
+        .parse::<PathBuf>()
+        .map_err(|e| format!("Unable to parse ssz-state: {}", e))?;
+
+    let genesis_time = matches
+        .value_of("genesis-time")
+        .ok_or_else(|| "genesis-time not specified")?
+        .parse::<u64>()
+        .map_err(|e| format!("Unable to parse genesis-time: {}", e))?;
+
+    let mut state: BeaconState<T> = {
+        let mut file = File::open(&path).map_err(|e| format!("Unable to open file: {}", e))?;
+
+        let mut ssz = vec![];
+
+        file.read_to_end(&mut ssz)
+            .map_err(|e| format!("Unable to read file: {}", e))?;
+
+        BeaconState::from_ssz_bytes(&ssz).map_err(|e| format!("Unable to decode SSZ: {:?}", e))?
+    };
+
+    state.genesis_time = genesis_time;
+
+    let mut file = File::create(path).map_err(|e| format!("Unable to create file: {}", e))?;
+
+    file.write_all(&state.as_ssz_bytes())
+        .map_err(|e| format!("Unable to write to file: {}", e))?;
+
+    Ok(())
+}

lcli/src/interop_genesis.rs

@@ -0,0 +1,65 @@
+use clap::ArgMatches;
+use environment::Environment;
+use eth2_testnet_config::Eth2TestnetConfig;
+use genesis::interop_genesis_state;
+use std::path::PathBuf;
+use std::time::{SystemTime, UNIX_EPOCH};
+use types::{test_utils::generate_deterministic_keypairs, Epoch, EthSpec, Fork};
+
+pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
+    let validator_count = matches
+        .value_of("validator-count")
+        .ok_or_else(|| "validator-count not specified")?
+        .parse::<usize>()
+        .map_err(|e| format!("Unable to parse validator-count: {}", e))?;
+
+    let genesis_time = if let Some(genesis_time) = matches.value_of("genesis-time") {
+        genesis_time
+            .parse::<u64>()
+            .map_err(|e| format!("Unable to parse genesis-time: {}", e))?
+    } else {
+        SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map_err(|e| format!("Unable to get time: {:?}", e))?
+            .as_secs()
+    };
+
+    let testnet_dir = matches
+        .value_of("testnet-dir")
+        .ok_or_else(|| ())
+        .and_then(|dir| dir.parse::<PathBuf>().map_err(|_| ()))
+        .unwrap_or_else(|_| {
+            dirs::home_dir()
+                .map(|home| home.join(".lighthouse").join("testnet"))
+                .expect("should locate home directory")
+        });
+
+    let mut eth2_testnet_config: Eth2TestnetConfig<T> =
+        Eth2TestnetConfig::load(testnet_dir.clone())?;
+
+    let mut spec = eth2_testnet_config
+        .yaml_config
+        .as_ref()
+        .ok_or_else(|| "The testnet directory must contain a spec config".to_string())?
+        .apply_to_chain_spec::<T>(&env.core_context().eth2_config.spec)
+        .ok_or_else(|| {
+            format!(
+                "The loaded config is not compatible with the {} spec",
+                &env.core_context().eth2_config.spec_constants
+            )
+        })?;
+
+    spec.genesis_fork = Fork {
+        previous_version: [0, 0, 0, 0],
+        current_version: [1, 3, 3, 7],
+        epoch: Epoch::new(0),
+    };
+
+    let keypairs = generate_deterministic_keypairs(validator_count);
+    let genesis_state = interop_genesis_state(&keypairs, genesis_time, &spec)?;
+
+    eth2_testnet_config.genesis_state = Some(genesis_state);
+    eth2_testnet_config.force_write_to_file(testnet_dir)?;
+
+    Ok(())
+}

lcli/src/main.rs

@@ -1,8 +1,10 @@
 #[macro_use]
 extern crate log;
 
+mod change_genesis_time;
 mod deploy_deposit_contract;
 mod eth1_genesis;
+mod interop_genesis;
 mod parse_hex;
 mod refund_deposit_contract;
 mod transition_blocks;
@@ -226,6 +228,59 @@ fn main() {
                         .help("The URL to the eth1 JSON-RPC http API."),
                 )
         )
+        .subcommand(
+            SubCommand::with_name("interop-genesis")
+                .about(
+                    "Produces an interop-compatible genesis state using deterministic keypairs",
+                )
+                .arg(
+                    Arg::with_name("testnet-dir")
+                        .short("d")
+                        .long("testnet-dir")
+                        .value_name("PATH")
+                        .takes_value(true)
+                        .help("The testnet dir. Defaults to ~/.lighthouse/testnet"),
+                )
+                .arg(
+                    Arg::with_name("validator-count")
+                        .long("validator-count")
+                        .index(1)
+                        .value_name("INTEGER")
+                        .takes_value(true)
+                        .default_value("1024")
+                        .help("The number of validators in the genesis state."),
+                )
+                .arg(
+                    Arg::with_name("genesis-time")
+                        .long("genesis-time")
+                        .short("t")
+                        .value_name("UNIX_EPOCH")
+                        .takes_value(true)
+                        .help("The value for state.genesis_time. Defaults to now."),
+                )
+        )
+        .subcommand(
+            SubCommand::with_name("change-genesis-time")
+                .about(
+                    "Loads a file with an SSZ-encoded BeaconState and modifies the genesis time.",
+                )
+                .arg(
+                    Arg::with_name("ssz-state")
+                        .index(1)
+                        .value_name("PATH")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The path to the SSZ file"),
+                )
+                .arg(
+                    Arg::with_name("genesis-time")
+                        .index(2)
+                        .value_name("UNIX_EPOCH")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The value for state.genesis_time."),
+                )
+        )
         .get_matches();
 
     macro_rules! run_with_spec {
@@ -306,6 +361,10 @@ fn run<T: EthSpec>(env_builder: EnvironmentBuilder<T>, matches: &ArgMatches) {
         }
         ("eth1-genesis", Some(matches)) => eth1_genesis::run::<T>(env, matches)
             .unwrap_or_else(|e| error!("Failed to run eth1-genesis command: {}", e)),
+        ("interop-genesis", Some(matches)) => interop_genesis::run::<T>(env, matches)
+            .unwrap_or_else(|e| error!("Failed to run interop-genesis command: {}", e)),
+        ("change-genesis-time", Some(matches)) => change_genesis_time::run::<T>(matches)
+            .unwrap_or_else(|e| error!("Failed to run change-genesis-time command: {}", e)),
         (other, _) => error!("Unknown subcommand {}. See --help.", other),
     }
 }