Post merge local testnets (#3807)

## Issue Addressed

N/A

## Proposed Changes

Modifies the local testnet scripts to start a network with genesis validators embedded into the genesis state. This allows us to start a local testnet without the need for deploying a deposit contract or depositing validators pre-genesis.

This also enables us to start a local test network at any fork we want without going through fork transitions. Also adds scripts to start multiple geth clients and peer them with each other and peer the geth clients with beacon nodes to start a post merge local testnet.

## Additional info

Adds a new lcli command `mnemonics-validators` that generates validator directories derived from a given mnemonic. 
Adds a new `derived-genesis-state` option to the `lcli new-testnet` command to generate a genesis state populated with validators derived from a mnemonic.

lcli/src/main.rs

@@ -10,6 +10,7 @@ mod generate_bootnode_enr;
 mod indexed_attestations;
 mod insecure_validators;
 mod interop_genesis;
+mod mnemonic_validators;
 mod new_testnet;
 mod parse_ssz;
 mod replace_state_pubkeys;
@@ -449,6 +450,22 @@ fn main() {
                             "If present, a interop-style genesis.ssz file will be generated.",
                         ),
                 )
+                .arg(
+                    Arg::with_name("derived-genesis-state")
+                        .long("derived-genesis-state")
+                        .takes_value(false)
+                        .help(
+                            "If present, a genesis.ssz file will be generated with keys generated from a given mnemonic.",
+                        ),
+                )
+                .arg(
+                    Arg::with_name("mnemonic-phrase")
+                        .long("mnemonic-phrase")
+                        .value_name("MNEMONIC_PHRASE")
+                        .takes_value(true)
+                        .requires("derived-genesis-state")
+                        .help("The mnemonic with which we generate the validator keys for a derived genesis state"),
+                )
                 .arg(
                     Arg::with_name("min-genesis-time")
                         .long("min-genesis-time")
@@ -568,14 +585,32 @@ fn main() {
                         ),
                 )
                 .arg(
-                    Arg::with_name("merge-fork-epoch")
-                        .long("merge-fork-epoch")
+                    Arg::with_name("bellatrix-fork-epoch")
+                        .long("bellatrix-fork-epoch")
                         .value_name("EPOCH")
                         .takes_value(true)
                         .help(
                             "The epoch at which to enable the Merge hard fork",
                         ),
                 )
+                .arg(
+                    Arg::with_name("capella-fork-epoch")
+                        .long("capella-fork-epoch")
+                        .value_name("EPOCH")
+                        .takes_value(true)
+                        .help(
+                            "The epoch at which to enable the Capella hard fork",
+                        ),
+                )
+                .arg(
+                    Arg::with_name("ttd")
+                        .long("ttd")
+                        .value_name("TTD")
+                        .takes_value(true)
+                        .help(
+                            "The terminal total difficulty",
+                        ),
+                )
                 .arg(
                     Arg::with_name("eth1-block-hash")
                         .long("eth1-block-hash")
@@ -695,6 +730,7 @@ fn main() {
                         .long("count")
                         .value_name("COUNT")
                         .takes_value(true)
+                        .required(true)
                         .help("Produces validators in the range of 0..count."),
                 )
                 .arg(
@@ -702,6 +738,7 @@ fn main() {
                         .long("base-dir")
                         .value_name("BASE_DIR")
                         .takes_value(true)
+                        .required(true)
                         .help("The base directory where validator keypairs and secrets are stored"),
                 )
                 .arg(
@@ -712,6 +749,43 @@ fn main() {
                         .help("The number of nodes to divide the validator keys to"),
                 )
         )
+        .subcommand(
+            SubCommand::with_name("mnemonic-validators")
+                .about("Produces validator directories by deriving the keys from \
+                        a mnemonic. For testing purposes only, DO NOT USE IN \
+                        PRODUCTION!")
+                .arg(
+                    Arg::with_name("count")
+                        .long("count")
+                        .value_name("COUNT")
+                        .takes_value(true)
+                        .required(true)
+                        .help("Produces validators in the range of 0..count."),
+                )
+                .arg(
+                    Arg::with_name("base-dir")
+                        .long("base-dir")
+                        .value_name("BASE_DIR")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The base directory where validator keypairs and secrets are stored"),
+                )
+                .arg(
+                    Arg::with_name("node-count")
+                        .long("node-count")
+                        .value_name("NODE_COUNT")
+                        .takes_value(true)
+                        .help("The number of nodes to divide the validator keys to"),
+                )
+                .arg(
+                    Arg::with_name("mnemonic-phrase")
+                        .long("mnemonic-phrase")
+                        .value_name("MNEMONIC_PHRASE")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The mnemonic with which we generate the validator keys"),
+                )
+        )
         .subcommand(
             SubCommand::with_name("indexed-attestations")
                 .about("Convert attestations to indexed form, using the committees from a state.")
@@ -853,6 +927,8 @@ fn run<T: EthSpec>(
             .map_err(|e| format!("Failed to run generate-bootnode-enr command: {}", e)),
         ("insecure-validators", Some(matches)) => insecure_validators::run(matches)
             .map_err(|e| format!("Failed to run insecure-validators command: {}", e)),
+        ("mnemonic-validators", Some(matches)) => mnemonic_validators::run(matches)
+            .map_err(|e| format!("Failed to run mnemonic-validators command: {}", e)),
         ("indexed-attestations", Some(matches)) => indexed_attestations::run::<T>(matches)
             .map_err(|e| format!("Failed to run indexed-attestations command: {}", e)),
         ("block-root", Some(matches)) => block_root::run::<T>(env, matches)

lcli/src/mnemonic_validators.rs

@@ -0,0 +1,104 @@
+use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
+use account_utils::random_password;
+use clap::ArgMatches;
+use eth2_wallet::bip39::Seed;
+use eth2_wallet::bip39::{Language, Mnemonic};
+use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType};
+use rayon::prelude::*;
+use std::fs;
+use std::path::PathBuf;
+use validator_dir::Builder as ValidatorBuilder;
+
+/// Generates validator directories with keys derived from the given mnemonic.
+pub fn generate_validator_dirs(
+    indices: &[usize],
+    mnemonic_phrase: &str,
+    validators_dir: PathBuf,
+    secrets_dir: PathBuf,
+) -> Result<(), String> {
+    if !validators_dir.exists() {
+        fs::create_dir_all(&validators_dir)
+            .map_err(|e| format!("Unable to create validators dir: {:?}", e))?;
+    }
+
+    if !secrets_dir.exists() {
+        fs::create_dir_all(&secrets_dir)
+            .map_err(|e| format!("Unable to create secrets dir: {:?}", e))?;
+    }
+    let mnemonic = Mnemonic::from_phrase(mnemonic_phrase, Language::English).map_err(|e| {
+        format!(
+            "Unable to derive mnemonic from string {:?}: {:?}",
+            mnemonic_phrase, e
+        )
+    })?;
+
+    let seed = Seed::new(&mnemonic, "");
+
+    let _: Vec<_> = indices
+        .par_iter()
+        .map(|index| {
+            let voting_password = random_password();
+
+            let derive = |key_type: KeyType, password: &[u8]| -> Result<Keystore, String> {
+                let (secret, path) = recover_validator_secret_from_mnemonic(
+                    seed.as_bytes(),
+                    *index as u32,
+                    key_type,
+                )
+                .map_err(|e| format!("Unable to recover validator keys: {:?}", e))?;
+
+                let keypair = keypair_from_secret(secret.as_bytes())
+                    .map_err(|e| format!("Unable build keystore: {:?}", e))?;
+
+                KeystoreBuilder::new(&keypair, password, format!("{}", path))
+                    .map_err(|e| format!("Unable build keystore: {:?}", e))?
+                    .build()
+                    .map_err(|e| format!("Unable build keystore: {:?}", e))
+            };
+
+            let voting_keystore = derive(KeyType::Voting, voting_password.as_bytes()).unwrap();
+
+            println!("Validator {}", index + 1);
+
+            ValidatorBuilder::new(validators_dir.clone())
+                .password_dir(secrets_dir.clone())
+                .store_withdrawal_keystore(false)
+                .voting_keystore(voting_keystore, voting_password.as_bytes())
+                .build()
+                .map_err(|e| format!("Unable to build validator: {:?}", e))
+                .unwrap()
+        })
+        .collect();
+
+    Ok(())
+}
+
+pub fn run(matches: &ArgMatches) -> Result<(), String> {
+    let validator_count: usize = clap_utils::parse_required(matches, "count")?;
+    let base_dir: PathBuf = clap_utils::parse_required(matches, "base-dir")?;
+    let node_count: Option<usize> = clap_utils::parse_optional(matches, "node-count")?;
+    let mnemonic_phrase: String = clap_utils::parse_required(matches, "mnemonic-phrase")?;
+    if let Some(node_count) = node_count {
+        let validators_per_node = validator_count / node_count;
+        let validator_range = (0..validator_count).collect::<Vec<_>>();
+        let indices_range = validator_range
+            .chunks(validators_per_node)
+            .collect::<Vec<_>>();
+
+        for (i, indices) in indices_range.iter().enumerate() {
+            let validators_dir = base_dir.join(format!("node_{}", i + 1)).join("validators");
+            let secrets_dir = base_dir.join(format!("node_{}", i + 1)).join("secrets");
+            generate_validator_dirs(indices, &mnemonic_phrase, validators_dir, secrets_dir)?;
+        }
+    } else {
+        let validators_dir = base_dir.join("validators");
+        let secrets_dir = base_dir.join("secrets");
+        generate_validator_dirs(
+            (0..validator_count).collect::<Vec<_>>().as_slice(),
+            &mnemonic_phrase,
+            validators_dir,
+            secrets_dir,
+        )?;
+    }
+    Ok(())
+}

lcli/src/new_testnet.rs

@@ -1,16 +1,26 @@
+use account_utils::eth2_keystore::keypair_from_secret;
 use clap::ArgMatches;
 use clap_utils::{parse_optional, parse_required, parse_ssz_optional};
 use eth2_network_config::Eth2NetworkConfig;
-use genesis::interop_genesis_state;
+use eth2_wallet::bip39::Seed;
+use eth2_wallet::bip39::{Language, Mnemonic};
+use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType};
+use ethereum_hashing::hash;
 use ssz::Decode;
 use ssz::Encode;
+use state_processing::process_activations;
+use state_processing::upgrade::{upgrade_to_altair, upgrade_to_bellatrix};
 use std::fs::File;
 use std::io::Read;
 use std::path::PathBuf;
+use std::str::FromStr;
 use std::time::{SystemTime, UNIX_EPOCH};
+use types::ExecutionBlockHash;
 use types::{
-    test_utils::generate_deterministic_keypairs, Address, Config, Epoch, EthSpec,
-    ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderMerge, ForkName,
+    test_utils::generate_deterministic_keypairs, Address, BeaconState, ChainSpec, Config, Epoch,
+    Eth1Data, EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella,
+    ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRefMut, ForkName, Hash256, Keypair,
+    PublicKey, Validator,
 };
 
 pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Result<(), String> {
@@ -67,63 +77,69 @@ pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul
         spec.altair_fork_epoch = Some(fork_epoch);
     }
 
-    if let Some(fork_epoch) = parse_optional(matches, "merge-fork-epoch")? {
+    if let Some(fork_epoch) = parse_optional(matches, "bellatrix-fork-epoch")? {
         spec.bellatrix_fork_epoch = Some(fork_epoch);
     }
 
-    let genesis_state_bytes = if matches.is_present("interop-genesis-state") {
-        let execution_payload_header: Option<ExecutionPayloadHeader<T>> =
-            parse_optional(matches, "execution-payload-header")?
-                .map(|filename: String| {
-                    let mut bytes = vec![];
-                    let mut file = File::open(filename.as_str())
-                        .map_err(|e| format!("Unable to open {}: {}", filename, e))?;
-                    file.read_to_end(&mut bytes)
-                        .map_err(|e| format!("Unable to read {}: {}", filename, e))?;
-                    let fork_name = spec.fork_name_at_epoch(Epoch::new(0));
-                    match fork_name {
-                        ForkName::Base | ForkName::Altair => Err(ssz::DecodeError::BytesInvalid(
-                            "genesis fork must be post-merge".to_string(),
-                        )),
-                        ForkName::Merge => {
-                            ExecutionPayloadHeaderMerge::<T>::from_ssz_bytes(bytes.as_slice())
-                                .map(ExecutionPayloadHeader::Merge)
-                        }
-                        ForkName::Capella => {
-                            ExecutionPayloadHeaderCapella::<T>::from_ssz_bytes(bytes.as_slice())
-                                .map(ExecutionPayloadHeader::Capella)
-                        }
+    if let Some(fork_epoch) = parse_optional(matches, "capella-fork-epoch")? {
+        spec.capella_fork_epoch = Some(fork_epoch);
+    }
+
+    if let Some(ttd) = parse_optional(matches, "ttd")? {
+        spec.terminal_total_difficulty = ttd;
+    }
+
+    let validator_count = parse_required(matches, "validator-count")?;
+    let execution_payload_header: Option<ExecutionPayloadHeader<T>> =
+        parse_optional(matches, "execution-payload-header")?
+            .map(|filename: String| {
+                let mut bytes = vec![];
+                let mut file = File::open(filename.as_str())
+                    .map_err(|e| format!("Unable to open {}: {}", filename, e))?;
+                file.read_to_end(&mut bytes)
+                    .map_err(|e| format!("Unable to read {}: {}", filename, e))?;
+                let fork_name = spec.fork_name_at_epoch(Epoch::new(0));
+                match fork_name {
+                    ForkName::Base | ForkName::Altair => Err(ssz::DecodeError::BytesInvalid(
+                        "genesis fork must be post-merge".to_string(),
+                    )),
+                    ForkName::Merge => {
+                        ExecutionPayloadHeaderMerge::<T>::from_ssz_bytes(bytes.as_slice())
+                            .map(ExecutionPayloadHeader::Merge)
                     }
-                    .map_err(|e| format!("SSZ decode failed: {:?}", e))
-                })
-                .transpose()?;
+                    ForkName::Capella => {
+                        ExecutionPayloadHeaderCapella::<T>::from_ssz_bytes(bytes.as_slice())
+                            .map(ExecutionPayloadHeader::Capella)
+                    }
+                }
+                .map_err(|e| format!("SSZ decode failed: {:?}", e))
+            })
+            .transpose()?;
 
-        let (eth1_block_hash, genesis_time) = if let Some(payload) =
-            execution_payload_header.as_ref()
-        {
-            let eth1_block_hash =
-                parse_optional(matches, "eth1-block-hash")?.unwrap_or_else(|| payload.block_hash());
-            let genesis_time =
-                parse_optional(matches, "genesis-time")?.unwrap_or_else(|| payload.timestamp());
-            (eth1_block_hash, genesis_time)
-        } else {
-            let eth1_block_hash = parse_required(matches, "eth1-block-hash").map_err(|_| {
-                "One of `--execution-payload-header` or `--eth1-block-hash` must be set".to_string()
-            })?;
-            let genesis_time = parse_optional(matches, "genesis-time")?.unwrap_or(
-                SystemTime::now()
-                    .duration_since(UNIX_EPOCH)
-                    .map_err(|e| format!("Unable to get time: {:?}", e))?
-                    .as_secs(),
-            );
-            (eth1_block_hash, genesis_time)
-        };
-
-        let validator_count = parse_required(matches, "validator-count")?;
+    let (eth1_block_hash, genesis_time) = if let Some(payload) = execution_payload_header.as_ref() {
+        let eth1_block_hash =
+            parse_optional(matches, "eth1-block-hash")?.unwrap_or_else(|| payload.block_hash());
+        let genesis_time =
+            parse_optional(matches, "genesis-time")?.unwrap_or_else(|| payload.timestamp());
+        (eth1_block_hash, genesis_time)
+    } else {
+        let eth1_block_hash = parse_required(matches, "eth1-block-hash").map_err(|_| {
+            "One of `--execution-payload-header` or `--eth1-block-hash` must be set".to_string()
+        })?;
+        let genesis_time = parse_optional(matches, "genesis-time")?.unwrap_or(
+            SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .map_err(|e| format!("Unable to get time: {:?}", e))?
+                .as_secs(),
+        );
+        (eth1_block_hash, genesis_time)
+    };
 
+    let genesis_state_bytes = if matches.is_present("interop-genesis-state") {
         let keypairs = generate_deterministic_keypairs(validator_count);
+        let keypairs: Vec<_> = keypairs.into_iter().map(|kp| (kp.clone(), kp)).collect();
 
-        let genesis_state = interop_genesis_state::<T>(
+        let genesis_state = initialize_state_with_validators::<T>(
             &keypairs,
             genesis_time,
             eth1_block_hash.into_root(),
@@ -131,6 +147,41 @@ pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul
             &spec,
         )?;
 
+        Some(genesis_state.as_ssz_bytes())
+    } else if matches.is_present("derived-genesis-state") {
+        let mnemonic_phrase: String = clap_utils::parse_required(matches, "mnemonic-phrase")?;
+        let mnemonic = Mnemonic::from_phrase(&mnemonic_phrase, Language::English).map_err(|e| {
+            format!(
+                "Unable to derive mnemonic from string {:?}: {:?}",
+                mnemonic_phrase, e
+            )
+        })?;
+        let seed = Seed::new(&mnemonic, "");
+        let keypairs = (0..validator_count as u32)
+            .map(|index| {
+                let (secret, _) =
+                    recover_validator_secret_from_mnemonic(seed.as_bytes(), index, KeyType::Voting)
+                        .unwrap();
+
+                let voting_keypair = keypair_from_secret(secret.as_bytes()).unwrap();
+
+                let (secret, _) = recover_validator_secret_from_mnemonic(
+                    seed.as_bytes(),
+                    index,
+                    KeyType::Withdrawal,
+                )
+                .unwrap();
+                let withdrawal_keypair = keypair_from_secret(secret.as_bytes()).unwrap();
+                (voting_keypair, withdrawal_keypair)
+            })
+            .collect::<Vec<_>>();
+        let genesis_state = initialize_state_with_validators::<T>(
+            &keypairs,
+            genesis_time,
+            eth1_block_hash.into_root(),
+            execution_payload_header,
+            &spec,
+        )?;
         Some(genesis_state.as_ssz_bytes())
     } else {
         None
@@ -145,3 +196,117 @@ pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul
 
     testnet.write_to_file(testnet_dir_path, overwrite_files)
 }
+
+/// Returns a `BeaconState` with the given validator keypairs embedded into the
+/// genesis state. This allows us to start testnets without having to deposit validators
+/// manually.
+///
+/// The optional `execution_payload_header` allows us to start a network from the bellatrix
+/// fork without the need to transition to altair and bellatrix.
+///
+/// We need to ensure that `eth1_block_hash` is equal to the genesis block hash that is
+/// generated from the execution side `genesis.json`.
+fn initialize_state_with_validators<T: EthSpec>(
+    keypairs: &[(Keypair, Keypair)], // Voting and Withdrawal keypairs
+    genesis_time: u64,
+    eth1_block_hash: Hash256,
+    execution_payload_header: Option<ExecutionPayloadHeader<T>>,
+    spec: &ChainSpec,
+) -> Result<BeaconState<T>, String> {
+    // If no header is provided, then start from a Bellatrix state by default
+    let default_header: ExecutionPayloadHeader<T> =
+        ExecutionPayloadHeader::Merge(ExecutionPayloadHeaderMerge {
+            block_hash: ExecutionBlockHash::from_root(eth1_block_hash),
+            parent_hash: ExecutionBlockHash::zero(),
+            ..ExecutionPayloadHeaderMerge::default()
+        });
+    let execution_payload_header = execution_payload_header.unwrap_or(default_header);
+    // Empty eth1 data
+    let eth1_data = Eth1Data {
+        block_hash: eth1_block_hash,
+        deposit_count: 0,
+        deposit_root: Hash256::from_str(
+            "0xd70a234731285c6804c2a4f56711ddb8c82c99740f207854891028af34e27e5e",
+        )
+        .unwrap(), // empty deposit tree root
+    };
+    let mut state = BeaconState::new(genesis_time, eth1_data, spec);
+
+    // Seed RANDAO with Eth1 entropy
+    state.fill_randao_mixes_with(eth1_block_hash);
+
+    for keypair in keypairs.iter() {
+        let withdrawal_credentials = |pubkey: &PublicKey| {
+            let mut credentials = hash(&pubkey.as_ssz_bytes());
+            credentials[0] = spec.bls_withdrawal_prefix_byte;
+            Hash256::from_slice(&credentials)
+        };
+        let amount = spec.max_effective_balance;
+        // Create a new validator.
+        let validator = Validator {
+            pubkey: keypair.0.pk.clone().into(),
+            withdrawal_credentials: withdrawal_credentials(&keypair.1.pk),
+            activation_eligibility_epoch: spec.far_future_epoch,
+            activation_epoch: spec.far_future_epoch,
+            exit_epoch: spec.far_future_epoch,
+            withdrawable_epoch: spec.far_future_epoch,
+            effective_balance: std::cmp::min(
+                amount - amount % (spec.effective_balance_increment),
+                spec.max_effective_balance,
+            ),
+            slashed: false,
+        };
+        state.validators_mut().push(validator).unwrap();
+        state.balances_mut().push(amount).unwrap();
+    }
+
+    process_activations(&mut state, spec).unwrap();
+
+    if spec
+        .altair_fork_epoch
+        .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch())
+    {
+        upgrade_to_altair(&mut state, spec).unwrap();
+
+        state.fork_mut().previous_version = spec.altair_fork_version;
+    }
+
+    // Similarly, perform an upgrade to the merge if configured from genesis.
+    if spec
+        .bellatrix_fork_epoch
+        .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch())
+    {
+        upgrade_to_bellatrix(&mut state, spec).unwrap();
+
+        // Remove intermediate Altair fork from `state.fork`.
+        state.fork_mut().previous_version = spec.bellatrix_fork_version;
+
+        // Override latest execution payload header.
+        // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/merge/beacon-chain.md#testing
+
+        // Currently, we only support starting from a bellatrix state
+        match state
+            .latest_execution_payload_header_mut()
+            .map_err(|e| format!("Failed to get execution payload header: {:?}", e))?
+        {
+            ExecutionPayloadHeaderRefMut::Merge(header_mut) => {
+                if let ExecutionPayloadHeader::Merge(eph) = execution_payload_header {
+                    *header_mut = eph;
+                } else {
+                    return Err("Execution payload header must be a bellatrix header".to_string());
+                }
+            }
+            ExecutionPayloadHeaderRefMut::Capella(_) => {
+                return Err("Cannot start genesis from a capella state".to_string())
+            }
+        }
+    }
+
+    // Now that we have our validators, initialize the caches (including the committees)
+    state.build_all_caches(spec).unwrap();
+
+    // Set genesis validators root for domain separation and chain versioning
+    *state.genesis_validators_root_mut() = state.update_validators_tree_hash_cache().unwrap();
+
+    Ok(state)
+}