Merge remote-tracking branch 'origin/unstable' into tree-states

lcli/src/block_root.rs

@@ -31,14 +31,19 @@ use clap::ArgMatches;
 use clap_utils::{parse_optional, parse_required};
 use environment::Environment;
 use eth2::{types::BlockId, BeaconNodeHttpClient, SensitiveUrl, Timeouts};
+use eth2_network_config::Eth2NetworkConfig;
 use std::path::PathBuf;
 use std::time::{Duration, Instant};
 use types::{EthSpec, FullPayload, SignedBeaconBlock};
 
 const HTTP_TIMEOUT: Duration = Duration::from_secs(5);
 
-pub fn run<T: EthSpec>(env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
-    let spec = &T::default_spec();
+pub fn run<T: EthSpec>(
+    env: Environment<T>,
+    network_config: Eth2NetworkConfig,
+    matches: &ArgMatches,
+) -> Result<(), String> {
+    let spec = &network_config.chain_spec::<T>()?;
     let executor = env.core_context().executor;
 
     /*

lcli/src/generate_bootnode_enr.rs

@@ -1,6 +1,7 @@
 use clap::ArgMatches;
 use lighthouse_network::{
-    discovery::{build_enr, CombinedKey, CombinedKeyExt, Keypair, ENR_FILENAME},
+    discovery::{build_enr, CombinedKey, CombinedKeyExt, ENR_FILENAME},
+    libp2p::identity::secp256k1,
     NetworkConfig, NETWORK_KEY_FILENAME,
 };
 use std::fs::File;
@@ -29,8 +30,8 @@ pub fn run<T: EthSpec>(matches: &ArgMatches) -> Result<(), String> {
     config.enr_udp4_port = Some(udp_port);
     config.enr_tcp6_port = Some(tcp_port);
 
-    let local_keypair = Keypair::generate_secp256k1();
-    let enr_key = CombinedKey::from_libp2p(&local_keypair)?;
+    let secp256k1_keypair = secp256k1::Keypair::generate();
+    let enr_key = CombinedKey::from_secp256k1(&secp256k1_keypair);
     let enr_fork_id = EnrForkId {
         fork_digest: ChainSpec::compute_fork_digest(genesis_fork_version, Hash256::zero()),
         next_fork_version: genesis_fork_version,
@@ -47,13 +48,10 @@ pub fn run<T: EthSpec>(matches: &ArgMatches) -> Result<(), String> {
         .write_all(enr.to_base64().as_bytes())
         .map_err(|e| format!("Unable to write ENR to {}: {:?}", ENR_FILENAME, e))?;
 
-    let secret_bytes = match local_keypair {
-        Keypair::Secp256k1(key) => key.secret().to_bytes(),
-        _ => return Err("Key is not a secp256k1 key".into()),
-    };
-
     let mut key_file = File::create(output_dir.join(NETWORK_KEY_FILENAME))
         .map_err(|e| format!("Unable to create {}: {:?}", NETWORK_KEY_FILENAME, e))?;
+
+    let secret_bytes = secp256k1_keypair.secret().to_bytes();
     key_file
         .write_all(&secret_bytes)
         .map_err(|e| format!("Unable to write key to {}: {:?}", NETWORK_KEY_FILENAME, e))?;

lcli/src/main.rs

@@ -16,11 +16,13 @@ mod parse_ssz;
 mod replace_state_pubkeys;
 mod skip_slots;
 mod state_diff;
+mod state_root;
 mod transition_blocks;
 
 use clap::{App, Arg, ArgMatches, SubCommand};
-use clap_utils::parse_path_with_default_in_home_dir;
+use clap_utils::parse_optional;
 use environment::{EnvironmentBuilder, LoggerConfig};
+use eth2_network_config::Eth2NetworkConfig;
 use parse_ssz::run_parse_ssz;
 use std::path::PathBuf;
 use std::process;
@@ -39,7 +41,6 @@ fn main() {
                 .long("spec")
                 .value_name("STRING")
                 .takes_value(true)
-                .required(true)
                 .possible_values(&["minimal", "mainnet", "gnosis"])
                 .default_value("mainnet")
                 .global(true),
@@ -51,7 +52,16 @@ fn main() {
                 .value_name("PATH")
                 .takes_value(true)
                 .global(true)
-                .help("The testnet dir. Defaults to ~/.lighthouse/testnet"),
+                .help("The testnet dir."),
+        )
+        .arg(
+            Arg::with_name("network")
+                .long("network")
+                .value_name("NAME")
+                .takes_value(true)
+                .global(true)
+                .help("The network to use. Defaults to mainnet.")
+                .conflicts_with("testnet-dir")
         )
         .subcommand(
             SubCommand::with_name("skip-slots")
@@ -127,7 +137,7 @@ fn main() {
                         .takes_value(true)
                         .conflicts_with("beacon-url")
                         .requires("block-path")
-                        .help("Path to load a BeaconState from file as SSZ."),
+                        .help("Path to load a BeaconState from as SSZ."),
                 )
                 .arg(
                     Arg::with_name("block-path")
@@ -136,7 +146,7 @@ fn main() {
                         .takes_value(true)
                         .conflicts_with("beacon-url")
                         .requires("pre-state-path")
-                        .help("Path to load a SignedBeaconBlock from file as SSZ."),
+                        .help("Path to load a SignedBeaconBlock from as SSZ."),
                 )
                 .arg(
                     Arg::with_name("post-state-output-path")
@@ -362,7 +372,6 @@ fn main() {
                         .index(2)
                         .value_name("BIP39_MNENMONIC")
                         .takes_value(true)
-                        .required(true)
                         .default_value(
                             "replace nephew blur decorate waste convince soup column \
                             orient excite play baby",
@@ -383,7 +392,6 @@ fn main() {
                         .help("The block hash used when generating an execution payload. This \
                             value is used for `execution_payload_header.block_hash` as well as \
                             `execution_payload_header.random`")
-                        .required(true)
                         .default_value(
                             "0x0000000000000000000000000000000000000000000000000000000000000000",
                         ),
@@ -401,7 +409,6 @@ fn main() {
                         .value_name("INTEGER")
                         .takes_value(true)
                         .help("The base fee per gas field in the execution payload generated.")
-                        .required(true)
                         .default_value("1000000000"),
                 )
                 .arg(
@@ -410,7 +417,6 @@ fn main() {
                         .value_name("INTEGER")
                         .takes_value(true)
                         .help("The gas limit field in the execution payload generated.")
-                        .required(true)
                         .default_value("30000000"),
                 )
                 .arg(
@@ -809,14 +815,14 @@ fn main() {
         )
         .subcommand(
             SubCommand::with_name("block-root")
-                .about("Computes the block root of some block")
+                .about("Computes the block root of some block.")
                 .arg(
                     Arg::with_name("block-path")
                         .long("block-path")
                         .value_name("PATH")
                         .takes_value(true)
                         .conflicts_with("beacon-url")
-                        .help("Path to load a SignedBeaconBlock from file as SSZ."),
+                        .help("Path to load a SignedBeaconBlock from as SSZ."),
                 )
                 .arg(
                     Arg::with_name("beacon-url")
@@ -858,6 +864,41 @@ fn main() {
                         .help("Path to second SSZ state"),
                 )
         )
+        .subcommand(
+            SubCommand::with_name("state-root")
+                .about("Computes the state root of some state.")
+                .arg(
+                    Arg::with_name("state-path")
+                        .long("state-path")
+                        .value_name("PATH")
+                        .takes_value(true)
+                        .conflicts_with("beacon-url")
+                        .help("Path to load a BeaconState from as SSZ."),
+                )
+                .arg(
+                    Arg::with_name("beacon-url")
+                        .long("beacon-url")
+                        .value_name("URL")
+                        .takes_value(true)
+                        .help("URL to a beacon-API provider."),
+                )
+                .arg(
+                    Arg::with_name("state-id")
+                        .long("state-id")
+                        .value_name("BLOCK_ID")
+                        .takes_value(true)
+                        .requires("beacon-url")
+                        .help("Identifier for a state as per beacon-API standards (slot, root, etc.)"),
+                )
+                .arg(
+                    Arg::with_name("runs")
+                        .long("runs")
+                        .value_name("INTEGER")
+                        .takes_value(true)
+                        .default_value("1")
+                        .help("Number of repeat runs, useful for benchmarking."),
+                )
+        )
         .get_matches();
 
     let result = matches
@@ -904,17 +945,44 @@ fn run<T: EthSpec>(
         .build()
         .map_err(|e| format!("should build env: {:?}", e))?;
 
-    let testnet_dir = parse_path_with_default_in_home_dir(
-        matches,
-        "testnet-dir",
-        PathBuf::from(directory::DEFAULT_ROOT_DIR).join("testnet"),
-    )?;
+    // Determine testnet-dir path or network name depending on CLI flags.
+    let (testnet_dir, network_name) =
+        if let Some(testnet_dir) = parse_optional::<PathBuf>(matches, "testnet-dir")? {
+            (Some(testnet_dir), None)
+        } else {
+            let network_name =
+                parse_optional(matches, "network")?.unwrap_or_else(|| "mainnet".to_string());
+            (None, Some(network_name))
+        };
+
+    // Lazily load either the testnet dir or the network config, as required.
+    // Some subcommands like new-testnet need the testnet dir but not the network config.
+    let get_testnet_dir = || testnet_dir.clone().ok_or("testnet-dir is required");
+    let get_network_config = || {
+        if let Some(testnet_dir) = &testnet_dir {
+            Eth2NetworkConfig::load(testnet_dir.clone()).map_err(|e| {
+                format!(
+                    "Unable to open testnet dir at {}: {}",
+                    testnet_dir.display(),
+                    e
+                )
+            })
+        } else {
+            let network_name = network_name.ok_or("no network name or testnet-dir provided")?;
+            Eth2NetworkConfig::constant(&network_name)?.ok_or("invalid network name".into())
+        }
+    };
 
     match matches.subcommand() {
-        ("transition-blocks", Some(matches)) => transition_blocks::run::<T>(env, matches)
-            .map_err(|e| format!("Failed to transition blocks: {}", e)),
+        ("transition-blocks", Some(matches)) => {
+            let network_config = get_network_config()?;
+            transition_blocks::run::<T>(env, network_config, matches)
+                .map_err(|e| format!("Failed to transition blocks: {}", e))
+        }
         ("skip-slots", Some(matches)) => {
-            skip_slots::run::<T>(env, matches).map_err(|e| format!("Failed to skip slots: {}", e))
+            let network_config = get_network_config()?;
+            skip_slots::run::<T>(env, network_config, matches)
+                .map_err(|e| format!("Failed to skip slots: {}", e))
         }
         ("pretty-ssz", Some(matches)) => {
             run_parse_ssz::<T>(matches).map_err(|e| format!("Failed to pretty print hex: {}", e))
@@ -923,22 +991,33 @@ fn run<T: EthSpec>(
             deploy_deposit_contract::run::<T>(env, matches)
                 .map_err(|e| format!("Failed to run deploy-deposit-contract command: {}", e))
         }
-        ("eth1-genesis", Some(matches)) => eth1_genesis::run::<T>(env, testnet_dir, matches)
-            .map_err(|e| format!("Failed to run eth1-genesis command: {}", e)),
-        ("interop-genesis", Some(matches)) => interop_genesis::run::<T>(testnet_dir, matches)
-            .map_err(|e| format!("Failed to run interop-genesis command: {}", e)),
+        ("eth1-genesis", Some(matches)) => {
+            let testnet_dir = get_testnet_dir()?;
+            eth1_genesis::run::<T>(env, testnet_dir, matches)
+                .map_err(|e| format!("Failed to run eth1-genesis command: {}", e))
+        }
+        ("interop-genesis", Some(matches)) => {
+            let testnet_dir = get_testnet_dir()?;
+            interop_genesis::run::<T>(testnet_dir, matches)
+                .map_err(|e| format!("Failed to run interop-genesis command: {}", e))
+        }
         ("change-genesis-time", Some(matches)) => {
+            let testnet_dir = get_testnet_dir()?;
             change_genesis_time::run::<T>(testnet_dir, matches)
                 .map_err(|e| format!("Failed to run change-genesis-time command: {}", e))
         }
         ("create-payload-header", Some(matches)) => create_payload_header::run::<T>(matches)
             .map_err(|e| format!("Failed to run create-payload-header command: {}", e)),
         ("replace-state-pubkeys", Some(matches)) => {
+            let testnet_dir = get_testnet_dir()?;
             replace_state_pubkeys::run::<T>(testnet_dir, matches)
                 .map_err(|e| format!("Failed to run replace-state-pubkeys command: {}", e))
         }
-        ("new-testnet", Some(matches)) => new_testnet::run::<T>(testnet_dir, matches)
-            .map_err(|e| format!("Failed to run new_testnet command: {}", e)),
+        ("new-testnet", Some(matches)) => {
+            let testnet_dir = get_testnet_dir()?;
+            new_testnet::run::<T>(testnet_dir, matches)
+                .map_err(|e| format!("Failed to run new_testnet command: {}", e))
+        }
         ("check-deposit-data", Some(matches)) => check_deposit_data::run(matches)
             .map_err(|e| format!("Failed to run check-deposit-data command: {}", e)),
         ("generate-bootnode-enr", Some(matches)) => generate_bootnode_enr::run::<T>(matches)
@@ -949,10 +1028,18 @@ fn run<T: EthSpec>(
             .map_err(|e| format!("Failed to run mnemonic-validators command: {}", e)),
         ("indexed-attestations", Some(matches)) => indexed_attestations::run::<T>(matches)
             .map_err(|e| format!("Failed to run indexed-attestations command: {}", e)),
-        ("block-root", Some(matches)) => block_root::run::<T>(env, matches)
-            .map_err(|e| format!("Failed to run block-root command: {}", e)),
         ("state-diff", Some(matches)) => state_diff::run::<T>(env, matches)
             .map_err(|e| format!("Failed to run state-diff command: {}", e)),
+        ("block-root", Some(matches)) => {
+            let network_config = get_network_config()?;
+            block_root::run::<T>(env, network_config, matches)
+                .map_err(|e| format!("Failed to run block-root command: {}", e))
+        }
+        ("state-root", Some(matches)) => {
+            let network_config = get_network_config()?;
+            state_root::run::<T>(env, network_config, matches)
+                .map_err(|e| format!("Failed to run state-root command: {}", e))
+        }
         (other, _) => Err(format!("Unknown subcommand {}. See --help.", other)),
     }
 }

lcli/src/skip_slots.rs

@@ -49,6 +49,7 @@ use clap::ArgMatches;
 use clap_utils::{parse_optional, parse_required};
 use environment::Environment;
 use eth2::{types::StateId, BeaconNodeHttpClient, SensitiveUrl, Timeouts};
+use eth2_network_config::Eth2NetworkConfig;
 use ssz::Encode;
 use state_processing::state_advance::{complete_state_advance, partial_state_advance};
 use state_processing::AllCaches;
@@ -60,8 +61,12 @@ use types::{BeaconState, EthSpec, Hash256};
 
 const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
 
-pub fn run<T: EthSpec>(env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
-    let spec = &T::default_spec();
+pub fn run<T: EthSpec>(
+    env: Environment<T>,
+    network_config: Eth2NetworkConfig,
+    matches: &ArgMatches,
+) -> Result<(), String> {
+    let spec = &network_config.chain_spec::<T>()?;
     let executor = env.core_context().executor;
 
     let output_path: Option<PathBuf> = parse_optional(matches, "output-path")?;

lcli/src/state_root.rs

@@ -0,0 +1,76 @@
+use crate::transition_blocks::load_from_ssz_with;
+use clap::ArgMatches;
+use clap_utils::{parse_optional, parse_required};
+use environment::Environment;
+use eth2::{types::StateId, BeaconNodeHttpClient, SensitiveUrl, Timeouts};
+use eth2_network_config::Eth2NetworkConfig;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+use types::{BeaconState, EthSpec};
+
+const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
+
+pub fn run<T: EthSpec>(
+    env: Environment<T>,
+    network_config: Eth2NetworkConfig,
+    matches: &ArgMatches,
+) -> Result<(), String> {
+    let executor = env.core_context().executor;
+
+    let spec = &network_config.chain_spec::<T>()?;
+
+    let state_path: Option<PathBuf> = parse_optional(matches, "state-path")?;
+    let beacon_url: Option<SensitiveUrl> = parse_optional(matches, "beacon-url")?;
+    let runs: usize = parse_required(matches, "runs")?;
+
+    info!(
+        "Using {} network ({} spec)",
+        spec.config_name.as_deref().unwrap_or("unknown"),
+        T::spec_name()
+    );
+    info!("Doing {} runs", runs);
+
+    let state = match (state_path, beacon_url) {
+        (Some(state_path), None) => {
+            info!("State path: {:?}", state_path);
+            load_from_ssz_with(&state_path, spec, BeaconState::from_ssz_bytes)?
+        }
+        (None, Some(beacon_url)) => {
+            let state_id: StateId = parse_required(matches, "state-id")?;
+            let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT));
+            executor
+                .handle()
+                .ok_or("shutdown in progress")?
+                .block_on(async move {
+                    client
+                        .get_debug_beacon_states::<T>(state_id)
+                        .await
+                        .map_err(|e| format!("Failed to download state: {:?}", e))
+                })
+                .map_err(|e| format!("Failed to complete task: {:?}", e))?
+                .ok_or_else(|| format!("Unable to locate state at {:?}", state_id))?
+                .data
+        }
+        _ => return Err("must supply either --state-path or --beacon-url".into()),
+    };
+
+    /*
+     * Perform the core "runs".
+     */
+    let mut state_root = None;
+    for i in 0..runs {
+        let mut state = state.clone();
+        let timer = Instant::now();
+        state_root = Some(
+            state
+                .update_tree_hash_cache()
+                .map_err(|e| format!("error computing state root: {e:?}"))?,
+        );
+        info!("Run {}: {:?}", i, timer.elapsed());
+    }
+
+    if let Some(state_root) = state_root {
+        info!("State root is {:?}", state_root);
+    }
+    Ok(())
+}

lcli/src/transition_blocks.rs

@@ -71,6 +71,7 @@ use eth2::{
     types::{BlockId, StateId},
     BeaconNodeHttpClient, SensitiveUrl, Timeouts,
 };
+use eth2_network_config::Eth2NetworkConfig;
 use ssz::Encode;
 use state_processing::{
     block_signature_verifier::BlockSignatureVerifier, per_block_processing, per_slot_processing,
@@ -94,8 +95,12 @@ struct Config {
     exclude_post_block_thc: bool,
 }
 
-pub fn run<T: EthSpec>(env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
-    let spec = &T::default_spec();
+pub fn run<T: EthSpec>(
+    env: Environment<T>,
+    network_config: Eth2NetworkConfig,
+    matches: &ArgMatches,
+) -> Result<(), String> {
+    let spec = &network_config.chain_spec::<T>()?;
     let executor = env.core_context().executor;
 
     /*