Directory Restructure (#1163)

* Move tests -> testing

* Directory restructure

* Update Cargo.toml during restructure

* Update Makefile during restructure

* Fix arbitrary path

testing/simulator/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "simulator"
+version = "0.2.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+node_test_rig = { path = "../node_test_rig" }
+types = { path = "../../consensus/types" }
+validator_client = { path = "../../validator_client" }
+parking_lot = "0.10.2"
+futures = "0.3.5"
+tokio = "0.2.20"
+eth1_test_rig = { path = "../eth1_test_rig" }
+env_logger = "0.7.1"
+clap = "2.33.0"
+rayon = "1.3.0"

testing/simulator/src/checks.rs

@@ -0,0 +1,128 @@
+use crate::local_network::LocalNetwork;
+use std::time::Duration;
+use types::{Epoch, EthSpec, Slot, Unsigned};
+
+/// Checks that all of the validators have on-boarded by the start of the second eth1 voting
+/// period.
+pub async fn verify_initial_validator_count<E: EthSpec>(
+    network: LocalNetwork<E>,
+    slot_duration: Duration,
+    initial_validator_count: usize,
+) -> Result<(), String> {
+    slot_delay(Slot::new(1), slot_duration).await;
+    verify_validator_count(network, initial_validator_count).await?;
+    Ok(())
+}
+
+/// Checks that all of the validators have on-boarded by the start of the second eth1 voting
+/// period.
+pub async fn verify_validator_onboarding<E: EthSpec>(
+    network: LocalNetwork<E>,
+    slot_duration: Duration,
+    expected_validator_count: usize,
+) -> Result<(), String> {
+    slot_delay(
+        Slot::new(E::SlotsPerEth1VotingPeriod::to_u64()),
+        slot_duration,
+    )
+    .await;
+    verify_validator_count(network, expected_validator_count).await?;
+    Ok(())
+}
+
+/// Checks that the chain has made the first possible finalization.
+///
+/// Intended to be run as soon as chain starts.
+pub async fn verify_first_finalization<E: EthSpec>(
+    network: LocalNetwork<E>,
+    slot_duration: Duration,
+) -> Result<(), String> {
+    epoch_delay(Epoch::new(4), slot_duration, E::slots_per_epoch()).await;
+    verify_all_finalized_at(network, Epoch::new(2)).await?;
+    Ok(())
+}
+
+/// Delays for `epochs`, plus half a slot extra.
+pub async fn epoch_delay(epochs: Epoch, slot_duration: Duration, slots_per_epoch: u64) {
+    let duration = slot_duration * (epochs.as_u64() * slots_per_epoch) as u32 + slot_duration / 2;
+    tokio::time::delay_for(duration).await
+}
+
+/// Delays for `slots`, plus half a slot extra.
+async fn slot_delay(slots: Slot, slot_duration: Duration) {
+    let duration = slot_duration * slots.as_u64() as u32 + slot_duration / 2;
+    tokio::time::delay_for(duration).await;
+}
+
+/// Verifies that all beacon nodes in the given network have a head state that has a finalized
+/// epoch of `epoch`.
+pub async fn verify_all_finalized_at<E: EthSpec>(
+    network: LocalNetwork<E>,
+    epoch: Epoch,
+) -> Result<(), String> {
+    let epochs = {
+        let mut epochs = Vec::new();
+        for remote_node in network.remote_nodes()? {
+            epochs.push(
+                remote_node
+                    .http
+                    .beacon()
+                    .get_head()
+                    .await
+                    .map(|head| head.finalized_slot.epoch(E::slots_per_epoch()))
+                    .map_err(|e| format!("Get head via http failed: {:?}", e))?,
+            );
+        }
+        epochs
+    };
+
+    if epochs.iter().any(|node_epoch| *node_epoch != epoch) {
+        Err(format!(
+            "Nodes are not finalized at epoch {}. Finalized epochs: {:?}",
+            epoch, epochs
+        ))
+    } else {
+        Ok(())
+    }
+}
+
+/// Verifies that all beacon nodes in the given `network` have a head state that contains
+/// `expected_count` validators.
+async fn verify_validator_count<E: EthSpec>(
+    network: LocalNetwork<E>,
+    expected_count: usize,
+) -> Result<(), String> {
+    let validator_counts = {
+        let mut validator_counts = Vec::new();
+        for remote_node in network.remote_nodes()? {
+            let beacon = remote_node.http.beacon();
+
+            let head = beacon
+                .get_head()
+                .await
+                .map_err(|e| format!("Get head via http failed: {:?}", e))?;
+
+            let vc = beacon
+                .get_state_by_root(head.state_root)
+                .await
+                .map(|(state, _root)| state)
+                .map_err(|e| format!("Get state root via http failed: {:?}", e))?
+                .validators
+                .len();
+            validator_counts.push(vc);
+        }
+        validator_counts
+    };
+
+    if validator_counts
+        .iter()
+        .any(|count| *count != expected_count)
+    {
+        Err(format!(
+            "Nodes do not all have {} validators in their state. Validator counts: {:?}",
+            expected_count, validator_counts
+        ))
+    } else {
+        Ok(())
+    }
+}

testing/simulator/src/cli.rs

@@ -0,0 +1,108 @@
+use clap::{App, Arg, SubCommand};
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new("simulator")
+        .version(crate_version!())
+        .author("Sigma Prime <contact@sigmaprime.io>")
+        .about("Options for interacting with simulator")
+        .subcommand(
+            SubCommand::with_name("eth1-sim")
+            .about(
+                "Lighthouse Beacon Chain Simulator creates `n` beacon node and validator clients, \
+                    each with `v` validators. A deposit contract is deployed at the start of the \
+                    simulation using a local `ganache-cli` instance (you must have `ganache-cli` \
+                    installed and avaliable on your path). All beacon nodes independently listen \
+                    for genesis from the deposit contract, then start operating. \
+
+                    As the simulation runs, there are checks made to ensure that all components \
+                    are running correctly. If any of these checks fail, the simulation will \
+                    exit immediately.",
+                    )
+                    .arg(Arg::with_name("nodes")
+                        .short("n")
+                        .long("nodes")
+                        .takes_value(true)
+                        .default_value("4")
+                        .help("Number of beacon nodes"))
+                    .arg(Arg::with_name("validators_per_node")
+                        .short("v")
+                        .long("validators_per_node")
+                        .takes_value(true)
+                        .default_value("20")
+                        .help("Number of validators"))
+                    .arg(Arg::with_name("speed_up_factor")
+                        .short("s")
+                        .long("speed_up_factor")
+                        .takes_value(true)
+                        .default_value("4")
+                        .help("Speed up factor"))
+                    .arg(Arg::with_name("end_after_checks")
+                        .short("e")
+                        .long("end_after_checks")
+                        .takes_value(false)
+                        .help("End after checks (default true)"))
+        )
+        .subcommand(
+            SubCommand::with_name("no-eth1-sim")
+            .about("Runs a simulator that bypasses the eth1 chain. Useful for faster testing of
+                components that don't rely upon eth1")
+                    .arg(Arg::with_name("nodes")
+                        .short("n")
+                        .long("nodes")
+                        .takes_value(true)
+                        .default_value("4")
+                        .help("Number of beacon nodes"))
+                    .arg(Arg::with_name("validators_per_node")
+                        .short("v")
+                        .long("validators_per_node")
+                        .takes_value(true)
+                        .default_value("20")
+                        .help("Number of validators"))
+                    .arg(Arg::with_name("speed_up_factor")
+                        .short("s")
+                        .long("speed_up_factor")
+                        .takes_value(true)
+                        .default_value("4")
+                        .help("Speed up factor"))
+                    .arg(Arg::with_name("end_after_checks")
+                        .short("e")
+                        .long("end_after_checks")
+                        .takes_value(false)
+                        .help("End after checks (default true)"))
+        )
+        .subcommand(
+            SubCommand::with_name("syncing-sim")
+                .about("Run the syncing simulation")
+                .arg(
+                    Arg::with_name("speedup")
+                        .short("s")
+                        .long("speedup")
+                        .takes_value(true)
+                        .default_value("15")
+                        .help("Speed up factor for eth1 blocks and slot production"),
+                )
+                .arg(
+                    Arg::with_name("initial_delay")
+                        .short("i")
+                        .long("initial_delay")
+                        .takes_value(true)
+                        .default_value("5")
+                        .help("Epoch delay for new beacon node to start syncing"),
+                )
+                .arg(
+                    Arg::with_name("sync_timeout")
+                        .long("sync_timeout")
+                        .takes_value(true)
+                        .default_value("10")
+                        .help("Number of epochs after which newly added beacon nodes must be synced"),
+                )
+                .arg(
+                    Arg::with_name("strategy")
+                        .long("strategy")
+                        .takes_value(true)
+                        .default_value("all")
+                        .possible_values(&["one-node", "two-nodes", "mixed", "all"])
+                        .help("Sync verification strategy to run."),
+                ),
+        )
+}

testing/simulator/src/eth1_sim.rs

@@ -0,0 +1,209 @@
+use crate::{checks, LocalNetwork, E};
+use clap::ArgMatches;
+use eth1_test_rig::GanacheEth1Instance;
+use futures::prelude::*;
+use node_test_rig::{
+    environment::EnvironmentBuilder, testing_client_config, ClientGenesis, ValidatorConfig,
+    ValidatorFiles,
+};
+use rayon::prelude::*;
+use std::net::{IpAddr, Ipv4Addr};
+use std::time::Duration;
+use tokio::time::{delay_until, Instant};
+
+pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
+    let node_count = value_t!(matches, "nodes", usize).expect("missing nodes default");
+    let validators_per_node = value_t!(matches, "validators_per_node", usize)
+        .expect("missing validators_per_node default");
+    let speed_up_factor =
+        value_t!(matches, "speed_up_factor", u64).expect("missing speed_up_factor default");
+    let mut end_after_checks = true;
+    if matches.is_present("end_after_checks") {
+        end_after_checks = false;
+    }
+
+    println!("Beacon Chain Simulator:");
+    println!(" nodes:{}", node_count);
+    println!(" validators_per_node:{}", validators_per_node);
+    println!(" end_after_checks:{}", end_after_checks);
+
+    // Generate the directories and keystores required for the validator clients.
+    let validator_files = (0..node_count)
+        .into_par_iter()
+        .map(|i| {
+            println!(
+                "Generating keystores for validator {} of {}",
+                i + 1,
+                node_count
+            );
+
+            let indices =
+                (i * validators_per_node..(i + 1) * validators_per_node).collect::<Vec<_>>();
+            ValidatorFiles::with_keystores(&indices).unwrap()
+        })
+        .collect::<Vec<_>>();
+
+    let expected_genesis_instant = Instant::now() + Duration::from_secs(60);
+
+    let log_level = "debug";
+    let log_format = None;
+
+    let mut env = EnvironmentBuilder::minimal()
+        .async_logger(log_level, log_format)?
+        .multi_threaded_tokio_runtime()?
+        .build()?;
+
+    let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor);
+
+    let spec = &mut env.eth2_config.spec;
+
+    spec.milliseconds_per_slot /= speed_up_factor;
+    spec.eth1_follow_distance = 16;
+    spec.min_genesis_delay = eth1_block_time.as_secs() * spec.eth1_follow_distance * 2;
+    spec.min_genesis_time = 0;
+    spec.min_genesis_active_validator_count = 64;
+    spec.seconds_per_eth1_block = 1;
+
+    let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+    let initial_validator_count = spec.min_genesis_active_validator_count as usize;
+    let total_validator_count = validators_per_node * node_count;
+    let deposit_amount = env.eth2_config.spec.max_effective_balance;
+
+    let context = env.core_context();
+
+    let main_future = async {
+        /*
+         * Deploy the deposit contract, spawn tasks to keep creating new blocks and deposit
+         * validators.
+         */
+        let ganache_eth1_instance = GanacheEth1Instance::new().await?;
+        let deposit_contract = ganache_eth1_instance.deposit_contract;
+        let ganache = ganache_eth1_instance.ganache;
+        let eth1_endpoint = ganache.endpoint();
+        let deposit_contract_address = deposit_contract.address();
+
+        // Start a timer that produces eth1 blocks on an interval.
+        tokio::spawn(async move {
+            let mut interval = tokio::time::interval(eth1_block_time);
+            while let Some(_) = interval.next().await {
+                let _ = ganache.evm_mine().await;
+            }
+        });
+
+        // Submit deposits to the deposit contract.
+        tokio::spawn(async move {
+            for i in 0..total_validator_count {
+                println!("Submitting deposit for validator {}...", i);
+                let _ = deposit_contract
+                    .deposit_deterministic_async::<E>(i, deposit_amount)
+                    .await;
+            }
+        });
+
+        let mut beacon_config = testing_client_config();
+
+        beacon_config.genesis = ClientGenesis::DepositContract;
+        beacon_config.eth1.endpoint = eth1_endpoint;
+        beacon_config.eth1.deposit_contract_address = deposit_contract_address;
+        beacon_config.eth1.deposit_contract_deploy_block = 0;
+        beacon_config.eth1.lowest_cached_block_number = 0;
+        beacon_config.eth1.follow_distance = 1;
+        beacon_config.dummy_eth1_backend = false;
+        beacon_config.sync_eth1_chain = true;
+
+        beacon_config.network.enr_address = Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
+
+        /*
+         * Create a new `LocalNetwork` with one beacon node.
+         */
+        let network = LocalNetwork::new(context, beacon_config.clone()).await?;
+        /*
+         * One by one, add beacon nodes to the network.
+         */
+
+        for _ in 0..node_count - 1 {
+            network.add_beacon_node(beacon_config.clone()).await?;
+        }
+
+        /*
+         * Create a future that will add validator clients to the network. Each validator client is
+         * attached to a single corresponding beacon node.
+         */
+        let add_validators_fut = async {
+            for (i, files) in validator_files.into_iter().enumerate() {
+                network
+                    .add_validator_client(
+                        ValidatorConfig {
+                            auto_register: true,
+                            ..ValidatorConfig::default()
+                        },
+                        i,
+                        files,
+                    )
+                    .await?;
+            }
+
+            Ok::<(), String>(())
+        };
+
+        /*
+         * Start the processes that will run checks on the network as it runs.
+         */
+
+        let checks_fut = async {
+            delay_until(expected_genesis_instant).await;
+
+            let (finalization, validator_count, onboarding) = futures::join!(
+                // Check that the chain finalizes at the first given opportunity.
+                checks::verify_first_finalization(network.clone(), slot_duration),
+                // Check that the chain starts with the expected validator count.
+                checks::verify_initial_validator_count(
+                    network.clone(),
+                    slot_duration,
+                    initial_validator_count,
+                ),
+                // Check that validators greater than `spec.min_genesis_active_validator_count` are
+                // onboarded at the first possible opportunity.
+                checks::verify_validator_onboarding(
+                    network.clone(),
+                    slot_duration,
+                    total_validator_count,
+                )
+            );
+
+            finalization?;
+            validator_count?;
+            onboarding?;
+
+            Ok::<(), String>(())
+        };
+
+        let (add_validators, checks) = futures::join!(add_validators_fut, checks_fut);
+
+        add_validators?;
+        checks?;
+
+        // The `final_future` either completes immediately or never completes, depending on the value
+        // of `end_after_checks`.
+
+        if !end_after_checks {
+            future::pending::<()>().await;
+        }
+        /*
+         * End the simulation by dropping the network. This will kill all running beacon nodes and
+         * validator clients.
+         */
+        println!(
+            "Simulation complete. Finished with {} beacon nodes and {} validator clients",
+            network.beacon_node_count(),
+            network.validator_client_count()
+        );
+
+        // Be explicit about dropping the network, as this kills all the nodes. This ensures
+        // all the checks have adequate time to pass.
+        drop(network);
+        Ok::<(), String>(())
+    };
+
+    Ok(env.runtime().block_on(main_future).unwrap())
+}

testing/simulator/src/local_network.rs

@@ -0,0 +1,164 @@
+use node_test_rig::{
+    environment::RuntimeContext, ClientConfig, LocalBeaconNode, LocalValidatorClient,
+    RemoteBeaconNode, ValidatorConfig, ValidatorFiles,
+};
+use parking_lot::RwLock;
+use std::ops::Deref;
+use std::sync::Arc;
+use types::{Epoch, EthSpec};
+
+const BOOTNODE_PORT: u16 = 42424;
+
+/// Helper struct to reduce `Arc` usage.
+pub struct Inner<E: EthSpec> {
+    context: RuntimeContext<E>,
+    beacon_nodes: RwLock<Vec<LocalBeaconNode<E>>>,
+    validator_clients: RwLock<Vec<LocalValidatorClient<E>>>,
+}
+
+/// Represents a set of interconnected `LocalBeaconNode` and `LocalValidatorClient`.
+///
+/// Provides functions to allow adding new beacon nodes and validators.
+pub struct LocalNetwork<E: EthSpec> {
+    inner: Arc<Inner<E>>,
+}
+
+impl<E: EthSpec> Clone for LocalNetwork<E> {
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<E: EthSpec> Deref for LocalNetwork<E> {
+    type Target = Inner<E>;
+
+    fn deref(&self) -> &Self::Target {
+        self.inner.deref()
+    }
+}
+
+impl<E: EthSpec> LocalNetwork<E> {
+    /// Creates a new network with a single `BeaconNode`.
+    pub async fn new(
+        context: RuntimeContext<E>,
+        mut beacon_config: ClientConfig,
+    ) -> Result<Self, String> {
+        beacon_config.network.discovery_port = BOOTNODE_PORT;
+        beacon_config.network.libp2p_port = BOOTNODE_PORT;
+        beacon_config.network.enr_udp_port = Some(BOOTNODE_PORT);
+        beacon_config.network.enr_tcp_port = Some(BOOTNODE_PORT);
+        let beacon_node =
+            LocalBeaconNode::production(context.service_context("boot_node".into()), beacon_config)
+                .await?;
+        Ok(Self {
+            inner: Arc::new(Inner {
+                context,
+                beacon_nodes: RwLock::new(vec![beacon_node]),
+                validator_clients: RwLock::new(vec![]),
+            }),
+        })
+    }
+
+    /// Returns the number of beacon nodes in the network.
+    ///
+    /// Note: does not count nodes that are external to this `LocalNetwork` that may have connected
+    /// (e.g., another Lighthouse process on the same machine.)
+    pub fn beacon_node_count(&self) -> usize {
+        self.beacon_nodes.read().len()
+    }
+
+    /// Returns the number of validator clients in the network.
+    ///
+    /// Note: does not count nodes that are external to this `LocalNetwork` that may have connected
+    /// (e.g., another Lighthouse process on the same machine.)
+    pub fn validator_client_count(&self) -> usize {
+        self.validator_clients.read().len()
+    }
+
+    /// Adds a beacon node to the network, connecting to the 0'th beacon node via ENR.
+    pub async fn add_beacon_node(&self, mut beacon_config: ClientConfig) -> Result<(), String> {
+        let self_1 = self.clone();
+        println!("Adding beacon node..");
+        {
+            let read_lock = self.beacon_nodes.read();
+
+            let boot_node = read_lock.first().expect("should have at least one node");
+
+            beacon_config.network.boot_nodes.push(
+                boot_node
+                    .client
+                    .enr()
+                    .expect("bootnode must have a network"),
+            );
+        }
+
+        let index = self.beacon_nodes.read().len();
+
+        let beacon_node = LocalBeaconNode::production(
+            self.context.service_context(format!("node_{}", index)),
+            beacon_config,
+        )
+        .await?;
+        self_1.beacon_nodes.write().push(beacon_node);
+        Ok(())
+    }
+
+    /// Adds a validator client to the network, connecting it to the beacon node with index
+    /// `beacon_node`.
+    pub async fn add_validator_client(
+        &self,
+        mut validator_config: ValidatorConfig,
+        beacon_node: usize,
+        validator_files: ValidatorFiles,
+    ) -> Result<(), String> {
+        let index = self.validator_clients.read().len();
+        let context = self.context.service_context(format!("validator_{}", index));
+        let self_1 = self.clone();
+        let socket_addr = {
+            let read_lock = self.beacon_nodes.read();
+            let beacon_node = read_lock
+                .get(beacon_node)
+                .ok_or_else(|| format!("No beacon node for index {}", beacon_node))?;
+            beacon_node
+                .client
+                .http_listen_addr()
+                .expect("Must have http started")
+        };
+
+        validator_config.http_server =
+            format!("http://{}:{}", socket_addr.ip(), socket_addr.port());
+        let validator_client = LocalValidatorClient::production_with_insecure_keypairs(
+            context,
+            validator_config,
+            validator_files,
+        )
+        .await?;
+        self_1.validator_clients.write().push(validator_client);
+        Ok(())
+    }
+
+    /// For all beacon nodes in `Self`, return a HTTP client to access each nodes HTTP API.
+    pub fn remote_nodes(&self) -> Result<Vec<RemoteBeaconNode<E>>, String> {
+        let beacon_nodes = self.beacon_nodes.read();
+
+        beacon_nodes
+            .iter()
+            .map(|beacon_node| beacon_node.remote_node())
+            .collect()
+    }
+
+    /// Return current epoch of bootnode.
+    pub async fn bootnode_epoch(&self) -> Result<Epoch, String> {
+        let nodes = self.remote_nodes().expect("Failed to get remote nodes");
+        let bootnode = nodes.first().expect("Should contain bootnode");
+        bootnode
+            .http
+            .beacon()
+            .get_head()
+            .await
+            .map_err(|e| format!("Cannot get head: {:?}", e))
+            .map(|head| head.finalized_slot.epoch(E::slots_per_epoch()))
+    }
+}

testing/simulator/src/main.rs

@@ -0,0 +1,65 @@
+//! This crate provides a simluation that creates `n` beacon node and validator clients, each with
+//! `v` validators. A deposit contract is deployed at the start of the simulation using a local
+//! `ganache-cli` instance (you must have `ganache-cli` installed and avaliable on your path). All
+//! beacon nodes independently listen for genesis from the deposit contract, then start operating.
+//!
+//! As the simulation runs, there are checks made to ensure that all components are running
+//! correctly. If any of these checks fail, the simulation will exit immediately.
+//!
+//! ## Future works
+//!
+//! Presently all the beacon nodes and validator clients all log to stdout. Additionally, the
+//! simulation uses `println` to communicate some info. It might be nice if the nodes logged to
+//! easy-to-find files and stdout only contained info from the simulation.
+//!
+
+#[macro_use]
+extern crate clap;
+
+mod checks;
+mod cli;
+mod eth1_sim;
+mod local_network;
+mod no_eth1_sim;
+mod sync_sim;
+
+use cli::cli_app;
+use env_logger::{Builder, Env};
+use local_network::LocalNetwork;
+use types::MinimalEthSpec;
+
+pub type E = MinimalEthSpec;
+
+fn main() {
+    // Debugging output for libp2p and external crates.
+    Builder::from_env(Env::default()).init();
+
+    let matches = cli_app().get_matches();
+    match matches.subcommand() {
+        ("eth1-sim", Some(matches)) => match eth1_sim::run_eth1_sim(matches) {
+            Ok(()) => println!("Simulation exited successfully"),
+            Err(e) => {
+                eprintln!("Simulation exited with error: {}", e);
+                std::process::exit(1)
+            }
+        },
+        ("no-eth1-sim", Some(matches)) => match no_eth1_sim::run_no_eth1_sim(matches) {
+            Ok(()) => println!("Simulation exited successfully"),
+            Err(e) => {
+                eprintln!("Simulation exited with error: {}", e);
+                std::process::exit(1)
+            }
+        },
+        ("syncing-sim", Some(matches)) => match sync_sim::run_syncing_sim(matches) {
+            Ok(()) => println!("Simulation exited successfully"),
+            Err(e) => {
+                eprintln!("Simulation exited with error: {}", e);
+                std::process::exit(1)
+            }
+        },
+        _ => {
+            eprintln!("Invalid subcommand. Use --help to see available options");
+            std::process::exit(1)
+        }
+    }
+}

testing/simulator/src/no_eth1_sim.rs

@@ -0,0 +1,158 @@
+use crate::{checks, LocalNetwork};
+use clap::ArgMatches;
+use futures::prelude::*;
+use node_test_rig::{
+    environment::EnvironmentBuilder, testing_client_config, ClientGenesis, ValidatorConfig,
+    ValidatorFiles,
+};
+use rayon::prelude::*;
+use std::net::{IpAddr, Ipv4Addr};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use tokio::time::{delay_until, Instant};
+
+pub fn run_no_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
+    let node_count = value_t!(matches, "nodes", usize).expect("missing nodes default");
+    let validators_per_node = value_t!(matches, "validators_per_node", usize)
+        .expect("missing validators_per_node default");
+    let speed_up_factor =
+        value_t!(matches, "speed_up_factor", u64).expect("missing speed_up_factor default");
+    let mut end_after_checks = true;
+    if matches.is_present("end_after_checks") {
+        end_after_checks = false;
+    }
+
+    println!("Beacon Chain Simulator:");
+    println!(" nodes:{}", node_count);
+    println!(" validators_per_node:{}", validators_per_node);
+    println!(" end_after_checks:{}", end_after_checks);
+
+    // Generate the directories and keystores required for the validator clients.
+    let validator_files = (0..node_count)
+        .into_par_iter()
+        .map(|i| {
+            println!(
+                "Generating keystores for validator {} of {}",
+                i + 1,
+                node_count
+            );
+
+            let indices =
+                (i * validators_per_node..(i + 1) * validators_per_node).collect::<Vec<_>>();
+            ValidatorFiles::with_keystores(&indices).unwrap()
+        })
+        .collect::<Vec<_>>();
+
+    let log_level = "debug";
+    let log_format = None;
+
+    let mut env = EnvironmentBuilder::mainnet()
+        .async_logger(log_level, log_format)?
+        .multi_threaded_tokio_runtime()?
+        .build()?;
+
+    let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor);
+
+    let spec = &mut env.eth2_config.spec;
+
+    spec.milliseconds_per_slot /= speed_up_factor;
+    spec.eth1_follow_distance = 16;
+    spec.min_genesis_delay = eth1_block_time.as_secs() * spec.eth1_follow_distance * 2;
+    spec.min_genesis_time = 0;
+    spec.min_genesis_active_validator_count = 64;
+    spec.seconds_per_eth1_block = 1;
+
+    let genesis_delay = Duration::from_secs(5);
+    let genesis_time = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_err(|_| "should get system time")?
+        + genesis_delay;
+    let genesis_instant = Instant::now() + genesis_delay;
+
+    let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+    let total_validator_count = validators_per_node * node_count;
+
+    let context = env.core_context();
+
+    let mut beacon_config = testing_client_config();
+
+    beacon_config.genesis = ClientGenesis::Interop {
+        validator_count: total_validator_count,
+        genesis_time: genesis_time.as_secs(),
+    };
+    beacon_config.dummy_eth1_backend = true;
+    beacon_config.sync_eth1_chain = true;
+
+    beacon_config.network.enr_address = Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
+
+    let main_future = async {
+        let network = LocalNetwork::new(context, beacon_config.clone()).await?;
+        /*
+         * One by one, add beacon nodes to the network.
+         */
+
+        for _ in 0..node_count - 1 {
+            network.add_beacon_node(beacon_config.clone()).await?;
+        }
+
+        /*
+         * Create a future that will add validator clients to the network. Each validator client is
+         * attached to a single corresponding beacon node.
+         */
+        let add_validators_fut = async {
+            for (i, files) in validator_files.into_iter().enumerate() {
+                network
+                    .add_validator_client(
+                        ValidatorConfig {
+                            auto_register: true,
+                            ..ValidatorConfig::default()
+                        },
+                        i,
+                        files,
+                    )
+                    .await?;
+            }
+
+            Ok::<(), String>(())
+        };
+
+        /*
+         * The processes that will run checks on the network as it runs.
+         */
+        let checks_fut = async {
+            delay_until(genesis_instant).await;
+
+            // Check that the chain finalizes at the first given opportunity.
+            checks::verify_first_finalization(network.clone(), slot_duration).await?;
+
+            Ok::<(), String>(())
+        };
+
+        let (add_validators, start_checks) = futures::join!(add_validators_fut, checks_fut);
+
+        add_validators?;
+        start_checks?;
+
+        // The `final_future` either completes immediately or never completes, depending on the value
+        // of `end_after_checks`.
+
+        if !end_after_checks {
+            future::pending::<()>().await;
+        }
+        /*
+         * End the simulation by dropping the network. This will kill all running beacon nodes and
+         * validator clients.
+         */
+        println!(
+            "Simulation complete. Finished with {} beacon nodes and {} validator clients",
+            network.beacon_node_count(),
+            network.validator_client_count()
+        );
+
+        // Be explicit about dropping the network, as this kills all the nodes. This ensures
+        // all the checks have adequate time to pass.
+        drop(network);
+        Ok::<(), String>(())
+    };
+
+    Ok(env.runtime().block_on(main_future).unwrap())
+}

testing/simulator/src/sync_sim.rs

@@ -0,0 +1,362 @@
+use crate::checks::{epoch_delay, verify_all_finalized_at};
+use crate::local_network::LocalNetwork;
+use clap::ArgMatches;
+use futures::prelude::*;
+use node_test_rig::ClientConfig;
+use node_test_rig::{
+    environment::EnvironmentBuilder, testing_client_config, ClientGenesis, ValidatorConfig,
+    ValidatorFiles,
+};
+use std::net::{IpAddr, Ipv4Addr};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use types::{Epoch, EthSpec};
+
+pub fn run_syncing_sim(matches: &ArgMatches) -> Result<(), String> {
+    let initial_delay = value_t!(matches, "initial_delay", u64).unwrap();
+    let sync_timeout = value_t!(matches, "sync_timeout", u64).unwrap();
+    let speed_up_factor = value_t!(matches, "speedup", u64).unwrap();
+    let strategy = value_t!(matches, "strategy", String).unwrap();
+
+    println!("Syncing Simulator:");
+    println!(" initial_delay:{}", initial_delay);
+    println!(" sync timeout: {}", sync_timeout);
+    println!(" speed up factor:{}", speed_up_factor);
+    println!(" strategy:{}", strategy);
+
+    let log_level = "debug";
+    let log_format = None;
+
+    syncing_sim(
+        speed_up_factor,
+        initial_delay,
+        sync_timeout,
+        strategy,
+        log_level,
+        log_format,
+    )
+}
+
+fn syncing_sim(
+    speed_up_factor: u64,
+    initial_delay: u64,
+    sync_timeout: u64,
+    strategy: String,
+    log_level: &str,
+    log_format: Option<&str>,
+) -> Result<(), String> {
+    let mut env = EnvironmentBuilder::minimal()
+        .async_logger(log_level, log_format)?
+        .multi_threaded_tokio_runtime()?
+        .build()?;
+
+    let spec = &mut env.eth2_config.spec;
+    let end_after_checks = true;
+    let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor);
+
+    spec.milliseconds_per_slot /= speed_up_factor;
+    spec.eth1_follow_distance = 16;
+    spec.min_genesis_delay = eth1_block_time.as_secs() * spec.eth1_follow_distance * 2;
+    spec.min_genesis_time = 0;
+    spec.min_genesis_active_validator_count = 64;
+    spec.seconds_per_eth1_block = 1;
+
+    let num_validators = 8;
+    let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+    let context = env.core_context();
+    let mut beacon_config = testing_client_config();
+
+    let genesis_time = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_err(|_| "should get system time")?
+        + Duration::from_secs(5);
+    beacon_config.genesis = ClientGenesis::Interop {
+        validator_count: num_validators,
+        genesis_time: genesis_time.as_secs(),
+    };
+    beacon_config.dummy_eth1_backend = true;
+    beacon_config.sync_eth1_chain = true;
+
+    beacon_config.network.enr_address = Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
+
+    // Generate the directories and keystores required for the validator clients.
+    let validator_indices = (0..num_validators).collect::<Vec<_>>();
+    let validator_files = ValidatorFiles::with_keystores(&validator_indices).unwrap();
+
+    let main_future = async {
+        /*
+         * Create a new `LocalNetwork` with one beacon node.
+         */
+        let network = LocalNetwork::new(context, beacon_config.clone()).await?;
+
+        /*
+         * Add a validator client which handles all validators from the genesis state.
+         */
+        network
+            .add_validator_client(ValidatorConfig::default(), 0, validator_files)
+            .await?;
+
+        // Check all syncing strategies one after other.
+        pick_strategy(
+            &strategy,
+            network.clone(),
+            beacon_config.clone(),
+            slot_duration,
+            initial_delay,
+            sync_timeout,
+        )
+        .await?;
+
+        // The `final_future` either completes immediately or never completes, depending on the value
+        // of `end_after_checks`.
+
+        if !end_after_checks {
+            future::pending::<()>().await;
+        }
+
+        /*
+         * End the simulation by dropping the network. This will kill all running beacon nodes and
+         * validator clients.
+         */
+        println!(
+            "Simulation complete. Finished with {} beacon nodes and {} validator clients",
+            network.beacon_node_count(),
+            network.validator_client_count()
+        );
+
+        // Be explicit about dropping the network, as this kills all the nodes. This ensures
+        // all the checks have adequate time to pass.
+        drop(network);
+        Ok::<(), String>(())
+    };
+
+    env.runtime().block_on(main_future)
+}
+
+pub async fn pick_strategy<E: EthSpec>(
+    strategy: &str,
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    match strategy {
+        "one-node" => {
+            verify_one_node_sync(
+                network,
+                beacon_config,
+                slot_duration,
+                initial_delay,
+                sync_timeout,
+            )
+            .await
+        }
+        "two-nodes" => {
+            verify_two_nodes_sync(
+                network,
+                beacon_config,
+                slot_duration,
+                initial_delay,
+                sync_timeout,
+            )
+            .await
+        }
+        "mixed" => {
+            verify_in_between_sync(
+                network,
+                beacon_config,
+                slot_duration,
+                initial_delay,
+                sync_timeout,
+            )
+            .await
+        }
+        "all" => {
+            verify_syncing(
+                network,
+                beacon_config,
+                slot_duration,
+                initial_delay,
+                sync_timeout,
+            )
+            .await
+        }
+        _ => Err("Invalid strategy".into()),
+    }
+}
+
+/// Verify one node added after `initial_delay` epochs is in sync
+/// after `sync_timeout` epochs.
+pub async fn verify_one_node_sync<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    let epoch_duration = slot_duration * (E::slots_per_epoch() as u32);
+    let network_c = network.clone();
+    // Delay for `initial_delay` epochs before adding another node to start syncing
+    epoch_delay(
+        Epoch::new(initial_delay),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .await;
+    // Add a beacon node
+    network.add_beacon_node(beacon_config).await?;
+    // Check every `epoch_duration` if nodes are synced
+    // limited to at most `sync_timeout` epochs
+    let mut interval = tokio::time::interval(epoch_duration);
+    let mut count = 0;
+    while let Some(_) = interval.next().await {
+        if count >= sync_timeout || !check_still_syncing(&network_c).await? {
+            break;
+        }
+        count += 1;
+    }
+    let epoch = network.bootnode_epoch().await?;
+    verify_all_finalized_at(network, epoch)
+        .map_err(|e| format!("One node sync error: {}", e))
+        .await
+}
+
+/// Verify two nodes added after `initial_delay` epochs are in sync
+/// after `sync_timeout` epochs.
+pub async fn verify_two_nodes_sync<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    let epoch_duration = slot_duration * (E::slots_per_epoch() as u32);
+    let network_c = network.clone();
+    // Delay for `initial_delay` epochs before adding another node to start syncing
+    epoch_delay(
+        Epoch::new(initial_delay),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .await;
+    // Add beacon nodes
+    network.add_beacon_node(beacon_config.clone()).await?;
+    network.add_beacon_node(beacon_config).await?;
+    // Check every `epoch_duration` if nodes are synced
+    // limited to at most `sync_timeout` epochs
+    let mut interval = tokio::time::interval(epoch_duration);
+    let mut count = 0;
+    while let Some(_) = interval.next().await {
+        if count >= sync_timeout || !check_still_syncing(&network_c).await? {
+            break;
+        }
+        count += 1;
+    }
+    let epoch = network.bootnode_epoch().await?;
+    verify_all_finalized_at(network, epoch)
+        .map_err(|e| format!("One node sync error: {}", e))
+        .await
+}
+
+/// Add 2 syncing nodes after `initial_delay` epochs,
+/// Add another node after `sync_timeout - 5` epochs and verify all are
+/// in sync after `sync_timeout + 5` epochs.
+pub async fn verify_in_between_sync<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    let epoch_duration = slot_duration * (E::slots_per_epoch() as u32);
+    let network_c = network.clone();
+    // Delay for `initial_delay` epochs before adding another node to start syncing
+    let config1 = beacon_config.clone();
+    epoch_delay(
+        Epoch::new(initial_delay),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .await;
+    // Add two beacon nodes
+    network.add_beacon_node(beacon_config.clone()).await?;
+    network.add_beacon_node(beacon_config).await?;
+    // Delay before adding additional syncing nodes.
+    epoch_delay(
+        Epoch::new(sync_timeout - 5),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .await;
+    // Add a beacon node
+    network.add_beacon_node(config1.clone()).await?;
+    // Check every `epoch_duration` if nodes are synced
+    // limited to at most `sync_timeout` epochs
+    let mut interval = tokio::time::interval(epoch_duration);
+    let mut count = 0;
+    while let Some(_) = interval.next().await {
+        if count >= sync_timeout || !check_still_syncing(&network_c).await? {
+            break;
+        }
+        count += 1;
+    }
+    let epoch = network.bootnode_epoch().await?;
+    verify_all_finalized_at(network, epoch)
+        .map_err(|e| format!("One node sync error: {}", e))
+        .await
+}
+
+/// Run syncing strategies one after other.
+pub async fn verify_syncing<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    verify_one_node_sync(
+        network.clone(),
+        beacon_config.clone(),
+        slot_duration,
+        initial_delay,
+        sync_timeout,
+    )
+    .await?;
+    println!("Completed one node sync");
+    verify_two_nodes_sync(
+        network.clone(),
+        beacon_config.clone(),
+        slot_duration,
+        initial_delay,
+        sync_timeout,
+    )
+    .await?;
+    println!("Completed two node sync");
+    verify_in_between_sync(
+        network,
+        beacon_config,
+        slot_duration,
+        initial_delay,
+        sync_timeout,
+    )
+    .await?;
+    println!("Completed in between sync");
+    Ok(())
+}
+
+pub async fn check_still_syncing<E: EthSpec>(network: &LocalNetwork<E>) -> Result<bool, String> {
+    // get syncing status of nodes
+    let mut status = Vec::new();
+    for remote_node in network.remote_nodes()? {
+        status.push(
+            remote_node
+                .http
+                .node()
+                .syncing_status()
+                .await
+                .map(|status| status.is_syncing)
+                .map_err(|e| format!("Get syncing status via http failed: {:?}", e))?,
+        )
+    }
+    Ok(status.iter().any(|is_syncing| *is_syncing))
+}