Merge remote-tracking branch 'origin/master' into v0.9

tests/eth1_test_rig/.gitignore

@@ -0,0 +1 @@
+contract/

tests/eth1_test_rig/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "eth1_test_rig"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+build = "build.rs"
+
+[build-dependencies]
+reqwest = "0.9.20"
+serde_json = "1.0"
+
+[dependencies]
+web3 = "0.8.0"
+tokio = "0.1.17"
+futures = "0.1.25"
+types = { path = "../../eth2/types"}
+eth2_ssz = { path = "../../eth2/utils/ssz"}
+serde_json = "1.0"

tests/eth1_test_rig/build.rs

@@ -0,0 +1,95 @@
+//! Downloads the ABI and bytecode for the deposit contract from the ethereum spec repository and
+//! stores them in a `contract/` directory in the crate root.
+//!
+//! These files are required for some `include_bytes` calls used in this crate.
+
+use reqwest::Response;
+use serde_json::Value;
+use std::env;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+
+const GITHUB_RAW: &str = "https://raw.githubusercontent.com";
+const SPEC_REPO: &str = "ethereum/eth2.0-specs";
+const SPEC_TAG: &str = "v0.8.3";
+const ABI_FILE: &str = "validator_registration.json";
+const BYTECODE_FILE: &str = "validator_registration.bytecode";
+
+fn main() {
+    match init_deposit_contract_abi() {
+        Ok(()) => (),
+        Err(e) => panic!(e),
+    }
+}
+
+/// Attempts to download the deposit contract ABI from github if a local copy is not already
+/// present.
+pub fn init_deposit_contract_abi() -> Result<(), String> {
+    let abi_file = abi_dir().join(format!("{}_{}", SPEC_TAG, ABI_FILE));
+    let bytecode_file = abi_dir().join(format!("{}_{}", SPEC_TAG, BYTECODE_FILE));
+
+    if abi_file.exists() {
+        // Nothing to do.
+    } else {
+        match download_abi() {
+            Ok(mut response) => {
+                let mut abi_file = File::create(abi_file)
+                    .map_err(|e| format!("Failed to create local abi file: {:?}", e))?;
+                let mut bytecode_file = File::create(bytecode_file)
+                    .map_err(|e| format!("Failed to create local bytecode file: {:?}", e))?;
+
+                let contract: Value = response
+                    .json()
+                    .map_err(|e| format!("Respsonse is not a valid json {:?}", e))?;
+
+                let abi = contract
+                    .get("abi")
+                    .ok_or(format!("Response does not contain key: abi"))?
+                    .to_string();
+                abi_file
+                    .write(abi.as_bytes())
+                    .map_err(|e| format!("Failed to write http response to abi file: {:?}", e))?;
+
+                let bytecode = contract
+                    .get("bytecode")
+                    .ok_or(format!("Response does not contain key: bytecode"))?
+                    .to_string();
+                bytecode_file.write(bytecode.as_bytes()).map_err(|e| {
+                    format!("Failed to write http response to bytecode file: {:?}", e)
+                })?;
+            }
+            Err(e) => {
+                return Err(format!(
+                    "No abi file found. Failed to download from github: {:?}",
+                    e
+                ))
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Attempts to download the deposit contract file from the Ethereum github.
+fn download_abi() -> Result<Response, String> {
+    reqwest::get(&format!(
+        "{}/{}/{}/deposit_contract/contracts/{}",
+        GITHUB_RAW, SPEC_REPO, SPEC_TAG, ABI_FILE
+    ))
+    .map_err(|e| format!("Failed to download deposit ABI from github: {:?}", e))
+}
+
+/// Returns the directory that will be used to store the deposit contract ABI.
+fn abi_dir() -> PathBuf {
+    let base = env::var("CARGO_MANIFEST_DIR")
+        .expect("should know manifest dir")
+        .parse::<PathBuf>()
+        .expect("should parse manifest dir as path")
+        .join("contract");
+
+    std::fs::create_dir_all(base.clone())
+        .expect("should be able to create abi directory in manifest");
+
+    base
+}

tests/eth1_test_rig/src/ganache.rs

@@ -0,0 +1,157 @@
+use futures::Future;
+use serde_json::json;
+use std::io::prelude::*;
+use std::io::BufReader;
+use std::net::TcpListener;
+use std::process::{Child, Command, Stdio};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use web3::{
+    transports::{EventLoopHandle, Http},
+    Transport, Web3,
+};
+
+/// How long we will wait for ganache to indicate that it is ready.
+const GANACHE_STARTUP_TIMEOUT_MILLIS: u64 = 10_000;
+
+/// Provides a dedicated `ganachi-cli` instance with a connected `Web3` instance.
+///
+/// Requires that `ganachi-cli` is installed and available on `PATH`.
+pub struct GanacheInstance {
+    pub port: u16,
+    child: Child,
+    _event_loop: Arc<EventLoopHandle>,
+    pub web3: Web3<Http>,
+}
+
+impl GanacheInstance {
+    /// Start a new `ganache-cli` process, waiting until it indicates that it is ready to accept
+    /// RPC connections.
+    pub fn new() -> Result<Self, String> {
+        let port = unused_port()?;
+
+        let mut child = Command::new("ganache-cli")
+            .stdout(Stdio::piped())
+            .arg("--defaultBalanceEther")
+            .arg("1000000000")
+            .arg("--gasLimit")
+            .arg("1000000000")
+            .arg("--accounts")
+            .arg("10")
+            .arg("--port")
+            .arg(format!("{}", port))
+            .arg("--mnemonic")
+            .arg("\"vast thought differ pull jewel broom cook wrist tribe word before omit\"")
+            .spawn()
+            .map_err(|e| {
+                format!(
+                    "Failed to start ganche-cli. \
+                     Is it ganache-cli installed and available on $PATH? Error: {:?}",
+                    e
+                )
+            })?;
+
+        let stdout = child
+            .stdout
+            .ok_or_else(|| "Unable to get stdout for ganache child process")?;
+
+        let start = Instant::now();
+        let mut reader = BufReader::new(stdout);
+        loop {
+            if start + Duration::from_millis(GANACHE_STARTUP_TIMEOUT_MILLIS) <= Instant::now() {
+                break Err(
+                    "Timed out waiting for ganache to start. Is ganache-cli installed?".to_string(),
+                );
+            }
+
+            let mut line = String::new();
+            if let Err(e) = reader.read_line(&mut line) {
+                break Err(format!("Failed to read line from ganache process: {:?}", e));
+            } else if line.starts_with("Listening on") {
+                break Ok(());
+            } else {
+                continue;
+            }
+        }?;
+
+        let (event_loop, transport) = Http::new(&endpoint(port)).map_err(|e| {
+            format!(
+                "Failed to start HTTP transport connected to ganache: {:?}",
+                e
+            )
+        })?;
+        let web3 = Web3::new(transport);
+
+        child.stdout = Some(reader.into_inner());
+
+        Ok(Self {
+            child,
+            port,
+            _event_loop: Arc::new(event_loop),
+            web3,
+        })
+    }
+
+    /// Returns the endpoint that this instance is listening on.
+    pub fn endpoint(&self) -> String {
+        endpoint(self.port)
+    }
+
+    /// Increase the timestamp on future blocks by `increase_by` seconds.
+    pub fn increase_time(&self, increase_by: u64) -> impl Future<Item = (), Error = String> {
+        self.web3
+            .transport()
+            .execute("evm_increaseTime", vec![json!(increase_by)])
+            .map(|_json_value| ())
+            .map_err(|e| format!("Failed to increase time on EVM (is this ganache?): {:?}", e))
+    }
+
+    /// Returns the current block number, as u64
+    pub fn block_number(&self) -> impl Future<Item = u64, Error = String> {
+        self.web3
+            .eth()
+            .block_number()
+            .map(|v| v.as_u64())
+            .map_err(|e| format!("Failed to get block number: {:?}", e))
+    }
+
+    /// Mines a single block.
+    pub fn evm_mine(&self) -> impl Future<Item = (), Error = String> {
+        self.web3
+            .transport()
+            .execute("evm_mine", vec![])
+            .map(|_| ())
+            .map_err(|_| {
+                "utils should mine new block with evm_mine (only works with ganache-cli!)"
+                    .to_string()
+            })
+    }
+}
+
+fn endpoint(port: u16) -> String {
+    format!("http://localhost:{}", port)
+}
+
+/// A bit of hack to find an unused TCP port.
+///
+/// Does not guarantee that the given port is unused after the function exists, just that it was
+/// unused before the function started (i.e., it does not reserve a port).
+pub fn unused_port() -> Result<u16, String> {
+    let listener = TcpListener::bind("127.0.0.1:0")
+        .map_err(|e| format!("Failed to create TCP listener to find unused port: {:?}", e))?;
+
+    let local_addr = listener.local_addr().map_err(|e| {
+        format!(
+            "Failed to read TCP listener local_addr to find unused port: {:?}",
+            e
+        )
+    })?;
+
+    Ok(local_addr.port())
+}
+
+impl Drop for GanacheInstance {
+    fn drop(&mut self) {
+        let _ = self.child.kill();
+    }
+}

tests/eth1_test_rig/src/lib.rs

@@ -0,0 +1,240 @@
+//! Provides utilities for deploying and manipulating the eth2 deposit contract on the eth1 chain.
+//!
+//! Presently used with [`ganache-cli`](https://github.com/trufflesuite/ganache-cli) to simulate
+//! the deposit contract for testing beacon node eth1 integration.
+//!
+//! Not tested to work with actual clients (e.g., geth). It should work fine, however there may be
+//! some initial issues.
+mod ganache;
+
+use futures::{stream, Future, IntoFuture, Stream};
+use ganache::GanacheInstance;
+use ssz::Encode;
+use std::time::{Duration, Instant};
+use tokio::{runtime::Runtime, timer::Delay};
+use types::DepositData;
+use types::{EthSpec, Hash256, Keypair, Signature};
+use web3::contract::{Contract, Options};
+use web3::transports::Http;
+use web3::types::{Address, U256};
+use web3::{Transport, Web3};
+
+pub const DEPLOYER_ACCOUNTS_INDEX: usize = 0;
+pub const DEPOSIT_ACCOUNTS_INDEX: usize = 0;
+
+const CONTRACT_DEPLOY_GAS: usize = 4_000_000;
+const DEPOSIT_GAS: usize = 4_000_000;
+
+// Deposit contract
+pub const ABI: &[u8] = include_bytes!("../contract/v0.8.3_validator_registration.json");
+pub const BYTECODE: &[u8] = include_bytes!("../contract/v0.8.3_validator_registration.bytecode");
+
+/// Provides a dedicated ganache-cli instance with the deposit contract already deployed.
+pub struct GanacheEth1Instance {
+    pub ganache: GanacheInstance,
+    pub deposit_contract: DepositContract,
+}
+
+impl GanacheEth1Instance {
+    pub fn new() -> impl Future<Item = Self, Error = String> {
+        GanacheInstance::new().into_future().and_then(|ganache| {
+            DepositContract::deploy(ganache.web3.clone(), 0).map(|deposit_contract| Self {
+                ganache,
+                deposit_contract,
+            })
+        })
+    }
+
+    pub fn endpoint(&self) -> String {
+        self.ganache.endpoint()
+    }
+
+    pub fn web3(&self) -> Web3<Http> {
+        self.ganache.web3.clone()
+    }
+}
+
+/// Deploys and provides functions for the eth2 deposit contract, deployed on the eth1 chain.
+#[derive(Clone, Debug)]
+pub struct DepositContract {
+    web3: Web3<Http>,
+    contract: Contract<Http>,
+}
+
+impl DepositContract {
+    pub fn deploy(
+        web3: Web3<Http>,
+        confirmations: usize,
+    ) -> impl Future<Item = Self, Error = String> {
+        let web3_1 = web3.clone();
+
+        deploy_deposit_contract(web3.clone(), confirmations)
+            .map_err(|e| {
+                format!(
+                    "Failed to deploy contract: {}. Is scripts/ganache_tests_node.sh running?.",
+                    e
+                )
+            })
+            .and_then(move |address| {
+                Contract::from_json(web3_1.eth(), address, ABI)
+                    .map_err(|e| format!("Failed to init contract: {:?}", e))
+            })
+            .map(|contract| Self { contract, web3 })
+    }
+
+    /// The deposit contract's address in `0x00ab...` format.
+    pub fn address(&self) -> String {
+        format!("0x{:x}", self.contract.address())
+    }
+
+    /// A helper to return a fully-formed `DepositData`. Does not submit the deposit data to the
+    /// smart contact.
+    pub fn deposit_helper<E: EthSpec>(
+        &self,
+        keypair: Keypair,
+        withdrawal_credentials: Hash256,
+        amount: u64,
+    ) -> DepositData {
+        let mut deposit = DepositData {
+            pubkey: keypair.pk.into(),
+            withdrawal_credentials,
+            amount,
+            signature: Signature::empty_signature().into(),
+        };
+
+        deposit.signature = deposit.create_signature(&keypair.sk, &E::default_spec());
+
+        deposit
+    }
+
+    /// Creates a random, valid deposit and submits it to the deposit contract.
+    ///
+    /// The keypairs are created randomly and destroyed.
+    pub fn deposit_random<E: EthSpec>(&self, runtime: &mut Runtime) -> Result<(), String> {
+        let keypair = Keypair::random();
+
+        let mut deposit = DepositData {
+            pubkey: keypair.pk.into(),
+            withdrawal_credentials: Hash256::zero(),
+            amount: 32_000_000_000,
+            signature: Signature::empty_signature().into(),
+        };
+
+        deposit.signature = deposit.create_signature(&keypair.sk, &E::default_spec());
+
+        self.deposit(runtime, deposit)
+    }
+
+    /// Perfoms a blocking deposit.
+    pub fn deposit(&self, runtime: &mut Runtime, deposit_data: DepositData) -> Result<(), String> {
+        runtime
+            .block_on(self.deposit_async(deposit_data))
+            .map_err(|e| format!("Deposit failed: {:?}", e))
+    }
+
+    /// Performs a non-blocking deposit.
+    pub fn deposit_async(
+        &self,
+        deposit_data: DepositData,
+    ) -> impl Future<Item = (), Error = String> {
+        let contract = self.contract.clone();
+
+        self.web3
+            .eth()
+            .accounts()
+            .map_err(|e| format!("Failed to get accounts: {:?}", e))
+            .and_then(|accounts| {
+                accounts
+                    .get(DEPOSIT_ACCOUNTS_INDEX)
+                    .cloned()
+                    .ok_or_else(|| "Insufficient accounts for deposit".to_string())
+            })
+            .and_then(move |from_address| {
+                let params = (
+                    deposit_data.pubkey.as_ssz_bytes(),
+                    deposit_data.withdrawal_credentials.as_ssz_bytes(),
+                    deposit_data.signature.as_ssz_bytes(),
+                );
+                let options = Options {
+                    gas: Some(U256::from(DEPOSIT_GAS)),
+                    value: Some(from_gwei(deposit_data.amount)),
+                    ..Options::default()
+                };
+                contract
+                    .call("deposit", params, from_address, options)
+                    .map_err(|e| format!("Failed to call deposit fn: {:?}", e))
+            })
+            .map(|_| ())
+    }
+
+    /// Peforms many deposits, each preceded by a delay.
+    pub fn deposit_multiple(
+        &self,
+        deposits: Vec<DelayThenDeposit>,
+    ) -> impl Future<Item = (), Error = String> {
+        let s = self.clone();
+        stream::unfold(deposits.into_iter(), move |mut deposit_iter| {
+            let s = s.clone();
+            match deposit_iter.next() {
+                Some(deposit) => Some(
+                    Delay::new(Instant::now() + deposit.delay)
+                        .map_err(|e| format!("Failed to execute delay: {:?}", e))
+                        .and_then(move |_| s.deposit_async(deposit.deposit))
+                        .map(move |yielded| (yielded, deposit_iter)),
+                ),
+                None => None,
+            }
+        })
+        .collect()
+        .map(|_| ())
+    }
+}
+
+/// Describes a deposit and a delay that should should precede it's submission to the deposit
+/// contract.
+#[derive(Clone)]
+pub struct DelayThenDeposit {
+    /// Wait this duration ...
+    pub delay: Duration,
+    /// ... then submit this deposit.
+    pub deposit: DepositData,
+}
+
+fn from_gwei(gwei: u64) -> U256 {
+    U256::from(gwei) * U256::exp10(9)
+}
+
+/// Deploys the deposit contract to the given web3 instance using the account with index
+/// `DEPLOYER_ACCOUNTS_INDEX`.
+fn deploy_deposit_contract<T: Transport>(
+    web3: Web3<T>,
+    confirmations: usize,
+) -> impl Future<Item = Address, Error = String> {
+    let bytecode = String::from_utf8_lossy(&BYTECODE);
+
+    web3.eth()
+        .accounts()
+        .map_err(|e| format!("Failed to get accounts: {:?}", e))
+        .and_then(|accounts| {
+            accounts
+                .get(DEPLOYER_ACCOUNTS_INDEX)
+                .cloned()
+                .ok_or_else(|| "Insufficient accounts for deployer".to_string())
+        })
+        .and_then(move |deploy_address| {
+            Contract::deploy(web3.eth(), &ABI)
+                .map_err(|e| format!("Unable to build contract deployer: {:?}", e))?
+                .confirmations(confirmations)
+                .options(Options {
+                    gas: Some(U256::from(CONTRACT_DEPLOY_GAS)),
+                    ..Options::default()
+                })
+                .execute(bytecode, (), deploy_address)
+                .map_err(|e| format!("Failed to execute deployment: {:?}", e))
+        })
+        .and_then(|pending_contract| {
+            pending_contract
+                .map(|contract| contract.address())
+                .map_err(|e| format!("Unable to resolve pending contract: {:?}", e))
+        })
+}

tests/node_test_rig/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "node_test_rig"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+environment = { path = "../../lighthouse/environment" }
+beacon_node = { path = "../../beacon_node" }
+types = { path = "../../eth2/types" }
+eth2_config = { path = "../../eth2/utils/eth2_config" }
+tempdir = "0.3"
+reqwest = "0.9"
+url = "1.2"
+serde = "1.0"
+futures = "0.1.25"
+genesis = { path = "../../beacon_node/genesis" }
+remote_beacon_node = { path = "../../eth2/utils/remote_beacon_node" }

tests/node_test_rig/src/lib.rs

@@ -0,0 +1,67 @@
+use beacon_node::{
+    beacon_chain::BeaconChainTypes, Client, ClientConfig, ClientGenesis, ProductionBeaconNode,
+    ProductionClient,
+};
+use environment::RuntimeContext;
+use futures::Future;
+use remote_beacon_node::RemoteBeaconNode;
+use tempdir::TempDir;
+use types::EthSpec;
+
+pub use environment;
+
+/// Provides a beacon node that is running in the current process. Useful for testing purposes.
+pub struct LocalBeaconNode<T> {
+    pub client: T,
+    pub datadir: TempDir,
+}
+
+impl<E: EthSpec> LocalBeaconNode<ProductionClient<E>> {
+    /// Starts a new, production beacon node.
+    pub fn production(context: RuntimeContext<E>) -> Self {
+        let (client_config, datadir) = testing_client_config();
+
+        let client = ProductionBeaconNode::new(context, client_config)
+            .wait()
+            .expect("should build production client")
+            .into_inner();
+
+        LocalBeaconNode { client, datadir }
+    }
+}
+
+impl<T: BeaconChainTypes> LocalBeaconNode<Client<T>> {
+    /// Returns a `RemoteBeaconNode` that can connect to `self`. Useful for testing the node as if
+    /// it were external this process.
+    pub fn remote_node(&self) -> Result<RemoteBeaconNode<T::EthSpec>, String> {
+        Ok(RemoteBeaconNode::new(
+            self.client
+                .http_listen_addr()
+                .ok_or_else(|| "A remote beacon node must have a http server".to_string())?,
+        )?)
+    }
+}
+
+fn testing_client_config() -> (ClientConfig, TempDir) {
+    // Creates a temporary directory that will be deleted once this `TempDir` is dropped.
+    let tempdir = TempDir::new("lighthouse_node_test_rig")
+        .expect("should create temp directory for client datadir");
+
+    let mut client_config = ClientConfig::default();
+
+    client_config.data_dir = tempdir.path().into();
+
+    // Setting ports to `0` means that the OS will choose some available port.
+    client_config.network.libp2p_port = 0;
+    client_config.network.discovery_port = 0;
+    client_config.rpc.port = 0;
+    client_config.rest_api.port = 0;
+    client_config.websocket_server.port = 0;
+
+    client_config.genesis = ClientGenesis::Interop {
+        validator_count: 8,
+        genesis_time: 13_371_337,
+    };
+
+    (client_config, tempdir)
+}