Validator client refactor (#618)

* Update to spec v0.9.0

* Update to v0.9.1

* Bump spec tags for v0.9.1

* Formatting, fix CI failures

* Resolve accidental KeyPair merge conflict

* Document new BeaconState functions

* Add `validator` changes from `validator-to-rest`

* Add initial (failing) REST api tests

* Fix signature parsing

* Add more tests

* Refactor http router

* Add working tests for publish beacon block

* Add validator duties tests

* Move account_manager under `lighthouse` binary

* Unify logfile handling in `environment` crate.

* Fix incorrect cache drops in `advance_caches`

* Update fork choice for v0.9.1

* Add `deposit_contract` crate

* Add progress on validator onboarding

* Add unfinished attesation code

* Update account manager CLI

* Write eth1 data file as hex string

* Integrate ValidatorDirectory with validator_client

* Move ValidatorDirectory into validator_client

* Clean up some FIXMEs

* Add beacon_chain_sim

* Fix a few docs/logs

* Expand `beacon_chain_sim`

* Fix spec for `beacon_chain_sim

* More testing for api

* Start work on attestation endpoint

* Reject empty attestations

* Allow attestations to genesis block

* Add working tests for `rest_api` validator endpoint

* Remove grpc from beacon_node

* Start heavy refactor of validator client

- Block production is working

* Prune old validator client files

* Start works on attestation service

* Add attestation service to validator client

* Use full pubkey for validator directories

* Add validator duties post endpoint

* Use par_iter for keypair generation

* Use bulk duties request in validator client

* Add version http endpoint tests

* Add interop keys and startup wait

* Ensure a prompt exit

* Add duties pruning

* Fix compile error in beacon node tests

* Add github workflow

* Modify rust.yaml

* Modify gitlab actions

* Add to CI file

* Add sudo to CI npm install

* Move cargo fmt to own job in tests

* Fix cargo fmt in CI

* Add rustup update before cargo fmt

* Change name of CI job

* Make other CI jobs require cargo fmt

* Add CI badge

* Remove gitlab and travis files

* Add different http timeout for debug

* Update docker file, use makefile in CI

* Use make in the dockerfile, skip the test

* Use the makefile for debug GI test

* Update book

* Tidy grpc and misc things

* Apply discv5 fixes

* Address other minor issues

* Fix warnings

* Attempt fix for addr parsing

* Tidy validator config, CLIs

* Tidy comments

* Tidy signing, reduce ForkService duplication

* Fail if skipping too many slots

* Set default recent genesis time to 0

* Add custom http timeout to validator

* Fix compile bug in node_test_rig

* Remove old bootstrap flag from val CLI

* Update docs

* Tidy val client

* Change val client log levels

* Add comments, more validity checks

* Fix compile error, add comments

* Undo changes to eth2-libp2p/src

* Reduce duplication of keypair generation

* Add more logging for validator duties

* Fix beacon_chain_sim, nitpicks

* Fix compile error, minor nits

* Address Michael's comments

tests/eth1_test_rig/Cargo.toml

@@ -4,16 +4,10 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
-build = "build.rs"
-
-[build-dependencies]
-reqwest = "0.9.20"
-serde_json = "1.0"
-
 [dependencies]
 web3 = "0.8.0"
 tokio = "0.1.17"
 futures = "0.1.25"
 types = { path = "../../eth2/types"}
-eth2_ssz = { path = "../../eth2/utils/ssz"}
 serde_json = "1.0"
+deposit_contract = { path = "../../eth2/utils/deposit_contract"}

tests/eth1_test_rig/build.rs

@@ -1,95 +0,0 @@
-//! Downloads the ABI and bytecode for the deposit contract from the ethereum spec repository and
-//! stores them in a `contract/` directory in the crate root.
-//!
-//! These files are required for some `include_bytes` calls used in this crate.
-
-use reqwest::Response;
-use serde_json::Value;
-use std::env;
-use std::fs::File;
-use std::io::Write;
-use std::path::PathBuf;
-
-const GITHUB_RAW: &str = "https://raw.githubusercontent.com";
-const SPEC_REPO: &str = "ethereum/eth2.0-specs";
-const SPEC_TAG: &str = "v0.8.3";
-const ABI_FILE: &str = "validator_registration.json";
-const BYTECODE_FILE: &str = "validator_registration.bytecode";
-
-fn main() {
-    match init_deposit_contract_abi() {
-        Ok(()) => (),
-        Err(e) => panic!(e),
-    }
-}
-
-/// Attempts to download the deposit contract ABI from github if a local copy is not already
-/// present.
-pub fn init_deposit_contract_abi() -> Result<(), String> {
-    let abi_file = abi_dir().join(format!("{}_{}", SPEC_TAG, ABI_FILE));
-    let bytecode_file = abi_dir().join(format!("{}_{}", SPEC_TAG, BYTECODE_FILE));
-
-    if abi_file.exists() {
-        // Nothing to do.
-    } else {
-        match download_abi() {
-            Ok(mut response) => {
-                let mut abi_file = File::create(abi_file)
-                    .map_err(|e| format!("Failed to create local abi file: {:?}", e))?;
-                let mut bytecode_file = File::create(bytecode_file)
-                    .map_err(|e| format!("Failed to create local bytecode file: {:?}", e))?;
-
-                let contract: Value = response
-                    .json()
-                    .map_err(|e| format!("Respsonse is not a valid json {:?}", e))?;
-
-                let abi = contract
-                    .get("abi")
-                    .ok_or(format!("Response does not contain key: abi"))?
-                    .to_string();
-                abi_file
-                    .write(abi.as_bytes())
-                    .map_err(|e| format!("Failed to write http response to abi file: {:?}", e))?;
-
-                let bytecode = contract
-                    .get("bytecode")
-                    .ok_or(format!("Response does not contain key: bytecode"))?
-                    .to_string();
-                bytecode_file.write(bytecode.as_bytes()).map_err(|e| {
-                    format!("Failed to write http response to bytecode file: {:?}", e)
-                })?;
-            }
-            Err(e) => {
-                return Err(format!(
-                    "No abi file found. Failed to download from github: {:?}",
-                    e
-                ))
-            }
-        }
-    }
-
-    Ok(())
-}
-
-/// Attempts to download the deposit contract file from the Ethereum github.
-fn download_abi() -> Result<Response, String> {
-    reqwest::get(&format!(
-        "{}/{}/{}/deposit_contract/contracts/{}",
-        GITHUB_RAW, SPEC_REPO, SPEC_TAG, ABI_FILE
-    ))
-    .map_err(|e| format!("Failed to download deposit ABI from github: {:?}", e))
-}
-
-/// Returns the directory that will be used to store the deposit contract ABI.
-fn abi_dir() -> PathBuf {
-    let base = env::var("CARGO_MANIFEST_DIR")
-        .expect("should know manifest dir")
-        .parse::<PathBuf>()
-        .expect("should parse manifest dir as path")
-        .join("contract");
-
-    std::fs::create_dir_all(base.clone())
-        .expect("should be able to create abi directory in manifest");
-
-    base
-}

tests/eth1_test_rig/src/lib.rs

@@ -7,28 +7,21 @@
 //! some initial issues.
 mod ganache;
 
+use deposit_contract::{eth1_tx_data, ABI, BYTECODE, CONTRACT_DEPLOY_GAS, DEPOSIT_GAS};
 use futures::{stream, Future, IntoFuture, Stream};
 use ganache::GanacheInstance;
-use ssz::Encode;
 use std::time::{Duration, Instant};
 use tokio::{runtime::Runtime, timer::Delay};
 use types::DepositData;
 use types::{EthSpec, Hash256, Keypair, Signature};
 use web3::contract::{Contract, Options};
 use web3::transports::Http;
-use web3::types::{Address, U256};
+use web3::types::{Address, TransactionRequest, U256};
 use web3::{Transport, Web3};
 
 pub const DEPLOYER_ACCOUNTS_INDEX: usize = 0;
 pub const DEPOSIT_ACCOUNTS_INDEX: usize = 0;
 
-const CONTRACT_DEPLOY_GAS: usize = 4_000_000;
-const DEPOSIT_GAS: usize = 4_000_000;
-
-// Deposit contract
-pub const ABI: &[u8] = include_bytes!("../contract/v0.8.3_validator_registration.json");
-pub const BYTECODE: &[u8] = include_bytes!("../contract/v0.8.3_validator_registration.bytecode");
-
 /// Provides a dedicated ganache-cli instance with the deposit contract already deployed.
 pub struct GanacheEth1Instance {
     pub ganache: GanacheInstance,
@@ -138,6 +131,7 @@ impl DepositContract {
         deposit_data: DepositData,
     ) -> impl Future<Item = (), Error = String> {
         let contract = self.contract.clone();
+        let web3_1 = self.web3.clone();
 
         self.web3
             .eth()
@@ -149,19 +143,27 @@ impl DepositContract {
                     .cloned()
                     .ok_or_else(|| "Insufficient accounts for deposit".to_string())
             })
-            .and_then(move |from_address| {
-                let params = (
-                    deposit_data.pubkey.as_ssz_bytes(),
-                    deposit_data.withdrawal_credentials.as_ssz_bytes(),
-                    deposit_data.signature.as_ssz_bytes(),
-                );
-                let options = Options {
+            .and_then(move |from| {
+                let tx_request = TransactionRequest {
+                    from,
+                    to: Some(contract.address()),
                     gas: Some(U256::from(DEPOSIT_GAS)),
+                    gas_price: None,
                     value: Some(from_gwei(deposit_data.amount)),
-                    ..Options::default()
+                    // Note: the reason we use this `TransactionRequest` instead of just using the
+                    // function in `self.contract` is so that the `eth1_tx_data` function gets used
+                    // during testing.
+                    //
+                    // It's important that `eth1_tx_data` stays correct and does not suffer from
+                    // code-rot.
+                    data: eth1_tx_data(&deposit_data).map(Into::into).ok(),
+                    nonce: None,
+                    condition: None,
                 };
-                contract
-                    .call("deposit", params, from_address, options)
+
+                web3_1
+                    .eth()
+                    .send_transaction(tx_request)
                     .map_err(|e| format!("Failed to call deposit fn: {:?}", e))
             })
             .map(|_| ())