Directory Restructure (#1163)

* Move tests -> testing

* Directory restructure

* Update Cargo.toml during restructure

* Update Makefile during restructure

* Fix arbitrary path

testing/ef_tests/.gitignore

@@ -0,0 +1 @@
+/eth2.0-spec-tests

testing/ef_tests/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "ef_tests"
+version = "0.2.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[features]
+# `ef_tests` feature must be enabled to actually run the tests
+ef_tests = []
+fake_crypto = ["bls/fake_crypto"]
+
+[dependencies]
+bls = { path = "../../crypto/bls" }
+compare_fields = { path = "../../common/compare_fields" }
+ethereum-types = "0.9.1"
+hex = "0.4.2"
+rayon = "1.3.0"
+serde = "1.0.110"
+serde_derive = "1.0.110"
+serde_repr = "0.1.5"
+serde_yaml = "0.8.11"
+eth2_ssz = "0.1.2"
+eth2_ssz_derive = "0.1.0"
+tree_hash = "0.1.0"
+tree_hash_derive = "0.2.0"
+cached_tree_hash = { path = "../../consensus/cached_tree_hash" }
+state_processing = { path = "../../consensus/state_processing" }
+swap_or_not_shuffle = { path = "../../consensus/swap_or_not_shuffle" }
+types = { path = "../../consensus/types" }
+walkdir = "2.3.1"

testing/ef_tests/Makefile

@@ -0,0 +1,28 @@
+# Bump the test tag here and in .gitlab-ci.yml and CI will take care of updating the cached tarballs
+TESTS_TAG := v0.11.1
+TESTS = general minimal mainnet
+TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS))
+
+REPO_NAME := eth2.0-spec-tests
+OUTPUT_DIR := ./$(REPO_NAME)
+
+BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(TESTS_TAG)
+
+$(OUTPUT_DIR): $(TARBALLS)
+	mkdir $(OUTPUT_DIR)
+	for test_tarball in $^; do \
+		tar -xzf $$test_tarball -C $(OUTPUT_DIR);\
+	done
+
+%-$(TESTS_TAG).tar.gz:
+	wget $(BASE_URL)/$*.tar.gz -O $@
+
+clean-test-files:
+	rm -rf $(OUTPUT_DIR)
+
+clean-archives:
+	rm -f $(TARBALLS)
+
+clean: clean-test-files clean-archives
+
+.PHONY: clean clean-archives clean-test-files

testing/ef_tests/README.md

@@ -0,0 +1,42 @@
+# Ethereum 2.0 Specification Tests
+
+This crate parses and executes the test vectors at [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests).
+
+Functionality is achieved only via the `$ cargo test --features ef_tests` command.
+
+## Running the Tests
+
+Because the test vectors are very large, we do not download or run them by default.
+To download them, run (in this directory):
+
+```
+$ make
+```
+
+_Note: this may download hundreds of MB of compressed archives from the
+[ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/),
+which may expand into several GB of files._
+
+If successful, you should now have the extracted tests in `./eth2.0-spec-tests`.
+
+Run them with:
+
+```
+$ cargo test --features ef_tests
+```
+
+The tests won't run without the `ef_tests` feature enabled (this is to ensure that a top-level
+`cargo test --all` won't fail on missing files).
+
+## Saving Space
+
+When you download the tests, the downloaded archives will be kept in addition to the extracted
+files. You have several options for saving space:
+
+1. Delete the archives (`make clean-archives`), and keep the extracted files. Suitable for everyday
+   use, just don't re-run `make` or it will redownload the archives.
+2. Delete the extracted files (`make clean-test-files`), and keep the archives. Suitable for CI, or
+   temporarily saving space. If you re-run `make` it will extract the archives rather than
+   redownloading them.
+3. Delete everything (`make clean`). Good for updating to a new version, or if you no longer wish to
+   run the EF tests.

testing/ef_tests/src/bls_setting.rs

@@ -0,0 +1,29 @@
+use self::BlsSetting::*;
+use crate::error::Error;
+use serde_repr::Deserialize_repr;
+
+#[derive(Deserialize_repr, Debug, Clone, Copy)]
+#[repr(u8)]
+pub enum BlsSetting {
+    Flexible = 0,
+    Required = 1,
+    Ignored = 2,
+}
+
+impl Default for BlsSetting {
+    fn default() -> Self {
+        Flexible
+    }
+}
+
+impl BlsSetting {
+    /// Check the BLS setting and skip the test if it isn't compatible with the crypto config.
+    pub fn check(self) -> Result<(), Error> {
+        match self {
+            Flexible => Ok(()),
+            Required if !cfg!(feature = "fake_crypto") => Ok(()),
+            Ignored if cfg!(feature = "fake_crypto") => Ok(()),
+            _ => Err(Error::SkippedBls),
+        }
+    }
+}

testing/ef_tests/src/case_result.rs

@@ -0,0 +1,122 @@
+use super::*;
+use compare_fields::{CompareFields, Comparison, FieldComparison};
+use std::fmt::Debug;
+use std::path::{Path, PathBuf};
+use types::BeaconState;
+
+pub const MAX_VALUE_STRING_LEN: usize = 500;
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct CaseResult {
+    pub case_index: usize,
+    pub desc: String,
+    pub path: PathBuf,
+    pub result: Result<(), Error>,
+}
+
+impl CaseResult {
+    pub fn new(
+        case_index: usize,
+        path: &Path,
+        case: &impl Case,
+        result: Result<(), Error>,
+    ) -> Self {
+        CaseResult {
+            case_index,
+            desc: case.description(),
+            path: path.into(),
+            result,
+        }
+    }
+}
+
+/// Same as `compare_result_detailed`, however it drops the caches on both states before
+/// comparison.
+pub fn compare_beacon_state_results_without_caches<T: EthSpec, E: Debug>(
+    result: &mut Result<BeaconState<T>, E>,
+    expected: &mut Option<BeaconState<T>>,
+) -> Result<(), Error> {
+    if let (Ok(ref mut result), Some(ref mut expected)) = (result.as_mut(), expected.as_mut()) {
+        result.drop_all_caches();
+        expected.drop_all_caches();
+    }
+
+    compare_result_detailed(&result, &expected)
+}
+
+/// Same as `compare_result`, however utilizes the `CompareFields` trait to give a list of
+/// mismatching fields when `Ok(result) != Some(expected)`.
+pub fn compare_result_detailed<T, E>(
+    result: &Result<T, E>,
+    expected: &Option<T>,
+) -> Result<(), Error>
+where
+    T: PartialEq<T> + Debug + CompareFields,
+    E: Debug,
+{
+    match (result, expected) {
+        (Ok(result), Some(expected)) => {
+            let mut mismatching_fields: Vec<Comparison> = expected
+                .compare_fields(result)
+                .into_iter()
+                // Filter all out all fields that are equal.
+                .filter(Comparison::not_equal)
+                .collect();
+
+            mismatching_fields
+                .iter_mut()
+                .for_each(|f| f.retain_children(FieldComparison::not_equal));
+
+            if !mismatching_fields.is_empty() {
+                Err(Error::NotEqual(format!(
+                    "Fields not equal (a = expected, b = result): {:#?}",
+                    mismatching_fields
+                )))
+            } else {
+                Ok(())
+            }
+        }
+        _ => compare_result(result, expected),
+    }
+}
+
+/// Compares `result` with `expected`.
+///
+/// If `expected.is_none()` then `result` is expected to be `Err`. Otherwise, `T` in `result` and
+/// `expected` must be equal.
+pub fn compare_result<T, E>(result: &Result<T, E>, expected: &Option<T>) -> Result<(), Error>
+where
+    T: PartialEq<T> + Debug,
+    E: Debug,
+{
+    match (result, expected) {
+        // Pass: The should have failed and did fail.
+        (Err(_), None) => Ok(()),
+        // Fail: The test failed when it should have produced a result (fail).
+        (Err(e), Some(expected)) => Err(Error::NotEqual(format!(
+            "Got {:?} | Expected {:?}",
+            e,
+            fmt_val(expected)
+        ))),
+        // Fail: The test produced a result when it should have failed (fail).
+        (Ok(result), None) => Err(Error::DidntFail(format!("Got {:?}", fmt_val(result)))),
+        // Potential Pass: The test should have produced a result, and it did.
+        (Ok(result), Some(expected)) => {
+            if result == expected {
+                Ok(())
+            } else {
+                Err(Error::NotEqual(format!(
+                    "Got {:?} | Expected {:?}",
+                    fmt_val(result),
+                    fmt_val(expected)
+                )))
+            }
+        }
+    }
+}
+
+fn fmt_val<T: Debug>(val: T) -> String {
+    let mut string = format!("{:?}", val);
+    string.truncate(MAX_VALUE_STRING_LEN);
+    string
+}

testing/ef_tests/src/cases.rs

@@ -0,0 +1,71 @@
+use super::*;
+use rayon::prelude::*;
+use std::fmt::Debug;
+use std::path::{Path, PathBuf};
+
+mod bls_aggregate_sigs;
+mod bls_aggregate_verify;
+mod bls_fast_aggregate_verify;
+mod bls_sign_msg;
+mod bls_verify_msg;
+mod common;
+mod epoch_processing;
+mod genesis_initialization;
+mod genesis_validity;
+mod operations;
+mod sanity_blocks;
+mod sanity_slots;
+mod shuffling;
+mod ssz_generic;
+mod ssz_static;
+
+pub use bls_aggregate_sigs::*;
+pub use bls_aggregate_verify::*;
+pub use bls_fast_aggregate_verify::*;
+pub use bls_sign_msg::*;
+pub use bls_verify_msg::*;
+pub use common::SszStaticType;
+pub use epoch_processing::*;
+pub use genesis_initialization::*;
+pub use genesis_validity::*;
+pub use operations::*;
+pub use sanity_blocks::*;
+pub use sanity_slots::*;
+pub use shuffling::*;
+pub use ssz_generic::*;
+pub use ssz_static::*;
+
+pub trait LoadCase: Sized {
+    /// Load the test case from a test case directory.
+    fn load_from_dir(_path: &Path) -> Result<Self, Error>;
+}
+
+pub trait Case: Debug + Sync {
+    /// An optional field for implementing a custom description.
+    ///
+    /// Defaults to "no description".
+    fn description(&self) -> String {
+        "no description".to_string()
+    }
+
+    /// Execute a test and return the result.
+    ///
+    /// `case_index` reports the index of the case in the set of test cases. It is not strictly
+    /// necessary, but it's useful when troubleshooting specific failing tests.
+    fn result(&self, case_index: usize) -> Result<(), Error>;
+}
+
+#[derive(Debug)]
+pub struct Cases<T> {
+    pub test_cases: Vec<(PathBuf, T)>,
+}
+
+impl<T: Case> Cases<T> {
+    pub fn test_results(&self) -> Vec<CaseResult> {
+        self.test_cases
+            .into_par_iter()
+            .enumerate()
+            .map(|(i, (ref path, ref tc))| CaseResult::new(i, path, tc, tc.result(i)))
+            .collect()
+    }
+}

testing/ef_tests/src/cases/bls_aggregate_sigs.rs

@@ -0,0 +1,36 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
+use bls::{AggregateSignature, Signature};
+use serde_derive::Deserialize;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsAggregateSigs {
+    pub input: Vec<String>,
+    pub output: String,
+}
+
+impl BlsCase for BlsAggregateSigs {}
+
+impl Case for BlsAggregateSigs {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let mut aggregate_signature = AggregateSignature::new();
+
+        for key_str in &self.input {
+            let sig = hex::decode(&key_str[2..])
+                .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+            let sig = Signature::from_bytes(&sig)
+                .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+            aggregate_signature.add(&sig);
+        }
+
+        let output_bytes = Some(
+            hex::decode(&self.output[2..])
+                .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?,
+        );
+        let aggregate_signature = Ok(aggregate_signature.as_bytes());
+
+        compare_result::<Vec<u8>, Vec<u8>>(&aggregate_signature, &output_bytes)
+    }
+}

testing/ef_tests/src/cases/bls_aggregate_verify.rs

@@ -0,0 +1,59 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
+use bls::{AggregateSignature, PublicKey};
+use serde_derive::Deserialize;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsAggregatePair {
+    pub pubkey: PublicKey,
+    pub message: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsAggregateVerifyInput {
+    pub pairs: Vec<BlsAggregatePair>,
+    pub signature: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsAggregateVerify {
+    pub input: BlsAggregateVerifyInput,
+    pub output: bool,
+}
+
+impl BlsCase for BlsAggregateVerify {}
+
+impl Case for BlsAggregateVerify {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let messages = self
+            .input
+            .pairs
+            .iter()
+            .map(|pair| {
+                hex::decode(&pair.message[2..])
+                    .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))
+            })
+            .collect::<Result<Vec<Vec<_>>, _>>()?;
+
+        let message_refs = messages
+            .iter()
+            .map(|x| x.as_slice())
+            .collect::<Vec<&[u8]>>();
+
+        let pubkey_refs = self
+            .input
+            .pairs
+            .iter()
+            .map(|p| &p.pubkey)
+            .collect::<Vec<_>>();
+
+        let signature_ok = hex::decode(&self.input.signature[2..])
+            .ok()
+            .and_then(|bytes: Vec<u8>| AggregateSignature::from_bytes(&bytes).ok())
+            .map(|signature| signature.verify_multiple(&message_refs, &pubkey_refs))
+            .unwrap_or(false);
+
+        compare_result::<bool, ()>(&Ok(signature_ok), &Some(self.output))
+    }
+}

testing/ef_tests/src/cases/bls_fast_aggregate_verify.rs

@@ -0,0 +1,50 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
+use bls::{AggregatePublicKey, AggregateSignature, PublicKey, PublicKeyBytes};
+use serde_derive::Deserialize;
+use std::convert::TryInto;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsFastAggregateVerifyInput {
+    pub pubkeys: Vec<PublicKeyBytes>,
+    pub message: String,
+    pub signature: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsFastAggregateVerify {
+    pub input: BlsFastAggregateVerifyInput,
+    pub output: bool,
+}
+
+impl BlsCase for BlsFastAggregateVerify {}
+
+impl Case for BlsFastAggregateVerify {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let message = hex::decode(&self.input.message[2..])
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+        let signature_ok = self
+            .input
+            .pubkeys
+            .iter()
+            .try_fold(
+                AggregatePublicKey::new(),
+                |mut agg, pkb| -> Option<AggregatePublicKey> {
+                    let pk: Result<PublicKey, ssz::DecodeError> = pkb.try_into();
+                    agg.add(&pk.ok()?);
+                    Some(agg)
+                },
+            )
+            .and_then(|aggregate_pubkey| {
+                hex::decode(&self.input.signature[2..])
+                    .ok()
+                    .and_then(|bytes: Vec<u8>| AggregateSignature::from_bytes(&bytes).ok())
+                    .map(|signature| signature.verify(&message, &aggregate_pubkey))
+            })
+            .unwrap_or(false);
+
+        compare_result::<bool, ()>(&Ok(signature_ok), &Some(self.output))
+    }
+}

testing/ef_tests/src/cases/bls_sign_msg.rs

@@ -0,0 +1,38 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
+use bls::{SecretKey, Signature};
+use serde_derive::Deserialize;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsSignInput {
+    pub privkey: String,
+    pub message: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsSign {
+    pub input: BlsSignInput,
+    pub output: String,
+}
+
+impl BlsCase for BlsSign {}
+
+impl Case for BlsSign {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        // Convert private_key and message to required types
+        let sk = hex::decode(&self.input.privkey[2..])
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+        let sk = SecretKey::from_bytes(&sk).unwrap();
+        let msg = hex::decode(&self.input.message[2..])
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+        let signature = Signature::new(&msg, &sk);
+
+        // Convert the output to one set of bytes
+        let decoded = hex::decode(&self.output[2..])
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+        compare_result::<Vec<u8>, Vec<u8>>(&Ok(signature.as_bytes()), &Some(decoded))
+    }
+}

testing/ef_tests/src/cases/bls_verify_msg.rs

@@ -0,0 +1,35 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
+use bls::{PublicKey, Signature, SignatureBytes};
+use serde_derive::Deserialize;
+use std::convert::TryInto;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsVerifyInput {
+    pub pubkey: PublicKey,
+    pub message: String,
+    pub signature: SignatureBytes,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsVerify {
+    pub input: BlsVerifyInput,
+    pub output: bool,
+}
+
+impl BlsCase for BlsVerify {}
+
+impl Case for BlsVerify {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let message = hex::decode(&self.input.message[2..])
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+        let signature_ok = (&self.input.signature)
+            .try_into()
+            .map(|signature: Signature| signature.verify(&message, &self.input.pubkey))
+            .unwrap_or(false);
+
+        compare_result::<bool, ()>(&Ok(signature_ok), &Some(self.output))
+    }
+}

testing/ef_tests/src/cases/common.rs

@@ -0,0 +1,72 @@
+use crate::cases::LoadCase;
+use crate::decode::yaml_decode_file;
+use crate::error::Error;
+use serde_derive::Deserialize;
+use ssz::{Decode, Encode};
+use ssz_derive::{Decode, Encode};
+use std::convert::TryFrom;
+use std::fmt::Debug;
+use std::path::Path;
+use tree_hash::TreeHash;
+
+/// Trait for all BLS cases to eliminate some boilerplate.
+pub trait BlsCase: serde::de::DeserializeOwned {}
+
+impl<T: BlsCase> LoadCase for T {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        yaml_decode_file(&path.join("data.yaml"))
+    }
+}
+
+/// Macro to wrap U128 and U256 so they deserialize correctly.
+macro_rules! uint_wrapper {
+    ($wrapper_name:ident, $wrapped_type:ty) => {
+        #[derive(Debug, Clone, Copy, Default, PartialEq, Decode, Encode, Deserialize)]
+        #[serde(try_from = "String")]
+        pub struct $wrapper_name {
+            pub x: $wrapped_type,
+        }
+
+        impl TryFrom<String> for $wrapper_name {
+            type Error = String;
+
+            fn try_from(s: String) -> Result<Self, Self::Error> {
+                <$wrapped_type>::from_dec_str(&s)
+                    .map(|x| Self { x })
+                    .map_err(|e| format!("{:?}", e))
+            }
+        }
+
+        impl tree_hash::TreeHash for $wrapper_name {
+            fn tree_hash_type() -> tree_hash::TreeHashType {
+                <$wrapped_type>::tree_hash_type()
+            }
+
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                self.x.tree_hash_packed_encoding()
+            }
+
+            fn tree_hash_packing_factor() -> usize {
+                <$wrapped_type>::tree_hash_packing_factor()
+            }
+
+            fn tree_hash_root(&self) -> tree_hash::Hash256 {
+                self.x.tree_hash_root()
+            }
+        }
+    };
+}
+
+uint_wrapper!(TestU128, ethereum_types::U128);
+uint_wrapper!(TestU256, ethereum_types::U256);
+
+/// Trait alias for all deez bounds
+pub trait SszStaticType:
+    serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync
+{
+}
+
+impl<T> SszStaticType for T where
+    T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync
+{
+}

testing/ef_tests/src/cases/epoch_processing.rs

@@ -0,0 +1,148 @@
+use super::*;
+use crate::bls_setting::BlsSetting;
+use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
+use crate::type_name;
+use crate::type_name::TypeName;
+use serde_derive::Deserialize;
+use state_processing::per_epoch_processing::{
+    errors::EpochProcessingError, process_final_updates, process_justification_and_finalization,
+    process_registry_updates, process_rewards_and_penalties, process_slashings,
+    validator_statuses::ValidatorStatuses,
+};
+use std::marker::PhantomData;
+use std::path::{Path, PathBuf};
+use types::{BeaconState, ChainSpec, EthSpec};
+
+#[derive(Debug, Clone, Default, Deserialize)]
+pub struct Metadata {
+    pub description: Option<String>,
+    pub bls_setting: Option<BlsSetting>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct EpochProcessing<E: EthSpec, T: EpochTransition<E>> {
+    pub path: PathBuf,
+    pub metadata: Metadata,
+    pub pre: BeaconState<E>,
+    pub post: Option<BeaconState<E>>,
+    #[serde(skip_deserializing)]
+    _phantom: PhantomData<T>,
+}
+
+pub trait EpochTransition<E: EthSpec>: TypeName + Debug + Sync {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError>;
+}
+
+#[derive(Debug)]
+pub struct JustificationAndFinalization;
+#[derive(Debug)]
+pub struct RewardsAndPenalties;
+#[derive(Debug)]
+pub struct RegistryUpdates;
+#[derive(Debug)]
+pub struct Slashings;
+#[derive(Debug)]
+pub struct FinalUpdates;
+
+type_name!(
+    JustificationAndFinalization,
+    "justification_and_finalization"
+);
+type_name!(RewardsAndPenalties, "rewards_and_penalties");
+type_name!(RegistryUpdates, "registry_updates");
+type_name!(Slashings, "slashings");
+type_name!(FinalUpdates, "final_updates");
+
+impl<E: EthSpec> EpochTransition<E> for JustificationAndFinalization {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        let mut validator_statuses = ValidatorStatuses::new(state, spec)?;
+        validator_statuses.process_attestations(state, spec)?;
+        process_justification_and_finalization(state, &validator_statuses.total_balances)
+    }
+}
+
+impl<E: EthSpec> EpochTransition<E> for RewardsAndPenalties {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        let mut validator_statuses = ValidatorStatuses::new(state, spec)?;
+        validator_statuses.process_attestations(state, spec)?;
+        process_rewards_and_penalties(state, &mut validator_statuses, spec)
+    }
+}
+
+impl<E: EthSpec> EpochTransition<E> for RegistryUpdates {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        process_registry_updates(state, spec)
+    }
+}
+
+impl<E: EthSpec> EpochTransition<E> for Slashings {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
+        validator_statuses.process_attestations(&state, spec)?;
+        process_slashings(
+            state,
+            validator_statuses.total_balances.current_epoch(),
+            spec,
+        )?;
+        Ok(())
+    }
+}
+
+impl<E: EthSpec> EpochTransition<E> for FinalUpdates {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        process_final_updates(state, spec)
+    }
+}
+
+impl<E: EthSpec, T: EpochTransition<E>> LoadCase for EpochProcessing<E, T> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let metadata_path = path.join("meta.yaml");
+        let metadata: Metadata = if metadata_path.is_file() {
+            yaml_decode_file(&metadata_path)?
+        } else {
+            Metadata::default()
+        };
+        let pre = ssz_decode_file(&path.join("pre.ssz"))?;
+        let post_file = path.join("post.ssz");
+        let post = if post_file.is_file() {
+            Some(ssz_decode_file(&post_file)?)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            path: path.into(),
+            metadata,
+            pre,
+            post,
+            _phantom: PhantomData,
+        })
+    }
+}
+
+impl<E: EthSpec, T: EpochTransition<E>> Case for EpochProcessing<E, T> {
+    fn description(&self) -> String {
+        self.metadata
+            .description
+            .clone()
+            .unwrap_or_else(String::new)
+    }
+
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let mut state = self.pre.clone();
+        let mut expected = self.post.clone();
+
+        let spec = &E::default_spec();
+
+        let mut result = (|| {
+            // Processing requires the committee caches.
+            state.build_all_committee_caches(spec)?;
+
+            T::run(&mut state, spec).map(|_| state)
+        })();
+
+        compare_beacon_state_results_without_caches(&mut result, &mut expected)
+    }
+}

testing/ef_tests/src/cases/genesis_initialization.rs

@@ -0,0 +1,62 @@
+use super::*;
+use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
+use serde_derive::Deserialize;
+use state_processing::initialize_beacon_state_from_eth1;
+use std::path::PathBuf;
+use types::{BeaconState, Deposit, EthSpec, Hash256};
+
+#[derive(Debug, Clone, Deserialize)]
+struct Metadata {
+    deposits_count: usize,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct GenesisInitialization<E: EthSpec> {
+    pub path: PathBuf,
+    pub eth1_block_hash: Hash256,
+    pub eth1_timestamp: u64,
+    pub deposits: Vec<Deposit>,
+    pub state: Option<BeaconState<E>>,
+}
+
+impl<E: EthSpec> LoadCase for GenesisInitialization<E> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let eth1_block_hash = ssz_decode_file(&path.join("eth1_block_hash.ssz"))?;
+        let eth1_timestamp = yaml_decode_file(&path.join("eth1_timestamp.yaml"))?;
+        let meta: Metadata = yaml_decode_file(&path.join("meta.yaml"))?;
+        let deposits: Vec<Deposit> = (0..meta.deposits_count)
+            .map(|i| {
+                let filename = format!("deposits_{}.ssz", i);
+                ssz_decode_file(&path.join(filename))
+            })
+            .collect::<Result<_, _>>()?;
+        let state = ssz_decode_file(&path.join("state.ssz"))?;
+
+        Ok(Self {
+            path: path.into(),
+            eth1_block_hash,
+            eth1_timestamp,
+            deposits,
+            state: Some(state),
+        })
+    }
+}
+
+impl<E: EthSpec> Case for GenesisInitialization<E> {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let spec = &E::default_spec();
+
+        let mut result = initialize_beacon_state_from_eth1(
+            self.eth1_block_hash,
+            self.eth1_timestamp,
+            self.deposits.clone(),
+            spec,
+        );
+
+        let mut expected = self.state.clone();
+
+        compare_beacon_state_results_without_caches(&mut result, &mut expected)
+    }
+}

testing/ef_tests/src/cases/genesis_validity.rs

@@ -0,0 +1,39 @@
+use super::*;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
+use serde_derive::Deserialize;
+use state_processing::is_valid_genesis_state;
+use std::path::Path;
+use types::{BeaconState, EthSpec};
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct GenesisValidity<E: EthSpec> {
+    pub genesis: BeaconState<E>,
+    pub is_valid: bool,
+}
+
+impl<E: EthSpec> LoadCase for GenesisValidity<E> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let genesis = ssz_decode_file(&path.join("genesis.ssz"))?;
+        let is_valid = yaml_decode_file(&path.join("is_valid.yaml"))?;
+
+        Ok(Self { genesis, is_valid })
+    }
+}
+
+impl<E: EthSpec> Case for GenesisValidity<E> {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let spec = &E::default_spec();
+
+        let is_valid = is_valid_genesis_state(&self.genesis, spec);
+
+        if is_valid == self.is_valid {
+            Ok(())
+        } else {
+            Err(Error::NotEqual(format!(
+                "Got {}, expected {}",
+                is_valid, self.is_valid
+            )))
+        }
+    }
+}

testing/ef_tests/src/cases/operations.rs

@@ -0,0 +1,180 @@
+use super::*;
+use crate::bls_setting::BlsSetting;
+use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
+use crate::type_name::TypeName;
+use serde_derive::Deserialize;
+use ssz::Decode;
+use state_processing::per_block_processing::{
+    errors::BlockProcessingError, process_attestations, process_attester_slashings,
+    process_block_header, process_deposits, process_exits, process_proposer_slashings,
+    VerifySignatures,
+};
+use std::fmt::Debug;
+use std::path::Path;
+use types::{
+    Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec,
+    ProposerSlashing, SignedVoluntaryExit,
+};
+
+#[derive(Debug, Clone, Default, Deserialize)]
+struct Metadata {
+    description: Option<String>,
+    bls_setting: Option<BlsSetting>,
+}
+
+#[derive(Debug, Clone)]
+pub struct Operations<E: EthSpec, O: Operation<E>> {
+    metadata: Metadata,
+    pub pre: BeaconState<E>,
+    pub operation: O,
+    pub post: Option<BeaconState<E>>,
+}
+
+pub trait Operation<E: EthSpec>: Decode + TypeName + Debug + Sync {
+    fn handler_name() -> String {
+        Self::name().to_lowercase()
+    }
+
+    fn filename() -> String {
+        format!("{}.ssz", Self::handler_name())
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError>;
+}
+
+impl<E: EthSpec> Operation<E> for Attestation<E> {
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_attestations(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for AttesterSlashing<E> {
+    fn handler_name() -> String {
+        "attester_slashing".into()
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_attester_slashings(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for Deposit {
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_deposits(state, &[self.clone()], spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for ProposerSlashing {
+    fn handler_name() -> String {
+        "proposer_slashing".into()
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_proposer_slashings(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for SignedVoluntaryExit {
+    fn handler_name() -> String {
+        "voluntary_exit".into()
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_exits(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for BeaconBlock<E> {
+    fn handler_name() -> String {
+        "block_header".into()
+    }
+
+    fn filename() -> String {
+        "block.ssz".into()
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        Ok(process_block_header(state, self, spec)?)
+    }
+}
+
+impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let metadata_path = path.join("meta.yaml");
+        let metadata: Metadata = if metadata_path.is_file() {
+            yaml_decode_file(&metadata_path)?
+        } else {
+            Metadata::default()
+        };
+        let pre = ssz_decode_file(&path.join("pre.ssz"))?;
+        let operation = ssz_decode_file(&path.join(O::filename()))?;
+        let post_filename = path.join("post.ssz");
+        let post = if post_filename.is_file() {
+            Some(ssz_decode_file(&post_filename)?)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            metadata,
+            pre,
+            operation,
+            post,
+        })
+    }
+}
+
+impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
+    fn description(&self) -> String {
+        self.metadata
+            .description
+            .clone()
+            .unwrap_or_else(String::new)
+    }
+
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        self.metadata.bls_setting.unwrap_or_default().check()?;
+
+        let spec = &E::default_spec();
+        let mut state = self.pre.clone();
+        let mut expected = self.post.clone();
+
+        // Processing requires the committee caches.
+        state
+            .build_all_committee_caches(spec)
+            .expect("committee caches OK");
+
+        let mut result = self.operation.apply_to(&mut state, spec).map(|()| state);
+
+        compare_beacon_state_results_without_caches(&mut result, &mut expected)
+    }
+}

testing/ef_tests/src/cases/sanity_blocks.rs

@@ -0,0 +1,126 @@
+use super::*;
+use crate::bls_setting::BlsSetting;
+use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
+use serde_derive::Deserialize;
+use state_processing::{
+    per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy,
+};
+use types::{BeaconState, EthSpec, RelativeEpoch, SignedBeaconBlock};
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct Metadata {
+    pub description: Option<String>,
+    pub bls_setting: Option<BlsSetting>,
+    pub blocks_count: usize,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct SanityBlocks<E: EthSpec> {
+    pub metadata: Metadata,
+    pub pre: BeaconState<E>,
+    pub blocks: Vec<SignedBeaconBlock<E>>,
+    pub post: Option<BeaconState<E>>,
+}
+
+impl<E: EthSpec> LoadCase for SanityBlocks<E> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let metadata: Metadata = yaml_decode_file(&path.join("meta.yaml"))?;
+        let pre = ssz_decode_file(&path.join("pre.ssz"))?;
+        let blocks: Vec<SignedBeaconBlock<E>> = (0..metadata.blocks_count)
+            .map(|i| {
+                let filename = format!("blocks_{}.ssz", i);
+                ssz_decode_file(&path.join(filename))
+            })
+            .collect::<Result<_, _>>()?;
+        let post_file = path.join("post.ssz");
+        let post = if post_file.is_file() {
+            Some(ssz_decode_file(&post_file)?)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            metadata,
+            pre,
+            blocks,
+            post,
+        })
+    }
+}
+
+impl<E: EthSpec> Case for SanityBlocks<E> {
+    fn description(&self) -> String {
+        self.metadata
+            .description
+            .clone()
+            .unwrap_or_else(String::new)
+    }
+
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        self.metadata.bls_setting.unwrap_or_default().check()?;
+
+        let mut bulk_state = self.pre.clone();
+        let mut expected = self.post.clone();
+        let spec = &E::default_spec();
+
+        // Processing requires the epoch cache.
+        bulk_state.build_all_caches(spec).unwrap();
+
+        // Spawning a second state to call the VerifyIndiviual strategy to avoid bitrot.
+        // See https://github.com/sigp/lighthouse/issues/742.
+        let mut indiv_state = bulk_state.clone();
+
+        let result = self
+            .blocks
+            .iter()
+            .try_for_each(|signed_block| {
+                let block = &signed_block.message;
+                while bulk_state.slot < block.slot {
+                    per_slot_processing(&mut bulk_state, None, spec).unwrap();
+                    per_slot_processing(&mut indiv_state, None, spec).unwrap();
+                }
+
+                bulk_state
+                    .build_committee_cache(RelativeEpoch::Current, spec)
+                    .unwrap();
+
+                indiv_state
+                    .build_committee_cache(RelativeEpoch::Current, spec)
+                    .unwrap();
+
+                per_block_processing(
+                    &mut indiv_state,
+                    signed_block,
+                    None,
+                    BlockSignatureStrategy::VerifyIndividual,
+                    spec,
+                )?;
+
+                per_block_processing(
+                    &mut bulk_state,
+                    signed_block,
+                    None,
+                    BlockSignatureStrategy::VerifyBulk,
+                    spec,
+                )?;
+
+                if block.state_root == bulk_state.canonical_root()
+                    && block.state_root == indiv_state.canonical_root()
+                {
+                    Ok(())
+                } else {
+                    Err(BlockProcessingError::StateRootMismatch)
+                }
+            })
+            .map(|_| (bulk_state, indiv_state));
+
+        let (mut bulk_result, mut indiv_result) = match result {
+            Err(e) => (Err(e.clone()), Err(e)),
+            Ok(res) => (Ok(res.0), Ok(res.1)),
+        };
+        compare_beacon_state_results_without_caches(&mut indiv_result, &mut expected)?;
+        compare_beacon_state_results_without_caches(&mut bulk_result, &mut expected)
+    }
+}

testing/ef_tests/src/cases/sanity_slots.rs

@@ -0,0 +1,74 @@
+use super::*;
+use crate::bls_setting::BlsSetting;
+use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
+use serde_derive::Deserialize;
+use state_processing::per_slot_processing;
+use types::{BeaconState, EthSpec};
+
+#[derive(Debug, Clone, Default, Deserialize)]
+pub struct Metadata {
+    pub description: Option<String>,
+    pub bls_setting: Option<BlsSetting>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct SanitySlots<E: EthSpec> {
+    pub metadata: Metadata,
+    pub pre: BeaconState<E>,
+    pub slots: u64,
+    pub post: Option<BeaconState<E>>,
+}
+
+impl<E: EthSpec> LoadCase for SanitySlots<E> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let metadata_path = path.join("meta.yaml");
+        let metadata: Metadata = if metadata_path.is_file() {
+            yaml_decode_file(&metadata_path)?
+        } else {
+            Metadata::default()
+        };
+        let pre = ssz_decode_file(&path.join("pre.ssz"))?;
+        let slots: u64 = yaml_decode_file(&path.join("slots.yaml"))?;
+        let post_file = path.join("post.ssz");
+        let post = if post_file.is_file() {
+            Some(ssz_decode_file(&post_file)?)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            metadata,
+            pre,
+            slots,
+            post,
+        })
+    }
+}
+
+impl<E: EthSpec> Case for SanitySlots<E> {
+    fn description(&self) -> String {
+        self.metadata
+            .description
+            .clone()
+            .unwrap_or_else(String::new)
+    }
+
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        self.metadata.bls_setting.unwrap_or_default().check()?;
+
+        let mut state = self.pre.clone();
+        let mut expected = self.post.clone();
+        let spec = &E::default_spec();
+
+        // Processing requires the epoch cache.
+        state.build_all_caches(spec).unwrap();
+
+        let mut result = (0..self.slots)
+            .try_for_each(|_| per_slot_processing(&mut state, None, spec).map(|_| ()))
+            .map(|_| state);
+
+        compare_beacon_state_results_without_caches(&mut result, &mut expected)
+    }
+}

testing/ef_tests/src/cases/shuffling.rs

@@ -0,0 +1,48 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::decode::yaml_decode_file;
+use serde_derive::Deserialize;
+use std::marker::PhantomData;
+use swap_or_not_shuffle::{compute_shuffled_index, shuffle_list};
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct Shuffling<T> {
+    pub seed: String,
+    pub count: usize,
+    pub mapping: Vec<usize>,
+    #[serde(skip)]
+    _phantom: PhantomData<T>,
+}
+
+impl<T: EthSpec> LoadCase for Shuffling<T> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        yaml_decode_file(&path.join("mapping.yaml"))
+    }
+}
+
+impl<T: EthSpec> Case for Shuffling<T> {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        if self.count == 0 {
+            compare_result::<_, Error>(&Ok(vec![]), &Some(self.mapping.clone()))?;
+        } else {
+            let spec = T::default_spec();
+            let seed = hex::decode(&self.seed[2..])
+                .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+            // Test compute_shuffled_index
+            let shuffling = (0..self.count)
+                .map(|i| {
+                    compute_shuffled_index(i, self.count, &seed, spec.shuffle_round_count).unwrap()
+                })
+                .collect();
+            compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?;
+
+            // Test "shuffle_list"
+            let input: Vec<usize> = (0..self.count).collect();
+            let shuffling = shuffle_list(input, spec.shuffle_round_count, &seed, false).unwrap();
+            compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?;
+        }
+
+        Ok(())
+    }
+}

testing/ef_tests/src/cases/ssz_generic.rs

@@ -0,0 +1,299 @@
+#![allow(non_snake_case)]
+
+use super::*;
+use crate::cases::common::{SszStaticType, TestU128, TestU256};
+use crate::cases::ssz_static::{check_serialization, check_tree_hash};
+use crate::decode::yaml_decode_file;
+use serde::{de::Error as SerdeError, Deserializer};
+use serde_derive::Deserialize;
+use ssz_derive::{Decode, Encode};
+use std::fs;
+use std::path::{Path, PathBuf};
+use tree_hash_derive::TreeHash;
+use types::typenum::*;
+use types::{BitList, BitVector, FixedVector, VariableList};
+
+#[derive(Debug, Clone, Deserialize)]
+struct Metadata {
+    root: String,
+    signing_root: Option<String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct SszGeneric {
+    path: PathBuf,
+    handler_name: String,
+    case_name: String,
+}
+
+impl LoadCase for SszGeneric {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let components = path
+            .components()
+            .map(|c| c.as_os_str().to_string_lossy().into_owned())
+            .rev()
+            .collect::<Vec<_>>();
+        // Test case name is last
+        let case_name = components[0].clone();
+        // Handler name is third last, before suite name and case name
+        let handler_name = components[2].clone();
+        Ok(Self {
+            path: path.into(),
+            handler_name,
+            case_name,
+        })
+    }
+}
+
+macro_rules! type_dispatch {
+    ($function:ident,
+     ($($arg:expr),*),
+     $base_ty:tt,
+     <$($param_ty:ty),*>,
+     [ $value:expr => primitive_type ] $($rest:tt)*) => {
+        match $value {
+            "bool" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* bool>, $($rest)*),
+            "uint8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u8>, $($rest)*),
+            "uint16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u16>, $($rest)*),
+            "uint32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u32>, $($rest)*),
+            "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*),
+            "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU128>, $($rest)*),
+            "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU256>, $($rest)*),
+            _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
+        }
+    };
+    ($function:ident,
+     ($($arg:expr),*),
+     $base_ty:tt,
+     <$($param_ty:ty),*>,
+     [ $value:expr => typenum ] $($rest:tt)*) => {
+        match $value {
+            // DO YOU LIKE NUMBERS?
+            "0" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U0>, $($rest)*),
+            "1" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1>, $($rest)*),
+            "2" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2>, $($rest)*),
+            "3" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U3>, $($rest)*),
+            "4" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4>, $($rest)*),
+            "5" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U5>, $($rest)*),
+            "6" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U6>, $($rest)*),
+            "7" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U7>, $($rest)*),
+            "8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8>, $($rest)*),
+            "9" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U9>, $($rest)*),
+            "16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U16>, $($rest)*),
+            "31" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U31>, $($rest)*),
+            "32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U32>, $($rest)*),
+            "64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U64>, $($rest)*),
+            "128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U128>, $($rest)*),
+            "256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U256>, $($rest)*),
+            "512" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U512>, $($rest)*),
+            "513" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U513>, $($rest)*),
+            "1024" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1024>, $($rest)*),
+            "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*),
+            "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*),
+            "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*),
+            _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
+        }
+    };
+    ($function:ident,
+     ($($arg:expr),*),
+     $base_ty:tt,
+     <$($param_ty:ty),*>,
+     [ $value:expr => test_container ] $($rest:tt)*) => {
+        match $value {
+            "SingleFieldTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SingleFieldTestStruct>, $($rest)*),
+            "SmallTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SmallTestStruct>, $($rest)*),
+            "FixedTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* FixedTestStruct>, $($rest)*),
+            "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*),
+            "ComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ComplexTestStruct>, $($rest)*),
+            "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*),
+            _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
+        }
+    };
+    // No base type: apply type params to function
+    ($function:ident, ($($arg:expr),*), _, <$($param_ty:ty),*>,) => {
+        $function::<$($param_ty),*>($($arg),*)
+    };
+    ($function:ident, ($($arg:expr),*), $base_type_name:ident, <$($param_ty:ty),*>,) => {
+        $function::<$base_type_name<$($param_ty),*>>($($arg),*)
+    }
+}
+
+impl Case for SszGeneric {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let parts = self.case_name.split('_').collect::<Vec<_>>();
+
+        match self.handler_name.as_str() {
+            "basic_vector" => {
+                let elem_ty = parts[1];
+                let length = parts[2];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    FixedVector,
+                    <>,
+                    [elem_ty => primitive_type]
+                    [length => typenum]
+                )?;
+            }
+            "bitlist" => {
+                let mut limit = parts[1];
+
+                // Test format is inconsistent, pretend the limit is 32 (arbitrary)
+                // https://github.com/ethereum/eth2.0-spec-tests
+                if limit == "no" {
+                    limit = "32";
+                }
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    BitList,
+                    <>,
+                    [limit => typenum]
+                )?;
+            }
+            "bitvector" => {
+                let length = parts[1];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    BitVector,
+                    <>,
+                    [length => typenum]
+                )?;
+            }
+            "boolean" => {
+                ssz_generic_test::<bool>(&self.path)?;
+            }
+            "uints" => {
+                let type_name = "uint".to_owned() + parts[1];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    _,
+                    <>,
+                    [type_name.as_str() => primitive_type]
+                )?;
+            }
+            "containers" => {
+                let type_name = parts[0];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    _,
+                    <>,
+                    [type_name => test_container]
+                )?;
+            }
+            _ => panic!("unsupported handler: {}", self.handler_name),
+        }
+        Ok(())
+    }
+}
+
+fn ssz_generic_test<T: SszStaticType>(path: &Path) -> Result<(), Error> {
+    let meta_path = path.join("meta.yaml");
+    let meta: Option<Metadata> = if meta_path.is_file() {
+        Some(yaml_decode_file(&meta_path)?)
+    } else {
+        None
+    };
+
+    let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists");
+
+    let value_path = path.join("value.yaml");
+    let value: Option<T> = if value_path.is_file() {
+        Some(yaml_decode_file(&value_path)?)
+    } else {
+        None
+    };
+
+    // Valid
+    // TODO: signing root (annoying because of traits)
+    if let Some(value) = value {
+        check_serialization(&value, &serialized)?;
+
+        if let Some(ref meta) = meta {
+            check_tree_hash(&meta.root, value.tree_hash_root().as_bytes())?;
+        }
+    }
+    // Invalid
+    else if let Ok(decoded) = T::from_ssz_bytes(&serialized) {
+        return Err(Error::DidntFail(format!(
+            "Decoded invalid bytes into: {:?}",
+            decoded
+        )));
+    }
+    Ok(())
+}
+
+// Containers for SSZ generic tests
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct SingleFieldTestStruct {
+    A: u8,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct SmallTestStruct {
+    A: u16,
+    B: u16,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct FixedTestStruct {
+    A: u8,
+    B: u64,
+    C: u32,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct VarTestStruct {
+    A: u16,
+    B: VariableList<u16, U1024>,
+    C: u8,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct ComplexTestStruct {
+    A: u16,
+    B: VariableList<u16, U128>,
+    C: u8,
+    #[serde(deserialize_with = "byte_list_from_hex_str")]
+    D: VariableList<u8, U256>,
+    E: VarTestStruct,
+    F: FixedVector<FixedTestStruct, U4>,
+    G: FixedVector<VarTestStruct, U2>,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct BitsStruct {
+    A: BitList<U5>,
+    B: BitVector<U2>,
+    C: BitVector<U1>,
+    D: BitList<U6>,
+    E: BitVector<U8>,
+}
+
+fn byte_list_from_hex_str<'de, D, N: Unsigned>(
+    deserializer: D,
+) -> Result<VariableList<u8, N>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s: String = serde::de::Deserialize::deserialize(deserializer)?;
+    let decoded: Vec<u8> = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?;
+
+    if decoded.len() > N::to_usize() {
+        Err(D::Error::custom(format!(
+            "Too many values for list, got: {}, limit: {}",
+            decoded.len(),
+            N::to_usize()
+        )))
+    } else {
+        Ok(decoded.into())
+    }
+}

testing/ef_tests/src/cases/ssz_static.rs

@@ -0,0 +1,105 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::cases::common::SszStaticType;
+use crate::decode::yaml_decode_file;
+use cached_tree_hash::{CacheArena, CachedTreeHash};
+use serde_derive::Deserialize;
+use std::fs;
+use std::marker::PhantomData;
+use types::Hash256;
+
+#[derive(Debug, Clone, Deserialize)]
+struct SszStaticRoots {
+    root: String,
+    signing_root: Option<String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct SszStatic<T> {
+    roots: SszStaticRoots,
+    serialized: Vec<u8>,
+    value: T,
+}
+
+#[derive(Debug, Clone)]
+pub struct SszStaticTHC<T, C> {
+    roots: SszStaticRoots,
+    serialized: Vec<u8>,
+    value: T,
+    _phantom: PhantomData<C>,
+}
+
+fn load_from_dir<T: SszStaticType>(path: &Path) -> Result<(SszStaticRoots, Vec<u8>, T), Error> {
+    let roots = yaml_decode_file(&path.join("roots.yaml"))?;
+    let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists");
+    let value = yaml_decode_file(&path.join("value.yaml"))?;
+
+    Ok((roots, serialized, value))
+}
+
+impl<T: SszStaticType> LoadCase for SszStatic<T> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        load_from_dir(path).map(|(roots, serialized, value)| Self {
+            roots,
+            serialized,
+            value,
+        })
+    }
+}
+
+impl<T: SszStaticType + CachedTreeHash<C>, C: Debug + Sync> LoadCase for SszStaticTHC<T, C> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        load_from_dir(path).map(|(roots, serialized, value)| Self {
+            roots,
+            serialized,
+            value,
+            _phantom: PhantomData,
+        })
+    }
+}
+
+pub fn check_serialization<T: SszStaticType>(value: &T, serialized: &[u8]) -> Result<(), Error> {
+    // Check serialization
+    let serialized_result = value.as_ssz_bytes();
+    compare_result::<usize, Error>(&Ok(value.ssz_bytes_len()), &Some(serialized.len()))?;
+    compare_result::<Vec<u8>, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?;
+
+    // Check deserialization
+    let deserialized_result = T::from_ssz_bytes(serialized);
+    compare_result(&deserialized_result, &Some(value.clone()))?;
+
+    Ok(())
+}
+
+pub fn check_tree_hash(expected_str: &str, actual_root: &[u8]) -> Result<(), Error> {
+    let expected_root = hex::decode(&expected_str[2..])
+        .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    let expected_root = Hash256::from_slice(&expected_root);
+    let tree_hash_root = Hash256::from_slice(actual_root);
+    compare_result::<Hash256, Error>(&Ok(tree_hash_root), &Some(expected_root))
+}
+
+impl<T: SszStaticType> Case for SszStatic<T> {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        check_serialization(&self.value, &self.serialized)?;
+        check_tree_hash(&self.roots.root, self.value.tree_hash_root().as_bytes())?;
+        Ok(())
+    }
+}
+
+impl<T: SszStaticType + CachedTreeHash<C>, C: Debug + Sync> Case for SszStaticTHC<T, C> {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        check_serialization(&self.value, &self.serialized)?;
+        check_tree_hash(&self.roots.root, self.value.tree_hash_root().as_bytes())?;
+
+        let arena = &mut CacheArena::default();
+        let mut cache = self.value.new_tree_hash_cache(arena);
+        let cached_tree_hash_root = self
+            .value
+            .recalculate_tree_hash_root(arena, &mut cache)
+            .unwrap();
+        check_tree_hash(&self.roots.root, cached_tree_hash_root.as_bytes())?;
+
+        Ok(())
+    }
+}

testing/ef_tests/src/decode.rs

@@ -0,0 +1,31 @@
+use super::*;
+use std::fs;
+use std::path::Path;
+
+pub fn yaml_decode<T: serde::de::DeserializeOwned>(string: &str) -> Result<T, Error> {
+    serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))
+}
+
+pub fn yaml_decode_file<T: serde::de::DeserializeOwned>(path: &Path) -> Result<T, Error> {
+    fs::read_to_string(path)
+        .map_err(|e| {
+            Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e))
+        })
+        .and_then(|s| yaml_decode(&s))
+}
+
+pub fn ssz_decode_file<T: ssz::Decode>(path: &Path) -> Result<T, Error> {
+    fs::read(path)
+        .map_err(|e| {
+            Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e))
+        })
+        .and_then(|s| {
+            T::from_ssz_bytes(&s).map_err(|e| {
+                Error::FailedToParseTest(format!(
+                    "Unable to parse SSZ at {}: {:?}",
+                    path.display(),
+                    e
+                ))
+            })
+        })
+}

testing/ef_tests/src/error.rs

@@ -0,0 +1,41 @@
+#[derive(Debug, PartialEq, Clone)]
+pub enum Error {
+    /// The value in the test didn't match our value.
+    NotEqual(String),
+    /// The test specified a failure and we did not experience one.
+    DidntFail(String),
+    /// Failed to parse the test (internal error).
+    FailedToParseTest(String),
+    /// Skipped the test because the BLS setting was mismatched.
+    SkippedBls,
+    /// Skipped the test because it's known to fail.
+    SkippedKnownFailure,
+}
+
+impl Error {
+    pub fn name(&self) -> &str {
+        match self {
+            Error::NotEqual(_) => "NotEqual",
+            Error::DidntFail(_) => "DidntFail",
+            Error::FailedToParseTest(_) => "FailedToParseTest",
+            Error::SkippedBls => "SkippedBls",
+            Error::SkippedKnownFailure => "SkippedKnownFailure",
+        }
+    }
+
+    pub fn message(&self) -> &str {
+        match self {
+            Error::NotEqual(m) => m.as_str(),
+            Error::DidntFail(m) => m.as_str(),
+            Error::FailedToParseTest(m) => m.as_str(),
+            _ => self.name(),
+        }
+    }
+
+    pub fn is_skipped(&self) -> bool {
+        match self {
+            Error::SkippedBls | Error::SkippedKnownFailure => true,
+            _ => false,
+        }
+    }
+}

testing/ef_tests/src/handler.rs

@@ -0,0 +1,294 @@
+use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation};
+use crate::type_name;
+use crate::type_name::TypeName;
+use cached_tree_hash::CachedTreeHash;
+use std::fmt::Debug;
+use std::fs;
+use std::marker::PhantomData;
+use std::path::PathBuf;
+use types::EthSpec;
+
+pub trait Handler {
+    type Case: Case + LoadCase;
+
+    fn config_name() -> &'static str {
+        "general"
+    }
+
+    fn fork_name() -> &'static str {
+        "phase0"
+    }
+
+    fn runner_name() -> &'static str;
+
+    fn handler_name() -> String;
+
+    fn run() {
+        let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("eth2.0-spec-tests")
+            .join("tests")
+            .join(Self::config_name())
+            .join(Self::fork_name())
+            .join(Self::runner_name())
+            .join(Self::handler_name());
+
+        // Iterate through test suites
+        let test_cases = fs::read_dir(&handler_path)
+            .expect("handler dir exists")
+            .flat_map(|entry| {
+                entry
+                    .ok()
+                    .filter(|e| e.file_type().map(|ty| ty.is_dir()).unwrap_or(false))
+            })
+            .flat_map(|suite| fs::read_dir(suite.path()).expect("suite dir exists"))
+            .flat_map(Result::ok)
+            .map(|test_case_dir| {
+                let path = test_case_dir.path();
+                let case = Self::Case::load_from_dir(&path).expect("test should load");
+                (path, case)
+            })
+            .collect();
+
+        let results = Cases { test_cases }.test_results();
+
+        let name = format!("{}/{}", Self::runner_name(), Self::handler_name());
+        crate::results::assert_tests_pass(&name, &handler_path, &results);
+    }
+}
+
+macro_rules! bls_handler {
+    ($runner_name: ident, $case_name:ident, $handler_name:expr) => {
+        pub struct $runner_name;
+
+        impl Handler for $runner_name {
+            type Case = cases::$case_name;
+
+            fn runner_name() -> &'static str {
+                "bls"
+            }
+
+            fn handler_name() -> String {
+                $handler_name.into()
+            }
+        }
+    };
+}
+
+bls_handler!(BlsAggregateSigsHandler, BlsAggregateSigs, "aggregate");
+bls_handler!(BlsSignMsgHandler, BlsSign, "sign");
+bls_handler!(BlsVerifyMsgHandler, BlsVerify, "verify");
+bls_handler!(
+    BlsAggregateVerifyHandler,
+    BlsAggregateVerify,
+    "aggregate_verify"
+);
+bls_handler!(
+    BlsFastAggregateVerifyHandler,
+    BlsFastAggregateVerify,
+    "fast_aggregate_verify"
+);
+
+/// Handler for SSZ types.
+pub struct SszStaticHandler<T, E>(PhantomData<(T, E)>);
+
+/// Handler for SSZ types that implement `CachedTreeHash`.
+pub struct SszStaticTHCHandler<T, C, E>(PhantomData<(T, C, E)>);
+
+impl<T, E> Handler for SszStaticHandler<T, E>
+where
+    T: cases::SszStaticType + TypeName,
+    E: TypeName,
+{
+    type Case = cases::SszStatic<T>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "ssz_static"
+    }
+
+    fn handler_name() -> String {
+        T::name().into()
+    }
+}
+
+impl<T, C, E> Handler for SszStaticTHCHandler<T, C, E>
+where
+    T: cases::SszStaticType + CachedTreeHash<C> + TypeName,
+    C: Debug + Sync,
+    E: TypeName,
+{
+    type Case = cases::SszStaticTHC<T, C>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "ssz_static"
+    }
+
+    fn handler_name() -> String {
+        T::name().into()
+    }
+}
+
+pub struct ShufflingHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for ShufflingHandler<E> {
+    type Case = cases::Shuffling<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "shuffling"
+    }
+
+    fn handler_name() -> String {
+        "core".into()
+    }
+}
+
+pub struct SanityBlocksHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for SanityBlocksHandler<E> {
+    type Case = cases::SanityBlocks<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "sanity"
+    }
+
+    fn handler_name() -> String {
+        "blocks".into()
+    }
+}
+
+pub struct SanitySlotsHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for SanitySlotsHandler<E> {
+    type Case = cases::SanitySlots<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "sanity"
+    }
+
+    fn handler_name() -> String {
+        "slots".into()
+    }
+}
+
+pub struct EpochProcessingHandler<E, T>(PhantomData<(E, T)>);
+
+impl<E: EthSpec + TypeName, T: EpochTransition<E>> Handler for EpochProcessingHandler<E, T> {
+    type Case = cases::EpochProcessing<E, T>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "epoch_processing"
+    }
+
+    fn handler_name() -> String {
+        T::name().into()
+    }
+}
+
+pub struct GenesisValidityHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for GenesisValidityHandler<E> {
+    type Case = cases::GenesisValidity<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "genesis"
+    }
+
+    fn handler_name() -> String {
+        "validity".into()
+    }
+}
+
+pub struct GenesisInitializationHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for GenesisInitializationHandler<E> {
+    type Case = cases::GenesisInitialization<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "genesis"
+    }
+
+    fn handler_name() -> String {
+        "initialization".into()
+    }
+}
+
+pub struct OperationsHandler<E, O>(PhantomData<(E, O)>);
+
+impl<E: EthSpec + TypeName, O: Operation<E>> Handler for OperationsHandler<E, O> {
+    type Case = cases::Operations<E, O>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "operations"
+    }
+
+    fn handler_name() -> String {
+        O::handler_name()
+    }
+}
+
+pub struct SszGenericHandler<H>(PhantomData<H>);
+
+impl<H: TypeName> Handler for SszGenericHandler<H> {
+    type Case = cases::SszGeneric;
+
+    fn config_name() -> &'static str {
+        "general"
+    }
+
+    fn runner_name() -> &'static str {
+        "ssz_generic"
+    }
+
+    fn handler_name() -> String {
+        H::name().into()
+    }
+}
+
+// Supported SSZ generic handlers
+pub struct BasicVector;
+type_name!(BasicVector, "basic_vector");
+pub struct Bitlist;
+type_name!(Bitlist, "bitlist");
+pub struct Bitvector;
+type_name!(Bitvector, "bitvector");
+pub struct Boolean;
+type_name!(Boolean, "boolean");
+pub struct Uints;
+type_name!(Uints, "uints");
+pub struct Containers;
+type_name!(Containers, "containers");

testing/ef_tests/src/lib.rs

@@ -0,0 +1,19 @@
+use types::EthSpec;
+
+pub use case_result::CaseResult;
+pub use cases::Case;
+pub use cases::{
+    FinalUpdates, JustificationAndFinalization, RegistryUpdates, RewardsAndPenalties, Slashings,
+};
+pub use error::Error;
+pub use handler::*;
+pub use type_name::TypeName;
+
+mod bls_setting;
+mod case_result;
+mod cases;
+mod decode;
+mod error;
+mod handler;
+mod results;
+mod type_name;

testing/ef_tests/src/results.rs

@@ -0,0 +1,92 @@
+use crate::case_result::CaseResult;
+use crate::error::Error;
+use std::path::Path;
+
+pub fn assert_tests_pass(handler_name: &str, path: &Path, results: &[CaseResult]) {
+    let (failed, skipped_bls, skipped_known_failures) = categorize_results(results);
+
+    if failed.len() + skipped_known_failures.len() > 0 {
+        print_results(
+            handler_name,
+            &failed,
+            &skipped_bls,
+            &skipped_known_failures,
+            &results,
+        );
+        if !failed.is_empty() {
+            panic!("Tests failed (see above)");
+        }
+    } else {
+        println!("Passed {} tests in {}", results.len(), path.display());
+    }
+}
+
+pub fn categorize_results(
+    results: &[CaseResult],
+) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) {
+    let mut failed = vec![];
+    let mut skipped_bls = vec![];
+    let mut skipped_known_failures = vec![];
+
+    for case in results {
+        match case.result.as_ref().err() {
+            Some(Error::SkippedBls) => skipped_bls.push(case),
+            Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case),
+            Some(_) => failed.push(case),
+            None => (),
+        }
+    }
+
+    (failed, skipped_bls, skipped_known_failures)
+}
+
+pub fn print_results(
+    handler_name: &str,
+    failed: &[&CaseResult],
+    skipped_bls: &[&CaseResult],
+    skipped_known_failures: &[&CaseResult],
+    results: &[CaseResult],
+) {
+    println!("--------------------------------------------------");
+    println!(
+        "Test {}",
+        if failed.is_empty() {
+            "Result"
+        } else {
+            "Failure"
+        }
+    );
+    println!("Title: {}", handler_name);
+    println!(
+        "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)",
+        results.len(),
+        failed.len(),
+        skipped_known_failures.len(),
+        skipped_bls.len(),
+        results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len()
+    );
+    println!();
+
+    for case in skipped_known_failures {
+        println!("-------");
+        println!(
+            "case ({}) from {} skipped because it's a known failure",
+            case.desc,
+            case.path.display()
+        );
+    }
+    for failure in failed {
+        let error = failure.result.clone().unwrap_err();
+
+        println!("-------");
+        println!(
+            "case {} ({}) from {} failed with {}:",
+            failure.case_index,
+            failure.desc,
+            failure.path.display(),
+            error.name()
+        );
+        println!("{}", error.message());
+    }
+    println!();
+}

testing/ef_tests/src/type_name.rs

@@ -0,0 +1,57 @@
+//! Mapping from types to canonical string identifiers used in testing.
+use types::*;
+
+pub trait TypeName {
+    fn name() -> &'static str;
+}
+
+#[macro_export]
+macro_rules! type_name {
+    ($typ:ident) => {
+        type_name!($typ, stringify!($typ));
+    };
+    ($typ:ident, $name:expr) => {
+        impl TypeName for $typ {
+            fn name() -> &'static str {
+                $name
+            }
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! type_name_generic {
+    ($typ:ident) => {
+        type_name_generic!($typ, stringify!($typ));
+    };
+    ($typ:ident, $name:expr) => {
+        impl<E: EthSpec> TypeName for $typ<E> {
+            fn name() -> &'static str {
+                $name
+            }
+        }
+    };
+}
+
+type_name!(MinimalEthSpec, "minimal");
+type_name!(MainnetEthSpec, "mainnet");
+
+type_name_generic!(Attestation);
+type_name!(AttestationData);
+type_name_generic!(AttesterSlashing);
+type_name_generic!(BeaconBlock);
+type_name_generic!(BeaconBlockBody);
+type_name!(BeaconBlockHeader);
+type_name_generic!(BeaconState);
+type_name!(Checkpoint);
+type_name!(Deposit);
+type_name!(DepositData);
+type_name!(Eth1Data);
+type_name!(Fork);
+type_name_generic!(HistoricalBatch);
+type_name_generic!(IndexedAttestation);
+type_name_generic!(PendingAttestation);
+type_name!(ProposerSlashing);
+type_name!(SignedVoluntaryExit);
+type_name!(Validator);
+type_name!(VoluntaryExit);

testing/ef_tests/tests/tests.rs

@@ -0,0 +1,230 @@
+#![cfg(feature = "ef_tests")]
+
+use ef_tests::*;
+use std::path::PathBuf;
+use types::*;
+
+// Check that the config from the Eth2.0 spec tests matches our minimal/mainnet config.
+fn config_test<E: EthSpec + TypeName>() {
+    let config_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("eth2.0-spec-tests")
+        .join("tests")
+        .join(E::name())
+        .join("config.yaml");
+    let yaml_config = YamlConfig::from_file(&config_path).expect("config file loads OK");
+    let spec = E::default_spec();
+    let yaml_from_spec = YamlConfig::from_spec::<E>(&spec);
+    assert_eq!(yaml_config.apply_to_chain_spec::<E>(&spec), Some(spec));
+    assert_eq!(yaml_from_spec, yaml_config);
+}
+
+#[test]
+fn mainnet_config_ok() {
+    config_test::<MainnetEthSpec>();
+}
+
+#[test]
+fn minimal_config_ok() {
+    config_test::<MinimalEthSpec>();
+}
+
+#[test]
+fn shuffling() {
+    ShufflingHandler::<MinimalEthSpec>::run();
+    ShufflingHandler::<MainnetEthSpec>::run();
+}
+
+#[test]
+fn operations_deposit() {
+    OperationsHandler::<MinimalEthSpec, Deposit>::run();
+    OperationsHandler::<MainnetEthSpec, Deposit>::run();
+}
+
+#[test]
+fn operations_exit() {
+    OperationsHandler::<MinimalEthSpec, SignedVoluntaryExit>::run();
+    OperationsHandler::<MainnetEthSpec, SignedVoluntaryExit>::run();
+}
+
+#[test]
+fn operations_proposer_slashing() {
+    OperationsHandler::<MinimalEthSpec, ProposerSlashing>::run();
+    OperationsHandler::<MainnetEthSpec, ProposerSlashing>::run();
+}
+
+#[test]
+fn operations_attester_slashing() {
+    OperationsHandler::<MinimalEthSpec, AttesterSlashing<_>>::run();
+    OperationsHandler::<MainnetEthSpec, AttesterSlashing<_>>::run();
+}
+
+#[test]
+fn operations_attestation() {
+    OperationsHandler::<MinimalEthSpec, Attestation<_>>::run();
+    OperationsHandler::<MainnetEthSpec, Attestation<_>>::run();
+}
+
+#[test]
+fn operations_block_header() {
+    OperationsHandler::<MinimalEthSpec, BeaconBlock<_>>::run();
+    OperationsHandler::<MainnetEthSpec, BeaconBlock<_>>::run();
+}
+
+#[test]
+fn sanity_blocks() {
+    SanityBlocksHandler::<MinimalEthSpec>::run();
+    SanityBlocksHandler::<MainnetEthSpec>::run();
+}
+
+#[test]
+fn sanity_slots() {
+    SanitySlotsHandler::<MinimalEthSpec>::run();
+    SanitySlotsHandler::<MainnetEthSpec>::run();
+}
+
+#[test]
+#[cfg(not(feature = "fake_crypto"))]
+fn bls_aggregate() {
+    BlsAggregateSigsHandler::run();
+}
+
+#[test]
+#[cfg(not(feature = "fake_crypto"))]
+fn bls_sign() {
+    BlsSignMsgHandler::run();
+}
+
+#[test]
+#[cfg(not(feature = "fake_crypto"))]
+fn bls_verify() {
+    BlsVerifyMsgHandler::run();
+}
+
+#[test]
+#[cfg(not(feature = "fake_crypto"))]
+fn bls_aggregate_verify() {
+    BlsAggregateVerifyHandler::run();
+}
+
+#[test]
+#[cfg(not(feature = "fake_crypto"))]
+fn bls_fast_aggregate_verify() {
+    BlsFastAggregateVerifyHandler::run();
+}
+
+#[cfg(feature = "fake_crypto")]
+macro_rules! ssz_static_test {
+    // Non-tree hash caching
+    ($test_name:ident, $typ:ident$(<$generics:tt>)?) => {
+        ssz_static_test!($test_name, SszStaticHandler, $typ$(<$generics>)?);
+    };
+    // Generic
+    ($test_name:ident, $handler:ident, $typ:ident<_>) => {
+        ssz_static_test!(
+            $test_name, $handler, {
+                ($typ<MinimalEthSpec>, MinimalEthSpec),
+                ($typ<MainnetEthSpec>, MainnetEthSpec)
+            }
+        );
+    };
+    // Non-generic
+    ($test_name:ident, $handler:ident, $typ:ident) => {
+        ssz_static_test!(
+            $test_name, $handler, {
+                ($typ, MinimalEthSpec),
+                ($typ, MainnetEthSpec)
+            }
+        );
+    };
+    // Base case
+    ($test_name:ident, $handler:ident, { $(($($typ:ty),+)),+ }) => {
+        #[test]
+        fn $test_name() {
+            $(
+                $handler::<$($typ),+>::run();
+            )+
+        }
+    };
+}
+
+#[cfg(feature = "fake_crypto")]
+mod ssz_static {
+    use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler};
+    use types::*;
+
+    ssz_static_test!(attestation, Attestation<_>);
+    ssz_static_test!(attestation_data, AttestationData);
+    ssz_static_test!(attester_slashing, AttesterSlashing<_>);
+    ssz_static_test!(beacon_block, BeaconBlock<_>);
+    ssz_static_test!(beacon_block_body, BeaconBlockBody<_>);
+    ssz_static_test!(beacon_block_header, BeaconBlockHeader);
+    ssz_static_test!(
+        beacon_state,
+        SszStaticTHCHandler, {
+            (BeaconState<MinimalEthSpec>, BeaconTreeHashCache, MinimalEthSpec),
+            (BeaconState<MainnetEthSpec>, BeaconTreeHashCache, MainnetEthSpec)
+        }
+    );
+    ssz_static_test!(checkpoint, Checkpoint);
+    ssz_static_test!(deposit, Deposit);
+    ssz_static_test!(deposit_data, DepositData);
+    ssz_static_test!(eth1_data, Eth1Data);
+    ssz_static_test!(fork, Fork);
+    ssz_static_test!(historical_batch, HistoricalBatch<_>);
+    ssz_static_test!(indexed_attestation, IndexedAttestation<_>);
+    ssz_static_test!(pending_attestation, PendingAttestation<_>);
+    ssz_static_test!(proposer_slashing, ProposerSlashing);
+    ssz_static_test!(validator, Validator);
+    ssz_static_test!(voluntary_exit, VoluntaryExit);
+}
+
+#[test]
+fn ssz_generic() {
+    SszGenericHandler::<BasicVector>::run();
+    SszGenericHandler::<Bitlist>::run();
+    SszGenericHandler::<Bitvector>::run();
+    SszGenericHandler::<Boolean>::run();
+    SszGenericHandler::<Uints>::run();
+    SszGenericHandler::<Containers>::run();
+}
+
+#[test]
+fn epoch_processing_justification_and_finalization() {
+    EpochProcessingHandler::<MinimalEthSpec, JustificationAndFinalization>::run();
+    EpochProcessingHandler::<MainnetEthSpec, JustificationAndFinalization>::run();
+}
+
+#[test]
+fn epoch_processing_rewards_and_penalties() {
+    EpochProcessingHandler::<MinimalEthSpec, RewardsAndPenalties>::run();
+    // Note: there are no reward and penalty tests for mainnet yet
+}
+
+#[test]
+fn epoch_processing_registry_updates() {
+    EpochProcessingHandler::<MinimalEthSpec, RegistryUpdates>::run();
+    EpochProcessingHandler::<MainnetEthSpec, RegistryUpdates>::run();
+}
+
+#[test]
+fn epoch_processing_slashings() {
+    EpochProcessingHandler::<MinimalEthSpec, Slashings>::run();
+    EpochProcessingHandler::<MainnetEthSpec, Slashings>::run();
+}
+
+#[test]
+fn epoch_processing_final_updates() {
+    EpochProcessingHandler::<MainnetEthSpec, FinalUpdates>::run();
+    EpochProcessingHandler::<MainnetEthSpec, FinalUpdates>::run();
+}
+
+#[test]
+fn genesis_initialization() {
+    GenesisInitializationHandler::<MinimalEthSpec>::run();
+}
+
+#[test]
+fn genesis_validity() {
+    GenesisValidityHandler::<MinimalEthSpec>::run();
+    // Note: there are no genesis validity tests for mainnet
+}

testing/eth1_test_rig/.gitignore

@@ -0,0 +1 @@
+contract/

testing/eth1_test_rig/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "eth1_test_rig"
+version = "0.2.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+web3 = "0.10.0"
+tokio = { version = "0.2.20", features = ["time"] }
+futures = { version = "0.3.5", features = ["compat"] }
+types = { path = "../../consensus/types"}
+serde_json = "1.0.52"
+deposit_contract = { path = "../../common/deposit_contract"}

testing/eth1_test_rig/src/ganache.rs

@@ -0,0 +1,163 @@
+use futures::compat::Future01CompatExt;
+use serde_json::json;
+use std::io::prelude::*;
+use std::io::BufReader;
+use std::net::TcpListener;
+use std::process::{Child, Command, Stdio};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use web3::{
+    transports::{EventLoopHandle, Http},
+    Transport, Web3,
+};
+
+/// How long we will wait for ganache to indicate that it is ready.
+const GANACHE_STARTUP_TIMEOUT_MILLIS: u64 = 10_000;
+
+/// Provides a dedicated `ganachi-cli` instance with a connected `Web3` instance.
+///
+/// Requires that `ganachi-cli` is installed and available on `PATH`.
+pub struct GanacheInstance {
+    pub port: u16,
+    child: Child,
+    _event_loop: Arc<EventLoopHandle>,
+    pub web3: Web3<Http>,
+}
+
+impl GanacheInstance {
+    /// Start a new `ganache-cli` process, waiting until it indicates that it is ready to accept
+    /// RPC connections.
+    pub fn new() -> Result<Self, String> {
+        let port = unused_port()?;
+
+        let mut child = Command::new("ganache-cli")
+            .stdout(Stdio::piped())
+            .arg("--defaultBalanceEther")
+            .arg("1000000000")
+            .arg("--gasLimit")
+            .arg("1000000000")
+            .arg("--accounts")
+            .arg("10")
+            .arg("--port")
+            .arg(format!("{}", port))
+            .arg("--mnemonic")
+            .arg("\"vast thought differ pull jewel broom cook wrist tribe word before omit\"")
+            .spawn()
+            .map_err(|e| {
+                format!(
+                    "Failed to start ganache-cli. \
+                     Is it ganache-cli installed and available on $PATH? Error: {:?}",
+                    e
+                )
+            })?;
+
+        let stdout = child
+            .stdout
+            .ok_or_else(|| "Unable to get stdout for ganache child process")?;
+
+        let start = Instant::now();
+        let mut reader = BufReader::new(stdout);
+        loop {
+            if start + Duration::from_millis(GANACHE_STARTUP_TIMEOUT_MILLIS) <= Instant::now() {
+                break Err(
+                    "Timed out waiting for ganache to start. Is ganache-cli installed?".to_string(),
+                );
+            }
+
+            let mut line = String::new();
+            if let Err(e) = reader.read_line(&mut line) {
+                break Err(format!("Failed to read line from ganache process: {:?}", e));
+            } else if line.starts_with("Listening on") {
+                break Ok(());
+            } else {
+                continue;
+            }
+        }?;
+
+        let (event_loop, transport) = Http::new(&endpoint(port)).map_err(|e| {
+            format!(
+                "Failed to start HTTP transport connected to ganache: {:?}",
+                e
+            )
+        })?;
+        let web3 = Web3::new(transport);
+
+        child.stdout = Some(reader.into_inner());
+
+        Ok(Self {
+            child,
+            port,
+            _event_loop: Arc::new(event_loop),
+            web3,
+        })
+    }
+
+    /// Returns the endpoint that this instance is listening on.
+    pub fn endpoint(&self) -> String {
+        endpoint(self.port)
+    }
+
+    /// Increase the timestamp on future blocks by `increase_by` seconds.
+    pub async fn increase_time(&self, increase_by: u64) -> Result<(), String> {
+        self.web3
+            .transport()
+            .execute("evm_increaseTime", vec![json!(increase_by)])
+            .compat()
+            .await
+            .map(|_json_value| ())
+            .map_err(|e| format!("Failed to increase time on EVM (is this ganache?): {:?}", e))
+    }
+
+    /// Returns the current block number, as u64
+    pub async fn block_number(&self) -> Result<u64, String> {
+        self.web3
+            .eth()
+            .block_number()
+            .compat()
+            .await
+            .map(|v| v.as_u64())
+            .map_err(|e| format!("Failed to get block number: {:?}", e))
+    }
+
+    /// Mines a single block.
+    pub async fn evm_mine(&self) -> Result<(), String> {
+        self.web3
+            .transport()
+            .execute("evm_mine", vec![])
+            .compat()
+            .await
+            .map(|_| ())
+            .map_err(|_| {
+                "utils should mine new block with evm_mine (only works with ganache-cli!)"
+                    .to_string()
+            })
+    }
+}
+
+fn endpoint(port: u16) -> String {
+    format!("http://localhost:{}", port)
+}
+
+/// A bit of hack to find an unused TCP port.
+///
+/// Does not guarantee that the given port is unused after the function exists, just that it was
+/// unused before the function started (i.e., it does not reserve a port).
+pub fn unused_port() -> Result<u16, String> {
+    let listener = TcpListener::bind("127.0.0.1:0")
+        .map_err(|e| format!("Failed to create TCP listener to find unused port: {:?}", e))?;
+
+    let local_addr = listener.local_addr().map_err(|e| {
+        format!(
+            "Failed to read TCP listener local_addr to find unused port: {:?}",
+            e
+        )
+    })?;
+
+    Ok(local_addr.port())
+}
+
+impl Drop for GanacheInstance {
+    fn drop(&mut self) {
+        let _ = self.child.kill();
+    }
+}

testing/eth1_test_rig/src/lib.rs

@@ -0,0 +1,298 @@
+//! Provides utilities for deploying and manipulating the eth2 deposit contract on the eth1 chain.
+//!
+//! Presently used with [`ganache-cli`](https://github.com/trufflesuite/ganache-cli) to simulate
+//! the deposit contract for testing beacon node eth1 integration.
+//!
+//! Not tested to work with actual clients (e.g., geth). It should work fine, however there may be
+//! some initial issues.
+mod ganache;
+
+use deposit_contract::{
+    encode_eth1_tx_data, testnet, ABI, BYTECODE, CONTRACT_DEPLOY_GAS, DEPOSIT_GAS,
+};
+use futures::compat::Future01CompatExt;
+use ganache::GanacheInstance;
+use std::time::Duration;
+use tokio::time::delay_for;
+use types::DepositData;
+use types::{test_utils::generate_deterministic_keypair, EthSpec, Hash256, Keypair, Signature};
+use web3::contract::{Contract, Options};
+use web3::transports::Http;
+use web3::types::{Address, TransactionRequest, U256};
+use web3::Web3;
+
+pub const DEPLOYER_ACCOUNTS_INDEX: usize = 0;
+pub const DEPOSIT_ACCOUNTS_INDEX: usize = 0;
+
+/// Provides a dedicated ganache-cli instance with the deposit contract already deployed.
+pub struct GanacheEth1Instance {
+    pub ganache: GanacheInstance,
+    pub deposit_contract: DepositContract,
+}
+
+impl GanacheEth1Instance {
+    pub async fn new() -> Result<Self, String> {
+        let ganache = GanacheInstance::new()?;
+        DepositContract::deploy(ganache.web3.clone(), 0, None)
+            .await
+            .map(|deposit_contract| Self {
+                ganache,
+                deposit_contract,
+            })
+    }
+
+    pub fn endpoint(&self) -> String {
+        self.ganache.endpoint()
+    }
+
+    pub fn web3(&self) -> Web3<Http> {
+        self.ganache.web3.clone()
+    }
+}
+
+/// Deploys and provides functions for the eth2 deposit contract, deployed on the eth1 chain.
+#[derive(Clone, Debug)]
+pub struct DepositContract {
+    web3: Web3<Http>,
+    contract: Contract<Http>,
+}
+
+impl DepositContract {
+    pub async fn deploy(
+        web3: Web3<Http>,
+        confirmations: usize,
+        password: Option<String>,
+    ) -> Result<Self, String> {
+        Self::deploy_bytecode(web3, confirmations, BYTECODE, ABI, password).await
+    }
+
+    pub async fn deploy_testnet(
+        web3: Web3<Http>,
+        confirmations: usize,
+        password: Option<String>,
+    ) -> Result<Self, String> {
+        Self::deploy_bytecode(
+            web3,
+            confirmations,
+            testnet::BYTECODE,
+            testnet::ABI,
+            password,
+        )
+        .await
+    }
+
+    async fn deploy_bytecode(
+        web3: Web3<Http>,
+        confirmations: usize,
+        bytecode: &[u8],
+        abi: &[u8],
+        password: Option<String>,
+    ) -> Result<Self, String> {
+        let address = deploy_deposit_contract(
+            web3.clone(),
+            confirmations,
+            bytecode.to_vec(),
+            abi.to_vec(),
+            password,
+        )
+        .await
+        .map_err(|e| {
+            format!(
+                "Failed to deploy contract: {}. Is scripts/ganache_tests_node.sh running?.",
+                e
+            )
+        })?;
+        Contract::from_json(web3.clone().eth(), address, ABI)
+            .map_err(|e| format!("Failed to init contract: {:?}", e))
+            .map(move |contract| Self { contract, web3 })
+    }
+
+    /// The deposit contract's address in `0x00ab...` format.
+    pub fn address(&self) -> String {
+        format!("0x{:x}", self.contract.address())
+    }
+
+    /// A helper to return a fully-formed `DepositData`. Does not submit the deposit data to the
+    /// smart contact.
+    pub fn deposit_helper<E: EthSpec>(
+        &self,
+        keypair: Keypair,
+        withdrawal_credentials: Hash256,
+        amount: u64,
+    ) -> DepositData {
+        let mut deposit = DepositData {
+            pubkey: keypair.pk.into(),
+            withdrawal_credentials,
+            amount,
+            signature: Signature::empty_signature().into(),
+        };
+
+        deposit.signature = deposit.create_signature(&keypair.sk, &E::default_spec());
+
+        deposit
+    }
+
+    /// Creates a random, valid deposit and submits it to the deposit contract.
+    ///
+    /// The keypairs are created randomly and destroyed.
+    pub async fn deposit_random<E: EthSpec>(&self) -> Result<(), String> {
+        let keypair = Keypair::random();
+
+        let mut deposit = DepositData {
+            pubkey: keypair.pk.into(),
+            withdrawal_credentials: Hash256::zero(),
+            amount: 32_000_000_000,
+            signature: Signature::empty_signature().into(),
+        };
+
+        deposit.signature = deposit.create_signature(&keypair.sk, &E::default_spec());
+
+        self.deposit(deposit).await
+    }
+
+    /// Perfoms a blocking deposit.
+    pub async fn deposit(&self, deposit_data: DepositData) -> Result<(), String> {
+        self.deposit_async(deposit_data)
+            .await
+            .map_err(|e| format!("Deposit failed: {:?}", e))
+    }
+
+    pub async fn deposit_deterministic_async<E: EthSpec>(
+        &self,
+        keypair_index: usize,
+        amount: u64,
+    ) -> Result<(), String> {
+        let keypair = generate_deterministic_keypair(keypair_index);
+
+        let mut deposit = DepositData {
+            pubkey: keypair.pk.into(),
+            withdrawal_credentials: Hash256::zero(),
+            amount,
+            signature: Signature::empty_signature().into(),
+        };
+
+        deposit.signature = deposit.create_signature(&keypair.sk, &E::default_spec());
+
+        self.deposit_async(deposit).await
+    }
+
+    /// Performs a non-blocking deposit.
+    pub async fn deposit_async(&self, deposit_data: DepositData) -> Result<(), String> {
+        let from = self
+            .web3
+            .eth()
+            .accounts()
+            .compat()
+            .await
+            .map_err(|e| format!("Failed to get accounts: {:?}", e))
+            .and_then(|accounts| {
+                accounts
+                    .get(DEPOSIT_ACCOUNTS_INDEX)
+                    .cloned()
+                    .ok_or_else(|| "Insufficient accounts for deposit".to_string())
+            })?;
+        let tx_request = TransactionRequest {
+            from,
+            to: Some(self.contract.address()),
+            gas: Some(U256::from(DEPOSIT_GAS)),
+            gas_price: None,
+            value: Some(from_gwei(deposit_data.amount)),
+            // Note: the reason we use this `TransactionRequest` instead of just using the
+            // function in `self.contract` is so that the `eth1_tx_data` function gets used
+            // during testing.
+            //
+            // It's important that `eth1_tx_data` stays correct and does not suffer from
+            // code-rot.
+            data: encode_eth1_tx_data(&deposit_data).map(Into::into).ok(),
+            nonce: None,
+            condition: None,
+        };
+
+        self.web3
+            .eth()
+            .send_transaction(tx_request)
+            .compat()
+            .await
+            .map_err(|e| format!("Failed to call deposit fn: {:?}", e))?;
+        Ok(())
+    }
+
+    /// Peforms many deposits, each preceded by a delay.
+    pub async fn deposit_multiple(&self, deposits: Vec<DelayThenDeposit>) -> Result<(), String> {
+        for deposit in deposits.into_iter() {
+            delay_for(deposit.delay).await;
+            self.deposit_async(deposit.deposit).await?;
+        }
+        Ok(())
+    }
+}
+
+/// Describes a deposit and a delay that should should precede it's submission to the deposit
+/// contract.
+#[derive(Clone)]
+pub struct DelayThenDeposit {
+    /// Wait this duration ...
+    pub delay: Duration,
+    /// ... then submit this deposit.
+    pub deposit: DepositData,
+}
+
+fn from_gwei(gwei: u64) -> U256 {
+    U256::from(gwei) * U256::exp10(9)
+}
+
+/// Deploys the deposit contract to the given web3 instance using the account with index
+/// `DEPLOYER_ACCOUNTS_INDEX`.
+async fn deploy_deposit_contract(
+    web3: Web3<Http>,
+    confirmations: usize,
+    bytecode: Vec<u8>,
+    abi: Vec<u8>,
+    password_opt: Option<String>,
+) -> Result<Address, String> {
+    let bytecode = String::from_utf8(bytecode).expect("bytecode must be valid utf8");
+
+    let from_address = web3
+        .eth()
+        .accounts()
+        .compat()
+        .await
+        .map_err(|e| format!("Failed to get accounts: {:?}", e))
+        .and_then(|accounts| {
+            accounts
+                .get(DEPLOYER_ACCOUNTS_INDEX)
+                .cloned()
+                .ok_or_else(|| "Insufficient accounts for deployer".to_string())
+        })?;
+
+    let deploy_address = if let Some(password) = password_opt {
+        let result = web3
+            .personal()
+            .unlock_account(from_address, &password, None)
+            .compat()
+            .await;
+        match result {
+            Ok(true) => return Ok(from_address),
+            Ok(false) => return Err("Eth1 node refused to unlock account".to_string()),
+            Err(e) => return Err(format!("Eth1 unlock request failed: {:?}", e)),
+        };
+    } else {
+        from_address
+    };
+
+    let pending_contract = Contract::deploy(web3.eth(), &abi)
+        .map_err(|e| format!("Unable to build contract deployer: {:?}", e))?
+        .confirmations(confirmations)
+        .options(Options {
+            gas: Some(U256::from(CONTRACT_DEPLOY_GAS)),
+            ..Options::default()
+        })
+        .execute(bytecode, (), deploy_address)
+        .map_err(|e| format!("Failed to execute deployment: {:?}", e))?;
+
+    pending_contract
+        .compat()
+        .await
+        .map(|contract| contract.address())
+        .map_err(|e| format!("Unable to resolve pending contract: {:?}", e))
+}

testing/node_test_rig/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "node_test_rig"
+version = "0.2.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+environment = { path = "../../lighthouse/environment" }
+beacon_node = { path = "../../beacon_node" }
+types = { path = "../../consensus/types" }
+eth2_config = { path = "../../common/eth2_config" }
+tempdir = "0.3.7"
+reqwest = "0.10.4"
+url = "2.1.1"
+serde = "1.0.110"
+futures = "0.3.5"
+genesis = { path = "../../beacon_node/genesis" }
+remote_beacon_node = { path = "../../common/remote_beacon_node" }
+validator_client = { path = "../../validator_client" }
+validator_dir = { path = "../../common/validator_dir", features = ["insecure_keys"] }

testing/node_test_rig/src/lib.rs

@@ -0,0 +1,185 @@
+//! Provides easy ways to run a beacon node or validator client in-process.
+//!
+//! Intended to be used for testing and simulation purposes. Not for production.
+
+use beacon_node::ProductionBeaconNode;
+use environment::RuntimeContext;
+use std::path::PathBuf;
+use std::time::{SystemTime, UNIX_EPOCH};
+use tempdir::TempDir;
+use types::EthSpec;
+use validator_client::ProductionValidatorClient;
+use validator_dir::insecure_keys::build_deterministic_validator_dirs;
+
+pub use beacon_node::{ClientConfig, ClientGenesis, ProductionClient};
+pub use environment;
+pub use remote_beacon_node::RemoteBeaconNode;
+pub use validator_client::Config as ValidatorConfig;
+
+/// Provides a beacon node that is running in the current process on a given tokio executor (it
+/// is _local_ to this process).
+///
+/// Intended for use in testing and simulation. Not for production.
+pub struct LocalBeaconNode<E: EthSpec> {
+    pub client: ProductionClient<E>,
+    pub datadir: TempDir,
+}
+
+impl<E: EthSpec> LocalBeaconNode<E> {
+    /// Starts a new, production beacon node on the tokio runtime in the given `context`.
+    ///
+    /// The node created is using the same types as the node we use in production.
+    pub async fn production(
+        context: RuntimeContext<E>,
+        mut client_config: ClientConfig,
+    ) -> Result<Self, String> {
+        // Creates a temporary directory that will be deleted once this `TempDir` is dropped.
+        let datadir = TempDir::new("lighthouse_node_test_rig")
+            .expect("should create temp directory for client datadir");
+
+        client_config.data_dir = datadir.path().into();
+        client_config.network.network_dir = PathBuf::from(datadir.path()).join("network");
+
+        ProductionBeaconNode::new(context, client_config)
+            .await
+            .map(move |client| Self {
+                client: client.into_inner(),
+                datadir,
+            })
+    }
+}
+
+impl<E: EthSpec> LocalBeaconNode<E> {
+    /// Returns a `RemoteBeaconNode` that can connect to `self`. Useful for testing the node as if
+    /// it were external this process.
+    pub fn remote_node(&self) -> Result<RemoteBeaconNode<E>, String> {
+        let socket_addr = self
+            .client
+            .http_listen_addr()
+            .ok_or_else(|| "A remote beacon node must have a http server".to_string())?;
+        Ok(RemoteBeaconNode::new(format!(
+            "http://{}:{}",
+            socket_addr.ip(),
+            socket_addr.port()
+        ))?)
+    }
+}
+
+pub fn testing_client_config() -> ClientConfig {
+    let mut client_config = ClientConfig::default();
+
+    // Setting ports to `0` means that the OS will choose some available port.
+    client_config.network.libp2p_port = 0;
+    client_config.network.discovery_port = 0;
+    client_config.rest_api.enabled = true;
+    client_config.rest_api.port = 0;
+    client_config.websocket_server.enabled = true;
+    client_config.websocket_server.port = 0;
+
+    client_config.dummy_eth1_backend = true;
+
+    let now = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .expect("should get system time")
+        .as_secs();
+
+    client_config.genesis = ClientGenesis::Interop {
+        validator_count: 8,
+        genesis_time: now,
+    };
+
+    client_config
+}
+
+/// Contains the directories for a `LocalValidatorClient`.
+///
+/// This struct is separate to `LocalValidatorClient` to allow for pre-computation of validator
+/// keypairs since the task is quite resource intensive.
+pub struct ValidatorFiles {
+    pub datadir: TempDir,
+    pub secrets_dir: TempDir,
+}
+
+impl ValidatorFiles {
+    /// Creates temporary data and secrets dirs.
+    pub fn new() -> Result<Self, String> {
+        let datadir = TempDir::new("lighthouse-validator-client")
+            .map_err(|e| format!("Unable to create VC data dir: {:?}", e))?;
+
+        let secrets_dir = TempDir::new("lighthouse-validator-client-secrets")
+            .map_err(|e| format!("Unable to create VC secrets dir: {:?}", e))?;
+
+        Ok(Self {
+            datadir,
+            secrets_dir,
+        })
+    }
+
+    /// Creates temporary data and secrets dirs, preloaded with keystores.
+    pub fn with_keystores(keypair_indices: &[usize]) -> Result<Self, String> {
+        let this = Self::new()?;
+
+        build_deterministic_validator_dirs(
+            this.datadir.path().into(),
+            this.secrets_dir.path().into(),
+            keypair_indices,
+        )
+        .map_err(|e| format!("Unable to build validator directories: {:?}", e))?;
+
+        Ok(this)
+    }
+}
+
+/// Provides a validator client that is running in the current process on a given tokio executor (it
+/// is _local_ to this process).
+///
+/// Intended for use in testing and simulation. Not for production.
+pub struct LocalValidatorClient<T: EthSpec> {
+    pub client: ProductionValidatorClient<T>,
+    pub files: ValidatorFiles,
+}
+
+impl<E: EthSpec> LocalValidatorClient<E> {
+    /// Creates a validator client with insecure deterministic keypairs. The validator directories
+    /// are created in a temp dir then removed when the process exits.
+    ///
+    /// The validator created is using the same types as the node we use in production.
+    pub async fn production_with_insecure_keypairs(
+        context: RuntimeContext<E>,
+        config: ValidatorConfig,
+        files: ValidatorFiles,
+    ) -> Result<Self, String> {
+        Self::new(context, config, files).await
+    }
+
+    /// Creates a validator client that attempts to read keys from the default data dir.
+    ///
+    /// - The validator created is using the same types as the node we use in production.
+    /// - It is recommended to use `production_with_insecure_keypairs` for testing.
+    pub async fn production(
+        context: RuntimeContext<E>,
+        config: ValidatorConfig,
+    ) -> Result<Self, String> {
+        let files = ValidatorFiles::new()?;
+
+        Self::new(context, config, files).await
+    }
+
+    async fn new(
+        context: RuntimeContext<E>,
+        mut config: ValidatorConfig,
+        files: ValidatorFiles,
+    ) -> Result<Self, String> {
+        config.data_dir = files.datadir.path().into();
+        config.secrets_dir = files.secrets_dir.path().into();
+
+        ProductionValidatorClient::new(context, config)
+            .await
+            .map(move |mut client| {
+                client
+                    .start_service()
+                    .expect("should start validator services");
+                Self { client, files }
+            })
+    }
+}

testing/simulator/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "simulator"
+version = "0.2.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+node_test_rig = { path = "../node_test_rig" }
+types = { path = "../../consensus/types" }
+validator_client = { path = "../../validator_client" }
+parking_lot = "0.10.2"
+futures = "0.3.5"
+tokio = "0.2.20"
+eth1_test_rig = { path = "../eth1_test_rig" }
+env_logger = "0.7.1"
+clap = "2.33.0"
+rayon = "1.3.0"

testing/simulator/src/checks.rs

@@ -0,0 +1,128 @@
+use crate::local_network::LocalNetwork;
+use std::time::Duration;
+use types::{Epoch, EthSpec, Slot, Unsigned};
+
+/// Checks that all of the validators have on-boarded by the start of the second eth1 voting
+/// period.
+pub async fn verify_initial_validator_count<E: EthSpec>(
+    network: LocalNetwork<E>,
+    slot_duration: Duration,
+    initial_validator_count: usize,
+) -> Result<(), String> {
+    slot_delay(Slot::new(1), slot_duration).await;
+    verify_validator_count(network, initial_validator_count).await?;
+    Ok(())
+}
+
+/// Checks that all of the validators have on-boarded by the start of the second eth1 voting
+/// period.
+pub async fn verify_validator_onboarding<E: EthSpec>(
+    network: LocalNetwork<E>,
+    slot_duration: Duration,
+    expected_validator_count: usize,
+) -> Result<(), String> {
+    slot_delay(
+        Slot::new(E::SlotsPerEth1VotingPeriod::to_u64()),
+        slot_duration,
+    )
+    .await;
+    verify_validator_count(network, expected_validator_count).await?;
+    Ok(())
+}
+
+/// Checks that the chain has made the first possible finalization.
+///
+/// Intended to be run as soon as chain starts.
+pub async fn verify_first_finalization<E: EthSpec>(
+    network: LocalNetwork<E>,
+    slot_duration: Duration,
+) -> Result<(), String> {
+    epoch_delay(Epoch::new(4), slot_duration, E::slots_per_epoch()).await;
+    verify_all_finalized_at(network, Epoch::new(2)).await?;
+    Ok(())
+}
+
+/// Delays for `epochs`, plus half a slot extra.
+pub async fn epoch_delay(epochs: Epoch, slot_duration: Duration, slots_per_epoch: u64) {
+    let duration = slot_duration * (epochs.as_u64() * slots_per_epoch) as u32 + slot_duration / 2;
+    tokio::time::delay_for(duration).await
+}
+
+/// Delays for `slots`, plus half a slot extra.
+async fn slot_delay(slots: Slot, slot_duration: Duration) {
+    let duration = slot_duration * slots.as_u64() as u32 + slot_duration / 2;
+    tokio::time::delay_for(duration).await;
+}
+
+/// Verifies that all beacon nodes in the given network have a head state that has a finalized
+/// epoch of `epoch`.
+pub async fn verify_all_finalized_at<E: EthSpec>(
+    network: LocalNetwork<E>,
+    epoch: Epoch,
+) -> Result<(), String> {
+    let epochs = {
+        let mut epochs = Vec::new();
+        for remote_node in network.remote_nodes()? {
+            epochs.push(
+                remote_node
+                    .http
+                    .beacon()
+                    .get_head()
+                    .await
+                    .map(|head| head.finalized_slot.epoch(E::slots_per_epoch()))
+                    .map_err(|e| format!("Get head via http failed: {:?}", e))?,
+            );
+        }
+        epochs
+    };
+
+    if epochs.iter().any(|node_epoch| *node_epoch != epoch) {
+        Err(format!(
+            "Nodes are not finalized at epoch {}. Finalized epochs: {:?}",
+            epoch, epochs
+        ))
+    } else {
+        Ok(())
+    }
+}
+
+/// Verifies that all beacon nodes in the given `network` have a head state that contains
+/// `expected_count` validators.
+async fn verify_validator_count<E: EthSpec>(
+    network: LocalNetwork<E>,
+    expected_count: usize,
+) -> Result<(), String> {
+    let validator_counts = {
+        let mut validator_counts = Vec::new();
+        for remote_node in network.remote_nodes()? {
+            let beacon = remote_node.http.beacon();
+
+            let head = beacon
+                .get_head()
+                .await
+                .map_err(|e| format!("Get head via http failed: {:?}", e))?;
+
+            let vc = beacon
+                .get_state_by_root(head.state_root)
+                .await
+                .map(|(state, _root)| state)
+                .map_err(|e| format!("Get state root via http failed: {:?}", e))?
+                .validators
+                .len();
+            validator_counts.push(vc);
+        }
+        validator_counts
+    };
+
+    if validator_counts
+        .iter()
+        .any(|count| *count != expected_count)
+    {
+        Err(format!(
+            "Nodes do not all have {} validators in their state. Validator counts: {:?}",
+            expected_count, validator_counts
+        ))
+    } else {
+        Ok(())
+    }
+}

testing/simulator/src/cli.rs

@@ -0,0 +1,108 @@
+use clap::{App, Arg, SubCommand};
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new("simulator")
+        .version(crate_version!())
+        .author("Sigma Prime <contact@sigmaprime.io>")
+        .about("Options for interacting with simulator")
+        .subcommand(
+            SubCommand::with_name("eth1-sim")
+            .about(
+                "Lighthouse Beacon Chain Simulator creates `n` beacon node and validator clients, \
+                    each with `v` validators. A deposit contract is deployed at the start of the \
+                    simulation using a local `ganache-cli` instance (you must have `ganache-cli` \
+                    installed and avaliable on your path). All beacon nodes independently listen \
+                    for genesis from the deposit contract, then start operating. \
+
+                    As the simulation runs, there are checks made to ensure that all components \
+                    are running correctly. If any of these checks fail, the simulation will \
+                    exit immediately.",
+                    )
+                    .arg(Arg::with_name("nodes")
+                        .short("n")
+                        .long("nodes")
+                        .takes_value(true)
+                        .default_value("4")
+                        .help("Number of beacon nodes"))
+                    .arg(Arg::with_name("validators_per_node")
+                        .short("v")
+                        .long("validators_per_node")
+                        .takes_value(true)
+                        .default_value("20")
+                        .help("Number of validators"))
+                    .arg(Arg::with_name("speed_up_factor")
+                        .short("s")
+                        .long("speed_up_factor")
+                        .takes_value(true)
+                        .default_value("4")
+                        .help("Speed up factor"))
+                    .arg(Arg::with_name("end_after_checks")
+                        .short("e")
+                        .long("end_after_checks")
+                        .takes_value(false)
+                        .help("End after checks (default true)"))
+        )
+        .subcommand(
+            SubCommand::with_name("no-eth1-sim")
+            .about("Runs a simulator that bypasses the eth1 chain. Useful for faster testing of
+                components that don't rely upon eth1")
+                    .arg(Arg::with_name("nodes")
+                        .short("n")
+                        .long("nodes")
+                        .takes_value(true)
+                        .default_value("4")
+                        .help("Number of beacon nodes"))
+                    .arg(Arg::with_name("validators_per_node")
+                        .short("v")
+                        .long("validators_per_node")
+                        .takes_value(true)
+                        .default_value("20")
+                        .help("Number of validators"))
+                    .arg(Arg::with_name("speed_up_factor")
+                        .short("s")
+                        .long("speed_up_factor")
+                        .takes_value(true)
+                        .default_value("4")
+                        .help("Speed up factor"))
+                    .arg(Arg::with_name("end_after_checks")
+                        .short("e")
+                        .long("end_after_checks")
+                        .takes_value(false)
+                        .help("End after checks (default true)"))
+        )
+        .subcommand(
+            SubCommand::with_name("syncing-sim")
+                .about("Run the syncing simulation")
+                .arg(
+                    Arg::with_name("speedup")
+                        .short("s")
+                        .long("speedup")
+                        .takes_value(true)
+                        .default_value("15")
+                        .help("Speed up factor for eth1 blocks and slot production"),
+                )
+                .arg(
+                    Arg::with_name("initial_delay")
+                        .short("i")
+                        .long("initial_delay")
+                        .takes_value(true)
+                        .default_value("5")
+                        .help("Epoch delay for new beacon node to start syncing"),
+                )
+                .arg(
+                    Arg::with_name("sync_timeout")
+                        .long("sync_timeout")
+                        .takes_value(true)
+                        .default_value("10")
+                        .help("Number of epochs after which newly added beacon nodes must be synced"),
+                )
+                .arg(
+                    Arg::with_name("strategy")
+                        .long("strategy")
+                        .takes_value(true)
+                        .default_value("all")
+                        .possible_values(&["one-node", "two-nodes", "mixed", "all"])
+                        .help("Sync verification strategy to run."),
+                ),
+        )
+}

testing/simulator/src/eth1_sim.rs

@@ -0,0 +1,209 @@
+use crate::{checks, LocalNetwork, E};
+use clap::ArgMatches;
+use eth1_test_rig::GanacheEth1Instance;
+use futures::prelude::*;
+use node_test_rig::{
+    environment::EnvironmentBuilder, testing_client_config, ClientGenesis, ValidatorConfig,
+    ValidatorFiles,
+};
+use rayon::prelude::*;
+use std::net::{IpAddr, Ipv4Addr};
+use std::time::Duration;
+use tokio::time::{delay_until, Instant};
+
+pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
+    let node_count = value_t!(matches, "nodes", usize).expect("missing nodes default");
+    let validators_per_node = value_t!(matches, "validators_per_node", usize)
+        .expect("missing validators_per_node default");
+    let speed_up_factor =
+        value_t!(matches, "speed_up_factor", u64).expect("missing speed_up_factor default");
+    let mut end_after_checks = true;
+    if matches.is_present("end_after_checks") {
+        end_after_checks = false;
+    }
+
+    println!("Beacon Chain Simulator:");
+    println!(" nodes:{}", node_count);
+    println!(" validators_per_node:{}", validators_per_node);
+    println!(" end_after_checks:{}", end_after_checks);
+
+    // Generate the directories and keystores required for the validator clients.
+    let validator_files = (0..node_count)
+        .into_par_iter()
+        .map(|i| {
+            println!(
+                "Generating keystores for validator {} of {}",
+                i + 1,
+                node_count
+            );
+
+            let indices =
+                (i * validators_per_node..(i + 1) * validators_per_node).collect::<Vec<_>>();
+            ValidatorFiles::with_keystores(&indices).unwrap()
+        })
+        .collect::<Vec<_>>();
+
+    let expected_genesis_instant = Instant::now() + Duration::from_secs(60);
+
+    let log_level = "debug";
+    let log_format = None;
+
+    let mut env = EnvironmentBuilder::minimal()
+        .async_logger(log_level, log_format)?
+        .multi_threaded_tokio_runtime()?
+        .build()?;
+
+    let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor);
+
+    let spec = &mut env.eth2_config.spec;
+
+    spec.milliseconds_per_slot /= speed_up_factor;
+    spec.eth1_follow_distance = 16;
+    spec.min_genesis_delay = eth1_block_time.as_secs() * spec.eth1_follow_distance * 2;
+    spec.min_genesis_time = 0;
+    spec.min_genesis_active_validator_count = 64;
+    spec.seconds_per_eth1_block = 1;
+
+    let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+    let initial_validator_count = spec.min_genesis_active_validator_count as usize;
+    let total_validator_count = validators_per_node * node_count;
+    let deposit_amount = env.eth2_config.spec.max_effective_balance;
+
+    let context = env.core_context();
+
+    let main_future = async {
+        /*
+         * Deploy the deposit contract, spawn tasks to keep creating new blocks and deposit
+         * validators.
+         */
+        let ganache_eth1_instance = GanacheEth1Instance::new().await?;
+        let deposit_contract = ganache_eth1_instance.deposit_contract;
+        let ganache = ganache_eth1_instance.ganache;
+        let eth1_endpoint = ganache.endpoint();
+        let deposit_contract_address = deposit_contract.address();
+
+        // Start a timer that produces eth1 blocks on an interval.
+        tokio::spawn(async move {
+            let mut interval = tokio::time::interval(eth1_block_time);
+            while let Some(_) = interval.next().await {
+                let _ = ganache.evm_mine().await;
+            }
+        });
+
+        // Submit deposits to the deposit contract.
+        tokio::spawn(async move {
+            for i in 0..total_validator_count {
+                println!("Submitting deposit for validator {}...", i);
+                let _ = deposit_contract
+                    .deposit_deterministic_async::<E>(i, deposit_amount)
+                    .await;
+            }
+        });
+
+        let mut beacon_config = testing_client_config();
+
+        beacon_config.genesis = ClientGenesis::DepositContract;
+        beacon_config.eth1.endpoint = eth1_endpoint;
+        beacon_config.eth1.deposit_contract_address = deposit_contract_address;
+        beacon_config.eth1.deposit_contract_deploy_block = 0;
+        beacon_config.eth1.lowest_cached_block_number = 0;
+        beacon_config.eth1.follow_distance = 1;
+        beacon_config.dummy_eth1_backend = false;
+        beacon_config.sync_eth1_chain = true;
+
+        beacon_config.network.enr_address = Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
+
+        /*
+         * Create a new `LocalNetwork` with one beacon node.
+         */
+        let network = LocalNetwork::new(context, beacon_config.clone()).await?;
+        /*
+         * One by one, add beacon nodes to the network.
+         */
+
+        for _ in 0..node_count - 1 {
+            network.add_beacon_node(beacon_config.clone()).await?;
+        }
+
+        /*
+         * Create a future that will add validator clients to the network. Each validator client is
+         * attached to a single corresponding beacon node.
+         */
+        let add_validators_fut = async {
+            for (i, files) in validator_files.into_iter().enumerate() {
+                network
+                    .add_validator_client(
+                        ValidatorConfig {
+                            auto_register: true,
+                            ..ValidatorConfig::default()
+                        },
+                        i,
+                        files,
+                    )
+                    .await?;
+            }
+
+            Ok::<(), String>(())
+        };
+
+        /*
+         * Start the processes that will run checks on the network as it runs.
+         */
+
+        let checks_fut = async {
+            delay_until(expected_genesis_instant).await;
+
+            let (finalization, validator_count, onboarding) = futures::join!(
+                // Check that the chain finalizes at the first given opportunity.
+                checks::verify_first_finalization(network.clone(), slot_duration),
+                // Check that the chain starts with the expected validator count.
+                checks::verify_initial_validator_count(
+                    network.clone(),
+                    slot_duration,
+                    initial_validator_count,
+                ),
+                // Check that validators greater than `spec.min_genesis_active_validator_count` are
+                // onboarded at the first possible opportunity.
+                checks::verify_validator_onboarding(
+                    network.clone(),
+                    slot_duration,
+                    total_validator_count,
+                )
+            );
+
+            finalization?;
+            validator_count?;
+            onboarding?;
+
+            Ok::<(), String>(())
+        };
+
+        let (add_validators, checks) = futures::join!(add_validators_fut, checks_fut);
+
+        add_validators?;
+        checks?;
+
+        // The `final_future` either completes immediately or never completes, depending on the value
+        // of `end_after_checks`.
+
+        if !end_after_checks {
+            future::pending::<()>().await;
+        }
+        /*
+         * End the simulation by dropping the network. This will kill all running beacon nodes and
+         * validator clients.
+         */
+        println!(
+            "Simulation complete. Finished with {} beacon nodes and {} validator clients",
+            network.beacon_node_count(),
+            network.validator_client_count()
+        );
+
+        // Be explicit about dropping the network, as this kills all the nodes. This ensures
+        // all the checks have adequate time to pass.
+        drop(network);
+        Ok::<(), String>(())
+    };
+
+    Ok(env.runtime().block_on(main_future).unwrap())
+}

testing/simulator/src/local_network.rs

@@ -0,0 +1,164 @@
+use node_test_rig::{
+    environment::RuntimeContext, ClientConfig, LocalBeaconNode, LocalValidatorClient,
+    RemoteBeaconNode, ValidatorConfig, ValidatorFiles,
+};
+use parking_lot::RwLock;
+use std::ops::Deref;
+use std::sync::Arc;
+use types::{Epoch, EthSpec};
+
+const BOOTNODE_PORT: u16 = 42424;
+
+/// Helper struct to reduce `Arc` usage.
+pub struct Inner<E: EthSpec> {
+    context: RuntimeContext<E>,
+    beacon_nodes: RwLock<Vec<LocalBeaconNode<E>>>,
+    validator_clients: RwLock<Vec<LocalValidatorClient<E>>>,
+}
+
+/// Represents a set of interconnected `LocalBeaconNode` and `LocalValidatorClient`.
+///
+/// Provides functions to allow adding new beacon nodes and validators.
+pub struct LocalNetwork<E: EthSpec> {
+    inner: Arc<Inner<E>>,
+}
+
+impl<E: EthSpec> Clone for LocalNetwork<E> {
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<E: EthSpec> Deref for LocalNetwork<E> {
+    type Target = Inner<E>;
+
+    fn deref(&self) -> &Self::Target {
+        self.inner.deref()
+    }
+}
+
+impl<E: EthSpec> LocalNetwork<E> {
+    /// Creates a new network with a single `BeaconNode`.
+    pub async fn new(
+        context: RuntimeContext<E>,
+        mut beacon_config: ClientConfig,
+    ) -> Result<Self, String> {
+        beacon_config.network.discovery_port = BOOTNODE_PORT;
+        beacon_config.network.libp2p_port = BOOTNODE_PORT;
+        beacon_config.network.enr_udp_port = Some(BOOTNODE_PORT);
+        beacon_config.network.enr_tcp_port = Some(BOOTNODE_PORT);
+        let beacon_node =
+            LocalBeaconNode::production(context.service_context("boot_node".into()), beacon_config)
+                .await?;
+        Ok(Self {
+            inner: Arc::new(Inner {
+                context,
+                beacon_nodes: RwLock::new(vec![beacon_node]),
+                validator_clients: RwLock::new(vec![]),
+            }),
+        })
+    }
+
+    /// Returns the number of beacon nodes in the network.
+    ///
+    /// Note: does not count nodes that are external to this `LocalNetwork` that may have connected
+    /// (e.g., another Lighthouse process on the same machine.)
+    pub fn beacon_node_count(&self) -> usize {
+        self.beacon_nodes.read().len()
+    }
+
+    /// Returns the number of validator clients in the network.
+    ///
+    /// Note: does not count nodes that are external to this `LocalNetwork` that may have connected
+    /// (e.g., another Lighthouse process on the same machine.)
+    pub fn validator_client_count(&self) -> usize {
+        self.validator_clients.read().len()
+    }
+
+    /// Adds a beacon node to the network, connecting to the 0'th beacon node via ENR.
+    pub async fn add_beacon_node(&self, mut beacon_config: ClientConfig) -> Result<(), String> {
+        let self_1 = self.clone();
+        println!("Adding beacon node..");
+        {
+            let read_lock = self.beacon_nodes.read();
+
+            let boot_node = read_lock.first().expect("should have at least one node");
+
+            beacon_config.network.boot_nodes.push(
+                boot_node
+                    .client
+                    .enr()
+                    .expect("bootnode must have a network"),
+            );
+        }
+
+        let index = self.beacon_nodes.read().len();
+
+        let beacon_node = LocalBeaconNode::production(
+            self.context.service_context(format!("node_{}", index)),
+            beacon_config,
+        )
+        .await?;
+        self_1.beacon_nodes.write().push(beacon_node);
+        Ok(())
+    }
+
+    /// Adds a validator client to the network, connecting it to the beacon node with index
+    /// `beacon_node`.
+    pub async fn add_validator_client(
+        &self,
+        mut validator_config: ValidatorConfig,
+        beacon_node: usize,
+        validator_files: ValidatorFiles,
+    ) -> Result<(), String> {
+        let index = self.validator_clients.read().len();
+        let context = self.context.service_context(format!("validator_{}", index));
+        let self_1 = self.clone();
+        let socket_addr = {
+            let read_lock = self.beacon_nodes.read();
+            let beacon_node = read_lock
+                .get(beacon_node)
+                .ok_or_else(|| format!("No beacon node for index {}", beacon_node))?;
+            beacon_node
+                .client
+                .http_listen_addr()
+                .expect("Must have http started")
+        };
+
+        validator_config.http_server =
+            format!("http://{}:{}", socket_addr.ip(), socket_addr.port());
+        let validator_client = LocalValidatorClient::production_with_insecure_keypairs(
+            context,
+            validator_config,
+            validator_files,
+        )
+        .await?;
+        self_1.validator_clients.write().push(validator_client);
+        Ok(())
+    }
+
+    /// For all beacon nodes in `Self`, return a HTTP client to access each nodes HTTP API.
+    pub fn remote_nodes(&self) -> Result<Vec<RemoteBeaconNode<E>>, String> {
+        let beacon_nodes = self.beacon_nodes.read();
+
+        beacon_nodes
+            .iter()
+            .map(|beacon_node| beacon_node.remote_node())
+            .collect()
+    }
+
+    /// Return current epoch of bootnode.
+    pub async fn bootnode_epoch(&self) -> Result<Epoch, String> {
+        let nodes = self.remote_nodes().expect("Failed to get remote nodes");
+        let bootnode = nodes.first().expect("Should contain bootnode");
+        bootnode
+            .http
+            .beacon()
+            .get_head()
+            .await
+            .map_err(|e| format!("Cannot get head: {:?}", e))
+            .map(|head| head.finalized_slot.epoch(E::slots_per_epoch()))
+    }
+}

testing/simulator/src/main.rs

@@ -0,0 +1,65 @@
+//! This crate provides a simluation that creates `n` beacon node and validator clients, each with
+//! `v` validators. A deposit contract is deployed at the start of the simulation using a local
+//! `ganache-cli` instance (you must have `ganache-cli` installed and avaliable on your path). All
+//! beacon nodes independently listen for genesis from the deposit contract, then start operating.
+//!
+//! As the simulation runs, there are checks made to ensure that all components are running
+//! correctly. If any of these checks fail, the simulation will exit immediately.
+//!
+//! ## Future works
+//!
+//! Presently all the beacon nodes and validator clients all log to stdout. Additionally, the
+//! simulation uses `println` to communicate some info. It might be nice if the nodes logged to
+//! easy-to-find files and stdout only contained info from the simulation.
+//!
+
+#[macro_use]
+extern crate clap;
+
+mod checks;
+mod cli;
+mod eth1_sim;
+mod local_network;
+mod no_eth1_sim;
+mod sync_sim;
+
+use cli::cli_app;
+use env_logger::{Builder, Env};
+use local_network::LocalNetwork;
+use types::MinimalEthSpec;
+
+pub type E = MinimalEthSpec;
+
+fn main() {
+    // Debugging output for libp2p and external crates.
+    Builder::from_env(Env::default()).init();
+
+    let matches = cli_app().get_matches();
+    match matches.subcommand() {
+        ("eth1-sim", Some(matches)) => match eth1_sim::run_eth1_sim(matches) {
+            Ok(()) => println!("Simulation exited successfully"),
+            Err(e) => {
+                eprintln!("Simulation exited with error: {}", e);
+                std::process::exit(1)
+            }
+        },
+        ("no-eth1-sim", Some(matches)) => match no_eth1_sim::run_no_eth1_sim(matches) {
+            Ok(()) => println!("Simulation exited successfully"),
+            Err(e) => {
+                eprintln!("Simulation exited with error: {}", e);
+                std::process::exit(1)
+            }
+        },
+        ("syncing-sim", Some(matches)) => match sync_sim::run_syncing_sim(matches) {
+            Ok(()) => println!("Simulation exited successfully"),
+            Err(e) => {
+                eprintln!("Simulation exited with error: {}", e);
+                std::process::exit(1)
+            }
+        },
+        _ => {
+            eprintln!("Invalid subcommand. Use --help to see available options");
+            std::process::exit(1)
+        }
+    }
+}

testing/simulator/src/no_eth1_sim.rs

@@ -0,0 +1,158 @@
+use crate::{checks, LocalNetwork};
+use clap::ArgMatches;
+use futures::prelude::*;
+use node_test_rig::{
+    environment::EnvironmentBuilder, testing_client_config, ClientGenesis, ValidatorConfig,
+    ValidatorFiles,
+};
+use rayon::prelude::*;
+use std::net::{IpAddr, Ipv4Addr};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use tokio::time::{delay_until, Instant};
+
+pub fn run_no_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
+    let node_count = value_t!(matches, "nodes", usize).expect("missing nodes default");
+    let validators_per_node = value_t!(matches, "validators_per_node", usize)
+        .expect("missing validators_per_node default");
+    let speed_up_factor =
+        value_t!(matches, "speed_up_factor", u64).expect("missing speed_up_factor default");
+    let mut end_after_checks = true;
+    if matches.is_present("end_after_checks") {
+        end_after_checks = false;
+    }
+
+    println!("Beacon Chain Simulator:");
+    println!(" nodes:{}", node_count);
+    println!(" validators_per_node:{}", validators_per_node);
+    println!(" end_after_checks:{}", end_after_checks);
+
+    // Generate the directories and keystores required for the validator clients.
+    let validator_files = (0..node_count)
+        .into_par_iter()
+        .map(|i| {
+            println!(
+                "Generating keystores for validator {} of {}",
+                i + 1,
+                node_count
+            );
+
+            let indices =
+                (i * validators_per_node..(i + 1) * validators_per_node).collect::<Vec<_>>();
+            ValidatorFiles::with_keystores(&indices).unwrap()
+        })
+        .collect::<Vec<_>>();
+
+    let log_level = "debug";
+    let log_format = None;
+
+    let mut env = EnvironmentBuilder::mainnet()
+        .async_logger(log_level, log_format)?
+        .multi_threaded_tokio_runtime()?
+        .build()?;
+
+    let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor);
+
+    let spec = &mut env.eth2_config.spec;
+
+    spec.milliseconds_per_slot /= speed_up_factor;
+    spec.eth1_follow_distance = 16;
+    spec.min_genesis_delay = eth1_block_time.as_secs() * spec.eth1_follow_distance * 2;
+    spec.min_genesis_time = 0;
+    spec.min_genesis_active_validator_count = 64;
+    spec.seconds_per_eth1_block = 1;
+
+    let genesis_delay = Duration::from_secs(5);
+    let genesis_time = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_err(|_| "should get system time")?
+        + genesis_delay;
+    let genesis_instant = Instant::now() + genesis_delay;
+
+    let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+    let total_validator_count = validators_per_node * node_count;
+
+    let context = env.core_context();
+
+    let mut beacon_config = testing_client_config();
+
+    beacon_config.genesis = ClientGenesis::Interop {
+        validator_count: total_validator_count,
+        genesis_time: genesis_time.as_secs(),
+    };
+    beacon_config.dummy_eth1_backend = true;
+    beacon_config.sync_eth1_chain = true;
+
+    beacon_config.network.enr_address = Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
+
+    let main_future = async {
+        let network = LocalNetwork::new(context, beacon_config.clone()).await?;
+        /*
+         * One by one, add beacon nodes to the network.
+         */
+
+        for _ in 0..node_count - 1 {
+            network.add_beacon_node(beacon_config.clone()).await?;
+        }
+
+        /*
+         * Create a future that will add validator clients to the network. Each validator client is
+         * attached to a single corresponding beacon node.
+         */
+        let add_validators_fut = async {
+            for (i, files) in validator_files.into_iter().enumerate() {
+                network
+                    .add_validator_client(
+                        ValidatorConfig {
+                            auto_register: true,
+                            ..ValidatorConfig::default()
+                        },
+                        i,
+                        files,
+                    )
+                    .await?;
+            }
+
+            Ok::<(), String>(())
+        };
+
+        /*
+         * The processes that will run checks on the network as it runs.
+         */
+        let checks_fut = async {
+            delay_until(genesis_instant).await;
+
+            // Check that the chain finalizes at the first given opportunity.
+            checks::verify_first_finalization(network.clone(), slot_duration).await?;
+
+            Ok::<(), String>(())
+        };
+
+        let (add_validators, start_checks) = futures::join!(add_validators_fut, checks_fut);
+
+        add_validators?;
+        start_checks?;
+
+        // The `final_future` either completes immediately or never completes, depending on the value
+        // of `end_after_checks`.
+
+        if !end_after_checks {
+            future::pending::<()>().await;
+        }
+        /*
+         * End the simulation by dropping the network. This will kill all running beacon nodes and
+         * validator clients.
+         */
+        println!(
+            "Simulation complete. Finished with {} beacon nodes and {} validator clients",
+            network.beacon_node_count(),
+            network.validator_client_count()
+        );
+
+        // Be explicit about dropping the network, as this kills all the nodes. This ensures
+        // all the checks have adequate time to pass.
+        drop(network);
+        Ok::<(), String>(())
+    };
+
+    Ok(env.runtime().block_on(main_future).unwrap())
+}

testing/simulator/src/sync_sim.rs

@@ -0,0 +1,362 @@
+use crate::checks::{epoch_delay, verify_all_finalized_at};
+use crate::local_network::LocalNetwork;
+use clap::ArgMatches;
+use futures::prelude::*;
+use node_test_rig::ClientConfig;
+use node_test_rig::{
+    environment::EnvironmentBuilder, testing_client_config, ClientGenesis, ValidatorConfig,
+    ValidatorFiles,
+};
+use std::net::{IpAddr, Ipv4Addr};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use types::{Epoch, EthSpec};
+
+pub fn run_syncing_sim(matches: &ArgMatches) -> Result<(), String> {
+    let initial_delay = value_t!(matches, "initial_delay", u64).unwrap();
+    let sync_timeout = value_t!(matches, "sync_timeout", u64).unwrap();
+    let speed_up_factor = value_t!(matches, "speedup", u64).unwrap();
+    let strategy = value_t!(matches, "strategy", String).unwrap();
+
+    println!("Syncing Simulator:");
+    println!(" initial_delay:{}", initial_delay);
+    println!(" sync timeout: {}", sync_timeout);
+    println!(" speed up factor:{}", speed_up_factor);
+    println!(" strategy:{}", strategy);
+
+    let log_level = "debug";
+    let log_format = None;
+
+    syncing_sim(
+        speed_up_factor,
+        initial_delay,
+        sync_timeout,
+        strategy,
+        log_level,
+        log_format,
+    )
+}
+
+fn syncing_sim(
+    speed_up_factor: u64,
+    initial_delay: u64,
+    sync_timeout: u64,
+    strategy: String,
+    log_level: &str,
+    log_format: Option<&str>,
+) -> Result<(), String> {
+    let mut env = EnvironmentBuilder::minimal()
+        .async_logger(log_level, log_format)?
+        .multi_threaded_tokio_runtime()?
+        .build()?;
+
+    let spec = &mut env.eth2_config.spec;
+    let end_after_checks = true;
+    let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor);
+
+    spec.milliseconds_per_slot /= speed_up_factor;
+    spec.eth1_follow_distance = 16;
+    spec.min_genesis_delay = eth1_block_time.as_secs() * spec.eth1_follow_distance * 2;
+    spec.min_genesis_time = 0;
+    spec.min_genesis_active_validator_count = 64;
+    spec.seconds_per_eth1_block = 1;
+
+    let num_validators = 8;
+    let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+    let context = env.core_context();
+    let mut beacon_config = testing_client_config();
+
+    let genesis_time = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_err(|_| "should get system time")?
+        + Duration::from_secs(5);
+    beacon_config.genesis = ClientGenesis::Interop {
+        validator_count: num_validators,
+        genesis_time: genesis_time.as_secs(),
+    };
+    beacon_config.dummy_eth1_backend = true;
+    beacon_config.sync_eth1_chain = true;
+
+    beacon_config.network.enr_address = Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
+
+    // Generate the directories and keystores required for the validator clients.
+    let validator_indices = (0..num_validators).collect::<Vec<_>>();
+    let validator_files = ValidatorFiles::with_keystores(&validator_indices).unwrap();
+
+    let main_future = async {
+        /*
+         * Create a new `LocalNetwork` with one beacon node.
+         */
+        let network = LocalNetwork::new(context, beacon_config.clone()).await?;
+
+        /*
+         * Add a validator client which handles all validators from the genesis state.
+         */
+        network
+            .add_validator_client(ValidatorConfig::default(), 0, validator_files)
+            .await?;
+
+        // Check all syncing strategies one after other.
+        pick_strategy(
+            &strategy,
+            network.clone(),
+            beacon_config.clone(),
+            slot_duration,
+            initial_delay,
+            sync_timeout,
+        )
+        .await?;
+
+        // The `final_future` either completes immediately or never completes, depending on the value
+        // of `end_after_checks`.
+
+        if !end_after_checks {
+            future::pending::<()>().await;
+        }
+
+        /*
+         * End the simulation by dropping the network. This will kill all running beacon nodes and
+         * validator clients.
+         */
+        println!(
+            "Simulation complete. Finished with {} beacon nodes and {} validator clients",
+            network.beacon_node_count(),
+            network.validator_client_count()
+        );
+
+        // Be explicit about dropping the network, as this kills all the nodes. This ensures
+        // all the checks have adequate time to pass.
+        drop(network);
+        Ok::<(), String>(())
+    };
+
+    env.runtime().block_on(main_future)
+}
+
+pub async fn pick_strategy<E: EthSpec>(
+    strategy: &str,
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    match strategy {
+        "one-node" => {
+            verify_one_node_sync(
+                network,
+                beacon_config,
+                slot_duration,
+                initial_delay,
+                sync_timeout,
+            )
+            .await
+        }
+        "two-nodes" => {
+            verify_two_nodes_sync(
+                network,
+                beacon_config,
+                slot_duration,
+                initial_delay,
+                sync_timeout,
+            )
+            .await
+        }
+        "mixed" => {
+            verify_in_between_sync(
+                network,
+                beacon_config,
+                slot_duration,
+                initial_delay,
+                sync_timeout,
+            )
+            .await
+        }
+        "all" => {
+            verify_syncing(
+                network,
+                beacon_config,
+                slot_duration,
+                initial_delay,
+                sync_timeout,
+            )
+            .await
+        }
+        _ => Err("Invalid strategy".into()),
+    }
+}
+
+/// Verify one node added after `initial_delay` epochs is in sync
+/// after `sync_timeout` epochs.
+pub async fn verify_one_node_sync<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    let epoch_duration = slot_duration * (E::slots_per_epoch() as u32);
+    let network_c = network.clone();
+    // Delay for `initial_delay` epochs before adding another node to start syncing
+    epoch_delay(
+        Epoch::new(initial_delay),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .await;
+    // Add a beacon node
+    network.add_beacon_node(beacon_config).await?;
+    // Check every `epoch_duration` if nodes are synced
+    // limited to at most `sync_timeout` epochs
+    let mut interval = tokio::time::interval(epoch_duration);
+    let mut count = 0;
+    while let Some(_) = interval.next().await {
+        if count >= sync_timeout || !check_still_syncing(&network_c).await? {
+            break;
+        }
+        count += 1;
+    }
+    let epoch = network.bootnode_epoch().await?;
+    verify_all_finalized_at(network, epoch)
+        .map_err(|e| format!("One node sync error: {}", e))
+        .await
+}
+
+/// Verify two nodes added after `initial_delay` epochs are in sync
+/// after `sync_timeout` epochs.
+pub async fn verify_two_nodes_sync<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    let epoch_duration = slot_duration * (E::slots_per_epoch() as u32);
+    let network_c = network.clone();
+    // Delay for `initial_delay` epochs before adding another node to start syncing
+    epoch_delay(
+        Epoch::new(initial_delay),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .await;
+    // Add beacon nodes
+    network.add_beacon_node(beacon_config.clone()).await?;
+    network.add_beacon_node(beacon_config).await?;
+    // Check every `epoch_duration` if nodes are synced
+    // limited to at most `sync_timeout` epochs
+    let mut interval = tokio::time::interval(epoch_duration);
+    let mut count = 0;
+    while let Some(_) = interval.next().await {
+        if count >= sync_timeout || !check_still_syncing(&network_c).await? {
+            break;
+        }
+        count += 1;
+    }
+    let epoch = network.bootnode_epoch().await?;
+    verify_all_finalized_at(network, epoch)
+        .map_err(|e| format!("One node sync error: {}", e))
+        .await
+}
+
+/// Add 2 syncing nodes after `initial_delay` epochs,
+/// Add another node after `sync_timeout - 5` epochs and verify all are
+/// in sync after `sync_timeout + 5` epochs.
+pub async fn verify_in_between_sync<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    let epoch_duration = slot_duration * (E::slots_per_epoch() as u32);
+    let network_c = network.clone();
+    // Delay for `initial_delay` epochs before adding another node to start syncing
+    let config1 = beacon_config.clone();
+    epoch_delay(
+        Epoch::new(initial_delay),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .await;
+    // Add two beacon nodes
+    network.add_beacon_node(beacon_config.clone()).await?;
+    network.add_beacon_node(beacon_config).await?;
+    // Delay before adding additional syncing nodes.
+    epoch_delay(
+        Epoch::new(sync_timeout - 5),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .await;
+    // Add a beacon node
+    network.add_beacon_node(config1.clone()).await?;
+    // Check every `epoch_duration` if nodes are synced
+    // limited to at most `sync_timeout` epochs
+    let mut interval = tokio::time::interval(epoch_duration);
+    let mut count = 0;
+    while let Some(_) = interval.next().await {
+        if count >= sync_timeout || !check_still_syncing(&network_c).await? {
+            break;
+        }
+        count += 1;
+    }
+    let epoch = network.bootnode_epoch().await?;
+    verify_all_finalized_at(network, epoch)
+        .map_err(|e| format!("One node sync error: {}", e))
+        .await
+}
+
+/// Run syncing strategies one after other.
+pub async fn verify_syncing<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_timeout: u64,
+) -> Result<(), String> {
+    verify_one_node_sync(
+        network.clone(),
+        beacon_config.clone(),
+        slot_duration,
+        initial_delay,
+        sync_timeout,
+    )
+    .await?;
+    println!("Completed one node sync");
+    verify_two_nodes_sync(
+        network.clone(),
+        beacon_config.clone(),
+        slot_duration,
+        initial_delay,
+        sync_timeout,
+    )
+    .await?;
+    println!("Completed two node sync");
+    verify_in_between_sync(
+        network,
+        beacon_config,
+        slot_duration,
+        initial_delay,
+        sync_timeout,
+    )
+    .await?;
+    println!("Completed in between sync");
+    Ok(())
+}
+
+pub async fn check_still_syncing<E: EthSpec>(network: &LocalNetwork<E>) -> Result<bool, String> {
+    // get syncing status of nodes
+    let mut status = Vec::new();
+    for remote_node in network.remote_nodes()? {
+        status.push(
+            remote_node
+                .http
+                .node()
+                .syncing_status()
+                .await
+                .map(|status| status.is_syncing)
+                .map_err(|e| format!("Get syncing status via http failed: {:?}", e))?,
+        )
+    }
+    Ok(status.iter().any(|is_syncing| *is_syncing))
+}

testing/state_transition_vectors/.gitignore

@@ -0,0 +1 @@
+/vectors/

testing/state_transition_vectors/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "state_transition_vectors"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+state_processing = { path = "../../consensus/state_processing" }
+types = { path = "../../consensus/types" }
+eth2_ssz = "0.1.2"

testing/state_transition_vectors/Makefile

@@ -0,0 +1,8 @@
+produce-vectors:
+	cargo run --release
+
+test:
+	cargo test --release
+
+clean:
+	rm -r vectors/

testing/state_transition_vectors/README.md

@@ -0,0 +1,72 @@
+# state_transition_vectors
+
+This crate contains test vectors for Lighthouse state transition functions.
+
+This crate serves two purposes:
+
+- Outputting the test vectors to disk via `make`.
+- Running the vectors against our code via `make test`.
+
+
+## Outputting vectors to disk
+
+Whilst we don't actually need to write the vectors to disk to test them, we
+provide this functionality so we can generate corpra for the fuzzer and also so
+they can be of use to other clients.
+
+To create the files in `./vectors` (directory relative to this crate), run:
+
+```bash
+make
+```
+
+This will produce a directory structure that looks roughly like this:
+
+```
+vectors
+└── exit
+    ├── invalid_bad_signature
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_duplicate
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_exit_already_initiated
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_future_exit_epoch
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_not_active_after_exit_epoch
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_not_active_before_activation_epoch
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_too_young_by_a_lot
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_too_young_by_one_epoch
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_validator_unknown
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── valid_genesis_epoch
+    │   ├── block.ssz
+    │   ├── post.ssz
+    │   └── pre.ssz
+    └── valid_previous_epoch
+        ├── block.ssz
+        ├── post.ssz
+        └── pre.ssz
+```

testing/state_transition_vectors/src/exit.rs

@@ -0,0 +1,346 @@
+use super::*;
+use state_processing::{
+    per_block_processing, per_block_processing::errors::ExitInvalid,
+    test_utils::BlockProcessingBuilder, BlockProcessingError, BlockSignatureStrategy,
+};
+use types::{BeaconBlock, BeaconState, Epoch, EthSpec, SignedBeaconBlock};
+
+// Default validator index to exit.
+pub const VALIDATOR_INDEX: u64 = 0;
+// Epoch that the state will be transitioned to by default, equal to PERSISTENT_COMMITTEE_PERIOD.
+pub const STATE_EPOCH: Epoch = Epoch::new(2048);
+
+struct ExitTest {
+    validator_index: u64,
+    exit_epoch: Epoch,
+    state_epoch: Epoch,
+    block_modifier: Box<dyn FnOnce(&mut BeaconBlock<E>)>,
+    builder_modifier: Box<dyn FnOnce(BlockProcessingBuilder<E>) -> BlockProcessingBuilder<E>>,
+    #[allow(dead_code)]
+    expected: Result<(), BlockProcessingError>,
+}
+
+impl Default for ExitTest {
+    fn default() -> Self {
+        Self {
+            validator_index: VALIDATOR_INDEX,
+            exit_epoch: STATE_EPOCH,
+            state_epoch: STATE_EPOCH,
+            block_modifier: Box::new(|_| ()),
+            builder_modifier: Box::new(|x| x),
+            expected: Ok(()),
+        }
+    }
+}
+
+impl ExitTest {
+    fn block_and_pre_state(self) -> (SignedBeaconBlock<E>, BeaconState<E>) {
+        let spec = &E::default_spec();
+
+        (self.builder_modifier)(
+            get_builder(spec, self.state_epoch.as_u64(), VALIDATOR_COUNT)
+                .insert_exit(self.validator_index, self.exit_epoch)
+                .modify(self.block_modifier),
+        )
+        .build(None, None)
+    }
+
+    fn process(
+        block: &SignedBeaconBlock<E>,
+        state: &mut BeaconState<E>,
+    ) -> Result<(), BlockProcessingError> {
+        per_block_processing(
+            state,
+            block,
+            None,
+            BlockSignatureStrategy::VerifyIndividual,
+            &E::default_spec(),
+        )
+    }
+
+    #[cfg(test)]
+    fn run(self) -> BeaconState<E> {
+        let spec = &E::default_spec();
+        let expected = self.expected.clone();
+        assert_eq!(STATE_EPOCH, spec.persistent_committee_period);
+
+        let (block, mut state) = self.block_and_pre_state();
+
+        let result = Self::process(&block, &mut state);
+
+        assert_eq!(result, expected);
+
+        state
+    }
+
+    fn test_vector(self, title: String) -> TestVector {
+        let (block, pre_state) = self.block_and_pre_state();
+        let mut post_state = pre_state.clone();
+        let (post_state, error) = match Self::process(&block, &mut post_state) {
+            Ok(_) => (Some(post_state), None),
+            Err(e) => (None, Some(format!("{:?}", e))),
+        };
+
+        TestVector {
+            title,
+            block,
+            pre_state,
+            post_state,
+            error,
+        }
+    }
+}
+
+vectors_and_tests!(
+    // Ensures we can process a valid exit,
+    valid_single_exit,
+    ExitTest::default(),
+    // Tests three exists in the same block.
+    valid_three_exits,
+    ExitTest {
+        builder_modifier: Box::new(|builder| {
+            builder
+                .insert_exit(1, STATE_EPOCH)
+                .insert_exit(2, STATE_EPOCH)
+        }),
+        ..ExitTest::default()
+    },
+    // Ensures that a validator cannot be exited twice in the same block.
+    invalid_duplicate,
+    ExitTest {
+        block_modifier: Box::new(|block| {
+            // Duplicate the exit
+            let exit = block.body.voluntary_exits[0].clone();
+            block.body.voluntary_exits.push(exit).unwrap();
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 1,
+            reason: ExitInvalid::AlreadyExited(0),
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // validator = state.validators[voluntary_exit.validator_index]
+    // ```
+    invalid_validator_unknown,
+    ExitTest {
+        block_modifier: Box::new(|block| {
+            block.body.voluntary_exits[0].message.validator_index = VALIDATOR_COUNT as u64;
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::ValidatorUnknown(VALIDATOR_COUNT as u64),
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify exit has not been initiated
+    // assert validator.exit_epoch == FAR_FUTURE_EPOCH
+    // ```
+    invalid_exit_already_initiated,
+    ExitTest {
+        builder_modifier: Box::new(|mut builder| {
+            builder.state.validators[0].exit_epoch = STATE_EPOCH + 1;
+            builder
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::AlreadyExited(0),
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify the validator is active
+    // assert is_active_validator(validator, get_current_epoch(state))
+    // ```
+    invalid_not_active_before_activation_epoch,
+    ExitTest {
+        builder_modifier: Box::new(|mut builder| {
+            builder.state.validators[0].activation_epoch = builder.spec.far_future_epoch;
+            builder
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::NotActive(0),
+        }),
+        ..ExitTest::default()
+    },
+    // Also tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify the validator is active
+    // assert is_active_validator(validator, get_current_epoch(state))
+    // ```
+    invalid_not_active_after_exit_epoch,
+    ExitTest {
+        builder_modifier: Box::new(|mut builder| {
+            builder.state.validators[0].exit_epoch = STATE_EPOCH;
+            builder
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::NotActive(0),
+        }),
+        ..ExitTest::default()
+    },
+    // Ensures we can process an exit from genesis.
+    valid_genesis_epoch,
+    ExitTest {
+        exit_epoch: Epoch::new(0),
+        ..ExitTest::default()
+    },
+    // Ensures we can process an exit from the previous epoch.
+    valid_previous_epoch,
+    ExitTest {
+        exit_epoch: STATE_EPOCH - 1,
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Exits must specify an epoch when they become valid; they are not
+    // # valid before then
+    // assert get_current_epoch(state) >= voluntary_exit.epoch
+    // ```
+    invalid_future_exit_epoch,
+    ExitTest {
+        exit_epoch: STATE_EPOCH + 1,
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::FutureEpoch {
+                state: STATE_EPOCH,
+                exit: STATE_EPOCH + 1,
+            },
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify the validator has been active long enough
+    // assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD
+    // ```
+    invalid_too_young_by_one_epoch,
+    ExitTest {
+        state_epoch: STATE_EPOCH - 1,
+        exit_epoch: STATE_EPOCH - 1,
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::TooYoungToExit {
+                current_epoch: STATE_EPOCH - 1,
+                earliest_exit_epoch: STATE_EPOCH,
+            },
+        }),
+        ..ExitTest::default()
+    },
+    // Also tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify the validator has been active long enough
+    // assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD
+    // ```
+    invalid_too_young_by_a_lot,
+    ExitTest {
+        state_epoch: Epoch::new(0),
+        exit_epoch: Epoch::new(0),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::TooYoungToExit {
+                current_epoch: Epoch::new(0),
+                earliest_exit_epoch: STATE_EPOCH,
+            },
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify signature
+    // domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT,
+    // voluntary_exit.epoch)
+    // signing_root = compute_signing_root(voluntary_exit, domain)
+    // assert bls.Verify(validator.pubkey, signing_root,
+    // signed_voluntary_exit.signature)
+    // ```
+    invalid_bad_signature,
+    ExitTest {
+        block_modifier: Box::new(|block| {
+            // Shift the validator index by 1 so that it's mismatched from the key that was
+            // used to sign.
+            block.body.voluntary_exits[0].message.validator_index = VALIDATOR_INDEX + 1;
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::BadSignature,
+        }),
+        ..ExitTest::default()
+    }
+);
+
+#[cfg(test)]
+mod custom_tests {
+    use super::*;
+
+    fn assert_exited(state: &BeaconState<E>, validator_index: usize) {
+        let spec = E::default_spec();
+
+        let validator = &state.validators[validator_index];
+        assert_eq!(
+            validator.exit_epoch,
+            // This is correct until we exceed the churn limit. If that happens, we
+            // need to introduce more complex logic.
+            state.current_epoch() + 1 + spec.max_seed_lookahead,
+            "exit epoch"
+        );
+        assert_eq!(
+            validator.withdrawable_epoch,
+            validator.exit_epoch + E::default_spec().min_validator_withdrawability_delay,
+            "withdrawable epoch"
+        );
+    }
+
+    #[test]
+    fn valid() {
+        let state = ExitTest::default().run();
+        assert_exited(&state, VALIDATOR_INDEX as usize);
+    }
+
+    #[test]
+    fn valid_three() {
+        let state = ExitTest {
+            builder_modifier: Box::new(|builder| {
+                builder
+                    .insert_exit(1, STATE_EPOCH)
+                    .insert_exit(2, STATE_EPOCH)
+            }),
+            ..ExitTest::default()
+        }
+        .run();
+
+        for i in &[VALIDATOR_INDEX, 1, 2] {
+            assert_exited(&state, *i as usize);
+        }
+    }
+}

testing/state_transition_vectors/src/macros.rs

@@ -0,0 +1,28 @@
+/// Provides:
+///
+/// - `fn vectors()`: allows for getting a `Vec<TestVector>` of all vectors for exporting.
+/// - `mod tests`: runs all the test vectors locally.
+macro_rules! vectors_and_tests {
+    ($($name: ident, $test: expr),*) => {
+        pub fn vectors() -> Vec<TestVector> {
+            let mut vec = vec![];
+
+            $(
+                vec.push($test.test_vector(stringify!($name).into()));
+            )*
+
+            vec
+        }
+
+        #[cfg(test)]
+        mod tests {
+            use super::*;
+            $(
+                #[test]
+                fn $name() {
+                    $test.run();
+                }
+            )*
+        }
+    };
+}

testing/state_transition_vectors/src/main.rs

@@ -0,0 +1,107 @@
+#[macro_use]
+mod macros;
+mod exit;
+
+use ssz::Encode;
+use state_processing::test_utils::BlockProcessingBuilder;
+use std::env;
+use std::fs::{self, File};
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::exit;
+use types::MainnetEthSpec;
+use types::{BeaconState, ChainSpec, EthSpec, SignedBeaconBlock};
+
+type E = MainnetEthSpec;
+
+pub const VALIDATOR_COUNT: usize = 64;
+
+/// The base output directory for test vectors.
+pub const BASE_VECTOR_DIR: &str = "vectors";
+
+/// Writes all known test vectors to `CARGO_MANIFEST_DIR/vectors`.
+fn main() {
+    match write_all_vectors() {
+        Ok(()) => exit(0),
+        Err(e) => {
+            eprintln!("Error: {}", e);
+            exit(1)
+        }
+    }
+}
+
+/// An abstract definition of a test vector that can be run as a test or exported to disk.
+pub struct TestVector {
+    pub title: String,
+    pub pre_state: BeaconState<E>,
+    pub block: SignedBeaconBlock<E>,
+    pub post_state: Option<BeaconState<E>>,
+    pub error: Option<String>,
+}
+
+/// Gets a `BlockProcessingBuilder` to be used in testing.
+fn get_builder(
+    spec: &ChainSpec,
+    epoch_offset: u64,
+    num_validators: usize,
+) -> BlockProcessingBuilder<MainnetEthSpec> {
+    // Set the state and block to be in the last slot of the `epoch_offset`th epoch.
+    let last_slot_of_epoch = (MainnetEthSpec::genesis_epoch() + epoch_offset)
+        .end_slot(MainnetEthSpec::slots_per_epoch());
+    BlockProcessingBuilder::new(num_validators, last_slot_of_epoch, &spec).build_caches()
+}
+
+/// Writes all vectors to file.
+fn write_all_vectors() -> Result<(), String> {
+    write_vectors_to_file("exit", &exit::vectors())
+}
+
+/// Writes a list of `vectors` to the `title` dir.
+fn write_vectors_to_file(title: &str, vectors: &[TestVector]) -> Result<(), String> {
+    let dir = env::var("CARGO_MANIFEST_DIR")
+        .map_err(|e| format!("Unable to find manifest dir: {:?}", e))?
+        .parse::<PathBuf>()
+        .map_err(|e| format!("Unable to parse manifest dir: {:?}", e))?
+        .join(BASE_VECTOR_DIR)
+        .join(title);
+
+    if dir.exists() {
+        fs::remove_dir_all(&dir).map_err(|e| format!("Unable to remove {:?}: {:?}", dir, e))?;
+    }
+    fs::create_dir_all(&dir).map_err(|e| format!("Unable to create {:?}: {:?}", dir, e))?;
+
+    for vector in vectors {
+        let dir = dir.clone().join(&vector.title);
+        if dir.exists() {
+            fs::remove_dir_all(&dir).map_err(|e| format!("Unable to remove {:?}: {:?}", dir, e))?;
+        }
+        fs::create_dir_all(&dir).map_err(|e| format!("Unable to create {:?}: {:?}", dir, e))?;
+
+        write_to_ssz_file(&dir.clone().join("pre.ssz"), &vector.pre_state)?;
+        write_to_ssz_file(&dir.clone().join("block.ssz"), &vector.block)?;
+        if let Some(post_state) = vector.post_state.as_ref() {
+            write_to_ssz_file(&dir.clone().join("post.ssz"), post_state)?;
+        }
+        if let Some(error) = vector.error.as_ref() {
+            write_to_file(&dir.clone().join("error.txt"), error.as_bytes())?;
+        }
+    }
+
+    Ok(())
+}
+
+/// Write some SSZ object to file.
+fn write_to_ssz_file<T: Encode>(path: &PathBuf, item: &T) -> Result<(), String> {
+    write_to_file(path, &item.as_ssz_bytes())
+}
+
+/// Write some bytes to file.
+fn write_to_file(path: &PathBuf, item: &[u8]) -> Result<(), String> {
+    File::create(path)
+        .map_err(|e| format!("Unable to create {:?}: {:?}", path, e))
+        .and_then(|mut file| {
+            file.write_all(item)
+                .map(|_| ())
+                .map_err(|e| format!("Unable to write to {:?}: {:?}", path, e))
+        })
+}