Update to spec v1.0.0-rc.0 and BLSv4 (#1765)

## Issue Addressed

Closes #1504 
Closes #1505
Replaces #1703
Closes #1707

## Proposed Changes

* Update BLST and Milagro to versions compatible with BLSv4 spec
* Update Lighthouse to spec v1.0.0-rc.0, and update EF test vectors
* Use the v1.0.0 constants for `MainnetEthSpec`.
* Rename `InteropEthSpec` -> `V012LegacyEthSpec`
    * Change all constants to suit the mainnet `v0.12.3` specification (i.e., Medalla).
* Deprecate the `--spec` flag for the `lighthouse` binary
    * This value is now obtained from the `config_name` field of the `YamlConfig`.
        * Built in testnet YAML files have been updated.
    * Ignore the `--spec` value, if supplied, log a warning that it will be deprecated
    * `lcli` still has the spec flag, that's fine because it's dev tooling.
* Remove the `E: EthSpec` from `YamlConfig`
    * This means we need to deser the genesis `BeaconState` on-demand, but this is fine.
* Swap the old "minimal", "mainnet" strings over to the new `EthSpecId` enum.
* Always require a `CONFIG_NAME` field in `YamlConfig` (it used to have a default).

## Additional Info

Lots of breaking changes, do not merge! ~~We will likely need a Lighthouse v0.4.0 branch, and possibly a long-term v0.3.0 branch to keep Medalla alive~~.

Co-authored-by: Kirk Baird <baird.k@outlook.com>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

.dockerignore

@@ -2,4 +2,3 @@ tests/ef_tests/eth2.0-spec-tests
 target/
 *.data
 *.tar.gz
-.git

.github/workflows/test-suite.yml

@@ -33,20 +33,6 @@ jobs:
       run: sudo npm install -g ganache-cli
     - name: Run tests in release
       run: make test-release
-  release-tests-and-install-macos:
-    name: release-tests-and-install-macos
-    runs-on: macos-latest
-    needs: cargo-fmt
-    steps:
-    - uses: actions/checkout@v1
-    - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
-    - name: Run tests in release
-      run: make test-release
-    - name: Install Lighthouse
-      run: make
   debug-tests-ubuntu:
     name: debug-tests-ubuntu
     runs-on: ubuntu-latest
@@ -115,6 +101,14 @@ jobs:
     - uses: actions/checkout@v1
     - name: Typecheck benchmark code without running it
       run: make check-benches
+  check-consensus:
+    name: check-consensus
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Typecheck consensus code in strict mode
+      run: make check-consensus
   clippy:
     name: clippy
     runs-on: ubuntu-latest
@@ -123,6 +117,8 @@ jobs:
     - uses: actions/checkout@v1
     - name: Lint code for quality and style with Clippy
       run: make lint
+    - name: Certify Cargo.lock freshness
+      run: git diff --exit-code Cargo.lock
   arbitrary-check:
     name: arbitrary-check
     runs-on: ubuntu-latest

.gitignore

@@ -6,3 +6,4 @@ target/
 flamegraph.svg
 perf.data*
 *.tar.gz
+/bin

Cargo.toml

@@ -7,8 +7,9 @@ members = [
     "beacon_node/client",
     "beacon_node/eth1",
     "beacon_node/eth2_libp2p",
+    "beacon_node/http_api",
+    "beacon_node/http_metrics",
     "beacon_node/network",
-    "beacon_node/rest_api",
     "beacon_node/store",
     "beacon_node/timer",
     "beacon_node/websocket_server",
@@ -20,6 +21,8 @@ members = [
     "common/compare_fields",
     "common/compare_fields_derive",
     "common/deposit_contract",
+    "common/directory",
+    "common/eth2",
     "common/eth2_config",
     "common/eth2_interop_keypairs",
     "common/eth2_testnet_config",
@@ -28,10 +31,11 @@ members = [
     "common/lighthouse_metrics",
     "common/lighthouse_version",
     "common/logging",
-    "common/remote_beacon_node",
-    "common/rest_types",
+    "common/lru_cache",
     "common/slot_clock",
     "common/test_random_derive",
+    "common/warp_utils",
+    "common/task_executor",
     "common/validator_dir",
 
     "consensus/cached_tree_hash",
@@ -42,7 +46,7 @@ members = [
     "consensus/ssz",
     "consensus/ssz_derive",
     "consensus/ssz_types",
-    "consensus/serde_hex",
+    "consensus/serde_utils",
     "consensus/state_processing",
     "consensus/swap_or_not_shuffle",
     "consensus/tree_hash",
@@ -77,4 +81,3 @@ eth2_ssz = { path = "consensus/ssz" }
 eth2_ssz_derive = { path = "consensus/ssz_derive" }
 eth2_ssz_types = { path = "consensus/ssz_types" }
 eth2_hashing = { path = "crypto/eth2_hashing" }
-leveldb-sys = { git = "https://github.com/michaelsproul/leveldb-sys", branch = "v2.0.6-cmake" }

Cross.toml

@@ -0,0 +1,4 @@
+[build.env]
+passthrough = [
+    "RUSTFLAGS",
+]

Makefile

@@ -2,6 +2,13 @@
 
 EF_TESTS = "testing/ef_tests"
 STATE_TRANSITION_VECTORS = "testing/state_transition_vectors"
+GIT_TAG := $(shell git describe --tags --candidates 1)
+BIN_DIR = "bin"
+
+X86_64_TAG = "x86_64-unknown-linux-gnu"
+BUILD_PATH_X86_64 = "target/$(X86_64_TAG)/release"
+AARCH64_TAG = "aarch64-unknown-linux-gnu"
+BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
 
 # Builds the Lighthouse binary in release (optimized).
 #
@@ -21,6 +28,53 @@ else
 	cargo install --path lcli --force --locked
 endif
 
+# The following commands use `cross` to build a cross-compile.
+#
+# These commands require that:
+#
+# - `cross` is installed (`cargo install cross`).
+# - Docker is running.
+# - The current user is in the `docker` group.
+#
+# The resulting binaries will be created in the `target/` directory.
+#
+# The *-portable options compile the blst library *without* the use of some
+# optimized CPU functions that may not be available on some systems. This
+# results in a more portable binary with ~20% slower BLS verification.
+build-x86_64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features modern
+build-x86_64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features portable
+build-aarch64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu
+build-aarch64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu --features portable
+
+# Create a `.tar.gz` containing a binary for a specific target.
+define tarball_release_binary
+	cp $(1)/lighthouse $(BIN_DIR)/lighthouse
+	cd $(BIN_DIR) && \
+		tar -czf lighthouse-$(GIT_TAG)-$(2)$(3).tar.gz lighthouse && \
+		rm lighthouse
+endef
+
+# Create a series of `.tar.gz` files in the BIN_DIR directory, each containing
+# a `lighthouse` binary for a different target.
+#
+# The current git tag will be used as the version in the output file names. You
+# will likely need to use `git tag` and create a semver tag (e.g., `v0.2.3`).
+build-release-tarballs:
+	[ -d $(BIN_DIR) ] || mkdir -p $(BIN_DIR)
+	$(MAKE) build-x86_64
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"")
+	$(MAKE) build-x86_64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"-portable")
+	$(MAKE) build-aarch64
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"")
+	$(MAKE) build-aarch64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"-portable")
+
+
 # Runs the full workspace tests in **release**, without downloading any additional
 # test vectors.
 test-release:
@@ -39,6 +93,10 @@ cargo-fmt:
 check-benches:
 	cargo check --all --benches
 
+# Typechecks consensus code *without* allowing deprecated legacy arithmetic
+check-consensus:
+	cargo check --manifest-path=consensus/state_processing/Cargo.toml --no-default-features
+
 # Runs only the ef-test vectors.
 run-ef-tests:
 	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests"
@@ -79,7 +137,11 @@ arbitrary-fuzz:
 # Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database)
 audit:
 	cargo install --force cargo-audit
-	cargo audit
+	# TODO: we should address this --ignore.
+	#
+	# Tracking issue:
+	# https://github.com/sigp/lighthouse/issues/1669
+	cargo audit --ignore RUSTSEC-2020-0043
 
 # Runs `cargo udeps` to check for unused dependencies
 udeps:

README.md

@@ -2,7 +2,7 @@
 
 An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prime.
 
-[![Build Status]][Build Link] [![Book Status]][Book Link] [![RustDoc Status]][RustDoc Link] [![Chat Badge]][Chat Link]
+[![Build Status]][Build Link] [![Book Status]][Book Link] [![Chat Badge]][Chat Link]
 
 [Build Status]: https://github.com/sigp/lighthouse/workflows/test-suite/badge.svg?branch=master
 [Build Link]: https://github.com/sigp/lighthouse/actions
@@ -10,26 +10,23 @@ An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prim
 [Chat Link]: https://discord.gg/cyAszAh
 [Book Status]:https://img.shields.io/badge/user--docs-master-informational
 [Book Link]: http://lighthouse-book.sigmaprime.io/
-[RustDoc Status]:https://img.shields.io/badge/code--docs-master-orange
-[RustDoc Link]: http://lighthouse-docs.sigmaprime.io/
 
 [Documentation](http://lighthouse-book.sigmaprime.io/)
 
-![terminalize](https://i.postimg.cc/kG11dpCW/lighthouse-cli-png.gif)
+![Banner](https://i.postimg.cc/hjdTGKPd/photo-2020-10-23-09-52-16.jpg)
 
 ## Overview
 
 Lighthouse is:
 
 - Fully open-source, licensed under Apache 2.0.
-- Security-focused. Fuzzing has begun and security reviews are planned
-	for late-2019.
+- Security-focused. Fuzzing has begun and security reviews are underway.
 - Built in [Rust](https://www.rust-lang.org/), a modern language providing unique safety guarantees and
 	excellent performance (comparable to C++).
 - Funded by various organisations, including Sigma Prime, the
 	Ethereum Foundation, ConsenSys and private individuals.
-- Actively involved in the specification and security analysis of the emerging
-    Ethereum 2.0 specification.
+- Actively involved in the specification and security analysis of the
+	Ethereum 2.0 specification.
 
 Like all Ethereum 2.0 clients, Lighthouse is a work-in-progress.
 
@@ -37,7 +34,7 @@ Like all Ethereum 2.0 clients, Lighthouse is a work-in-progress.
 
 Current development overview:
 
-- Specification `v0.12.1` implemented, optimized and passing test vectors.
+- Specification `v1.0.0` implemented, optimized and passing test vectors.
 - Rust-native libp2p with Gossipsub and Discv5.
 - RESTful JSON API via HTTP server.
 - Events via WebSocket.
@@ -50,8 +47,8 @@ Current development overview:
 - ~~**Q1 2020**: `lighthouse-0.1.0` release: All major phase 0 features implemented.~~
 - ~~**Q2 2020**: Public, multi-client testnet with user-facing functionality.~~
 - ~~**Q2 2020**: Third-party security review.~~
-- **Q3 2020**: Additional third-party security reviews.
-- **Q3 2020**: Long-lived, multi-client Beacon Chain testnet
+- ~~**Q4 2020**: Long-lived, multi-client Beacon Chain testnet~~
+- **Q4 2020**: Additional third-party security reviews.
 - **Q4 2020**: Production Beacon Chain (tentative).
 
 
@@ -60,9 +57,6 @@ Current development overview:
 The [Lighthouse Book](http://lighthouse-book.sigmaprime.io/) contains information
 for testnet users and developers.
 
-Code documentation is generated via `cargo doc` and hosted at
-[lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/).
-
 If you'd like some background on Sigma Prime, please see the [Lighthouse Update
 \#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or
 [sigmaprime.io](https://sigmaprime.io).

account_manager/Cargo.toml

@@ -1,33 +1,33 @@
 [package]
 name = "account_manager"
-version = "0.0.1"
+version = "0.3.2"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
 edition = "2018"
 
 [dependencies]
 bls = { path = "../crypto/bls" }
-clap = "2.33.0"
+clap = "2.33.3"
 slog = "2.5.2"
-slog-term = "2.5.0"
+slog-term = "2.6.0"
 slog-async = "2.5.0"
 types = { path = "../consensus/types" }
-state_processing = { path = "../consensus/state_processing" }
-dirs = "2.0.2"
+dirs = "3.0.1"
 environment = { path = "../lighthouse/environment" }
 deposit_contract = { path = "../common/deposit_contract" }
-libc = "0.2.65"
+libc = "0.2.79"
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
 hex = "0.4.2"
-rayon = "1.3.0"
+rayon = "1.4.1"
 eth2_testnet_config = { path = "../common/eth2_testnet_config" }
-web3 = "0.11.0"
 futures = { version = "0.3.5", features = ["compat"] }
 clap_utils = { path = "../common/clap_utils" }
+directory = { path = "../common/directory" }
 eth2_wallet = { path = "../crypto/eth2_wallet" }
 eth2_wallet_manager = { path = "../common/eth2_wallet_manager" }
-rand = "0.7.2"
+rand = "0.7.3"
 validator_dir = { path = "../common/validator_dir" }
-tokio = { version = "0.2.21", features = ["full"] }
+tokio = { version = "0.2.22", features = ["full"] }
 eth2_keystore = { path = "../crypto/eth2_keystore" }
 account_utils = { path = "../common/account_utils" }
+slashing_protection = { path = "../validator_client/slashing_protection" }

account_manager/src/common.rs

@@ -1,21 +1,67 @@
-use clap::ArgMatches;
-use std::fs::create_dir_all;
-use std::path::{Path, PathBuf};
+use account_utils::PlainText;
+use account_utils::{read_input_from_user, strip_off_newlines};
+use eth2_wallet::bip39::{Language, Mnemonic};
+use std::fs;
+use std::path::PathBuf;
+use std::str::from_utf8;
+use std::thread::sleep;
+use std::time::Duration;
 
-pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> {
-    let path = path.as_ref();
+pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
+pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:";
 
-    if !path.exists() {
-        create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?;
+pub fn read_mnemonic_from_cli(
+    mnemonic_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<Mnemonic, String> {
+    let mnemonic = match mnemonic_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .and_then(|bytes| {
+                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
+                let phrase = from_utf8(&bytes_no_newlines.as_ref())
+                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
+                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
+                    format!(
+                        "Unable to derive mnemonic from string {:?}: {:?}",
+                        phrase, e
+                    )
+                })
+            })?,
+        None => loop {
+            eprintln!("");
+            eprintln!("{}", MNEMONIC_PROMPT);
+
+            let mnemonic = read_input_from_user(stdin_inputs)?;
+
+            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
+                Ok(mnemonic_m) => {
+                    eprintln!("Valid mnemonic provided.");
+                    eprintln!("");
+                    sleep(Duration::from_secs(1));
+                    break mnemonic_m;
+                }
+                Err(_) => {
+                    eprintln!("Invalid mnemonic");
+                }
+            }
+        },
+    };
+    Ok(mnemonic)
+}
+
+/// Reads in a wallet name from the user. If the `--wallet-name` flag is provided, use it. Otherwise
+/// read from an interactive prompt using tty unless the `--stdin-inputs` flag is provided.
+pub fn read_wallet_name_from_cli(
+    wallet_name: Option<String>,
+    stdin_inputs: bool,
+) -> Result<String, String> {
+    match wallet_name {
+        Some(name) => Ok(name),
+        None => {
+            eprintln!("{}", WALLET_NAME_PROMPT);
+
+            read_input_from_user(stdin_inputs)
+        }
     }
-
-    Ok(())
-}
-
-pub fn base_wallet_dir(matches: &ArgMatches, arg: &'static str) -> Result<PathBuf, String> {
-    clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        arg,
-        PathBuf::new().join(".lighthouse").join("wallets"),
-    )
 }

account_manager/src/lib.rs

@@ -10,7 +10,7 @@ use types::EthSpec;
 pub const CMD: &str = "account_manager";
 pub const SECRETS_DIR_FLAG: &str = "secrets-dir";
 pub const VALIDATOR_DIR_FLAG: &str = "validator-dir";
-pub const BASE_DIR_FLAG: &str = "base-dir";
+pub const WALLETS_DIR_FLAG: &str = "wallets-dir";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)

account_manager/src/validator/create.rs

@@ -1,9 +1,16 @@
-use crate::{common::ensure_dir_exists, SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG};
-use account_utils::{random_password, strip_off_newlines, validator_definitions};
+use crate::common::read_wallet_name_from_cli;
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use crate::{SECRETS_DIR_FLAG, WALLETS_DIR_FLAG};
+use account_utils::{
+    random_password, read_password_from_user, strip_off_newlines, validator_definitions, PlainText,
+};
 use clap::{App, Arg, ArgMatches};
+use directory::{
+    ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_SECRET_DIR, DEFAULT_WALLET_DIR,
+};
 use environment::Environment;
-use eth2_wallet::PlainText;
 use eth2_wallet_manager::WalletManager;
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
 use std::ffi::OsStr;
 use std::fs;
 use std::path::{Path, PathBuf};
@@ -11,13 +18,13 @@ use types::EthSpec;
 use validator_dir::Builder as ValidatorDirBuilder;
 
 pub const CMD: &str = "create";
-pub const BASE_DIR_FLAG: &str = "base-dir";
 pub const WALLET_NAME_FLAG: &str = "wallet-name";
-pub const WALLET_PASSPHRASE_FLAG: &str = "wallet-passphrase";
+pub const WALLET_PASSWORD_FLAG: &str = "wallet-password";
 pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
 pub const STORE_WITHDRAW_FLAG: &str = "store-withdrawal-keystore";
 pub const COUNT_FLAG: &str = "count";
 pub const AT_MOST_FLAG: &str = "at-most";
+pub const WALLET_PASSWORD_PROMPT: &str = "Enter your wallet's password:";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
@@ -30,26 +37,22 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .long(WALLET_NAME_FLAG)
                 .value_name("WALLET_NAME")
                 .help("Use the wallet identified by this name")
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
-            Arg::with_name(WALLET_PASSPHRASE_FLAG)
-                .long(WALLET_PASSPHRASE_FLAG)
+            Arg::with_name(WALLET_PASSWORD_FLAG)
+                .long(WALLET_PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help("A path to a file containing the password which will unlock the wallet.")
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
+            Arg::with_name(WALLETS_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
+                .value_name(WALLETS_DIR_FLAG)
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{testnet}/wallets")
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .arg(
             Arg::with_name(SECRETS_DIR_FLAG)
@@ -57,8 +60,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .value_name("SECRETS_DIR")
                 .help(
                     "The path where the validator keystore passwords will be stored. \
-                    Defaults to ~/.lighthouse/secrets",
+                    Defaults to ~/.lighthouse/{testnet}/secrets",
                 )
+                .conflicts_with("datadir")
                 .takes_value(true),
         )
         .arg(
@@ -99,28 +103,35 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .conflicts_with("count")
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
 }
 
 pub fn cli_run<T: EthSpec>(
     matches: &ArgMatches,
     mut env: Environment<T>,
-    wallet_base_dir: PathBuf,
+    validator_dir: PathBuf,
 ) -> Result<(), String> {
     let spec = env.core_context().eth2_config.spec;
 
-    let name: String = clap_utils::parse_required(matches, WALLET_NAME_FLAG)?;
-    let wallet_password_path: PathBuf =
-        clap_utils::parse_required(matches, WALLET_PASSPHRASE_FLAG)?;
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let secrets_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        SECRETS_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("secrets"),
-    )?;
+    let name: Option<String> = clap_utils::parse_optional(matches, WALLET_NAME_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
+
     let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
         .unwrap_or_else(|| spec.max_effective_balance);
     let count: Option<usize> = clap_utils::parse_optional(matches, COUNT_FLAG)?;
@@ -129,6 +140,9 @@ pub fn cli_run<T: EthSpec>(
     ensure_dir_exists(&validator_dir)?;
     ensure_dir_exists(&secrets_dir)?;
 
+    eprintln!("secrets-dir path {:?}", secrets_dir);
+    eprintln!("wallets-dir path {:?}", wallet_base_dir);
+
     let starting_validator_count = existing_validator_count(&validator_dir)?;
 
     let n = match (count, at_most) {
@@ -152,17 +166,29 @@ pub fn cli_run<T: EthSpec>(
         return Ok(());
     }
 
-    let wallet_password = fs::read(&wallet_password_path)
-        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
-        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
+    let wallet_password_path: Option<PathBuf> =
+        clap_utils::parse_optional(matches, WALLET_PASSWORD_FLAG)?;
+
+    let wallet_name = read_wallet_name_from_cli(name, stdin_inputs)?;
+    let wallet_password = read_wallet_password_from_cli(wallet_password_path, stdin_inputs)?;
 
     let mgr = WalletManager::open(&wallet_base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     let mut wallet = mgr
-        .wallet_by_name(&name)
+        .wallet_by_name(&wallet_name)
         .map_err(|e| format!("Unable to open wallet: {:?}", e))?;
 
+    let slashing_protection_path = validator_dir.join(SLASHING_PROTECTION_FILENAME);
+    let slashing_protection =
+        SlashingDatabase::open_or_create(&slashing_protection_path).map_err(|e| {
+            format!(
+                "Unable to open or create slashing protection database at {}: {:?}",
+                slashing_protection_path.display(),
+                e
+            )
+        })?;
+
     for i in 0..n {
         let voting_password = random_password();
         let withdrawal_password = random_password();
@@ -175,9 +201,25 @@ pub fn cli_run<T: EthSpec>(
             )
             .map_err(|e| format!("Unable to create validator keys: {:?}", e))?;
 
-        let voting_pubkey = keystores.voting.pubkey().to_string();
+        let voting_pubkey = keystores.voting.public_key().ok_or_else(|| {
+            format!(
+                "Keystore public key is invalid: {}",
+                keystores.voting.pubkey()
+            )
+        })?;
 
-        ValidatorDirBuilder::new(validator_dir.clone(), secrets_dir.clone())
+        slashing_protection
+            .register_validator(&voting_pubkey)
+            .map_err(|e| {
+                format!(
+                    "Error registering validator {}: {:?}",
+                    voting_pubkey.to_hex_string(),
+                    e
+                )
+            })?;
+
+        ValidatorDirBuilder::new(validator_dir.clone())
+            .password_dir(secrets_dir.clone())
             .voting_keystore(keystores.voting, voting_password.as_bytes())
             .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
             .create_eth1_tx_data(deposit_gwei, &spec)
@@ -185,7 +227,7 @@ pub fn cli_run<T: EthSpec>(
             .build()
             .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
 
-        println!("{}/{}\t0x{}", i + 1, n, voting_pubkey);
+        println!("{}/{}\t{}", i + 1, n, voting_pubkey.to_hex_string());
     }
 
     Ok(())
@@ -193,15 +235,40 @@ pub fn cli_run<T: EthSpec>(
 
 /// Returns the number of validators that exist in the given `validator_dir`.
 ///
-/// This function just assumes all files and directories, excluding the validator definitions YAML,
-/// are validator directories, making it likely to return a higher number than accurate
-/// but never a lower one.
+/// This function just assumes all files and directories, excluding the validator definitions YAML
+/// and slashing protection database are validator directories, making it likely to return a higher
+/// number than accurate but never a lower one.
 fn existing_validator_count<P: AsRef<Path>>(validator_dir: P) -> Result<usize, String> {
     fs::read_dir(validator_dir.as_ref())
         .map(|iter| {
             iter.filter_map(|e| e.ok())
-                .filter(|e| e.file_name() != OsStr::new(validator_definitions::CONFIG_FILENAME))
+                .filter(|e| {
+                    e.file_name() != OsStr::new(validator_definitions::CONFIG_FILENAME)
+                        && e.file_name()
+                            != OsStr::new(slashing_protection::SLASHING_PROTECTION_FILENAME)
+                })
                 .count()
         })
         .map_err(|e| format!("Unable to read {:?}: {}", validator_dir.as_ref(), e))
 }
+
+/// Used when a user is accessing an existing wallet. Read in a wallet password from a file if the password file
+/// path is provided. Otherwise, read from an interactive prompt using tty unless the `--stdin-inputs`
+/// flag is provided.
+pub fn read_wallet_password_from_cli(
+    password_file_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<PlainText, String> {
+    match password_file_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .map(|bytes| strip_off_newlines(bytes).into()),
+        None => {
+            eprintln!("");
+            eprintln!("{}", WALLET_PASSWORD_PROMPT);
+            let password =
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+            Ok(password)
+        }
+    }
+}

account_manager/src/validator/deposit.rs

@@ -1,405 +0,0 @@
-use crate::VALIDATOR_DIR_FLAG;
-use clap::{App, Arg, ArgMatches};
-use deposit_contract::DEPOSIT_GAS;
-use environment::Environment;
-use futures::{
-    compat::Future01CompatExt,
-    stream::{FuturesUnordered, StreamExt},
-};
-use slog::{info, Logger};
-use state_processing::per_block_processing::verify_deposit_signature;
-use std::path::PathBuf;
-use tokio::time::{delay_until, Duration, Instant};
-use types::EthSpec;
-use validator_dir::{Eth1DepositData, Manager as ValidatorManager, ValidatorDir};
-use web3::{
-    transports::Http,
-    transports::Ipc,
-    types::{Address, SyncInfo, SyncState, TransactionRequest, U256},
-    Transport, Web3,
-};
-
-pub const CMD: &str = "deposit";
-pub const VALIDATOR_FLAG: &str = "validator";
-pub const ETH1_IPC_FLAG: &str = "eth1-ipc";
-pub const ETH1_HTTP_FLAG: &str = "eth1-http";
-pub const FROM_ADDRESS_FLAG: &str = "from-address";
-pub const CONFIRMATION_COUNT_FLAG: &str = "confirmation-count";
-pub const CONFIRMATION_BATCH_SIZE_FLAG: &str = "confirmation-batch-size";
-
-const GWEI: u64 = 1_000_000_000;
-
-const SYNCING_STATE_RETRY_DELAY: Duration = Duration::from_secs(2);
-
-const CONFIRMATIONS_POLL_TIME: Duration = Duration::from_secs(2);
-
-pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
-    App::new("deposit")
-        .about(
-            "Submits a deposit to an Eth1 validator registration contract via an IPC endpoint \
-            of an Eth1 client (e.g., Geth, OpenEthereum, etc.). The validators must already \
-            have been created and exist on the file-system. The process will exit immediately \
-            with an error if any error occurs. After each deposit is submitted to the Eth1 \
-            node, a file will be saved in the validator directory with the transaction hash. \
-            If confirmations are set to non-zero then the application will wait for confirmations \
-            before saving the transaction hash and moving onto the next batch of deposits. \
-            The deposit contract address will be determined by the --testnet-dir flag on the \
-            primary Lighthouse binary.",
-        )
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to the validator client data directory. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name(VALIDATOR_FLAG)
-                .long(VALIDATOR_FLAG)
-                .value_name("VALIDATOR_NAME")
-                .help(
-                    "The name of the directory in --data-dir for which to deposit. \
-                    Set to 'all' to deposit all validators in the --data-dir.",
-                )
-                .takes_value(true)
-                .required(true),
-        )
-        .arg(
-            Arg::with_name(ETH1_IPC_FLAG)
-                .long(ETH1_IPC_FLAG)
-                .value_name("ETH1_IPC_PATH")
-                .help("Path to an Eth1 JSON-RPC IPC endpoint")
-                .takes_value(true)
-                .required(false),
-        )
-        .arg(
-            Arg::with_name(ETH1_HTTP_FLAG)
-                .long(ETH1_HTTP_FLAG)
-                .value_name("ETH1_HTTP_URL")
-                .help("URL to an Eth1 JSON-RPC endpoint")
-                .takes_value(true)
-                .required(false),
-        )
-        .arg(
-            Arg::with_name(FROM_ADDRESS_FLAG)
-                .long(FROM_ADDRESS_FLAG)
-                .value_name("FROM_ETH1_ADDRESS")
-                .help(
-                    "The address that will submit the eth1 deposit. \
-                    Must be unlocked on the node at --eth1-ipc.",
-                )
-                .takes_value(true)
-                .required(true),
-        )
-        .arg(
-            Arg::with_name(CONFIRMATION_COUNT_FLAG)
-                .long(CONFIRMATION_COUNT_FLAG)
-                .value_name("CONFIRMATION_COUNT")
-                .help(
-                    "The number of Eth1 block confirmations required \
-                    before a transaction is considered complete. Set to \
-                    0 for no confirmations.",
-                )
-                .takes_value(true)
-                .default_value("1"),
-        )
-        .arg(
-            Arg::with_name(CONFIRMATION_BATCH_SIZE_FLAG)
-                .long(CONFIRMATION_BATCH_SIZE_FLAG)
-                .value_name("BATCH_SIZE")
-                .help(
-                    "Perform BATCH_SIZE deposits and wait for confirmations \
-                    in parallel. Useful for achieving faster bulk deposits.",
-                )
-                .takes_value(true)
-                .default_value("10"),
-        )
-}
-
-#[allow(clippy::too_many_arguments)]
-fn send_deposit_transactions<T1, T2: 'static>(
-    mut env: Environment<T1>,
-    log: Logger,
-    mut eth1_deposit_datas: Vec<(ValidatorDir, Eth1DepositData)>,
-    from_address: Address,
-    deposit_contract: Address,
-    transport: T2,
-    confirmation_count: usize,
-    confirmation_batch_size: usize,
-) -> Result<(), String>
-where
-    T1: EthSpec,
-    T2: Transport + std::marker::Send,
-    <T2 as web3::Transport>::Out: std::marker::Send,
-{
-    let web3 = Web3::new(transport);
-    let spec = env.eth2_config.spec.clone();
-
-    let deposits_fut = async {
-        poll_until_synced(web3.clone(), log.clone()).await?;
-
-        for chunk in eth1_deposit_datas.chunks_mut(confirmation_batch_size) {
-            let futures = FuturesUnordered::default();
-
-            for (ref mut validator_dir, eth1_deposit_data) in chunk.iter_mut() {
-                verify_deposit_signature(&eth1_deposit_data.deposit_data, &spec).map_err(|e| {
-                    format!(
-                        "Deposit for {:?} fails verification, \
-                         are you using the correct testnet configuration?\nError: {:?}",
-                        eth1_deposit_data.deposit_data.pubkey, e
-                    )
-                })?;
-
-                let web3 = web3.clone();
-                let log = log.clone();
-                futures.push(async move {
-                    let tx_hash = web3
-                        .send_transaction_with_confirmation(
-                            TransactionRequest {
-                                from: from_address,
-                                to: Some(deposit_contract),
-                                gas: Some(DEPOSIT_GAS.into()),
-                                gas_price: None,
-                                value: Some(from_gwei(eth1_deposit_data.deposit_data.amount)),
-                                data: Some(eth1_deposit_data.rlp.clone().into()),
-                                nonce: None,
-                                condition: None,
-                            },
-                            CONFIRMATIONS_POLL_TIME,
-                            confirmation_count,
-                        )
-                        .compat()
-                        .await
-                        .map_err(|e| format!("Failed to send transaction: {:?}", e))?;
-
-                    info!(
-                        log,
-                        "Submitted deposit";
-                        "tx_hash" => format!("{:?}", tx_hash),
-                    );
-
-                    validator_dir
-                        .save_eth1_deposit_tx_hash(&format!("{:?}", tx_hash))
-                        .map_err(|e| {
-                            format!("Failed to save tx hash {:?} to disk: {:?}", tx_hash, e)
-                        })?;
-
-                    Ok::<(), String>(())
-                });
-            }
-
-            futures
-                .collect::<Vec<_>>()
-                .await
-                .into_iter()
-                .collect::<Result<_, _>>()?;
-        }
-
-        Ok::<(), String>(())
-    };
-
-    env.runtime().block_on(deposits_fut)?;
-
-    Ok(())
-}
-
-pub fn cli_run<T: EthSpec>(
-    matches: &ArgMatches<'_>,
-    mut env: Environment<T>,
-) -> Result<(), String> {
-    let log = env.core_context().log().clone();
-
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let validator: String = clap_utils::parse_required(matches, VALIDATOR_FLAG)?;
-    let eth1_ipc_path: Option<PathBuf> = clap_utils::parse_optional(matches, ETH1_IPC_FLAG)?;
-    let eth1_http_url: Option<String> = clap_utils::parse_optional(matches, ETH1_HTTP_FLAG)?;
-    let from_address: Address = clap_utils::parse_required(matches, FROM_ADDRESS_FLAG)?;
-    let confirmation_count: usize = clap_utils::parse_required(matches, CONFIRMATION_COUNT_FLAG)?;
-    let confirmation_batch_size: usize =
-        clap_utils::parse_required(matches, CONFIRMATION_BATCH_SIZE_FLAG)?;
-
-    let manager = ValidatorManager::open(&data_dir)
-        .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
-
-    let validators = match validator.as_ref() {
-        "all" => manager
-            .open_all_validators()
-            .map_err(|e| format!("Unable to read all validators: {:?}", e)),
-        name => {
-            let path = manager
-                .directory_names()
-                .map_err(|e| {
-                    format!(
-                        "Unable to read --{} directory names: {:?}",
-                        VALIDATOR_DIR_FLAG, e
-                    )
-                })?
-                .get(name)
-                .ok_or_else(|| format!("Unknown validator:  {}", name))?
-                .clone();
-
-            manager
-                .open_validator(&path)
-                .map_err(|e| format!("Unable to open {}: {:?}", name, e))
-                .map(|v| vec![v])
-        }
-    }?;
-
-    let eth1_deposit_datas = validators
-        .into_iter()
-        .filter(|v| !v.eth1_deposit_tx_hash_exists())
-        .map(|v| match v.eth1_deposit_data() {
-            Ok(Some(data)) => Ok((v, data)),
-            Ok(None) => Err(format!(
-                "Validator is missing deposit data file: {:?}",
-                v.dir()
-            )),
-            Err(e) => Err(format!(
-                "Unable to read deposit data for {:?}: {:?}",
-                v.dir(),
-                e
-            )),
-        })
-        .collect::<Result<Vec<_>, _>>()?;
-
-    let total_gwei: u64 = eth1_deposit_datas
-        .iter()
-        .map(|(_, d)| d.deposit_data.amount)
-        .sum();
-
-    if eth1_deposit_datas.is_empty() {
-        info!(log, "No validators to deposit");
-
-        return Ok(());
-    }
-
-    info!(
-        log,
-        "Starting deposits";
-        "deposit_count" => eth1_deposit_datas.len(),
-        "total_eth" => total_gwei / GWEI,
-    );
-
-    let deposit_contract = env
-        .testnet
-        .as_ref()
-        .ok_or_else(|| "Unable to run account manager without a testnet dir".to_string())?
-        .deposit_contract_address()
-        .map_err(|e| format!("Unable to parse deposit contract address: {}", e))?;
-
-    if deposit_contract == Address::zero() {
-        return Err("Refusing to deposit to the zero address. Check testnet configuration.".into());
-    }
-
-    match (eth1_ipc_path, eth1_http_url) {
-        (Some(_), Some(_)) => Err(format!(
-            "error: Cannot supply both --{} and --{}",
-            ETH1_IPC_FLAG, ETH1_HTTP_FLAG
-        )),
-        (None, None) => Err(format!(
-            "error: Must supply one of --{} or --{}",
-            ETH1_IPC_FLAG, ETH1_HTTP_FLAG
-        )),
-        (Some(ipc_path), None) => {
-            let (_event_loop_handle, ipc_transport) = Ipc::new(ipc_path)
-                .map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?;
-            send_deposit_transactions(
-                env,
-                log,
-                eth1_deposit_datas,
-                from_address,
-                deposit_contract,
-                ipc_transport,
-                confirmation_count,
-                confirmation_batch_size,
-            )
-        }
-        (None, Some(http_url)) => {
-            let (_event_loop_handle, http_transport) = Http::new(http_url.as_str())
-                .map_err(|e| format!("Unable to connect to eth1 http RPC: {:?}", e))?;
-            send_deposit_transactions(
-                env,
-                log,
-                eth1_deposit_datas,
-                from_address,
-                deposit_contract,
-                http_transport,
-                confirmation_count,
-                confirmation_batch_size,
-            )
-        }
-    }
-}
-
-/// Converts gwei to wei.
-fn from_gwei(gwei: u64) -> U256 {
-    U256::from(gwei) * U256::exp10(9)
-}
-
-/// Run a poll on the `eth_syncing` endpoint, blocking until the node is synced.
-async fn poll_until_synced<T>(web3: Web3<T>, log: Logger) -> Result<(), String>
-where
-    T: Transport + Send + 'static,
-    <T as Transport>::Out: Send,
-{
-    loop {
-        let sync_state = web3
-            .clone()
-            .eth()
-            .syncing()
-            .compat()
-            .await
-            .map_err(|e| format!("Unable to read syncing state from eth1 node: {:?}", e))?;
-
-        match sync_state {
-            SyncState::Syncing(SyncInfo {
-                current_block,
-                highest_block,
-                ..
-            }) => {
-                info!(
-                    log,
-                    "Waiting for eth1 node to sync";
-                    "est_highest_block" => format!("{}", highest_block),
-                    "current_block" => format!("{}", current_block),
-                );
-
-                delay_until(Instant::now() + SYNCING_STATE_RETRY_DELAY).await;
-            }
-            SyncState::NotSyncing => {
-                let block_number = web3
-                    .clone()
-                    .eth()
-                    .block_number()
-                    .compat()
-                    .await
-                    .map_err(|e| format!("Unable to read block number from eth1 node: {:?}", e))?;
-
-                if block_number > 0.into() {
-                    info!(
-                        log,
-                        "Eth1 node is synced";
-                        "head_block" => format!("{}", block_number),
-                    );
-                    break;
-                } else {
-                    delay_until(Instant::now() + SYNCING_STATE_RETRY_DELAY).await;
-                    info!(
-                        log,
-                        "Waiting for eth1 node to sync";
-                        "current_block" => 0,
-                    );
-                }
-            }
-        }
-    }
-
-    Ok(())
-}

account_manager/src/validator/import.rs

@@ -1,4 +1,4 @@
-use crate::{common::ensure_dir_exists, VALIDATOR_DIR_FLAG};
+use crate::wallet::create::STDIN_INPUTS_FLAG;
 use account_utils::{
     eth2_keystore::Keystore,
     read_password_from_user,
@@ -6,8 +6,10 @@ use account_utils::{
         recursively_find_voting_keystores, ValidatorDefinition, ValidatorDefinitions,
         CONFIG_FILENAME,
     },
+    ZeroizeString,
 };
 use clap::{App, Arg, ArgMatches};
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
 use std::fs;
 use std::path::PathBuf;
 use std::thread::sleep;
@@ -16,7 +18,7 @@ use std::time::Duration;
 pub const CMD: &str = "import";
 pub const KEYSTORE_FLAG: &str = "keystore";
 pub const DIR_FLAG: &str = "directory";
-pub const STDIN_PASSWORD_FLAG: &str = "stdin-passwords";
+pub const REUSE_PASSWORD_FLAG: &str = "reuse-password";
 
 pub const PASSWORD_PROMPT: &str = "Enter the keystore password, or press enter to omit it:";
 pub const KEYSTORE_REUSE_WARNING: &str = "DO NOT USE THE ORIGINAL KEYSTORES TO VALIDATE WITH \
@@ -54,37 +56,36 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
         )
         .arg(
-            Arg::with_name(STDIN_PASSWORD_FLAG)
-                .long(STDIN_PASSWORD_FLAG)
-                .help("If present, read passwords from stdin instead of tty."),
+            Arg::with_name(REUSE_PASSWORD_FLAG)
+                .long(REUSE_PASSWORD_FLAG)
+                .help("If present, the same password will be used for all imported keystores."),
         )
 }
 
-pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
     let keystore: Option<PathBuf> = clap_utils::parse_optional(matches, KEYSTORE_FLAG)?;
     let keystores_dir: Option<PathBuf> = clap_utils::parse_optional(matches, DIR_FLAG)?;
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let stdin_password = matches.is_present(STDIN_PASSWORD_FLAG);
-
-    ensure_dir_exists(&validator_dir)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+    let reuse_password = matches.is_present(REUSE_PASSWORD_FLAG);
 
     let mut defs = ValidatorDefinitions::open_or_create(&validator_dir)
         .map_err(|e| format!("Unable to open {}: {:?}", CONFIG_FILENAME, e))?;
 
+    let slashing_protection_path = validator_dir.join(SLASHING_PROTECTION_FILENAME);
+    let slashing_protection =
+        SlashingDatabase::open_or_create(&slashing_protection_path).map_err(|e| {
+            format!(
+                "Unable to open or create slashing protection database at {}: {:?}",
+                slashing_protection_path.display(),
+                e
+            )
+        })?;
+
     // Collect the paths for the keystores that should be imported.
     let keystore_paths = match (keystore, keystores_dir) {
         (Some(keystore), None) => vec![keystore],
@@ -115,10 +116,13 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
     //
     // - Obtain the keystore password, if the user desires.
     // - Copy the keystore into the `validator_dir`.
+    // - Register the voting key with the slashing protection database.
     // - Add the keystore to the validator definitions file.
     //
     // Skip keystores that already exist, but exit early if any operation fails.
+    // Reuses the same password for all keystores if the `REUSE_PASSWORD_FLAG` flag is set.
     let mut num_imported_keystores = 0;
+    let mut previous_password: Option<ZeroizeString> = None;
     for src_keystore in &keystore_paths {
         let keystore = Keystore::from_json_file(src_keystore)
             .map_err(|e| format!("Unable to read keystore JSON {:?}: {:?}", src_keystore, e))?;
@@ -136,10 +140,14 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         );
 
         let password_opt = loop {
+            if let Some(password) = previous_password.clone() {
+                eprintln!("Reuse previous password.");
+                break Some(password);
+            }
             eprintln!("");
             eprintln!("{}", PASSWORD_PROMPT);
 
-            let password = read_password_from_user(stdin_password)?;
+            let password = read_password_from_user(stdin_inputs)?;
 
             if password.as_ref().is_empty() {
                 eprintln!("Continuing without password.");
@@ -152,6 +160,9 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
                     eprintln!("Password is correct.");
                     eprintln!("");
                     sleep(Duration::from_secs(1)); // Provides nicer UX.
+                    if reuse_password {
+                        previous_password = Some(password.clone());
+                    }
                     break Some(password);
                 }
                 Err(eth2_keystore::Error::InvalidPassword) => {
@@ -186,6 +197,20 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         fs::copy(&src_keystore, &dest_keystore)
             .map_err(|e| format!("Unable to copy keystore: {:?}", e))?;
 
+        // Register with slashing protection.
+        let voting_pubkey = keystore
+            .public_key()
+            .ok_or_else(|| format!("Keystore public key is invalid: {}", keystore.pubkey()))?;
+        slashing_protection
+            .register_validator(&voting_pubkey)
+            .map_err(|e| {
+                format!(
+                    "Error registering validator {}: {:?}",
+                    voting_pubkey.to_hex_string(),
+                    e
+                )
+            })?;
+
         eprintln!("Successfully imported keystore.");
         num_imported_keystores += 1;
 

account_manager/src/validator/list.rs

@@ -1,38 +1,21 @@
 use crate::VALIDATOR_DIR_FLAG;
-use clap::{App, Arg, ArgMatches};
+use clap::App;
 use std::path::PathBuf;
 use validator_dir::Manager as ValidatorManager;
 
 pub const CMD: &str = "list";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
-    App::new(CMD)
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to search for validator directories. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
-        .about("Lists the names of all validators.")
+    App::new(CMD).about("Lists the names of all validators.")
 }
 
-pub fn cli_run(matches: &ArgMatches<'_>) -> Result<(), String> {
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-
-    let mgr = ValidatorManager::open(&data_dir)
+pub fn cli_run(validator_dir: PathBuf) -> Result<(), String> {
+    let mgr = ValidatorManager::open(&validator_dir)
         .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
 
     for (name, _path) in mgr
         .directory_names()
-        .map_err(|e| format!("Unable to list wallets: {:?}", e))?
+        .map_err(|e| format!("Unable to list validators: {:?}", e))?
     {
         println!("{}", name)
     }

account_manager/src/validator/mod.rs

@@ -1,11 +1,14 @@
 pub mod create;
-pub mod deposit;
 pub mod import;
 pub mod list;
+pub mod recover;
+pub mod slashing_protection;
 
-use crate::common::base_wallet_dir;
+use crate::VALIDATOR_DIR_FLAG;
 use clap::{App, Arg, ArgMatches};
+use directory::{parse_path_or_default_with_flag, DEFAULT_VALIDATOR_DIR};
 use environment::Environment;
+use std::path::PathBuf;
 use types::EthSpec;
 
 pub const CMD: &str = "validator";
@@ -14,26 +17,40 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Provides commands for managing Eth2 validators.")
         .arg(
-            Arg::with_name("base-dir")
-                .long("base-dir")
-                .value_name("BASE_DIRECTORY")
-                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets")
-                .takes_value(true),
+            Arg::with_name(VALIDATOR_DIR_FLAG)
+                .long(VALIDATOR_DIR_FLAG)
+                .value_name("VALIDATOR_DIRECTORY")
+                .help(
+                    "The path to search for validator directories. \
+                    Defaults to ~/.lighthouse/{testnet}/validators",
+                )
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .subcommand(create::cli_app())
-        .subcommand(deposit::cli_app())
         .subcommand(import::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
+        .subcommand(slashing_protection::cli_app())
 }
 
 pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
-    let base_wallet_dir = base_wallet_dir(matches, "base-dir")?;
+    let validator_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_VALIDATOR_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, VALIDATOR_DIR_FLAG, DEFAULT_VALIDATOR_DIR)?
+    };
+    eprintln!("validator-dir path: {:?}", validator_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, base_wallet_dir),
-        (deposit::CMD, Some(matches)) => deposit::cli_run::<T>(matches, env),
-        (import::CMD, Some(matches)) => import::cli_run(matches),
-        (list::CMD, Some(matches)) => list::cli_run(matches),
+        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, validator_base_dir),
+        (import::CMD, Some(matches)) => import::cli_run(matches, validator_base_dir),
+        (list::CMD, Some(_)) => list::cli_run(validator_base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, validator_base_dir),
+        (slashing_protection::CMD, Some(matches)) => {
+            slashing_protection::cli_run(matches, env, validator_base_dir)
+        }
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/validator/recover.rs

@@ -0,0 +1,145 @@
+use super::create::STORE_WITHDRAW_FLAG;
+use crate::common::read_mnemonic_from_cli;
+use crate::validator::create::COUNT_FLAG;
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use crate::SECRETS_DIR_FLAG;
+use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
+use account_utils::random_password;
+use clap::{App, Arg, ArgMatches};
+use directory::ensure_dir_exists;
+use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR};
+use eth2_wallet::bip39::Seed;
+use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType, ValidatorKeystores};
+use std::path::PathBuf;
+use validator_dir::Builder as ValidatorDirBuilder;
+pub const CMD: &str = "recover";
+pub const FIRST_INDEX_FLAG: &str = "first-index";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Recovers validator private keys given a BIP-39 mnemonic phrase. \
+            If you did not specify a `--first-index` or count `--count`, by default this will \
+            only recover the keys associated with the validator at index 0 for an HD wallet \
+            in accordance with the EIP-2333 spec.")
+        .arg(
+            Arg::with_name(FIRST_INDEX_FLAG)
+                .long(FIRST_INDEX_FLAG)
+                .value_name("FIRST_INDEX")
+                .help("The first of consecutive key indexes you wish to recover.")
+                .takes_value(true)
+                .required(false)
+                .default_value("0"),
+        )
+        .arg(
+            Arg::with_name(COUNT_FLAG)
+                .long(COUNT_FLAG)
+                .value_name("COUNT")
+                .help("The number of validator keys you wish to recover. Counted consecutively from the provided `--first_index`.")
+                .takes_value(true)
+                .required(false)
+                .default_value("1"),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help(
+                    "If present, the mnemonic will be read in from this file.",
+                )
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name(SECRETS_DIR_FLAG)
+                .long(SECRETS_DIR_FLAG)
+                .value_name("SECRETS_DIR")
+                .help(
+                    "The path where the validator keystore passwords will be stored. \
+                    Defaults to ~/.lighthouse/{testnet}/secrets",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STORE_WITHDRAW_FLAG)
+                .long(STORE_WITHDRAW_FLAG)
+                .help(
+                    "If present, the withdrawal keystore will be stored alongside the voting \
+                    keypair. It is generally recommended to *not* store the withdrawal key and \
+                    instead generate them from the wallet seed when required.",
+                ),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
+    let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
+    let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?;
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    ensure_dir_exists(&validator_dir)?;
+    ensure_dir_exists(&secrets_dir)?;
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+
+    let seed = Seed::new(&mnemonic, "");
+
+    for index in first_index..first_index + count {
+        let voting_password = random_password();
+        let withdrawal_password = random_password();
+
+        let derive = |key_type: KeyType, password: &[u8]| -> Result<Keystore, String> {
+            let (secret, path) =
+                recover_validator_secret_from_mnemonic(seed.as_bytes(), index, key_type)
+                    .map_err(|e| format!("Unable to recover validator keys: {:?}", e))?;
+
+            let keypair = keypair_from_secret(secret.as_bytes())
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?;
+
+            KeystoreBuilder::new(&keypair, password, format!("{}", path))
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?
+                .build()
+                .map_err(|e| format!("Unable build keystore: {:?}", e))
+        };
+
+        let keystores = ValidatorKeystores {
+            voting: derive(KeyType::Voting, voting_password.as_bytes())?,
+            withdrawal: derive(KeyType::Withdrawal, withdrawal_password.as_bytes())?,
+        };
+
+        let voting_pubkey = keystores.voting.pubkey().to_string();
+
+        ValidatorDirBuilder::new(validator_dir.clone())
+            .password_dir(secrets_dir.clone())
+            .voting_keystore(keystores.voting, voting_password.as_bytes())
+            .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
+            .store_withdrawal_keystore(matches.is_present(STORE_WITHDRAW_FLAG))
+            .build()
+            .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
+
+        println!(
+            "{}/{}\tIndex: {}\t0x{}",
+            index - first_index,
+            count - first_index,
+            index,
+            voting_pubkey
+        );
+    }
+
+    Ok(())
+}

account_manager/src/validator/slashing_protection.rs

@@ -0,0 +1,137 @@
+use clap::{App, Arg, ArgMatches};
+use environment::Environment;
+use slashing_protection::{
+    interchange::Interchange, SlashingDatabase, SLASHING_PROTECTION_FILENAME,
+};
+use std::fs::File;
+use std::path::PathBuf;
+use types::{BeaconState, EthSpec};
+
+pub const CMD: &str = "slashing-protection";
+pub const IMPORT_CMD: &str = "import";
+pub const EXPORT_CMD: &str = "export";
+
+pub const IMPORT_FILE_ARG: &str = "IMPORT-FILE";
+pub const EXPORT_FILE_ARG: &str = "EXPORT-FILE";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Import or export slashing protection data to or from another client")
+        .subcommand(
+            App::new(IMPORT_CMD)
+                .about("Import an interchange file")
+                .arg(
+                    Arg::with_name(IMPORT_FILE_ARG)
+                        .takes_value(true)
+                        .value_name("FILE")
+                        .help("The slashing protection interchange file to import (.json)"),
+                ),
+        )
+        .subcommand(
+            App::new(EXPORT_CMD)
+                .about("Export an interchange file")
+                .arg(
+                    Arg::with_name(EXPORT_FILE_ARG)
+                        .takes_value(true)
+                        .value_name("FILE")
+                        .help("The filename to export the interchange file to"),
+                ),
+        )
+}
+
+pub fn cli_run<T: EthSpec>(
+    matches: &ArgMatches<'_>,
+    env: Environment<T>,
+    validator_base_dir: PathBuf,
+) -> Result<(), String> {
+    let slashing_protection_db_path = validator_base_dir.join(SLASHING_PROTECTION_FILENAME);
+
+    let testnet_config = env
+        .testnet
+        .ok_or_else(|| "Unable to get testnet configuration from the environment".to_string())?;
+
+    let genesis_validators_root = testnet_config
+        .beacon_state::<T>()
+        .map(|state: BeaconState<T>| state.genesis_validators_root)
+        .map_err(|e| {
+            format!(
+                "Unable to get genesis state, has genesis occurred? Detail: {:?}",
+                e
+            )
+        })?;
+
+    match matches.subcommand() {
+        (IMPORT_CMD, Some(matches)) => {
+            let import_filename: PathBuf = clap_utils::parse_required(&matches, IMPORT_FILE_ARG)?;
+            let import_file = File::open(&import_filename).map_err(|e| {
+                format!(
+                    "Unable to open import file at {}: {:?}",
+                    import_filename.display(),
+                    e
+                )
+            })?;
+
+            let interchange = Interchange::from_json_reader(&import_file)
+                .map_err(|e| format!("Error parsing file for import: {:?}", e))?;
+
+            let slashing_protection_database =
+                SlashingDatabase::open_or_create(&slashing_protection_db_path).map_err(|e| {
+                    format!(
+                        "Unable to open database at {}: {:?}",
+                        slashing_protection_db_path.display(),
+                        e
+                    )
+                })?;
+
+            slashing_protection_database
+                .import_interchange_info(&interchange, genesis_validators_root)
+                .map_err(|e| {
+                    format!(
+                        "Error during import, no data imported: {:?}\n\
+                         IT IS NOT SAFE TO START VALIDATING",
+                        e
+                    )
+                })?;
+
+            eprintln!("Import completed successfully");
+
+            Ok(())
+        }
+        (EXPORT_CMD, Some(matches)) => {
+            let export_filename: PathBuf = clap_utils::parse_required(&matches, EXPORT_FILE_ARG)?;
+
+            if !slashing_protection_db_path.exists() {
+                return Err(format!(
+                    "No slashing protection database exists at: {}",
+                    slashing_protection_db_path.display()
+                ));
+            }
+
+            let slashing_protection_database = SlashingDatabase::open(&slashing_protection_db_path)
+                .map_err(|e| {
+                    format!(
+                        "Unable to open database at {}: {:?}",
+                        slashing_protection_db_path.display(),
+                        e
+                    )
+                })?;
+
+            let interchange = slashing_protection_database
+                .export_interchange_info(genesis_validators_root)
+                .map_err(|e| format!("Error during export: {:?}", e))?;
+
+            let output_file = File::create(export_filename)
+                .map_err(|e| format!("Error creating output file: {:?}", e))?;
+
+            interchange
+                .write_to(&output_file)
+                .map_err(|e| format!("Error writing output file: {:?}", e))?;
+
+            eprintln!("Export completed successfully");
+
+            Ok(())
+        }
+        ("", _) => Err("No subcommand provided, see --help for options".to_string()),
+        (command, _) => Err(format!("No such subcommand `{}`", command)),
+    }
+}

account_manager/src/wallet/create.rs

@@ -1,13 +1,17 @@
-use crate::BASE_DIR_FLAG;
-use account_utils::{random_password, strip_off_newlines};
+use crate::common::read_wallet_name_from_cli;
+use crate::WALLETS_DIR_FLAG;
+use account_utils::{
+    is_password_sufficiently_complex, random_password, read_password_from_user, strip_off_newlines,
+};
 use clap::{App, Arg, ArgMatches};
 use eth2_wallet::{
     bip39::{Language, Mnemonic, MnemonicType},
     PlainText,
 };
-use eth2_wallet_manager::{WalletManager, WalletType};
+use eth2_wallet_manager::{LockedWallet, WalletManager, WalletType};
 use std::ffi::OsStr;
-use std::fs::{self, File};
+use std::fs;
+use std::fs::File;
 use std::io::prelude::*;
 use std::os::unix::fs::PermissionsExt;
 use std::path::{Path, PathBuf};
@@ -15,9 +19,21 @@ use std::path::{Path, PathBuf};
 pub const CMD: &str = "create";
 pub const HD_TYPE: &str = "hd";
 pub const NAME_FLAG: &str = "name";
-pub const PASSPHRASE_FLAG: &str = "passphrase-file";
+pub const PASSWORD_FLAG: &str = "password-file";
 pub const TYPE_FLAG: &str = "type";
 pub const MNEMONIC_FLAG: &str = "mnemonic-output-path";
+pub const STDIN_INPUTS_FLAG: &str = "stdin-inputs";
+pub const MNEMONIC_LENGTH_FLAG: &str = "mnemonic-length";
+pub const MNEMONIC_TYPES: &[MnemonicType] = &[
+    MnemonicType::Words12,
+    MnemonicType::Words15,
+    MnemonicType::Words18,
+    MnemonicType::Words21,
+    MnemonicType::Words24,
+];
+pub const NEW_WALLET_PASSWORD_PROMPT: &str =
+    "Enter a password for your new wallet that is at least 12 characters long:";
+pub const RETYPE_PASSWORD_PROMPT: &str = "Please re-enter your wallet's new password:";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
@@ -30,12 +46,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     "The wallet will be created with this name. It is not allowed to \
                             create two wallets with the same name for the same --base-dir.",
                 )
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
-            Arg::with_name(PASSPHRASE_FLAG)
-                .long(PASSPHRASE_FLAG)
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help(
                     "A path to a file containing the password which will unlock the wallet. \
@@ -43,8 +58,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     saved at that path. To avoid confusion, if the file does not already \
                     exist it must include a '.pass' suffix.",
                 )
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
             Arg::with_name(TYPE_FLAG)
@@ -67,56 +81,47 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 )
                 .takes_value(true)
         )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_LENGTH_FLAG)
+                .long(MNEMONIC_LENGTH_FLAG)
+                .value_name("MNEMONIC_LENGTH")
+                .help("The number of words to use for the mnemonic phrase.")
+                .takes_value(true)
+                .validator(|len| {
+                    match len.parse::<usize>().ok().and_then(|words| MnemonicType::for_word_count(words).ok()) {
+                        Some(_) => Ok(()),
+                        None => Err(format!("Mnemonic length must be one of {}", MNEMONIC_TYPES.iter().map(|t| t.word_count().to_string()).collect::<Vec<_>>().join(", "))),
+                    }
+                })
+                .default_value("24"),
+        )
 }
 
-pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
-    let name: String = clap_utils::parse_required(matches, NAME_FLAG)?;
-    let wallet_password_path: PathBuf = clap_utils::parse_required(matches, PASSPHRASE_FLAG)?;
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
     let mnemonic_output_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
-    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
-
-    let wallet_type = match type_field.as_ref() {
-        HD_TYPE => WalletType::Hd,
-        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
-    };
-
-    let mgr = WalletManager::open(&base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
 
     // Create a new random mnemonic.
     //
     // The `tiny-bip39` crate uses `thread_rng()` for this entropy.
-    let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English);
+    let mnemonic_length = clap_utils::parse_required(matches, MNEMONIC_LENGTH_FLAG)?;
+    let mnemonic = Mnemonic::new(
+        MnemonicType::for_word_count(mnemonic_length).expect("Mnemonic length already validated"),
+        Language::English,
+    );
 
-    // Create a random password if the file does not exist.
-    if !wallet_password_path.exists() {
-        // To prevent users from accidentally supplying their password to the PASSPHRASE_FLAG and
-        // create a file with that name, we require that the password has a .pass suffix.
-        if wallet_password_path.extension() != Some(&OsStr::new("pass")) {
-            return Err(format!(
-                "Only creates a password file if that file ends in .pass: {:?}",
-                wallet_password_path
-            ));
-        }
-
-        create_with_600_perms(&wallet_password_path, random_password().as_bytes())
-            .map_err(|e| format!("Unable to write to {:?}: {:?}", wallet_password_path, e))?;
-    }
-
-    let wallet_password = fs::read(&wallet_password_path)
-        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
-        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
-
-    let wallet = mgr
-        .create_wallet(name, wallet_type, &mnemonic, wallet_password.as_bytes())
-        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)?;
 
     if let Some(path) = mnemonic_output_path {
         create_with_600_perms(&path, mnemonic.phrase().as_bytes())
             .map_err(|e| format!("Unable to write mnemonic to {:?}: {:?}", path, e))?;
     }
 
-    println!("Your wallet's 12-word BIP-39 mnemonic is:");
+    println!("Your wallet's {}-word BIP-39 mnemonic is:", mnemonic_length);
     println!();
     println!("\t{}", mnemonic.phrase());
     println!();
@@ -140,6 +145,99 @@ pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
     Ok(())
 }
 
+pub fn create_wallet_from_mnemonic(
+    matches: &ArgMatches,
+    wallet_base_dir: &Path,
+    mnemonic: &Mnemonic,
+) -> Result<LockedWallet, String> {
+    let name: Option<String> = clap_utils::parse_optional(matches, NAME_FLAG)?;
+    let wallet_password_path: Option<PathBuf> = clap_utils::parse_optional(matches, PASSWORD_FLAG)?;
+    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    let wallet_type = match type_field.as_ref() {
+        HD_TYPE => WalletType::Hd,
+        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
+    };
+
+    let mgr = WalletManager::open(&wallet_base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
+
+    let wallet_password: PlainText = match wallet_password_path {
+        Some(path) => {
+            // Create a random password if the file does not exist.
+            if !path.exists() {
+                // To prevent users from accidentally supplying their password to the PASSWORD_FLAG and
+                // create a file with that name, we require that the password has a .pass suffix.
+                if path.extension() != Some(&OsStr::new("pass")) {
+                    return Err(format!(
+                        "Only creates a password file if that file ends in .pass: {:?}",
+                        path
+                    ));
+                }
+
+                create_with_600_perms(&path, random_password().as_bytes())
+                    .map_err(|e| format!("Unable to write to {:?}: {:?}", path, e))?;
+            }
+            read_new_wallet_password_from_cli(Some(path), stdin_inputs)?
+        }
+        None => read_new_wallet_password_from_cli(None, stdin_inputs)?,
+    };
+
+    let wallet_name = read_wallet_name_from_cli(name, stdin_inputs)?;
+
+    let wallet = mgr
+        .create_wallet(
+            wallet_name,
+            wallet_type,
+            &mnemonic,
+            wallet_password.as_bytes(),
+        )
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    Ok(wallet)
+}
+
+/// Used when a user is creating a new wallet. Read in a wallet password from a file if the password file
+/// path is provided. Otherwise, read from an interactive prompt using tty unless the `--stdin-inputs`
+/// flag is provided. This verifies the password complexity and verifies the password is correctly re-entered.
+pub fn read_new_wallet_password_from_cli(
+    password_file_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<PlainText, String> {
+    match password_file_path {
+        Some(path) => {
+            let password: PlainText = fs::read(&path)
+                .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+                .map(|bytes| strip_off_newlines(bytes).into())?;
+
+            // Ensure the password meets the minimum requirements.
+            is_password_sufficiently_complex(password.as_bytes())?;
+            Ok(password)
+        }
+        None => loop {
+            eprintln!("");
+            eprintln!("{}", NEW_WALLET_PASSWORD_PROMPT);
+            let password =
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+
+            // Ensure the password meets the minimum requirements.
+            match is_password_sufficiently_complex(password.as_bytes()) {
+                Ok(_) => {
+                    eprintln!("{}", RETYPE_PASSWORD_PROMPT);
+                    let retyped_password =
+                        PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+                    if retyped_password == password {
+                        break Ok(password);
+                    } else {
+                        eprintln!("Passwords do not match.");
+                    }
+                }
+                Err(message) => eprintln!("{}", message),
+            }
+        },
+    }
+}
+
 /// Creates a file with `600 (-rw-------)` permissions.
 pub fn create_with_600_perms<P: AsRef<Path>>(path: P, bytes: &[u8]) -> Result<(), String> {
     let path = path.as_ref();

account_manager/src/wallet/list.rs

@@ -1,4 +1,4 @@
-use crate::BASE_DIR_FLAG;
+use crate::WALLETS_DIR_FLAG;
 use clap::App;
 use eth2_wallet_manager::WalletManager;
 use std::path::PathBuf;
@@ -9,9 +9,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD).about("Lists the names of all wallets.")
 }
 
-pub fn cli_run(base_dir: PathBuf) -> Result<(), String> {
-    let mgr = WalletManager::open(&base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+pub fn cli_run(wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mgr = WalletManager::open(&wallet_base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     for (name, _uuid) in mgr
         .wallets()

account_manager/src/wallet/mod.rs

@@ -1,11 +1,11 @@
 pub mod create;
 pub mod list;
+pub mod recover;
 
-use crate::{
-    common::{base_wallet_dir, ensure_dir_exists},
-    BASE_DIR_FLAG,
-};
+use crate::WALLETS_DIR_FLAG;
 use clap::{App, Arg, ArgMatches};
+use directory::{ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_WALLET_DIR};
+use std::path::PathBuf;
 
 pub const CMD: &str = "wallet";
 
@@ -13,23 +13,33 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Manage wallets, from which validator keys can be derived.")
         .arg(
-            Arg::with_name(BASE_DIR_FLAG)
-                .long(BASE_DIR_FLAG)
-                .value_name("BASE_DIRECTORY")
-                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets")
-                .takes_value(true),
+            Arg::with_name(WALLETS_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
+                .value_name("WALLETS_DIRECTORY")
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{testnet}/wallets")
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .subcommand(create::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
 }
 
 pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
-    let base_dir = base_wallet_dir(matches, BASE_DIR_FLAG)?;
-    ensure_dir_exists(&base_dir)?;
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    ensure_dir_exists(&wallet_base_dir)?;
+
+    eprintln!("wallet-dir path: {:?}", wallet_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run(matches, base_dir),
-        (list::CMD, Some(_)) => list::cli_run(base_dir),
+        (create::CMD, Some(matches)) => create::cli_run(matches, wallet_base_dir),
+        (list::CMD, Some(_)) => list::cli_run(wallet_base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, wallet_base_dir),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/wallet/recover.rs

@@ -0,0 +1,84 @@
+use crate::common::read_mnemonic_from_cli;
+use crate::wallet::create::{create_wallet_from_mnemonic, STDIN_INPUTS_FLAG};
+use crate::wallet::create::{HD_TYPE, NAME_FLAG, PASSWORD_FLAG, TYPE_FLAG};
+use clap::{App, Arg, ArgMatches};
+use std::path::PathBuf;
+
+pub const CMD: &str = "recover";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Recovers an EIP-2386 wallet from a given a BIP-39 mnemonic phrase.")
+        .arg(
+            Arg::with_name(NAME_FLAG)
+                .long(NAME_FLAG)
+                .value_name("WALLET_NAME")
+                .help(
+                    "The wallet will be created with this name. It is not allowed to \
+                            create two wallets with the same name for the same --base-dir.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
+                .value_name("PASSWORD_FILE_PATH")
+                .help(
+                    "This will be the new password for your recovered wallet. \
+                    A path to a file containing the password which will unlock the wallet. \
+                    If the file does not exist, a random password will be generated and \
+                    saved at that path. To avoid confusion, if the file does not already \
+                    exist it must include a '.pass' suffix.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help("If present, the mnemonic will be read in from this file.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(TYPE_FLAG)
+                .long(TYPE_FLAG)
+                .value_name("WALLET_TYPE")
+                .help(
+                    "The type of wallet to create. Only HD (hierarchical-deterministic) \
+                            wallets are supported presently..",
+                )
+                .takes_value(true)
+                .possible_values(&[HD_TYPE])
+                .default_value(HD_TYPE),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+
+    let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+    println!("Your wallet has been successfully recovered.");
+    println!();
+    println!("Your wallet's UUID is:");
+    println!();
+    println!("\t{}", wallet.wallet().uuid());
+    println!();
+    println!("You do not need to backup your UUID or keep it secret.");
+
+    Ok(())
+}

beacon_node/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "beacon_node"
-version = "0.1.2"
-authors = ["Paul Hauner <paul@paulhauner.com>", "Sigma Prime <contact@sigmaprime.io>"]
+version = "0.3.2"
+authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
 edition = "2018"
 
 [lib]
@@ -20,23 +20,26 @@ beacon_chain = { path = "beacon_chain" }
 types = { path = "../consensus/types" }
 store = { path = "./store" }
 client = { path = "client" }
-clap = "2.33.0"
+clap = "2.33.3"
 rand = "0.7.3"
 slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_trace"] }
-slog-term = "2.5.0"
+slog-term = "2.6.0"
 slog-async = "2.5.0"
-ctrlc = { version = "3.1.4", features = ["termination"] }
-tokio = { version = "0.2.21", features = ["time"] }
+ctrlc = { version = "3.1.6", features = ["termination"] }
+tokio = { version = "0.2.22", features = ["time"] }
 exit-future = "0.2.0"
-dirs = "2.0.2"
+dirs = "3.0.1"
 logging = { path = "../common/logging" }
+directory = {path = "../common/directory"}
 futures = "0.3.5"
 environment = { path = "../lighthouse/environment" }
+task_executor = { path = "../common/task_executor" }
 genesis = { path = "genesis" }
 eth2_testnet_config = { path = "../common/eth2_testnet_config" }
 eth2_libp2p = { path = "./eth2_libp2p" }
 eth2_ssz = "0.1.2"
-serde = "1.0.110"
+serde = "1.0.116"
 clap_utils = { path = "../common/clap_utils" }
-hyper = "0.13.5"
+hyper = "0.13.8"
 lighthouse_version = { path = "../common/lighthouse_version" }
+hex = "0.4.2"

beacon_node/beacon_chain/Cargo.toml

@@ -1,16 +1,19 @@
 [package]
 name = "beacon_chain"
-version = "0.1.2"
-authors = ["Paul Hauner <paul@paulhauner.com>"]
+version = "0.2.0"
+authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com>"]
 edition = "2018"
 
 [features]
 default = ["participation_metrics"]
 write_ssz_files = []  # Writes debugging .ssz files to /tmp during block processing.
 participation_metrics = []  # Exposes validator participation metrics to Prometheus.
+test_logger = [] # Print log output to stderr when running tests instead of dropping it
 
 [dev-dependencies]
 int_to_bytes = { path = "../../consensus/int_to_bytes" }
+maplit = "1.0.2"
+environment = { path = "../../lighthouse/environment" }
 
 [dependencies]
 eth2_config = { path = "../../common/eth2_config" }
@@ -18,41 +21,43 @@ merkle_proof = { path = "../../consensus/merkle_proof" }
 store = { path = "../store" }
 parking_lot = "0.11.0"
 lazy_static = "1.4.0"
-smallvec = "1.4.1"
+smallvec = "1.4.2"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-log = "0.4.8"
+log = "0.4.11"
 operation_pool = { path = "../operation_pool" }
-rayon = "1.3.0"
-serde = "1.0.110"
-serde_derive = "1.0.110"
-serde_yaml = "0.8.11"
-serde_json = "1.0.52"
+rayon = "1.4.1"
+serde = "1.0.116"
+serde_derive = "1.0.116"
+serde_yaml = "0.8.13"
+serde_json = "1.0.58"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
-slog-term = "2.6.0"
-sloggers = "1.0.0"
+sloggers = "1.0.1"
 slot_clock = { path = "../../common/slot_clock" }
 eth2_hashing = "0.1.0"
 eth2_ssz = "0.1.2"
 eth2_ssz_types = { path = "../../consensus/ssz_types" }
 eth2_ssz_derive = "0.1.0"
 state_processing = { path = "../../consensus/state_processing" }
-tree_hash = "0.1.0"
+tree_hash = "0.1.1"
 types = { path = "../../consensus/types" }
-tokio = "0.2.21"
+tokio = "0.2.22"
 eth1 = { path = "../eth1" }
 websocket_server = { path = "../websocket_server" }
 futures = "0.3.5"
 genesis = { path = "../genesis" }
-integer-sqrt = "0.1.3"
+integer-sqrt = "0.1.5"
 rand = "0.7.3"
+rand_core = "0.5.1"
 proto_array = { path = "../../consensus/proto_array" }
-lru = "0.5.1"
+lru = "0.6.0"
 tempfile = "3.1.0"
-bitvec = "0.17.4"
+bitvec = "0.19.3"
 bls = { path = "../../crypto/bls" }
 safe_arith = { path = "../../consensus/safe_arith" }
 fork_choice = { path = "../../consensus/fork_choice" }
-environment = { path = "../../lighthouse/environment" }
+task_executor = { path = "../../common/task_executor" }
 bus = "2.2.3"
 derivative = "2.1.1"
 itertools = "0.9.0"
+regex = "1.3.9"
+exit-future = "0.2.0"

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -28,8 +28,7 @@
 
 use crate::{
     beacon_chain::{
-        ATTESTATION_CACHE_LOCK_TIMEOUT, HEAD_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY,
-        VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT,
+        HEAD_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY, VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT,
     },
     metrics,
     observed_attestations::ObserveOutcome,
@@ -37,12 +36,11 @@ use crate::{
     BeaconChain, BeaconChainError, BeaconChainTypes,
 };
 use bls::verify_signature_sets;
-use slog::debug;
+use proto_array::Block as ProtoBlock;
 use slot_clock::SlotClock;
 use state_processing::{
     common::get_indexed_attestation,
     per_block_processing::errors::AttestationValidationError,
-    per_slot_processing,
     signature_sets::{
         indexed_attestation_signature_set_from_pubkeys,
         signed_aggregate_selection_proof_signature_set, signed_aggregate_signature_set,
@@ -52,7 +50,7 @@ use std::borrow::Cow;
 use tree_hash::TreeHash;
 use types::{
     Attestation, BeaconCommittee, CommitteeIndex, Epoch, EthSpec, Hash256, IndexedAttestation,
-    RelativeEpoch, SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
+    SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
 };
 
 /// Returned when an attestation was not successfully verified. It might not have been verified for
@@ -220,6 +218,27 @@ pub enum Error {
     ///
     /// The peer has sent an invalid message.
     Invalid(AttestationValidationError),
+    /// The attestation head block is too far behind the attestation slot, causing many skip slots.
+    /// This is deemed a DoS risk.
+    TooManySkippedSlots {
+        head_block_slot: Slot,
+        attestation_slot: Slot,
+    },
+    /// The attestation has an invalid target epoch.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    InvalidTargetEpoch { slot: Slot, epoch: Epoch },
+    /// The attestation references an invalid target block.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    InvalidTargetRoot {
+        attestation: Hash256,
+        expected: Option<Hash256>,
+    },
     /// There was an error whilst processing the attestation. It is not known if it is valid or invalid.
     ///
     /// ## Peer scoring
@@ -245,6 +264,7 @@ pub struct VerifiedAggregatedAttestation<T: BeaconChainTypes> {
 pub struct VerifiedUnaggregatedAttestation<T: BeaconChainTypes> {
     attestation: Attestation<T::EthSpec>,
     indexed_attestation: IndexedAttestation<T::EthSpec>,
+    subnet_id: SubnetId,
 }
 
 /// Custom `Clone` implementation is to avoid the restrictive trait bounds applied by the usual derive
@@ -254,6 +274,7 @@ impl<T: BeaconChainTypes> Clone for VerifiedUnaggregatedAttestation<T> {
         Self {
             attestation: self.attestation.clone(),
             indexed_attestation: self.indexed_attestation.clone(),
+            subnet_id: self.subnet_id,
         }
     }
 }
@@ -319,6 +340,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         }?;
 
         // Ensure the block being voted for (attestation.data.beacon_block_root) passes validation.
+        // Don't enforce the skip slot restriction for aggregates.
         //
         // This indirectly checks to see if the `attestation.data.beacon_block_root` is in our fork
         // choice. Any known, non-finalized, processed block should be in fork choice, so this
@@ -327,7 +349,16 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         //
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
-        verify_head_block_is_known(chain, &attestation)?;
+        let head_block = verify_head_block_is_known(chain, &attestation, None)?;
+
+        // Check the attestation target root is consistent with the head root.
+        //
+        // This check is not in the specification, however we guard against it since it opens us up
+        // to weird edge cases during verification.
+        //
+        // Whilst this attestation *technically* could be used to add value to a block, it is
+        // invalid in the spirit of the protocol. Here we choose safety over profit.
+        verify_attestation_target_root::<T::EthSpec>(&head_block, &attestation)?;
 
         // Ensure that the attestation has participants.
         if attestation.aggregation_bits.is_zero() {
@@ -405,6 +436,11 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
     pub fn attestation(&self) -> &Attestation<T::EthSpec> {
         &self.signed_aggregate.message.aggregate
     }
+
+    /// Returns the underlying `signed_aggregate`.
+    pub fn aggregate(&self) -> &SignedAggregateAndProof<T::EthSpec> {
+        &self.signed_aggregate
+    }
 }
 
 impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
@@ -415,9 +451,19 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
     /// verify that it was received on the correct subnet.
     pub fn verify(
         attestation: Attestation<T::EthSpec>,
-        subnet_id: SubnetId,
+        subnet_id: Option<SubnetId>,
         chain: &BeaconChain<T>,
     ) -> Result<Self, Error> {
+        let attestation_epoch = attestation.data.slot.epoch(T::EthSpec::slots_per_epoch());
+
+        // Check the attestation's epoch matches its target.
+        if attestation_epoch != attestation.data.target.epoch {
+            return Err(Error::InvalidTargetEpoch {
+                slot: attestation.data.slot,
+                epoch: attestation.data.target.epoch,
+            });
+        }
+
         // Ensure attestation is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (within a
         // MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance).
         //
@@ -426,14 +472,20 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 
         // Check to ensure that the attestation is "unaggregated". I.e., it has exactly one
         // aggregation bit set.
-        let num_aggreagtion_bits = attestation.aggregation_bits.num_set_bits();
-        if num_aggreagtion_bits != 1 {
-            return Err(Error::NotExactlyOneAggregationBitSet(num_aggreagtion_bits));
+        let num_aggregation_bits = attestation.aggregation_bits.num_set_bits();
+        if num_aggregation_bits != 1 {
+            return Err(Error::NotExactlyOneAggregationBitSet(num_aggregation_bits));
         }
 
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
-        verify_head_block_is_known(chain, &attestation)?;
+        //
+        // Enforce a maximum skip distance for unaggregated attestations.
+        let head_block =
+            verify_head_block_is_known(chain, &attestation, chain.config.import_max_skip_slots)?;
+
+        // Check the attestation target root is consistent with the head root.
+        verify_attestation_target_root::<T::EthSpec>(&head_block, &attestation)?;
 
         let (indexed_attestation, committees_per_slot) =
             obtain_indexed_attestation_and_committees_per_slot(chain, &attestation)?;
@@ -445,13 +497,15 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         )
         .map_err(BeaconChainError::from)?;
 
-        // Ensure the attestation is from the correct subnet.
-        if subnet_id != expected_subnet_id {
-            return Err(Error::InvalidSubnetId {
-                received: subnet_id,
-                expected: expected_subnet_id,
-            });
-        }
+        // If a subnet was specified, ensure that subnet is correct.
+        if let Some(subnet_id) = subnet_id {
+            if subnet_id != expected_subnet_id {
+                return Err(Error::InvalidSubnetId {
+                    received: subnet_id,
+                    expected: expected_subnet_id,
+                });
+            }
+        };
 
         let validator_index = *indexed_attestation
             .attesting_indices
@@ -496,6 +550,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         Ok(Self {
             attestation,
             indexed_attestation,
+            subnet_id: expected_subnet_id,
         })
     }
 
@@ -504,6 +559,11 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         chain.add_to_naive_aggregation_pool(self)
     }
 
+    /// Returns the correct subnet for the attestation.
+    pub fn subnet_id(&self) -> SubnetId {
+        self.subnet_id
+    }
+
     /// Returns the wrapped `attestation`.
     pub fn attestation(&self) -> &Attestation<T::EthSpec> {
         &self.attestation
@@ -519,6 +579,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 }
 
 /// Returns `Ok(())` if the `attestation.data.beacon_block_root` is known to this chain.
+/// You can use this `shuffling_id` to read from the shuffling cache.
 ///
 /// The block root may not be known for two reasons:
 ///
@@ -531,13 +592,24 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 fn verify_head_block_is_known<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
-) -> Result<(), Error> {
-    if chain
+    max_skip_slots: Option<u64>,
+) -> Result<ProtoBlock, Error> {
+    if let Some(block) = chain
         .fork_choice
         .read()
-        .contains_block(&attestation.data.beacon_block_root)
+        .get_block(&attestation.data.beacon_block_root)
     {
-        Ok(())
+        // Reject any block that exceeds our limit on skipped slots.
+        if let Some(max_skip_slots) = max_skip_slots {
+            if attestation.data.slot > block.slot + max_skip_slots {
+                return Err(Error::TooManySkippedSlots {
+                    head_block_slot: block.slot,
+                    attestation_slot: attestation.data.slot,
+                });
+            }
+        }
+
+        Ok(block)
     } else {
         Err(Error::UnknownHeadBlock {
             beacon_block_root: attestation.data.beacon_block_root,
@@ -623,6 +695,57 @@ pub fn verify_attestation_signature<T: BeaconChainTypes>(
     }
 }
 
+/// Verifies that the `attestation.data.target.root` is indeed the target root of the block at
+/// `attestation.data.beacon_block_root`.
+pub fn verify_attestation_target_root<T: EthSpec>(
+    head_block: &ProtoBlock,
+    attestation: &Attestation<T>,
+) -> Result<(), Error> {
+    // Check the attestation target root.
+    let head_block_epoch = head_block.slot.epoch(T::slots_per_epoch());
+    let attestation_epoch = attestation.data.slot.epoch(T::slots_per_epoch());
+    if head_block_epoch > attestation_epoch {
+        // The epoch references an invalid head block from a future epoch.
+        //
+        // This check is not in the specification, however we guard against it since it opens us up
+        // to weird edge cases during verification.
+        //
+        // Whilst this attestation *technically* could be used to add value to a block, it is
+        // invalid in the spirit of the protocol. Here we choose safety over profit.
+        //
+        // Reference:
+        // https://github.com/ethereum/eth2.0-specs/pull/2001#issuecomment-699246659
+        return Err(Error::InvalidTargetRoot {
+            attestation: attestation.data.target.root,
+            // It is not clear what root we should expect in this case, since the attestation is
+            // fundamentally invalid.
+            expected: None,
+        });
+    } else {
+        let target_root = if head_block_epoch == attestation_epoch {
+            // If the block is in the same epoch as the attestation, then use the target root
+            // from the block.
+            head_block.target_root
+        } else {
+            // If the head block is from a previous epoch then skip slots will cause the head block
+            // root to become the target block root.
+            //
+            // We know the head block is from a previous epoch due to a previous check.
+            head_block.root
+        };
+
+        // Reject any attestation with an invalid target root.
+        if target_root != attestation.data.target.root {
+            return Err(Error::InvalidTargetRoot {
+                attestation: attestation.data.target.root,
+                expected: Some(target_root),
+            });
+        }
+    }
+
+    Ok(())
+}
+
 /// Verifies all the signatures in a `SignedAggregateAndProof` using BLS batch verification. This
 /// includes three signatures:
 ///
@@ -692,14 +815,14 @@ type CommitteesPerSlot = u64;
 
 /// Returns the `indexed_attestation` and committee count per slot for the `attestation` using the
 /// public keys cached in the `chain`.
-pub fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
+fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
 ) -> Result<(IndexedAttestation<T::EthSpec>, CommitteesPerSlot), Error> {
     map_attestation_committee(chain, attestation, |(committee, committees_per_slot)| {
         get_indexed_attestation(committee.committee, &attestation)
             .map(|attestation| (attestation, committees_per_slot))
-            .map_err(|e| BeaconChainError::from(e).into())
+            .map_err(Error::Invalid)
     })
 }
 
@@ -712,8 +835,8 @@ pub fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
 ///
 /// If the committee for `attestation` isn't found in the `shuffling_cache`, we will read a state
 /// from disk and then update the `shuffling_cache`.
-pub fn map_attestation_committee<'a, T, F, R>(
-    chain: &'a BeaconChain<T>,
+fn map_attestation_committee<T, F, R>(
+    chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
     map_fn: F,
 ) -> Result<R, Error>
@@ -731,99 +854,23 @@ where
     // processing an attestation that does not include our latest finalized block in its chain.
     //
     // We do not delay consideration for later, we simply drop the attestation.
-    let target_block = chain
-        .fork_choice
-        .read()
-        .get_block(&target.root)
-        .ok_or_else(|| Error::UnknownTargetRoot(target.root))?;
-
-    // Obtain the shuffling cache, timing how long we wait.
-    let cache_wait_timer =
-        metrics::start_timer(&metrics::ATTESTATION_PROCESSING_SHUFFLING_CACHE_WAIT_TIMES);
-
-    let mut shuffling_cache = chain
-        .shuffling_cache
-        .try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
-        .ok_or_else(|| BeaconChainError::AttestationCacheLockTimeout)?;
-
-    metrics::stop_timer(cache_wait_timer);
-
-    if let Some(committee_cache) = shuffling_cache.get(attestation_epoch, target.root) {
-        let committees_per_slot = committee_cache.committees_per_slot();
-        committee_cache
-            .get_beacon_committee(attestation.data.slot, attestation.data.index)
-            .map(|committee| map_fn((committee, committees_per_slot)))
-            .unwrap_or_else(|| {
-                Err(Error::NoCommitteeForSlotAndIndex {
-                    slot: attestation.data.slot,
-                    index: attestation.data.index,
-                })
-            })
-    } else {
-        // Drop the shuffling cache to avoid holding the lock for any longer than
-        // required.
-        drop(shuffling_cache);
-
-        debug!(
-            chain.log,
-            "Attestation processing cache miss";
-            "attn_epoch" => attestation_epoch.as_u64(),
-            "target_block_epoch" => target_block.slot.epoch(T::EthSpec::slots_per_epoch()).as_u64(),
-        );
-
-        let state_read_timer =
-            metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_READ_TIMES);
-
-        let mut state = chain
-            .get_state(&target_block.state_root, Some(target_block.slot))?
-            .ok_or_else(|| BeaconChainError::MissingBeaconState(target_block.state_root))?;
-
-        metrics::stop_timer(state_read_timer);
-        let state_skip_timer =
-            metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_SKIP_TIMES);
-
-        while state.current_epoch() + 1 < attestation_epoch {
-            // Here we tell `per_slot_processing` to skip hashing the state and just
-            // use the zero hash instead.
-            //
-            // The state roots are not useful for the shuffling, so there's no need to
-            // compute them.
-            per_slot_processing(&mut state, Some(Hash256::zero()), &chain.spec)
-                .map_err(BeaconChainError::from)?;
-        }
-
-        metrics::stop_timer(state_skip_timer);
-        let committee_building_timer =
-            metrics::start_timer(&metrics::ATTESTATION_PROCESSING_COMMITTEE_BUILDING_TIMES);
-
-        let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), attestation_epoch)
-            .map_err(BeaconChainError::IncorrectStateForAttestation)?;
-
-        state
-            .build_committee_cache(relative_epoch, &chain.spec)
-            .map_err(BeaconChainError::from)?;
-
-        let committee_cache = state
-            .committee_cache(relative_epoch)
-            .map_err(BeaconChainError::from)?;
-
-        chain
-            .shuffling_cache
-            .try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
-            .ok_or_else(|| BeaconChainError::AttestationCacheLockTimeout)?
-            .insert(attestation_epoch, target.root, committee_cache);
-
-        metrics::stop_timer(committee_building_timer);
-
-        let committees_per_slot = committee_cache.committees_per_slot();
-        committee_cache
-            .get_beacon_committee(attestation.data.slot, attestation.data.index)
-            .map(|committee| map_fn((committee, committees_per_slot)))
-            .unwrap_or_else(|| {
-                Err(Error::NoCommitteeForSlotAndIndex {
-                    slot: attestation.data.slot,
-                    index: attestation.data.index,
-                })
-            })
+    if !chain.fork_choice.read().contains_block(&target.root) {
+        return Err(Error::UnknownTargetRoot(target.root));
     }
+
+    chain
+        .with_committee_cache(target.root, attestation_epoch, |committee_cache| {
+            let committees_per_slot = committee_cache.committees_per_slot();
+
+            Ok(committee_cache
+                .get_beacon_committee(attestation.data.slot, attestation.data.index)
+                .map(|committee| map_fn((committee, committees_per_slot)))
+                .unwrap_or_else(|| {
+                    Err(Error::NoCommitteeForSlotAndIndex {
+                        slot: attestation.data.slot,
+                        index: attestation.data.index,
+                    })
+                }))
+        })
+        .map_err(BeaconChainError::from)?
 }

beacon_node/beacon_chain/src/block_verification.rs

@@ -48,6 +48,7 @@ use crate::{
     },
     metrics, BeaconChain, BeaconChainError, BeaconChainTypes, BeaconSnapshot,
 };
+use fork_choice::{ForkChoice, ForkChoiceStore};
 use parking_lot::RwLockReadGuard;
 use slog::{error, Logger};
 use slot_clock::SlotClock;
@@ -62,7 +63,7 @@ use std::borrow::Cow;
 use std::convert::TryFrom;
 use std::fs;
 use std::io::Write;
-use store::{Error as DBError, HotStateSummary, StoreOp};
+use store::{Error as DBError, HotColdDB, HotStateSummary, KeyValueStore, StoreOp};
 use tree_hash::TreeHash;
 use types::{
     BeaconBlock, BeaconState, BeaconStateError, ChainSpec, CloneConfig, EthSpec, Hash256,
@@ -83,14 +84,16 @@ const WRITE_BLOCK_PROCESSING_SSZ: bool = cfg!(feature = "write_ssz_files");
 /// - The block is malformed/invalid (indicated by all results other than `BeaconChainError`.
 /// - We encountered an error whilst trying to verify the block (a `BeaconChainError`).
 #[derive(Debug)]
-pub enum BlockError {
+pub enum BlockError<T: EthSpec> {
     /// The parent block was unknown.
     ///
     /// ## Peer scoring
     ///
     /// It's unclear if this block is valid, but it cannot be processed without already knowing
     /// its parent.
-    ParentUnknown(Hash256),
+    ParentUnknown(Box<SignedBeaconBlock<T>>),
+    /// The block skips too many slots and is a DoS risk.
+    TooManySkippedSlots { parent_slot: Slot, block_slot: Slot },
     /// The block slot is greater than the present slot.
     ///
     /// ## Peer scoring
@@ -118,6 +121,13 @@ pub enum BlockError {
         block_slot: Slot,
         finalized_slot: Slot,
     },
+    /// The block conflicts with finalization, no need to propagate.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this block is valid, but it conflicts with finality and shouldn't be
+    /// imported.
+    NotFinalizedDescendant { block_parent_root: Hash256 },
     /// Block is already known, no need to re-import.
     ///
     /// ## Peer scoring
@@ -197,9 +207,27 @@ pub enum BlockError {
     /// We were unable to process this block due to an internal error. It's unclear if the block is
     /// valid.
     BeaconChainError(BeaconChainError),
+    /// There was an error whilst verifying weak subjectivity. This block conflicts with the
+    /// configured weak subjectivity checkpoint and was not imported.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The block is invalid and the peer is faulty.
+    WeakSubjectivityConflict,
 }
 
-impl From<BlockSignatureVerifierError> for BlockError {
+impl<T: EthSpec> std::fmt::Display for BlockError<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BlockError::ParentUnknown(block) => {
+                write!(f, "ParentUnknown(parent_root:{})", block.parent_root())
+            }
+            other => write!(f, "{:?}", other),
+        }
+    }
+}
+
+impl<T: EthSpec> From<BlockSignatureVerifierError> for BlockError<T> {
     fn from(e: BlockSignatureVerifierError) -> Self {
         match e {
             // Make a special distinction for `IncorrectBlockProposer` since it indicates an
@@ -216,25 +244,25 @@ impl From<BlockSignatureVerifierError> for BlockError {
     }
 }
 
-impl From<BeaconChainError> for BlockError {
+impl<T: EthSpec> From<BeaconChainError> for BlockError<T> {
     fn from(e: BeaconChainError) -> Self {
         BlockError::BeaconChainError(e)
     }
 }
 
-impl From<BeaconStateError> for BlockError {
+impl<T: EthSpec> From<BeaconStateError> for BlockError<T> {
     fn from(e: BeaconStateError) -> Self {
         BlockError::BeaconChainError(BeaconChainError::BeaconStateError(e))
     }
 }
 
-impl From<SlotProcessingError> for BlockError {
+impl<T: EthSpec> From<SlotProcessingError> for BlockError<T> {
     fn from(e: SlotProcessingError) -> Self {
         BlockError::BeaconChainError(BeaconChainError::SlotProcessingError(e))
     }
 }
 
-impl From<DBError> for BlockError {
+impl<T: EthSpec> From<DBError> for BlockError<T> {
     fn from(e: DBError) -> Self {
         BlockError::BeaconChainError(BeaconChainError::DBError(e))
     }
@@ -251,15 +279,17 @@ impl From<DBError> for BlockError {
 /// The given `chain_segment` must span no more than two epochs, otherwise an error will be
 /// returned.
 pub fn signature_verify_chain_segment<T: BeaconChainTypes>(
-    chain_segment: Vec<(Hash256, SignedBeaconBlock<T::EthSpec>)>,
+    mut chain_segment: Vec<(Hash256, SignedBeaconBlock<T::EthSpec>)>,
     chain: &BeaconChain<T>,
-) -> Result<Vec<SignatureVerifiedBlock<T>>, BlockError> {
-    let (mut parent, slot) = if let Some(block) = chain_segment.first().map(|(_, block)| block) {
-        let parent = load_parent(&block.message, chain)?;
-        (parent, block.slot())
-    } else {
+) -> Result<Vec<SignatureVerifiedBlock<T>>, BlockError<T::EthSpec>> {
+    if chain_segment.is_empty() {
         return Ok(vec![]);
-    };
+    }
+
+    let (first_root, first_block) = chain_segment.remove(0);
+    let (mut parent, first_block) = load_parent(first_block, chain)?;
+    let slot = first_block.slot();
+    chain_segment.insert(0, (first_root, first_block));
 
     let highest_slot = chain_segment
         .last()
@@ -333,7 +363,7 @@ pub struct FullyVerifiedBlock<'a, T: BeaconChainTypes> {
     pub block_root: Hash256,
     pub state: BeaconState<T::EthSpec>,
     pub parent_block: SignedBeaconBlock<T::EthSpec>,
-    pub intermediate_states: Vec<StoreOp<'a, T::EthSpec>>,
+    pub confirmation_db_batch: Vec<StoreOp<'a, T::EthSpec>>,
 }
 
 /// Implemented on types that can be converted into a `FullyVerifiedBlock`.
@@ -343,7 +373,7 @@ pub trait IntoFullyVerifiedBlock<T: BeaconChainTypes> {
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError>;
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>>;
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec>;
 }
@@ -356,7 +386,7 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
     pub fn new(
         block: SignedBeaconBlock<T::EthSpec>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, BlockError> {
+    ) -> Result<Self, BlockError<T::EthSpec>> {
         // Do not gossip or process blocks from future slots.
         let present_slot_with_tolerance = chain
             .slot_clock
@@ -369,9 +399,22 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
             });
         }
 
+        let block_root = get_block_root(&block);
+
         // Do not gossip a block from a finalized slot.
         check_block_against_finalized_slot(&block.message, chain)?;
 
+        // Check if the block is already known. We know it is post-finalization, so it is
+        // sufficient to check the fork choice.
+        //
+        // In normal operation this isn't necessary, however it is useful immediately after a
+        // reboot if the `observed_block_producers` cache is empty. In that case, without this
+        // check, we will load the parent and state from disk only to find out later that we
+        // already know this block.
+        if chain.fork_choice.read().contains_block(&block_root) {
+            return Err(BlockError::BlockIsAlreadyKnown);
+        }
+
         // Check that we have not already received a block with a valid signature for this slot.
         if chain
             .observed_block_producers
@@ -384,8 +427,19 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
             });
         }
 
-        let mut parent = load_parent(&block.message, chain)?;
-        let block_root = get_block_root(&block);
+        // Do not process a block that doesn't descend from the finalized root.
+        //
+        // We check this *before* we load the parent so that we can return a more detailed error.
+        let block = check_block_is_finalized_descendant::<T, _>(
+            block,
+            &chain.fork_choice.read(),
+            &chain.store,
+        )?;
+
+        let (mut parent, block) = load_parent(block, chain)?;
+
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
 
         let state = cheap_state_advance_to_obtain_committees(
             &mut parent.beacon_state,
@@ -453,7 +507,7 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for GossipVerifiedBlock<T> {
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError> {
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
         let fully_verified = SignatureVerifiedBlock::from_gossip_verified_block(self, chain)?;
         fully_verified.into_fully_verified_block(chain)
     }
@@ -471,8 +525,12 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
     pub fn new(
         block: SignedBeaconBlock<T::EthSpec>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, BlockError> {
-        let mut parent = load_parent(&block.message, chain)?;
+    ) -> Result<Self, BlockError<T::EthSpec>> {
+        let (mut parent, block) = load_parent(block, chain)?;
+
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
+
         let block_root = get_block_root(&block);
 
         let state = cheap_state_advance_to_obtain_committees(
@@ -503,7 +561,7 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
     pub fn from_gossip_verified_block(
         from: GossipVerifiedBlock<T>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, BlockError> {
+    ) -> Result<Self, BlockError<T::EthSpec>> {
         let mut parent = from.parent;
         let block = from.block;
 
@@ -536,12 +594,12 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignatureVerifiedBlock<T
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError> {
-        let block = self.block;
-        let parent = self
-            .parent
-            .map(Result::Ok)
-            .unwrap_or_else(|| load_parent(&block.message, chain))?;
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
+        let (parent, block) = if let Some(parent) = self.parent {
+            (parent, self.block)
+        } else {
+            load_parent(self.block, chain)?
+        };
 
         FullyVerifiedBlock::from_signature_verified_components(
             block,
@@ -562,7 +620,7 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignedBeaconBlock<T::Eth
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError> {
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
         SignatureVerifiedBlock::new(self, chain)?.into_fully_verified_block(chain)
     }
 
@@ -584,7 +642,7 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
         block_root: Hash256,
         parent: BeaconSnapshot<T::EthSpec>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, BlockError> {
+    ) -> Result<Self, BlockError<T::EthSpec>> {
         // Reject any block if its parent is not known to fork choice.
         //
         // A block that is not in fork choice is either:
@@ -600,9 +658,12 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
             .read()
             .contains_block(&block.parent_root())
         {
-            return Err(BlockError::ParentUnknown(block.parent_root()));
+            return Err(BlockError::ParentUnknown(Box::new(block)));
         }
 
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
+
         /*
          *  Perform cursory checks to see if the block is even worth processing.
          */
@@ -615,9 +676,9 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
 
         let catchup_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CATCHUP_STATE);
 
-        // Keep a batch of any states that were "skipped" (block-less) in between the parent state
-        // slot and the block slot. These will be stored in the database.
-        let mut intermediate_states: Vec<StoreOp<T::EthSpec>> = Vec::new();
+        // Stage a batch of operations to be completed atomically if this block is imported
+        // successfully.
+        let mut confirmation_db_batch = vec![];
 
         // The block must have a higher slot than its parent.
         if block.slot() <= parent.beacon_state.slot {
@@ -641,15 +702,36 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
                 // processing, but we get early access to it.
                 let state_root = state.update_tree_hash_cache()?;
 
-                let op = if state.slot % T::EthSpec::slots_per_epoch() == 0 {
-                    StoreOp::PutState(state_root.into(), Cow::Owned(state.clone()))
+                // Store the state immediately, marking it as temporary, and staging the deletion
+                // of its temporary status as part of the larger atomic operation.
+                let txn_lock = chain.store.hot_db.begin_rw_transaction();
+                let state_already_exists =
+                    chain.store.load_hot_state_summary(&state_root)?.is_some();
+
+                let state_batch = if state_already_exists {
+                    // If the state exists, it could be temporary or permanent, but in neither case
+                    // should we rewrite it or store a new temporary flag for it. We *will* stage
+                    // the temporary flag for deletion because it's OK to double-delete the flag,
+                    // and we don't mind if another thread gets there first.
+                    vec![]
                 } else {
-                    StoreOp::PutStateSummary(
-                        state_root.into(),
-                        HotStateSummary::new(&state_root, &state)?,
-                    )
+                    vec![
+                        if state.slot % T::EthSpec::slots_per_epoch() == 0 {
+                            StoreOp::PutState(state_root, &state)
+                        } else {
+                            StoreOp::PutStateSummary(
+                                state_root,
+                                HotStateSummary::new(&state_root, &state)?,
+                            )
+                        },
+                        StoreOp::PutStateTemporaryFlag(state_root),
+                    ]
                 };
-                intermediate_states.push(op);
+                chain.store.do_atomically(state_batch)?;
+                drop(txn_lock);
+
+                confirmation_db_batch.push(StoreOp::DeleteStateTemporaryFlag(state_root));
+
                 state_root
             };
 
@@ -737,11 +819,35 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
             block_root,
             state,
             parent_block: parent.beacon_block,
-            intermediate_states,
+            confirmation_db_batch,
         })
     }
 }
 
+/// Check that the count of skip slots between the block and its parent does not exceed our maximum
+/// value.
+///
+/// Whilst this is not part of the specification, we include this to help prevent us from DoS
+/// attacks. In times of dire network circumstance, the user can configure the
+/// `import_max_skip_slots` value.
+fn check_block_skip_slots<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    parent: &BeaconBlock<T::EthSpec>,
+    block: &BeaconBlock<T::EthSpec>,
+) -> Result<(), BlockError<T::EthSpec>> {
+    // Reject any block that exceeds our limit on skipped slots.
+    if let Some(max_skip_slots) = chain.config.import_max_skip_slots {
+        if block.slot > parent.slot + max_skip_slots {
+            return Err(BlockError::TooManySkippedSlots {
+                parent_slot: parent.slot,
+                block_slot: block.slot,
+            });
+        }
+    }
+
+    Ok(())
+}
+
 /// Returns `Ok(())` if the block is later than the finalized slot on `chain`.
 ///
 /// Returns an error if the block is earlier or equal to the finalized slot, or there was an error
@@ -749,7 +855,7 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
 fn check_block_against_finalized_slot<T: BeaconChainTypes>(
     block: &BeaconBlock<T::EthSpec>,
     chain: &BeaconChain<T>,
-) -> Result<(), BlockError> {
+) -> Result<(), BlockError<T::EthSpec>> {
     let finalized_slot = chain
         .head_info()?
         .finalized_checkpoint
@@ -766,6 +872,36 @@ fn check_block_against_finalized_slot<T: BeaconChainTypes>(
     }
 }
 
+/// Returns `Ok(block)` if the block descends from the finalized root.
+pub fn check_block_is_finalized_descendant<T: BeaconChainTypes, F: ForkChoiceStore<T::EthSpec>>(
+    block: SignedBeaconBlock<T::EthSpec>,
+    fork_choice: &ForkChoice<F, T::EthSpec>,
+    store: &HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>,
+) -> Result<SignedBeaconBlock<T::EthSpec>, BlockError<T::EthSpec>> {
+    if fork_choice.is_descendant_of_finalized(block.parent_root()) {
+        Ok(block)
+    } else {
+        // If fork choice does *not* consider the parent to be a descendant of the finalized block,
+        // then there are two more cases:
+        //
+        // 1. We have the parent stored in our database. Because fork-choice has confirmed the
+        //    parent is *not* in our post-finalization DAG, all other blocks must be either
+        //    pre-finalization or conflicting with finalization.
+        // 2. The parent is unknown to us, we probably want to download it since it might actually
+        //    descend from the finalized root.
+        if store
+            .item_exists::<SignedBeaconBlock<T::EthSpec>>(&block.parent_root())
+            .map_err(|e| BlockError::BeaconChainError(e.into()))?
+        {
+            Err(BlockError::NotFinalizedDescendant {
+                block_parent_root: block.parent_root(),
+            })
+        } else {
+            Err(BlockError::ParentUnknown(Box::new(block)))
+        }
+    }
+}
+
 /// Performs simple, cheap checks to ensure that the block is relevant to be imported.
 ///
 /// `Ok(block_root)` is returned if the block passes these checks and should progress with
@@ -777,7 +913,7 @@ pub fn check_block_relevancy<T: BeaconChainTypes>(
     signed_block: &SignedBeaconBlock<T::EthSpec>,
     block_root: Option<Hash256>,
     chain: &BeaconChain<T>,
-) -> Result<Hash256, BlockError> {
+) -> Result<Hash256, BlockError<T::EthSpec>> {
     let block = &signed_block.message;
 
     // Do not process blocks from the future.
@@ -830,12 +966,11 @@ pub fn get_block_root<E: EthSpec>(block: &SignedBeaconBlock<E>) -> Hash256 {
 ///
 /// Returns `Err(BlockError::ParentUnknown)` if the parent is not found, or if an error occurs
 /// whilst attempting the operation.
+#[allow(clippy::type_complexity)]
 fn load_parent<T: BeaconChainTypes>(
-    block: &BeaconBlock<T::EthSpec>,
+    block: SignedBeaconBlock<T::EthSpec>,
     chain: &BeaconChain<T>,
-) -> Result<BeaconSnapshot<T::EthSpec>, BlockError> {
-    let db_read_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_READ);
-
+) -> Result<(BeaconSnapshot<T::EthSpec>, SignedBeaconBlock<T::EthSpec>), BlockError<T::EthSpec>> {
     // Reject any block if its parent is not known to fork choice.
     //
     // A block that is not in fork choice is either:
@@ -846,50 +981,61 @@ fn load_parent<T: BeaconChainTypes>(
     //  because it will revert finalization. Note that the finalized block is stored in fork
     //  choice, so we will not reject any child of the finalized block (this is relevant during
     //  genesis).
-    if !chain.fork_choice.read().contains_block(&block.parent_root) {
-        return Err(BlockError::ParentUnknown(block.parent_root));
+    if !chain
+        .fork_choice
+        .read()
+        .contains_block(&block.parent_root())
+    {
+        return Err(BlockError::ParentUnknown(Box::new(block)));
     }
 
-    // Load the parent block and state from disk, returning early if it's not available.
-    let result = chain
+    let db_read_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_READ);
+
+    let result = if let Some(snapshot) = chain
         .snapshot_cache
         .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
-        .and_then(|mut snapshot_cache| snapshot_cache.try_remove(block.parent_root))
-        .map(|snapshot| Ok(Some(snapshot)))
-        .unwrap_or_else(|| {
-            // Load the blocks parent block from the database, returning invalid if that block is not
-            // found.
-            //
-            // We don't return a DBInconsistent error here since it's possible for a block to
-            // exist in fork choice but not in the database yet. In such a case we simply
-            // indicate that we don't yet know the parent.
-            let parent_block = if let Some(block) = chain.get_block(&block.parent_root)? {
-                block
-            } else {
-                return Ok(None);
-            };
+        .and_then(|mut snapshot_cache| snapshot_cache.try_remove(block.parent_root()))
+    {
+        Ok((snapshot, block))
+    } else {
+        // Load the blocks parent block from the database, returning invalid if that block is not
+        // found.
+        //
+        // We don't return a DBInconsistent error here since it's possible for a block to
+        // exist in fork choice but not in the database yet. In such a case we simply
+        // indicate that we don't yet know the parent.
+        let root = block.parent_root();
+        let parent_block = chain
+            .get_block(&block.parent_root())
+            .map_err(BlockError::BeaconChainError)?
+            .ok_or_else(|| {
+                // Return a `MissingBeaconBlock` error instead of a `ParentUnknown` error since
+                // we've already checked fork choice for this block.
+                //
+                // It's an internal error if the block exists in fork choice but not in the
+                // database.
+                BlockError::from(BeaconChainError::MissingBeaconBlock(block.parent_root()))
+            })?;
 
-            // Load the parent blocks state from the database, returning an error if it is not found.
-            // It is an error because if we know the parent block we should also know the parent state.
-            let parent_state_root = parent_block.state_root();
-            let parent_state = chain
-                .get_state(&parent_state_root, Some(parent_block.slot()))?
-                .ok_or_else(|| {
-                    BeaconChainError::DBInconsistent(format!(
-                        "Missing state {:?}",
-                        parent_state_root
-                    ))
-                })?;
+        // Load the parent blocks state from the database, returning an error if it is not found.
+        // It is an error because if we know the parent block we should also know the parent state.
+        let parent_state_root = parent_block.state_root();
+        let parent_state = chain
+            .get_state(&parent_state_root, Some(parent_block.slot()))?
+            .ok_or_else(|| {
+                BeaconChainError::DBInconsistent(format!("Missing state {:?}", parent_state_root))
+            })?;
 
-            Ok(Some(BeaconSnapshot {
+        Ok((
+            BeaconSnapshot {
                 beacon_block: parent_block,
-                beacon_block_root: block.parent_root,
+                beacon_block_root: root,
                 beacon_state: parent_state,
                 beacon_state_root: parent_state_root,
-            }))
-        })
-        .map_err(BlockError::BeaconChainError)?
-        .ok_or_else(|| BlockError::ParentUnknown(block.parent_root));
+            },
+            block,
+        ))
+    };
 
     metrics::stop_timer(db_read_timer);
 
@@ -911,7 +1057,7 @@ fn cheap_state_advance_to_obtain_committees<'a, E: EthSpec>(
     state: &'a mut BeaconState<E>,
     block_slot: Slot,
     spec: &ChainSpec,
-) -> Result<Cow<'a, BeaconState<E>>, BlockError> {
+) -> Result<Cow<'a, BeaconState<E>>, BlockError<E>> {
     let block_epoch = block_slot.epoch(E::slots_per_epoch());
 
     if state.current_epoch() == block_epoch {
@@ -943,7 +1089,7 @@ fn cheap_state_advance_to_obtain_committees<'a, E: EthSpec>(
 /// Obtains a read-locked `ValidatorPubkeyCache` from the `chain`.
 fn get_validator_pubkey_cache<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
-) -> Result<RwLockReadGuard<ValidatorPubkeyCache>, BlockError> {
+) -> Result<RwLockReadGuard<ValidatorPubkeyCache>, BlockError<T::EthSpec>> {
     chain
         .validator_pubkey_cache
         .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)

beacon_node/beacon_chain/src/builder.rs

@@ -4,22 +4,24 @@ use crate::beacon_chain::{
 use crate::eth1_chain::{CachingEth1Backend, SszEth1};
 use crate::events::NullEventHandler;
 use crate::head_tracker::HeadTracker;
-use crate::migrate::Migrate;
+use crate::migrate::{BackgroundMigrator, MigratorConfig};
 use crate::persisted_beacon_chain::PersistedBeaconChain;
 use crate::persisted_fork_choice::PersistedForkChoice;
 use crate::shuffling_cache::ShufflingCache;
 use crate::snapshot_cache::{SnapshotCache, DEFAULT_SNAPSHOT_CACHE_SIZE};
 use crate::timeout_rw_lock::TimeoutRwLock;
 use crate::validator_pubkey_cache::ValidatorPubkeyCache;
+use crate::ChainConfig;
 use crate::{
     BeaconChain, BeaconChainTypes, BeaconForkChoiceStore, BeaconSnapshot, Eth1Chain,
     Eth1ChainBackend, EventHandler,
 };
 use eth1::Config as Eth1Config;
 use fork_choice::ForkChoice;
+use futures::channel::mpsc::Sender;
 use operation_pool::{OperationPool, PersistedOperationPool};
 use parking_lot::RwLock;
-use slog::{info, Logger};
+use slog::{crit, info, Logger};
 use slot_clock::{SlotClock, TestingSlotClock};
 use std::marker::PhantomData;
 use std::path::PathBuf;
@@ -35,17 +37,8 @@ pub const PUBKEY_CACHE_FILENAME: &str = "pubkey_cache.ssz";
 
 /// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing
 /// functionality and only exists to satisfy the type system.
-pub struct Witness<
-    TStoreMigrator,
-    TSlotClock,
-    TEth1Backend,
-    TEthSpec,
-    TEventHandler,
-    THotStore,
-    TColdStore,
->(
+pub struct Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>(
     PhantomData<(
-        TStoreMigrator,
         TSlotClock,
         TEth1Backend,
         TEthSpec,
@@ -55,21 +48,11 @@ pub struct Witness<
     )>,
 );
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    BeaconChainTypes
-    for Witness<
-        TStoreMigrator,
-        TSlotClock,
-        TEth1Backend,
-        TEthSpec,
-        TEventHandler,
-        THotStore,
-        TColdStore,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore> BeaconChainTypes
+    for Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -77,7 +60,6 @@ where
 {
     type HotStore = THotStore;
     type ColdStore = TColdStore;
-    type StoreMigrator = TStoreMigrator;
     type SlotClock = TSlotClock;
     type Eth1Chain = TEth1Backend;
     type EthSpec = TEthSpec;
@@ -95,42 +77,37 @@ where
 pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     #[allow(clippy::type_complexity)]
     store: Option<Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>>,
-    store_migrator: Option<T::StoreMigrator>,
-    canonical_head: Option<BeaconSnapshot<T::EthSpec>>,
-    /// The finalized checkpoint to anchor the chain. May be genesis or a higher
-    /// checkpoint.
-    pub finalized_snapshot: Option<BeaconSnapshot<T::EthSpec>>,
+    store_migrator_config: Option<MigratorConfig>,
+    pub genesis_time: Option<u64>,
     genesis_block_root: Option<Hash256>,
+    genesis_state_root: Option<Hash256>,
+    #[allow(clippy::type_complexity)]
+    fork_choice: Option<
+        ForkChoice<BeaconForkChoiceStore<T::EthSpec, T::HotStore, T::ColdStore>, T::EthSpec>,
+    >,
     op_pool: Option<OperationPool<T::EthSpec>>,
     eth1_chain: Option<Eth1Chain<T::Eth1Chain, T::EthSpec>>,
     event_handler: Option<T::EventHandler>,
     slot_clock: Option<T::SlotClock>,
+    shutdown_sender: Option<Sender<&'static str>>,
     head_tracker: Option<HeadTracker>,
     data_dir: Option<PathBuf>,
     pubkey_cache_path: Option<PathBuf>,
     validator_pubkey_cache: Option<ValidatorPubkeyCache>,
     spec: ChainSpec,
+    chain_config: ChainConfig,
     disabled_forks: Vec<String>,
     log: Option<Logger>,
     graffiti: Graffiti,
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
     BeaconChainBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+        Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>,
     >
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -143,20 +120,23 @@ where
     pub fn new(_eth_spec_instance: TEthSpec) -> Self {
         Self {
             store: None,
-            store_migrator: None,
-            canonical_head: None,
-            finalized_snapshot: None,
+            store_migrator_config: None,
+            genesis_time: None,
             genesis_block_root: None,
+            genesis_state_root: None,
+            fork_choice: None,
             op_pool: None,
             eth1_chain: None,
             event_handler: None,
             slot_clock: None,
+            shutdown_sender: None,
             head_tracker: None,
             pubkey_cache_path: None,
             data_dir: None,
             disabled_forks: Vec::new(),
             validator_pubkey_cache: None,
             spec: TEthSpec::default_spec(),
+            chain_config: ChainConfig::default(),
             log: None,
             graffiti: Graffiti::default(),
         }
@@ -171,6 +151,15 @@ where
         self
     }
 
+    /// Sets the maximum number of blocks that will be skipped when processing
+    /// some consensus messages.
+    ///
+    /// Set to `None` for no limit.
+    pub fn import_max_skip_slots(mut self, n: Option<u64>) -> Self {
+        self.chain_config.import_max_skip_slots = n;
+        self
+    }
+
     /// Sets the store (database).
     ///
     /// Should generally be called early in the build chain.
@@ -179,9 +168,9 @@ where
         self
     }
 
-    /// Sets the store migrator.
-    pub fn store_migrator(mut self, store_migrator: TStoreMigrator) -> Self {
-        self.store_migrator = Some(store_migrator);
+    /// Sets the store migrator config (optional).
+    pub fn store_migrator_config(mut self, config: MigratorConfig) -> Self {
+        self.store_migrator_config = Some(config);
         self
     }
 
@@ -216,7 +205,7 @@ where
             .ok_or_else(|| "get_persisted_eth1_backend requires a store.".to_string())?;
 
         store
-            .get_item::<SszEth1>(&Hash256::from_slice(&ETH1_CACHE_DB_KEY))
+            .get_item::<SszEth1>(&ETH1_CACHE_DB_KEY)
             .map_err(|e| format!("DB error whilst reading eth1 cache: {:?}", e))
     }
 
@@ -228,7 +217,7 @@ where
             .ok_or_else(|| "store_contains_beacon_chain requires a store.".to_string())?;
 
         Ok(store
-            .get_item::<PersistedBeaconChain>(&Hash256::from_slice(&BEACON_CHAIN_DB_KEY))
+            .get_item::<PersistedBeaconChain>(&BEACON_CHAIN_DB_KEY)
             .map_err(|e| format!("DB error when reading persisted beacon chain: {:?}", e))?
             .is_some())
     }
@@ -259,67 +248,58 @@ where
             .ok_or_else(|| "resume_from_db requires a store.".to_string())?;
 
         let chain = store
-            .get_item::<PersistedBeaconChain>(&Hash256::from_slice(&BEACON_CHAIN_DB_KEY))
+            .get_item::<PersistedBeaconChain>(&BEACON_CHAIN_DB_KEY)
             .map_err(|e| format!("DB error when reading persisted beacon chain: {:?}", e))?
             .ok_or_else(|| {
                 "No persisted beacon chain found in store. Try purging the beacon chain database."
                     .to_string()
             })?;
 
-        self.genesis_block_root = Some(chain.genesis_block_root);
-        self.head_tracker = Some(
-            HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
-                .map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
-        );
+        let persisted_fork_choice = store
+            .get_item::<PersistedForkChoice>(&FORK_CHOICE_DB_KEY)
+            .map_err(|e| format!("DB error when reading persisted fork choice: {:?}", e))?
+            .ok_or_else(|| "No persisted fork choice present in database.".to_string())?;
 
-        let head_block_root = chain.canonical_head_block_root;
-        let head_block = store
-            .get_item::<SignedBeaconBlock<TEthSpec>>(&head_block_root)
-            .map_err(|e| format!("DB error when reading head block: {:?}", e))?
-            .ok_or_else(|| "Head block not found in store".to_string())?;
-        let head_state_root = head_block.state_root();
-        let head_state = store
-            .get_state(&head_state_root, Some(head_block.slot()))
-            .map_err(|e| format!("DB error when reading head state: {:?}", e))?
-            .ok_or_else(|| "Head state not found in store".to_string())?;
+        let fc_store = BeaconForkChoiceStore::from_persisted(
+            persisted_fork_choice.fork_choice_store,
+            store.clone(),
+        )
+        .map_err(|e| format!("Unable to load ForkChoiceStore: {:?}", e))?;
+
+        let fork_choice =
+            ForkChoice::from_persisted(persisted_fork_choice.fork_choice, fc_store)
+                .map_err(|e| format!("Unable to parse persisted fork choice from disk: {:?}", e))?;
+
+        let genesis_block = store
+            .get_item::<SignedBeaconBlock<TEthSpec>>(&chain.genesis_block_root)
+            .map_err(|e| format!("DB error when reading genesis block: {:?}", e))?
+            .ok_or_else(|| "Genesis block not found in store".to_string())?;
+        let genesis_state = store
+            .get_state(&genesis_block.state_root(), Some(genesis_block.slot()))
+            .map_err(|e| format!("DB error when reading genesis state: {:?}", e))?
+            .ok_or_else(|| "Genesis block not found in store".to_string())?;
+
+        self.genesis_time = Some(genesis_state.genesis_time);
 
         self.op_pool = Some(
             store
-                .get_item::<PersistedOperationPool<TEthSpec>>(&Hash256::from_slice(&OP_POOL_DB_KEY))
+                .get_item::<PersistedOperationPool<TEthSpec>>(&OP_POOL_DB_KEY)
                 .map_err(|e| format!("DB error whilst reading persisted op pool: {:?}", e))?
                 .map(PersistedOperationPool::into_operation_pool)
                 .unwrap_or_else(OperationPool::new),
         );
 
-        let finalized_block_root = head_state.finalized_checkpoint.root;
-        let finalized_block = store
-            .get_item::<SignedBeaconBlock<TEthSpec>>(&finalized_block_root)
-            .map_err(|e| format!("DB error when reading finalized block: {:?}", e))?
-            .ok_or_else(|| "Finalized block not found in store".to_string())?;
-        let finalized_state_root = finalized_block.state_root();
-        let finalized_state = store
-            .get_state(&finalized_state_root, Some(finalized_block.slot()))
-            .map_err(|e| format!("DB error when reading finalized state: {:?}", e))?
-            .ok_or_else(|| "Finalized state not found in store".to_string())?;
-
-        self.finalized_snapshot = Some(BeaconSnapshot {
-            beacon_block_root: finalized_block_root,
-            beacon_block: finalized_block,
-            beacon_state_root: finalized_state_root,
-            beacon_state: finalized_state,
-        });
-
-        self.canonical_head = Some(BeaconSnapshot {
-            beacon_block_root: head_block_root,
-            beacon_block: head_block,
-            beacon_state_root: head_state_root,
-            beacon_state: head_state,
-        });
-
         let pubkey_cache = ValidatorPubkeyCache::load_from_file(pubkey_cache_path)
             .map_err(|e| format!("Unable to open persisted pubkey cache: {:?}", e))?;
 
+        self.genesis_block_root = Some(chain.genesis_block_root);
+        self.genesis_state_root = Some(genesis_block.state_root());
+        self.head_tracker = Some(
+            HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
+                .map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
+        );
         self.validator_pubkey_cache = Some(pubkey_cache);
+        self.fork_choice = Some(fork_choice);
 
         Ok(self)
     }
@@ -343,6 +323,7 @@ where
         let beacon_state_root = beacon_block.message.state_root;
         let beacon_block_root = beacon_block.canonical_root();
 
+        self.genesis_state_root = Some(beacon_state_root);
         self.genesis_block_root = Some(beacon_block_root);
 
         store
@@ -362,12 +343,25 @@ where
                 )
             })?;
 
-        self.finalized_snapshot = Some(BeaconSnapshot {
+        let genesis = BeaconSnapshot {
             beacon_block_root,
             beacon_block,
             beacon_state_root,
             beacon_state,
-        });
+        };
+
+        let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis);
+
+        let fork_choice = ForkChoice::from_genesis(
+            fc_store,
+            genesis.beacon_block_root,
+            &genesis.beacon_block.message,
+            &genesis.beacon_state,
+        )
+        .map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?;
+
+        self.fork_choice = Some(fork_choice);
+        self.genesis_time = Some(genesis.beacon_state.genesis_time);
 
         Ok(self.empty_op_pool())
     }
@@ -394,6 +388,12 @@ where
         self
     }
 
+    /// Sets a `Sender` to allow the beacon chain to send shutdown signals.
+    pub fn shutdown_sender(mut self, sender: Sender<&'static str>) -> Self {
+        self.shutdown_sender = Some(sender);
+        self
+    }
+
     /// Creates a new, empty operation pool.
     fn empty_op_pool(mut self) -> Self {
         self.op_pool = Some(OperationPool::new());
@@ -406,6 +406,12 @@ where
         self
     }
 
+    /// Sets the `ChainConfig` that determines `BeaconChain` runtime behaviour.
+    pub fn chain_config(mut self, config: ChainConfig) -> Self {
+        self.chain_config = config;
+        self
+    }
+
     /// Consumes `self`, returning a `BeaconChain` if all required parameters have been supplied.
     ///
     /// An error will be returned at runtime if all required parameters have not been configured.
@@ -417,15 +423,7 @@ where
         self,
     ) -> Result<
         BeaconChain<
-            Witness<
-                TStoreMigrator,
-                TSlotClock,
-                TEth1Backend,
-                TEthSpec,
-                TEventHandler,
-                THotStore,
-                TColdStore,
-            >,
+            Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>,
         >,
         String,
     > {
@@ -439,23 +437,76 @@ where
             .store
             .clone()
             .ok_or_else(|| "Cannot build without a store.".to_string())?;
+        let mut fork_choice = self
+            .fork_choice
+            .ok_or_else(|| "Cannot build without fork choice.".to_string())?;
+        let genesis_block_root = self
+            .genesis_block_root
+            .ok_or_else(|| "Cannot build without a genesis block root".to_string())?;
+        let genesis_state_root = self
+            .genesis_state_root
+            .ok_or_else(|| "Cannot build without a genesis state root".to_string())?;
 
-        // If this beacon chain is being loaded from disk, use the stored head. Otherwise, just use
-        // the finalized checkpoint (which is probably genesis).
-        let mut canonical_head = if let Some(head) = self.canonical_head {
-            head
+        let current_slot = if slot_clock
+            .is_prior_to_genesis()
+            .ok_or_else(|| "Unable to read slot clock".to_string())?
+        {
+            self.spec.genesis_slot
         } else {
-            self.finalized_snapshot
-                .ok_or_else(|| "Cannot build without a state".to_string())?
+            slot_clock
+                .now()
+                .ok_or_else(|| "Unable to read slot".to_string())?
         };
 
+        let head_block_root = fork_choice
+            .get_head(current_slot)
+            .map_err(|e| format!("Unable to get fork choice head: {:?}", e))?;
+
+        let head_block = store
+            .get_item::<SignedBeaconBlock<TEthSpec>>(&head_block_root)
+            .map_err(|e| format!("DB error when reading head block: {:?}", e))?
+            .ok_or_else(|| "Head block not found in store".to_string())?;
+        let head_state_root = head_block.state_root();
+        let head_state = store
+            .get_state(&head_state_root, Some(head_block.slot()))
+            .map_err(|e| format!("DB error when reading head state: {:?}", e))?
+            .ok_or_else(|| "Head state not found in store".to_string())?;
+
+        let mut canonical_head = BeaconSnapshot {
+            beacon_block_root: head_block_root,
+            beacon_block: head_block,
+            beacon_state_root: head_state_root,
+            beacon_state: head_state,
+        };
+
+        if canonical_head.beacon_block.state_root() != canonical_head.beacon_state_root {
+            return Err("beacon_block.state_root != beacon_state".to_string());
+        }
+
         canonical_head
             .beacon_state
             .build_all_caches(&self.spec)
             .map_err(|e| format!("Failed to build state caches: {:?}", e))?;
 
-        if canonical_head.beacon_block.state_root() != canonical_head.beacon_state_root {
-            return Err("beacon_block.state_root != beacon_state".to_string());
+        // Perform a check to ensure that the finalization points of the head and fork choice are
+        // consistent.
+        //
+        // This is a sanity check to detect database corruption.
+        let fc_finalized = fork_choice.finalized_checkpoint();
+        let head_finalized = canonical_head.beacon_state.finalized_checkpoint;
+        if fc_finalized != head_finalized {
+            if head_finalized.root == Hash256::zero()
+                && head_finalized.epoch == fc_finalized.epoch
+                && fc_finalized.root == genesis_block_root
+            {
+                // This is a legal edge-case encountered during genesis.
+            } else {
+                return Err(format!(
+                    "Database corrupt: fork choice is finalized at {:?} whilst head is finalized at \
+                    {:?}",
+                    fc_finalized, head_finalized
+                ));
+            }
         }
 
         let pubkey_cache_path = self
@@ -467,32 +518,19 @@ where
                 .map_err(|e| format!("Unable to init validator pubkey cache: {:?}", e))
         })?;
 
-        let persisted_fork_choice = store
-            .get_item::<PersistedForkChoice>(&Hash256::from_slice(&FORK_CHOICE_DB_KEY))
-            .map_err(|e| format!("DB error when reading persisted fork choice: {:?}", e))?;
-
-        let fork_choice = if let Some(persisted) = persisted_fork_choice {
-            let fc_store =
-                BeaconForkChoiceStore::from_persisted(persisted.fork_choice_store, store.clone())
-                    .map_err(|e| format!("Unable to load ForkChoiceStore: {:?}", e))?;
-
-            ForkChoice::from_persisted(persisted.fork_choice, fc_store)
-                .map_err(|e| format!("Unable to parse persisted fork choice from disk: {:?}", e))?
-        } else {
-            let genesis = &canonical_head;
-
-            let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), genesis);
-
-            ForkChoice::from_genesis(fc_store, &genesis.beacon_block.message)
-                .map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?
-        };
+        let migrator_config = self.store_migrator_config.unwrap_or_default();
+        let store_migrator = BackgroundMigrator::new(
+            store.clone(),
+            migrator_config,
+            genesis_block_root,
+            log.clone(),
+        );
 
         let beacon_chain = BeaconChain {
             spec: self.spec,
+            config: self.chain_config,
             store,
-            store_migrator: self
-                .store_migrator
-                .ok_or_else(|| "Cannot build without store migrator".to_string())?,
+            store_migrator,
             slot_clock,
             op_pool: self
                 .op_pool
@@ -514,9 +552,8 @@ where
             eth1_chain: self.eth1_chain,
             genesis_validators_root: canonical_head.beacon_state.genesis_validators_root,
             canonical_head: TimeoutRwLock::new(canonical_head.clone()),
-            genesis_block_root: self
-                .genesis_block_root
-                .ok_or_else(|| "Cannot build without a genesis block root".to_string())?,
+            genesis_block_root,
+            genesis_state_root,
             fork_choice: RwLock::new(fork_choice),
             event_handler: self
                 .event_handler
@@ -529,6 +566,9 @@ where
             shuffling_cache: TimeoutRwLock::new(ShufflingCache::new()),
             validator_pubkey_cache: TimeoutRwLock::new(validator_pubkey_cache),
             disabled_forks: self.disabled_forks,
+            shutdown_sender: self
+                .shutdown_sender
+                .ok_or_else(|| "Cannot build without a shutdown sender.".to_string())?,
             log: log.clone(),
             graffiti: self.graffiti,
         };
@@ -537,6 +577,27 @@ where
             .head()
             .map_err(|e| format!("Failed to get head: {:?}", e))?;
 
+        // Only perform the check if it was configured.
+        if let Some(wss_checkpoint) = beacon_chain.config.weak_subjectivity_checkpoint {
+            if let Err(e) = beacon_chain.verify_weak_subjectivity_checkpoint(
+                wss_checkpoint,
+                head.beacon_block_root,
+                &head.beacon_state,
+            ) {
+                crit!(
+                    log,
+                    "Weak subjectivity checkpoint verification failed on startup!";
+                    "head_block_root" => format!("{}", head.beacon_block_root),
+                    "head_slot" => format!("{}", head.beacon_block.slot()),
+                    "finalized_epoch" => format!("{}", head.beacon_state.finalized_checkpoint.epoch),
+                    "wss_checkpoint_epoch" => format!("{}", wss_checkpoint.epoch),
+                    "error" => format!("{:?}", e),
+                );
+                crit!(log, "You must use the `--purge-db` flag to clear the database and restart sync. You may be on a hostile network.");
+                return Err(format!("Weak subjectivity verification failed: {:?}", e));
+            }
+        }
+
         info!(
             log,
             "Beacon chain initialized";
@@ -549,10 +610,9 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
     BeaconChainBuilder<
         Witness<
-            TStoreMigrator,
             TSlotClock,
             CachingEth1Backend<TEthSpec>,
             TEthSpec,
@@ -564,7 +624,6 @@ impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -590,22 +649,13 @@ where
     }
 }
 
-impl<TStoreMigrator, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
     BeaconChainBuilder<
-        Witness<
-            TStoreMigrator,
-            TestingSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+        Witness<TestingSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>,
     >
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -615,11 +665,8 @@ where
     /// Requires the state to be initialized.
     pub fn testing_slot_clock(self, slot_duration: Duration) -> Result<Self, String> {
         let genesis_time = self
-            .finalized_snapshot
-            .as_ref()
-            .ok_or_else(|| "testing_slot_clock requires an initialized state")?
-            .beacon_state
-            .genesis_time;
+            .genesis_time
+            .ok_or_else(|| "testing_slot_clock requires an initialized state")?;
 
         let slot_clock = TestingSlotClock::new(
             Slot::new(0),
@@ -631,10 +678,9 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
+impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
     BeaconChainBuilder<
         Witness<
-            TStoreMigrator,
             TSlotClock,
             TEth1Backend,
             TEthSpec,
@@ -646,7 +692,6 @@ impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -678,7 +723,6 @@ fn genesis_block<T: EthSpec>(
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::migrate::NullMigrator;
     use eth2_hashing::hash;
     use genesis::{generate_deterministic_keypairs, interop_genesis_state};
     use sloggers::{null::NullLoggerBuilder, Build};
@@ -718,10 +762,11 @@ mod test {
         )
         .expect("should create interop genesis state");
 
+        let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+
         let chain = BeaconChainBuilder::new(MinimalEthSpec)
             .logger(log.clone())
             .store(Arc::new(store))
-            .store_migrator(NullMigrator)
             .data_dir(data_dir.path().to_path_buf())
             .genesis_state(genesis_state)
             .expect("should build state using recent genesis")
@@ -730,6 +775,7 @@ mod test {
             .null_event_handler()
             .testing_slot_clock(Duration::from_secs(1))
             .expect("should configure testing slot clock")
+            .shutdown_sender(shutdown_tx)
             .build()
             .expect("should build");
 

beacon_node/beacon_chain/src/chain_config.rs

@@ -0,0 +1,24 @@
+use serde_derive::{Deserialize, Serialize};
+use types::Checkpoint;
+
+#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
+pub struct ChainConfig {
+    /// Maximum number of slots to skip when importing a consensus message (e.g., block,
+    /// attestation, etc).
+    ///
+    /// If `None`, there is no limit.
+    pub import_max_skip_slots: Option<u64>,
+    /// A user-input `Checkpoint` that must exist in the beacon chain's sync path.
+    ///
+    /// If `None`, there is no weak subjectivity verification.
+    pub weak_subjectivity_checkpoint: Option<Checkpoint>,
+}
+
+impl Default for ChainConfig {
+    fn default() -> Self {
+        Self {
+            import_max_skip_slots: None,
+            weak_subjectivity_checkpoint: None,
+        }
+    }
+}

beacon_node/beacon_chain/src/errors.rs

@@ -1,9 +1,11 @@
 use crate::beacon_chain::ForkChoiceError;
 use crate::eth1_chain::Error as Eth1ChainError;
+use crate::migrate::PruningError;
 use crate::naive_aggregation_pool::Error as NaiveAggregationError;
 use crate::observed_attestations::Error as ObservedAttestationsError;
 use crate::observed_attesters::Error as ObservedAttestersError;
 use crate::observed_block_producers::Error as ObservedBlockProducersError;
+use futures::channel::mpsc::TrySendError;
 use operation_pool::OpPoolError;
 use safe_arith::ArithError;
 use ssz_types::Error as SszTypesError;
@@ -61,6 +63,7 @@ pub enum BeaconChainError {
         requested_slot: Slot,
         max_task_runtime: Duration,
     },
+    MissingFinalizedStateRoot(Slot),
     /// Returned when an internal check fails, indicating corrupt data.
     InvariantViolated(String),
     SszTypesError(SszTypesError),
@@ -79,7 +82,14 @@ pub enum BeaconChainError {
     ObservedAttestationsError(ObservedAttestationsError),
     ObservedAttestersError(ObservedAttestersError),
     ObservedBlockProducersError(ObservedBlockProducersError),
+    PruningError(PruningError),
     ArithError(ArithError),
+    InvalidShufflingId {
+        shuffling_epoch: Epoch,
+        head_block_epoch: Epoch,
+    },
+    WeakSubjectivtyVerificationFailure,
+    WeakSubjectivtyShutdownError(TrySendError<&'static str>),
 }
 
 easy_from_to!(SlotProcessingError, BeaconChainError);
@@ -94,6 +104,7 @@ easy_from_to!(ObservedAttestationsError, BeaconChainError);
 easy_from_to!(ObservedAttestersError, BeaconChainError);
 easy_from_to!(ObservedBlockProducersError, BeaconChainError);
 easy_from_to!(BlockSignatureVerifierError, BeaconChainError);
+easy_from_to!(PruningError, BeaconChainError);
 easy_from_to!(ArithError, BeaconChainError);
 
 #[derive(Debug)]

beacon_node/beacon_chain/src/eth1_chain.rs

@@ -1,5 +1,4 @@
 use crate::metrics;
-use environment::TaskExecutor;
 use eth1::{Config as Eth1Config, Eth1Block, Service as HttpService};
 use eth2_hashing::hash;
 use slog::{debug, error, trace, Logger};
@@ -11,6 +10,7 @@ use std::collections::HashMap;
 use std::iter::DoubleEndedIterator;
 use std::marker::PhantomData;
 use store::{DBColumn, Error as StoreError, StoreItem};
+use task_executor::TaskExecutor;
 use types::{
     BeaconState, BeaconStateError, ChainSpec, Deposit, Eth1Data, EthSpec, Hash256, Slot, Unsigned,
     DEPOSIT_TREE_DEPTH,

beacon_node/beacon_chain/src/head_tracker.rs

@@ -15,7 +15,7 @@ pub enum Error {
 /// In order for this struct to be effective, every single block that is imported must be
 /// registered here.
 #[derive(Default, Debug)]
-pub struct HeadTracker(RwLock<HashMap<Hash256, Slot>>);
+pub struct HeadTracker(pub RwLock<HashMap<Hash256, Slot>>);
 
 impl HeadTracker {
     /// Register a block with `Self`, so it may or may not be included in a `Self::heads` call.
@@ -29,13 +29,6 @@ impl HeadTracker {
         map.insert(block_root, slot);
     }
 
-    /// Removes abandoned head.
-    pub fn remove_head(&self, block_root: Hash256) {
-        let mut map = self.0.write();
-        debug_assert!(map.contains_key(&block_root));
-        map.remove(&block_root);
-    }
-
     /// Returns true iff `block_root` is a recognized head.
     pub fn contains_head(&self, block_root: Hash256) -> bool {
         self.0.read().contains_key(&block_root)
@@ -53,14 +46,7 @@ impl HeadTracker {
     /// Returns a `SszHeadTracker`, which contains all necessary information to restore the state
     /// of `Self` at some later point.
     pub fn to_ssz_container(&self) -> SszHeadTracker {
-        let (roots, slots) = self
-            .0
-            .read()
-            .iter()
-            .map(|(hash, slot)| (*hash, *slot))
-            .unzip();
-
-        SszHeadTracker { roots, slots }
+        SszHeadTracker::from_map(&*self.0.read())
     }
 
     /// Creates a new `Self` from the given `SszHeadTracker`, restoring `Self` to the same state of
@@ -103,6 +89,13 @@ pub struct SszHeadTracker {
     slots: Vec<Slot>,
 }
 
+impl SszHeadTracker {
+    pub fn from_map(map: &HashMap<Hash256, Slot>) -> Self {
+        let (roots, slots) = map.iter().map(|(hash, slot)| (*hash, *slot)).unzip();
+        SszHeadTracker { roots, slots }
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

beacon_node/beacon_chain/src/lib.rs

@@ -1,13 +1,11 @@
 #![recursion_limit = "128"] // For lazy-static
-#[macro_use]
-extern crate lazy_static;
-
 pub mod attestation_verification;
 mod beacon_chain;
 mod beacon_fork_choice_store;
 mod beacon_snapshot;
 mod block_verification;
 pub mod builder;
+pub mod chain_config;
 mod errors;
 pub mod eth1_chain;
 pub mod events;
@@ -32,6 +30,7 @@ pub use self::beacon_chain::{
     ForkChoiceError, StateSkipConfig,
 };
 pub use self::beacon_snapshot::BeaconSnapshot;
+pub use self::chain_config::ChainConfig;
 pub use self::errors::{BeaconChainError, BlockProductionError};
 pub use attestation_verification::Error as AttestationError;
 pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError};

beacon_node/beacon_chain/src/metrics.rs

@@ -1,4 +1,5 @@
 use crate::{BeaconChain, BeaconChainTypes};
+use lazy_static::lazy_static;
 pub use lighthouse_metrics::*;
 use slot_clock::SlotClock;
 use types::{BeaconState, Epoch, EthSpec, Hash256, Slot};

beacon_node/beacon_chain/src/migrate.rs

@@ -1,5 +1,7 @@
+use crate::beacon_chain::BEACON_CHAIN_DB_KEY;
 use crate::errors::BeaconChainError;
-use crate::head_tracker::HeadTracker;
+use crate::head_tracker::{HeadTracker, SszHeadTracker};
+use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_BLOCK_ROOT};
 use parking_lot::Mutex;
 use slog::{debug, warn, Logger};
 use std::collections::{HashMap, HashSet};
@@ -7,296 +9,213 @@ use std::mem;
 use std::sync::mpsc;
 use std::sync::Arc;
 use std::thread;
-use store::hot_cold_store::{process_finalization, HotColdDBError};
-use store::iter::{ParentRootBlockIterator, RootsIterator};
-use store::{Error, ItemStore, StoreOp};
+use store::hot_cold_store::{migrate_database, HotColdDBError};
+use store::iter::RootsIterator;
+use store::{Error, ItemStore, StoreItem, StoreOp};
 pub use store::{HotColdDB, MemoryStore};
-use types::*;
-use types::{BeaconState, EthSpec, Hash256, Slot};
+use types::{
+    BeaconState, BeaconStateError, BeaconStateHash, Checkpoint, Epoch, EthSpec, Hash256,
+    SignedBeaconBlockHash, Slot,
+};
 
-/// Trait for migration processes that update the database upon finalization.
-pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
-    Send + Sync + 'static
-{
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self;
+/// The background migrator runs a thread to perform pruning and migrate state from the hot
+/// to the cold database.
+pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
+    db: Arc<HotColdDB<E, Hot, Cold>>,
+    #[allow(clippy::type_complexity)]
+    tx_thread: Option<
+        Mutex<(
+            mpsc::Sender<MigrationNotification<E>>,
+            thread::JoinHandle<()>,
+        )>,
+    >,
+    latest_checkpoint: Arc<Mutex<Checkpoint>>,
+    /// Genesis block root, for persisting the `PersistedBeaconChain`.
+    genesis_block_root: Hash256,
+    log: Logger,
+}
 
-    fn process_finalization(
-        &self,
-        _state_root: Hash256,
-        _new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
-        _head_tracker: Arc<HeadTracker>,
-        _old_finalized_block_hash: SignedBeaconBlockHash,
-        _new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct MigratorConfig {
+    pub blocking: bool,
+}
+
+impl MigratorConfig {
+    pub fn blocking(mut self) -> Self {
+        self.blocking = true;
+        self
+    }
+}
+
+/// Pruning can be successful, or in rare cases deferred to a later point.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PruningOutcome {
+    Successful,
+    DeferredConcurrentMutation,
+}
+
+/// Logic errors that can occur during pruning, none of these should ever happen.
+#[derive(Debug)]
+pub enum PruningError {
+    IncorrectFinalizedState {
+        state_slot: Slot,
+        new_finalized_slot: Slot,
+    },
+    MissingInfoForCanonicalChain {
+        slot: Slot,
+    },
+    UnexpectedEqualStateRoots,
+    UnexpectedUnequalStateRoots,
+}
+
+/// Message sent to the migration thread containing the information it needs to run.
+pub struct MigrationNotification<E: EthSpec> {
+    finalized_state_root: BeaconStateHash,
+    finalized_state: BeaconState<E>,
+    finalized_checkpoint: Checkpoint,
+    head_tracker: Arc<HeadTracker>,
+    latest_checkpoint: Arc<Mutex<Checkpoint>>,
+    genesis_block_root: Hash256,
+}
+
+impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
+    /// Create a new `BackgroundMigrator` and spawn its thread if necessary.
+    pub fn new(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        config: MigratorConfig,
+        genesis_block_root: Hash256,
+        log: Logger,
+    ) -> Self {
+        let tx_thread = if config.blocking {
+            None
+        } else {
+            Some(Mutex::new(Self::spawn_thread(db.clone(), log.clone())))
+        };
+        let latest_checkpoint = Arc::new(Mutex::new(Checkpoint {
+            root: Hash256::zero(),
+            epoch: Epoch::new(0),
+        }));
+        Self {
+            db,
+            tx_thread,
+            latest_checkpoint,
+            genesis_block_root,
+            log,
+        }
     }
 
-    /// Traverses live heads and prunes blocks and states of chains that we know can't be built
-    /// upon because finalization would prohibit it.  This is an optimisation intended to save disk
-    /// space.
+    /// Process a finalized checkpoint from the `BeaconChain`.
     ///
-    /// Assumptions:
-    ///  * It is called after every finalization.
-    fn prune_abandoned_forks(
-        store: Arc<HotColdDB<E, Hot, Cold>>,
+    /// If successful, all forks descending from before the `finalized_checkpoint` will be
+    /// pruned, and the split point of the database will be advanced to the slot of the finalized
+    /// checkpoint.
+    pub fn process_finalization(
+        &self,
+        finalized_state_root: BeaconStateHash,
+        finalized_state: BeaconState<E>,
+        finalized_checkpoint: Checkpoint,
         head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_slot: Slot,
     ) -> Result<(), BeaconChainError> {
-        // There will never be any blocks to prune if there is only a single head in the chain.
-        if head_tracker.heads().len() == 1 {
-            return Ok(());
-        }
+        let notif = MigrationNotification {
+            finalized_state_root,
+            finalized_state,
+            finalized_checkpoint,
+            head_tracker,
+            latest_checkpoint: self.latest_checkpoint.clone(),
+            genesis_block_root: self.genesis_block_root,
+        };
 
-        let old_finalized_slot = store
-            .get_block(&old_finalized_block_hash.into())?
-            .ok_or_else(|| BeaconChainError::MissingBeaconBlock(old_finalized_block_hash.into()))?
-            .slot();
+        // Async path, on the background thread.
+        if let Some(tx_thread) = &self.tx_thread {
+            let (ref mut tx, ref mut thread) = *tx_thread.lock();
 
-        // Collect hashes from new_finalized_block back to old_finalized_block (inclusive)
-        let mut found_block = false; // hack for `take_until`
-        let newly_finalized_blocks: HashMap<SignedBeaconBlockHash, Slot> =
-            ParentRootBlockIterator::new(&*store, new_finalized_block_hash.into())
-                .take_while(|result| match result {
-                    Ok((block_hash, _)) => {
-                        if found_block {
-                            false
-                        } else {
-                            found_block |= *block_hash == old_finalized_block_hash.into();
-                            true
-                        }
-                    }
-                    Err(_) => true,
-                })
-                .map(|result| result.map(|(block_hash, block)| (block_hash.into(), block.slot())))
-                .collect::<Result<_, _>>()?;
+            // Restart the background thread if it has crashed.
+            if let Err(tx_err) = tx.send(notif) {
+                let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
 
-        // We don't know which blocks are shared among abandoned chains, so we buffer and delete
-        // everything in one fell swoop.
-        let mut abandoned_blocks: HashSet<SignedBeaconBlockHash> = HashSet::new();
-        let mut abandoned_states: HashSet<(Slot, BeaconStateHash)> = HashSet::new();
-        let mut abandoned_heads: HashSet<Hash256> = HashSet::new();
+                *tx = new_tx;
+                let old_thread = mem::replace(thread, new_thread);
 
-        for (head_hash, head_slot) in head_tracker.heads() {
-            let mut potentially_abandoned_head: Option<Hash256> = Some(head_hash);
-            let mut potentially_abandoned_blocks: Vec<(
-                Slot,
-                Option<SignedBeaconBlockHash>,
-                Option<BeaconStateHash>,
-            )> = Vec::new();
-
-            let head_state_hash = store
-                .get_block(&head_hash)?
-                .ok_or_else(|| BeaconStateError::MissingBeaconBlock(head_hash.into()))?
-                .state_root();
-
-            let iter = std::iter::once(Ok((head_hash, head_state_hash, head_slot)))
-                .chain(RootsIterator::from_block(Arc::clone(&store), head_hash)?);
-            for maybe_tuple in iter {
-                let (block_hash, state_hash, slot) = maybe_tuple?;
-                if slot < old_finalized_slot {
-                    // We must assume here any candidate chains include old_finalized_block_hash,
-                    // i.e. there aren't any forks starting at a block that is a strict ancestor of
-                    // old_finalized_block_hash.
-                    break;
+                // Join the old thread, which will probably have panicked, or may have
+                // halted normally just now as a result of us dropping the old `mpsc::Sender`.
+                if let Err(thread_err) = old_thread.join() {
+                    warn!(
+                        self.log,
+                        "Migration thread died, so it was restarted";
+                        "reason" => format!("{:?}", thread_err)
+                    );
                 }
-                match newly_finalized_blocks.get(&block_hash.into()).copied() {
-                    // Block is not finalized, mark it and its state for deletion
-                    None => {
-                        potentially_abandoned_blocks.push((
-                            slot,
-                            Some(block_hash.into()),
-                            Some(state_hash.into()),
-                        ));
-                    }
-                    Some(finalized_slot) => {
-                        // Block root is finalized, and we have reached the slot it was finalized
-                        // at: we've hit a shared part of the chain.
-                        if finalized_slot == slot {
-                            // The first finalized block of a candidate chain lies after (in terms
-                            // of slots order) the newly finalized block.  It's not a candidate for
-                            // prunning.
-                            if finalized_slot == new_finalized_slot {
-                                potentially_abandoned_blocks.clear();
-                                potentially_abandoned_head.take();
-                            }
 
-                            break;
-                        }
-                        // Block root is finalized, but we're at a skip slot: delete the state only.
-                        else {
-                            potentially_abandoned_blocks.push((
-                                slot,
-                                None,
-                                Some(state_hash.into()),
-                            ));
-                        }
-                    }
-                }
-            }
-
-            abandoned_heads.extend(potentially_abandoned_head.into_iter());
-            if !potentially_abandoned_blocks.is_empty() {
-                abandoned_blocks.extend(
-                    potentially_abandoned_blocks
-                        .iter()
-                        .filter_map(|(_, maybe_block_hash, _)| *maybe_block_hash),
-                );
-                abandoned_states.extend(potentially_abandoned_blocks.iter().filter_map(
-                    |(slot, _, maybe_state_hash)| match maybe_state_hash {
-                        None => None,
-                        Some(state_hash) => Some((*slot, *state_hash)),
-                    },
-                ));
+                // Retry at most once, we could recurse but that would risk overflowing the stack.
+                let _ = tx.send(tx_err.0);
             }
         }
-
-        let batch: Vec<StoreOp<E>> = abandoned_blocks
-            .into_iter()
-            .map(StoreOp::DeleteBlock)
-            .chain(
-                abandoned_states
-                    .into_iter()
-                    .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash, slot)),
-            )
-            .collect();
-        store.do_atomically(batch)?;
-        for head_hash in abandoned_heads.into_iter() {
-            head_tracker.remove_head(head_hash);
+        // Synchronous path, on the current thread.
+        else {
+            Self::run_migration(self.db.clone(), notif, &self.log)
         }
 
         Ok(())
     }
-}
 
-/// Migrator that does nothing, for stores that don't need migration.
-pub struct NullMigrator;
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold> for NullMigrator {
-    fn new(_: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
-        NullMigrator
-    }
-}
-
-/// Migrator that immediately calls the store's migration function, blocking the current execution.
-///
-/// Mostly useful for tests.
-pub struct BlockingMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
-    db: Arc<HotColdDB<E, Hot, Cold>>,
-}
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
-    for BlockingMigrator<E, Hot, Cold>
-{
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
-        BlockingMigrator { db }
-    }
-
-    fn process_finalization(
-        &self,
-        state_root: Hash256,
-        new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
-        head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
+    /// Perform the actual work of `process_finalization`.
+    fn run_migration(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        notif: MigrationNotification<E>,
+        log: &Logger,
     ) {
-        if let Err(e) = process_finalization(self.db.clone(), state_root, &new_finalized_state) {
-            // This migrator is only used for testing, so we just log to stderr without a logger.
-            eprintln!("Migration error: {:?}", e);
-        }
+        let mut latest_checkpoint = notif.latest_checkpoint.lock();
+        let finalized_state_root = notif.finalized_state_root;
+        let finalized_state = notif.finalized_state;
 
-        if let Err(e) = Self::prune_abandoned_forks(
-            self.db.clone(),
-            head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_state.slot,
-        ) {
-            eprintln!("Pruning error: {:?}", e);
-        }
-    }
-}
-
-type MpscSender<E> = mpsc::Sender<(
-    Hash256,
-    BeaconState<E>,
-    Arc<HeadTracker>,
-    SignedBeaconBlockHash,
-    SignedBeaconBlockHash,
-    Slot,
-)>;
-
-/// Migrator that runs a background thread to migrate state from the hot to the cold database.
-pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
-    db: Arc<HotColdDB<E, Hot, Cold>>,
-    tx_thread: Mutex<(MpscSender<E>, thread::JoinHandle<()>)>,
-    log: Logger,
-}
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
-    for BackgroundMigrator<E, Hot, Cold>
-{
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self {
-        let tx_thread = Mutex::new(Self::spawn_thread(db.clone(), log.clone()));
-        Self { db, tx_thread, log }
-    }
-
-    /// Perform the freezing operation on the database,
-    fn process_finalization(
-        &self,
-        finalized_state_root: Hash256,
-        new_finalized_state: BeaconState<E>,
-        max_finality_distance: u64,
-        head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
-        if !self.needs_migration(new_finalized_state.slot, max_finality_distance) {
-            return;
-        }
-
-        let (ref mut tx, ref mut thread) = *self.tx_thread.lock();
-
-        let new_finalized_slot = new_finalized_state.slot;
-        if let Err(tx_err) = tx.send((
+        match Self::prune_abandoned_forks(
+            db.clone(),
+            notif.head_tracker,
             finalized_state_root,
-            new_finalized_state,
-            head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_slot,
-        )) {
-            let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
-
-            drop(mem::replace(tx, new_tx));
-            let old_thread = mem::replace(thread, new_thread);
-
-            // Join the old thread, which will probably have panicked, or may have
-            // halted normally just now as a result of us dropping the old `mpsc::Sender`.
-            if let Err(thread_err) = old_thread.join() {
+            &finalized_state,
+            *latest_checkpoint,
+            notif.finalized_checkpoint,
+            notif.genesis_block_root,
+            log,
+        ) {
+            Ok(PruningOutcome::DeferredConcurrentMutation) => {
                 warn!(
-                    self.log,
-                    "Migration thread died, so it was restarted";
-                    "reason" => format!("{:?}", thread_err)
+                    log,
+                    "Pruning deferred because of a concurrent mutation";
+                    "message" => "this is expected only very rarely!"
+                );
+                return;
+            }
+            Ok(PruningOutcome::Successful) => {
+                // Update the migrator's idea of the latest checkpoint only if the
+                // pruning process was successful.
+                *latest_checkpoint = notif.finalized_checkpoint;
+            }
+            Err(e) => {
+                warn!(log, "Block pruning failed"; "error" => format!("{:?}", e));
+                return;
+            }
+        };
+
+        match migrate_database(db, finalized_state_root.into(), &finalized_state) {
+            Ok(()) => {}
+            Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
+                debug!(
+                    log,
+                    "Database migration postponed, unaligned finalized block";
+                    "slot" => slot.as_u64()
                 );
             }
-
-            // Retry at most once, we could recurse but that would risk overflowing the stack.
-            let _ = tx.send(tx_err.0);
-        }
-    }
-}
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
-    /// Return true if a migration needs to be performed, given a new `finalized_slot`.
-    fn needs_migration(&self, finalized_slot: Slot, max_finality_distance: u64) -> bool {
-        let finality_distance = finalized_slot - self.db.get_split_slot();
-        finality_distance > max_finality_distance
+            Err(e) => {
+                warn!(
+                    log,
+                    "Database migration failed";
+                    "error" => format!("{:?}", e)
+                );
+            }
+        };
     }
 
-    #[allow(clippy::type_complexity)]
     /// Spawn a new child thread to run the migration process.
     ///
     /// Return a channel handle for sending new finalized states to the thread.
@@ -304,58 +223,243 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         db: Arc<HotColdDB<E, Hot, Cold>>,
         log: Logger,
     ) -> (
-        mpsc::Sender<(
-            Hash256,
-            BeaconState<E>,
-            Arc<HeadTracker>,
-            SignedBeaconBlockHash,
-            SignedBeaconBlockHash,
-            Slot,
-        )>,
+        mpsc::Sender<MigrationNotification<E>>,
         thread::JoinHandle<()>,
     ) {
         let (tx, rx) = mpsc::channel();
         let thread = thread::spawn(move || {
-            while let Ok((
-                state_root,
-                state,
-                head_tracker,
-                old_finalized_block_hash,
-                new_finalized_block_hash,
-                new_finalized_slot,
-            )) = rx.recv()
-            {
-                match process_finalization(db.clone(), state_root, &state) {
-                    Ok(()) => {}
-                    Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
-                        debug!(
-                            log,
-                            "Database migration postponed, unaligned finalized block";
-                            "slot" => slot.as_u64()
-                        );
-                    }
-                    Err(e) => {
-                        warn!(
-                            log,
-                            "Database migration failed";
-                            "error" => format!("{:?}", e)
-                        );
-                    }
-                };
-
-                match Self::prune_abandoned_forks(
-                    db.clone(),
-                    head_tracker,
-                    old_finalized_block_hash,
-                    new_finalized_block_hash,
-                    new_finalized_slot,
-                ) {
-                    Ok(()) => {}
-                    Err(e) => warn!(log, "Block pruning failed: {:?}", e),
-                }
+            while let Ok(notif) = rx.recv() {
+                Self::run_migration(db.clone(), notif, &log);
             }
         });
-
         (tx, thread)
     }
+
+    /// Traverses live heads and prunes blocks and states of chains that we know can't be built
+    /// upon because finalization would prohibit it. This is an optimisation intended to save disk
+    /// space.
+    #[allow(clippy::too_many_arguments)]
+    fn prune_abandoned_forks(
+        store: Arc<HotColdDB<E, Hot, Cold>>,
+        head_tracker: Arc<HeadTracker>,
+        new_finalized_state_hash: BeaconStateHash,
+        new_finalized_state: &BeaconState<E>,
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+        genesis_block_root: Hash256,
+        log: &Logger,
+    ) -> Result<PruningOutcome, BeaconChainError> {
+        let old_finalized_slot = old_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_slot = new_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_block_hash = new_finalized_checkpoint.root.into();
+
+        // The finalized state must be for the epoch boundary slot, not the slot of the finalized
+        // block.
+        if new_finalized_state.slot != new_finalized_slot {
+            return Err(PruningError::IncorrectFinalizedState {
+                state_slot: new_finalized_state.slot,
+                new_finalized_slot,
+            }
+            .into());
+        }
+
+        debug!(
+            log,
+            "Starting database pruning";
+            "old_finalized_epoch" => old_finalized_checkpoint.epoch,
+            "old_finalized_root" => format!("{:?}", old_finalized_checkpoint.root),
+            "new_finalized_epoch" => new_finalized_checkpoint.epoch,
+            "new_finalized_root" => format!("{:?}", new_finalized_checkpoint.root),
+        );
+
+        // For each slot between the new finalized checkpoint and the old finalized checkpoint,
+        // collect the beacon block root and state root of the canonical chain.
+        let newly_finalized_chain: HashMap<Slot, (SignedBeaconBlockHash, BeaconStateHash)> =
+            std::iter::once(Ok((
+                new_finalized_slot,
+                (new_finalized_block_hash, new_finalized_state_hash),
+            )))
+            .chain(
+                RootsIterator::new(store.clone(), new_finalized_state).map(|res| {
+                    res.map(|(block_root, state_root, slot)| {
+                        (slot, (block_root.into(), state_root.into()))
+                    })
+                }),
+            )
+            .take_while(|res| {
+                res.as_ref()
+                    .map_or(true, |(slot, _)| *slot >= old_finalized_slot)
+            })
+            .collect::<Result<_, _>>()?;
+
+        // We don't know which blocks are shared among abandoned chains, so we buffer and delete
+        // everything in one fell swoop.
+        let mut abandoned_blocks: HashSet<SignedBeaconBlockHash> = HashSet::new();
+        let mut abandoned_states: HashSet<(Slot, BeaconStateHash)> = HashSet::new();
+        let mut abandoned_heads: HashSet<Hash256> = HashSet::new();
+
+        let heads = head_tracker.heads();
+        debug!(log, "Pruning {} heads", heads.len());
+
+        for (head_hash, head_slot) in heads {
+            let mut potentially_abandoned_head = Some(head_hash);
+            let mut potentially_abandoned_blocks = vec![];
+
+            let head_state_hash = store
+                .get_block(&head_hash)?
+                .ok_or_else(|| BeaconStateError::MissingBeaconBlock(head_hash.into()))?
+                .state_root();
+
+            // Iterate backwards from this head, staging blocks and states for deletion.
+            let iter = std::iter::once(Ok((head_hash, head_state_hash, head_slot)))
+                .chain(RootsIterator::from_block(store.clone(), head_hash)?);
+
+            for maybe_tuple in iter {
+                let (block_root, state_root, slot) = maybe_tuple?;
+                let block_root = SignedBeaconBlockHash::from(block_root);
+                let state_root = BeaconStateHash::from(state_root);
+
+                match newly_finalized_chain.get(&slot) {
+                    // If there's no information about a slot on the finalized chain, then
+                    // it should be because it's ahead of the new finalized slot. Stage
+                    // the fork's block and state for possible deletion.
+                    None => {
+                        if slot > new_finalized_slot {
+                            potentially_abandoned_blocks.push((
+                                slot,
+                                Some(block_root),
+                                Some(state_root),
+                            ));
+                        } else if slot >= old_finalized_slot {
+                            return Err(PruningError::MissingInfoForCanonicalChain { slot }.into());
+                        } else {
+                            // We must assume here any candidate chains include the old finalized
+                            // checkpoint, i.e. there aren't any forks starting at a block that is a
+                            // strict ancestor of old_finalized_checkpoint.
+                            warn!(
+                                log,
+                                "Found a chain that should already have been pruned";
+                                "head_block_root" => format!("{:?}", head_hash),
+                                "head_slot" => head_slot,
+                            );
+                            potentially_abandoned_head.take();
+                            break;
+                        }
+                    }
+                    Some((finalized_block_root, finalized_state_root)) => {
+                        // This fork descends from a newly finalized block, we can stop.
+                        if block_root == *finalized_block_root {
+                            // Sanity check: if the slot and block root match, then the
+                            // state roots should match too.
+                            if state_root != *finalized_state_root {
+                                return Err(PruningError::UnexpectedUnequalStateRoots.into());
+                            }
+
+                            // If the fork descends from the whole finalized chain,
+                            // do not prune it. Otherwise continue to delete all
+                            // of the blocks and states that have been staged for
+                            // deletion so far.
+                            if slot == new_finalized_slot {
+                                potentially_abandoned_blocks.clear();
+                                potentially_abandoned_head.take();
+                            }
+                            // If there are skipped slots on the fork to be pruned, then
+                            // we will have just staged the common block for deletion.
+                            // Unstage it.
+                            else {
+                                for (_, block_root, _) in
+                                    potentially_abandoned_blocks.iter_mut().rev()
+                                {
+                                    if block_root.as_ref() == Some(finalized_block_root) {
+                                        *block_root = None;
+                                    } else {
+                                        break;
+                                    }
+                                }
+                            }
+                            break;
+                        } else {
+                            if state_root == *finalized_state_root {
+                                return Err(PruningError::UnexpectedEqualStateRoots.into());
+                            }
+                            potentially_abandoned_blocks.push((
+                                slot,
+                                Some(block_root),
+                                Some(state_root),
+                            ));
+                        }
+                    }
+                }
+            }
+
+            if let Some(abandoned_head) = potentially_abandoned_head {
+                debug!(
+                    log,
+                    "Pruning head";
+                    "head_block_root" => format!("{:?}", abandoned_head),
+                    "head_slot" => head_slot,
+                );
+                abandoned_heads.insert(abandoned_head);
+                abandoned_blocks.extend(
+                    potentially_abandoned_blocks
+                        .iter()
+                        .filter_map(|(_, maybe_block_hash, _)| *maybe_block_hash),
+                );
+                abandoned_states.extend(potentially_abandoned_blocks.iter().filter_map(
+                    |(slot, _, maybe_state_hash)| maybe_state_hash.map(|sr| (*slot, sr)),
+                ));
+            }
+        }
+
+        // Update the head tracker before the database, so that we maintain the invariant
+        // that a block present in the head tracker is present in the database.
+        // See https://github.com/sigp/lighthouse/issues/1557
+        let mut head_tracker_lock = head_tracker.0.write();
+
+        // Check that all the heads to be deleted are still present. The absence of any
+        // head indicates a race, that will likely resolve itself, so we defer pruning until
+        // later.
+        for head_hash in &abandoned_heads {
+            if !head_tracker_lock.contains_key(head_hash) {
+                return Ok(PruningOutcome::DeferredConcurrentMutation);
+            }
+        }
+
+        // Then remove them for real.
+        for head_hash in abandoned_heads {
+            head_tracker_lock.remove(&head_hash);
+        }
+
+        let batch: Vec<StoreOp<E>> = abandoned_blocks
+            .into_iter()
+            .map(Into::into)
+            .map(StoreOp::DeleteBlock)
+            .chain(
+                abandoned_states
+                    .into_iter()
+                    .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash.into(), Some(slot))),
+            )
+            .collect();
+
+        let mut kv_batch = store.convert_to_kv_batch(&batch)?;
+
+        // Persist the head in case the process is killed or crashes here. This prevents
+        // the head tracker reverting after our mutation above.
+        let persisted_head = PersistedBeaconChain {
+            _canonical_head_block_root: DUMMY_CANONICAL_HEAD_BLOCK_ROOT,
+            genesis_block_root,
+            ssz_head_tracker: SszHeadTracker::from_map(&*head_tracker_lock),
+        };
+        drop(head_tracker_lock);
+        kv_batch.push(persisted_head.as_kv_store_op(BEACON_CHAIN_DB_KEY));
+
+        store.hot_db.do_atomically(kv_batch)?;
+        debug!(log, "Database pruning complete");
+
+        Ok(PruningOutcome::Successful)
+    }
 }

beacon_node/beacon_chain/src/naive_aggregation_pool.rs

@@ -1,8 +1,9 @@
 use crate::metrics;
-use parking_lot::RwLock;
 use std::collections::HashMap;
-use types::{Attestation, AttestationData, EthSpec, Slot};
+use tree_hash::TreeHash;
+use types::{Attestation, AttestationData, EthSpec, Hash256, Slot};
 
+type AttestationDataRoot = Hash256;
 /// The number of slots that will be stored in the pool.
 ///
 /// For example, if `SLOTS_RETAINED == 3` and the pool is pruned at slot `6`, then all attestations
@@ -54,7 +55,7 @@ pub enum Error {
 /// A collection of `Attestation` objects, keyed by their `attestation.data`. Enforces that all
 /// `attestation` are from the same slot.
 struct AggregatedAttestationMap<E: EthSpec> {
-    map: HashMap<AttestationData, Attestation<E>>,
+    map: HashMap<AttestationDataRoot, Attestation<E>>,
 }
 
 impl<E: EthSpec> AggregatedAttestationMap<E> {
@@ -88,7 +89,9 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
             return Err(Error::MoreThanOneAggregationBitSet(set_bits.len()));
         }
 
-        if let Some(existing_attestation) = self.map.get_mut(&a.data) {
+        let attestation_data_root = a.data.tree_hash_root();
+
+        if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) {
             if existing_attestation
                 .aggregation_bits
                 .get(committee_index)
@@ -108,7 +111,7 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
                 ));
             }
 
-            self.map.insert(a.data.clone(), a.clone());
+            self.map.insert(attestation_data_root, a.clone());
             Ok(InsertOutcome::NewAttestationData { committee_index })
         }
     }
@@ -116,8 +119,18 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
     /// Returns an aggregated `Attestation` with the given `data`, if any.
     ///
     /// The given `a.data.slot` must match the slot that `self` was initialized with.
-    pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
-        Ok(self.map.get(data).cloned())
+    pub fn get(&self, data: &AttestationData) -> Option<Attestation<E>> {
+        self.map.get(&data.tree_hash_root()).cloned()
+    }
+
+    /// Returns an aggregated `Attestation` with the given `root`, if any.
+    pub fn get_by_root(&self, root: &AttestationDataRoot) -> Option<&Attestation<E>> {
+        self.map.get(root)
+    }
+
+    /// Iterate all attestations in `self`.
+    pub fn iter(&self) -> impl Iterator<Item = &Attestation<E>> {
+        self.map.iter().map(|(_key, attestation)| attestation)
     }
 
     pub fn len(&self) -> usize {
@@ -147,15 +160,15 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
 /// than that will also be refused. Pruning is done automatically based upon the attestations it
 /// receives and it can be triggered manually.
 pub struct NaiveAggregationPool<E: EthSpec> {
-    lowest_permissible_slot: RwLock<Slot>,
-    maps: RwLock<HashMap<Slot, AggregatedAttestationMap<E>>>,
+    lowest_permissible_slot: Slot,
+    maps: HashMap<Slot, AggregatedAttestationMap<E>>,
 }
 
 impl<E: EthSpec> Default for NaiveAggregationPool<E> {
     fn default() -> Self {
         Self {
-            lowest_permissible_slot: RwLock::new(Slot::new(0)),
-            maps: RwLock::new(HashMap::new()),
+            lowest_permissible_slot: Slot::new(0),
+            maps: HashMap::new(),
         }
     }
 }
@@ -168,10 +181,10 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
     ///
     /// The pool may be pruned if the given `attestation.data` has a slot higher than any
     /// previously seen.
-    pub fn insert(&self, attestation: &Attestation<E>) -> Result<InsertOutcome, Error> {
+    pub fn insert(&mut self, attestation: &Attestation<E>) -> Result<InsertOutcome, Error> {
         let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_INSERT);
         let slot = attestation.data.slot;
-        let lowest_permissible_slot: Slot = *self.lowest_permissible_slot.read();
+        let lowest_permissible_slot = self.lowest_permissible_slot;
 
         // Reject any attestations that are too old.
         if slot < lowest_permissible_slot {
@@ -183,16 +196,16 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
 
         let lock_timer =
             metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_MAPS_WRITE_LOCK);
-        let mut maps = self.maps.write();
         drop(lock_timer);
 
-        let outcome = if let Some(map) = maps.get_mut(&slot) {
+        let outcome = if let Some(map) = self.maps.get_mut(&slot) {
             map.insert(attestation)
         } else {
             let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CREATE_MAP);
             // To avoid re-allocations, try and determine a rough initial capacity for the new item
             // by obtaining the mean size of all items in earlier epoch.
-            let (count, sum) = maps
+            let (count, sum) = self
+                .maps
                 .iter()
                 // Only include epochs that are less than the given slot in the average. This should
                 // generally avoid including recent epochs that are still "filling up".
@@ -205,30 +218,40 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
 
             let mut item = AggregatedAttestationMap::new(initial_capacity);
             let outcome = item.insert(attestation);
-            maps.insert(slot, item);
+            self.maps.insert(slot, item);
 
             outcome
         };
 
-        drop(maps);
         self.prune(slot);
 
         outcome
     }
 
     /// Returns an aggregated `Attestation` with the given `data`, if any.
-    pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
+    pub fn get(&self, data: &AttestationData) -> Option<Attestation<E>> {
+        self.maps.get(&data.slot).and_then(|map| map.get(data))
+    }
+
+    /// Returns an aggregated `Attestation` with the given `data`, if any.
+    pub fn get_by_slot_and_root(
+        &self,
+        slot: Slot,
+        root: &AttestationDataRoot,
+    ) -> Option<Attestation<E>> {
         self.maps
-            .read()
-            .iter()
-            .find(|(slot, _map)| **slot == data.slot)
-            .map(|(_slot, map)| map.get(data))
-            .unwrap_or_else(|| Ok(None))
+            .get(&slot)
+            .and_then(|map| map.get_by_root(root).cloned())
+    }
+
+    /// Iterate all attestations in all slots of `self`.
+    pub fn iter(&self) -> impl Iterator<Item = &Attestation<E>> {
+        self.maps.iter().map(|(_slot, map)| map.iter()).flatten()
     }
 
     /// Removes any attestations with a slot lower than `current_slot` and bars any future
     /// attestations with a slot lower than `current_slot - SLOTS_RETAINED`.
-    pub fn prune(&self, current_slot: Slot) {
+    pub fn prune(&mut self, current_slot: Slot) {
         let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_PRUNE);
 
         // Taking advantage of saturating subtraction on `Slot`.
@@ -236,30 +259,34 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
 
         // No need to prune if the lowest permissible slot has not changed and the queue length is
         // less than the maximum
-        if *self.lowest_permissible_slot.read() == lowest_permissible_slot
-            && self.maps.read().len() <= SLOTS_RETAINED
+        if self.lowest_permissible_slot == lowest_permissible_slot
+            && self.maps.len() <= SLOTS_RETAINED
         {
             return;
         }
 
-        *self.lowest_permissible_slot.write() = lowest_permissible_slot;
-        let mut maps = self.maps.write();
+        self.lowest_permissible_slot = lowest_permissible_slot;
 
         // Remove any maps that are definitely expired.
-        maps.retain(|slot, _map| *slot >= lowest_permissible_slot);
+        self.maps
+            .retain(|slot, _map| *slot >= lowest_permissible_slot);
 
         // If we have too many maps, remove the lowest amount to ensure we only have
         // `SLOTS_RETAINED` left.
-        if maps.len() > SLOTS_RETAINED {
-            let mut slots = maps.iter().map(|(slot, _map)| *slot).collect::<Vec<_>>();
+        if self.maps.len() > SLOTS_RETAINED {
+            let mut slots = self
+                .maps
+                .iter()
+                .map(|(slot, _map)| *slot)
+                .collect::<Vec<_>>();
             // Sort is generally pretty slow, however `SLOTS_RETAINED` is quite low so it should be
             // negligible.
             slots.sort_unstable();
             slots
                 .into_iter()
-                .take(maps.len().saturating_sub(SLOTS_RETAINED))
+                .take(self.maps.len().saturating_sub(SLOTS_RETAINED))
                 .for_each(|slot| {
-                    maps.remove(&slot);
+                    self.maps.remove(&slot);
                 })
         }
     }
@@ -304,7 +331,7 @@ mod tests {
     fn single_attestation() {
         let mut a = get_attestation(Slot::new(0));
 
-        let pool = NaiveAggregationPool::default();
+        let mut pool = NaiveAggregationPool::default();
 
         assert_eq!(
             pool.insert(&a),
@@ -327,8 +354,7 @@ mod tests {
 
         let retrieved = pool
             .get(&a.data)
-            .expect("should not error while getting attestation")
-            .expect("should get an attestation");
+            .expect("should not error while getting attestation");
         assert_eq!(
             retrieved, a,
             "retrieved attestation should equal the one inserted"
@@ -352,7 +378,7 @@ mod tests {
         sign(&mut a_0, 0, genesis_validators_root);
         sign(&mut a_1, 1, genesis_validators_root);
 
-        let pool = NaiveAggregationPool::default();
+        let mut pool = NaiveAggregationPool::default();
 
         assert_eq!(
             pool.insert(&a_0),
@@ -367,8 +393,7 @@ mod tests {
 
         let retrieved = pool
             .get(&a_0.data)
-            .expect("should not error while getting attestation")
-            .expect("should get an attestation");
+            .expect("should not error while getting attestation");
 
         let mut a_01 = a_0.clone();
         a_01.aggregate(&a_1);
@@ -397,8 +422,7 @@ mod tests {
 
         assert_eq!(
             pool.get(&a_0.data)
-                .expect("should not error while getting attestation")
-                .expect("should get an attestation"),
+                .expect("should not error while getting attestation"),
             retrieved,
             "should not have aggregated different attestation data"
         );
@@ -409,7 +433,7 @@ mod tests {
         let mut base = get_attestation(Slot::new(0));
         sign(&mut base, 0, Hash256::random());
 
-        let pool = NaiveAggregationPool::default();
+        let mut pool = NaiveAggregationPool::default();
 
         for i in 0..SLOTS_RETAINED * 2 {
             let slot = Slot::from(i);
@@ -424,22 +448,16 @@ mod tests {
 
             if i < SLOTS_RETAINED {
                 let len = i + 1;
-                assert_eq!(
-                    pool.maps.read().len(),
-                    len,
-                    "the pool should have length {}",
-                    len
-                );
+                assert_eq!(pool.maps.len(), len, "the pool should have length {}", len);
             } else {
                 assert_eq!(
-                    pool.maps.read().len(),
+                    pool.maps.len(),
                     SLOTS_RETAINED,
                     "the pool should have length SLOTS_RETAINED"
                 );
 
                 let mut pool_slots = pool
                     .maps
-                    .read()
                     .iter()
                     .map(|(slot, _map)| *slot)
                     .collect::<Vec<_>>();
@@ -463,7 +481,7 @@ mod tests {
         let mut base = get_attestation(Slot::new(0));
         sign(&mut base, 0, Hash256::random());
 
-        let pool = NaiveAggregationPool::default();
+        let mut pool = NaiveAggregationPool::default();
 
         for i in 0..=MAX_ATTESTATIONS_PER_SLOT {
             let mut a = base.clone();

beacon_node/beacon_chain/src/persisted_beacon_chain.rs

@@ -4,9 +4,19 @@ use ssz_derive::{Decode, Encode};
 use store::{DBColumn, Error as StoreError, StoreItem};
 use types::Hash256;
 
+/// Dummy value to use for the canonical head block root, see below.
+pub const DUMMY_CANONICAL_HEAD_BLOCK_ROOT: Hash256 = Hash256::repeat_byte(0xff);
+
 #[derive(Clone, Encode, Decode)]
 pub struct PersistedBeaconChain {
-    pub canonical_head_block_root: Hash256,
+    /// This value is ignored to resolve the issue described here:
+    ///
+    /// https://github.com/sigp/lighthouse/pull/1639
+    ///
+    /// Its removal is tracked here:
+    ///
+    /// https://github.com/sigp/lighthouse/issues/1784
+    pub _canonical_head_block_root: Hash256,
     pub genesis_block_root: Hash256,
     pub ssz_head_tracker: SszHeadTracker,
 }

beacon_node/beacon_chain/src/shuffling_cache.rs

@@ -1,6 +1,6 @@
 use crate::metrics;
 use lru::LruCache;
-use types::{beacon_state::CommitteeCache, Epoch, Hash256};
+use types::{beacon_state::CommitteeCache, Epoch, Hash256, ShufflingId};
 
 /// The size of the LRU cache that stores committee caches for quicker verification.
 ///
@@ -14,7 +14,7 @@ const CACHE_SIZE: usize = 16;
 /// It has been named `ShufflingCache` because `CommitteeCacheCache` is a bit weird and looks like
 /// a find/replace error.
 pub struct ShufflingCache {
-    cache: LruCache<(Epoch, Hash256), CommitteeCache>,
+    cache: LruCache<ShufflingId, CommitteeCache>,
 }
 
 impl ShufflingCache {
@@ -24,8 +24,8 @@ impl ShufflingCache {
         }
     }
 
-    pub fn get(&mut self, epoch: Epoch, root: Hash256) -> Option<&CommitteeCache> {
-        let opt = self.cache.get(&(epoch, root));
+    pub fn get(&mut self, key: &ShufflingId) -> Option<&CommitteeCache> {
+        let opt = self.cache.get(key);
 
         if opt.is_some() {
             metrics::inc_counter(&metrics::SHUFFLING_CACHE_HITS);
@@ -36,11 +36,37 @@ impl ShufflingCache {
         opt
     }
 
-    pub fn insert(&mut self, epoch: Epoch, root: Hash256, committee_cache: &CommitteeCache) {
-        let key = (epoch, root);
+    pub fn contains(&self, key: &ShufflingId) -> bool {
+        self.cache.contains(key)
+    }
 
+    pub fn insert(&mut self, key: ShufflingId, committee_cache: &CommitteeCache) {
         if !self.cache.contains(&key) {
             self.cache.put(key, committee_cache.clone());
         }
     }
 }
+
+/// Contains the shuffling IDs for a beacon block.
+pub struct BlockShufflingIds {
+    pub current: ShufflingId,
+    pub next: ShufflingId,
+    pub block_root: Hash256,
+}
+
+impl BlockShufflingIds {
+    /// Returns the shuffling ID for the given epoch.
+    ///
+    /// Returns `None` if `epoch` is prior to `self.current.shuffling_epoch`.
+    pub fn id_for_epoch(&self, epoch: Epoch) -> Option<ShufflingId> {
+        if epoch == self.current.shuffling_epoch {
+            Some(self.current.clone())
+        } else if epoch == self.next.shuffling_epoch {
+            Some(self.next.clone())
+        } else if epoch > self.next.shuffling_epoch {
+            Some(ShufflingId::from_components(epoch, self.block_root))
+        } else {
+            None
+        }
+    }
+}

beacon_node/beacon_chain/src/validator_pubkey_cache.rs

@@ -39,13 +39,6 @@ impl ValidatorPubkeyCache {
         state: &BeaconState<T>,
         persistence_path: P,
     ) -> Result<Self, BeaconChainError> {
-        if persistence_path.as_ref().exists() {
-            return Err(BeaconChainError::ValidatorPubkeyCacheFileError(format!(
-                "Persistence file already exists: {:?}",
-                persistence_path.as_ref()
-            )));
-        }
-
         let mut cache = Self {
             persitence_file: ValidatorPubkeyCacheFile::create(persistence_path)?,
             pubkeys: vec![],
@@ -159,8 +152,9 @@ impl ValidatorPubkeyCacheFile {
     /// Creates a file for reading and writing.
     pub fn create<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
         OpenOptions::new()
-            .create_new(true)
+            .create(true)
             .write(true)
+            .truncate(true)
             .open(path)
             .map(Self)
             .map_err(Error::Io)

beacon_node/beacon_chain/tests/attestation_production.rs

@@ -26,7 +26,7 @@ lazy_static! {
 fn produces_attestations() {
     let num_blocks_produced = MainnetEthSpec::slots_per_epoch() * 4;
 
-    let harness = BeaconChainHarness::new(
+    let harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[..].to_vec(),
         StoreConfig::default(),
@@ -55,7 +55,7 @@ fn produces_attestations() {
     // Test all valid committee indices for all slots in the chain.
     for slot in 0..=current_slot.as_u64() + MainnetEthSpec::slots_per_epoch() * 3 {
         let slot = Slot::from(slot);
-        let state = chain
+        let mut state = chain
             .state_at_slot(slot, StateSkipConfig::WithStateRoots)
             .expect("should get state");
 
@@ -81,6 +81,9 @@ fn produces_attestations() {
                 .expect("should get target block root")
         };
 
+        state
+            .build_committee_cache(RelativeEpoch::Current, &harness.chain.spec)
+            .unwrap();
         let committee_cache = state
             .committee_cache(RelativeEpoch::Current)
             .expect("should get committee_cache");

beacon_node/beacon_chain/tests/attestation_verification.rs

@@ -5,17 +5,19 @@ extern crate lazy_static;
 
 use beacon_chain::{
     attestation_verification::Error as AttnError,
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
     BeaconChain, BeaconChainTypes,
 };
 use int_to_bytes::int_to_bytes32;
-use state_processing::per_slot_processing;
+use state_processing::{
+    per_block_processing::errors::AttestationValidationError, per_slot_processing,
+};
 use store::config::StoreConfig;
 use tree_hash::TreeHash;
 use types::{
-    test_utils::generate_deterministic_keypair, AggregateSignature, Attestation, EthSpec, Hash256,
-    Keypair, MainnetEthSpec, SecretKey, SelectionProof, SignedAggregateAndProof, SignedBeaconBlock,
-    SubnetId, Unsigned,
+    test_utils::generate_deterministic_keypair, AggregateSignature, Attestation, BeaconStateError,
+    BitList, EthSpec, Hash256, Keypair, MainnetEthSpec, SecretKey, SelectionProof,
+    SignedAggregateAndProof, SignedBeaconBlock, SubnetId, Unsigned,
 };
 
 pub type E = MainnetEthSpec;
@@ -30,7 +32,7 @@ lazy_static! {
 }
 
 /// Returns a beacon chain harness.
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<E>> {
     let harness = BeaconChainHarness::new_with_target_aggregators(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
@@ -185,7 +187,6 @@ fn get_non_aggregator<T: BeaconChainTypes>(
 #[test]
 fn aggregated_gossip_verification() {
     let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -197,7 +198,7 @@ fn aggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
+    let current_slot = harness.chain.slot().expect("should get slot");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -275,6 +276,21 @@ fn aggregated_gossip_verification() {
             && earliest_permissible_slot == current_slot - E::slots_per_epoch() - 1
     );
 
+    /*
+     * This is not in the specification for aggregate attestations (only unaggregates), but we
+     * check it anyway to avoid weird edge cases.
+     */
+    let unknown_root = Hash256::from_low_u64_le(424242);
+    assert_invalid!(
+        "attestation with invalid target root",
+        {
+            let mut a = valid_aggregate.clone();
+            a.message.aggregate.data.target.root = unknown_root;
+            a
+        },
+        AttnError::InvalidTargetRoot { .. }
+    );
+
     /*
      * The following test ensures:
      *
@@ -533,7 +549,6 @@ fn aggregated_gossip_verification() {
 #[test]
 fn unaggregated_gossip_verification() {
     let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -545,8 +560,8 @@ fn unaggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -568,7 +583,7 @@ fn unaggregated_gossip_verification() {
                 matches!(
                     harness
                         .chain
-                        .verify_unaggregated_attestation_for_gossip($attn_getter, $subnet_getter)
+                        .verify_unaggregated_attestation_for_gossip($attn_getter, Some($subnet_getter))
                         .err()
                         .expect(&format!(
                             "{} should error during verify_unaggregated_attestation_for_gossip",
@@ -582,6 +597,31 @@ fn unaggregated_gossip_verification() {
         };
     }
 
+    /*
+     * The following test ensures:
+     *
+     * Spec v0.12.3
+     *
+     * The committee index is within the expected range -- i.e. `data.index <
+     * get_committee_count_per_slot(state, data.target.epoch)`.
+     */
+    assert_invalid!(
+        "attestation with invalid committee index",
+        {
+            let mut a = valid_attestation.clone();
+            a.data.index = harness
+                .chain
+                .head()
+                .unwrap()
+                .beacon_state
+                .get_committee_count_at_slot(a.data.slot)
+                .unwrap();
+            a
+        },
+        subnet_id,
+        AttnError::NoCommitteeForSlotAndIndex { .. }
+    );
+
     /*
      * The following test ensures:
      *
@@ -642,6 +682,7 @@ fn unaggregated_gossip_verification() {
         {
             let mut a = valid_attestation.clone();
             a.data.slot = early_slot;
+            a.data.target.epoch = early_slot.epoch(E::slots_per_epoch());
             a
         },
         subnet_id,
@@ -654,6 +695,27 @@ fn unaggregated_gossip_verification() {
         if attestation_slot == early_slot && earliest_permissible_slot == current_slot - E::slots_per_epoch() - 1
     );
 
+    /*
+     * The following test ensures:
+     *
+     * Spec v0.12.3
+     *
+     * The attestation's epoch matches its target -- i.e. `attestation.data.target.epoch ==
+     *   compute_epoch_at_slot(attestation.data.slot)`
+     *
+     */
+
+    assert_invalid!(
+        "attestation with invalid target epoch",
+        {
+            let mut a = valid_attestation.clone();
+            a.data.target.epoch += 1;
+            a
+        },
+        subnet_id,
+        AttnError::InvalidTargetEpoch { .. }
+    );
+
     /*
      * The following two tests ensure:
      *
@@ -694,6 +756,32 @@ fn unaggregated_gossip_verification() {
         AttnError::NotExactlyOneAggregationBitSet(2)
     );
 
+    /*
+     * The following test ensures:
+     *
+     * Spec v0.12.3
+     *
+     * The number of aggregation bits matches the committee size -- i.e.
+     *   `len(attestation.aggregation_bits) == len(get_beacon_committee(state, data.slot,
+     *   data.index))`.
+     */
+    assert_invalid!(
+        "attestation with invalid bitfield",
+        {
+            let mut a = valid_attestation.clone();
+            let bits = a.aggregation_bits.iter().collect::<Vec<_>>();
+            a.aggregation_bits = BitList::with_capacity(bits.len() + 1).unwrap();
+            for (i, bit) in bits.into_iter().enumerate() {
+                a.aggregation_bits.set(i, bit).unwrap();
+            }
+            a
+        },
+        subnet_id,
+        AttnError::Invalid(AttestationValidationError::BeaconStateError(
+            BeaconStateError::InvalidBitfield
+        ))
+    );
+
     /*
      * The following test ensures that:
      *
@@ -717,6 +805,26 @@ fn unaggregated_gossip_verification() {
         if beacon_block_root == unknown_root
     );
 
+    /*
+     * The following test ensures that:
+     *
+     * Spec v0.12.3
+     *
+     * The attestation's target block is an ancestor of the block named in the LMD vote
+     */
+
+    let unknown_root = Hash256::from_low_u64_le(424242);
+    assert_invalid!(
+        "attestation with invalid target root",
+        {
+            let mut a = valid_attestation.clone();
+            a.data.target.root = unknown_root;
+            a
+        },
+        subnet_id,
+        AttnError::InvalidTargetRoot { .. }
+    );
+
     /*
      * The following test ensures that:
      *
@@ -742,7 +850,7 @@ fn unaggregated_gossip_verification() {
 
     harness
         .chain
-        .verify_unaggregated_attestation_for_gossip(valid_attestation.clone(), subnet_id)
+        .verify_unaggregated_attestation_for_gossip(valid_attestation.clone(), Some(subnet_id))
         .expect("valid attestation should be verified");
 
     /*
@@ -773,7 +881,6 @@ fn unaggregated_gossip_verification() {
 #[test]
 fn attestation_that_skips_epochs() {
     let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -782,16 +889,18 @@ fn attestation_that_skips_epochs() {
         AttestationStrategy::SomeValidators(vec![]),
     );
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     let earlier_slot = (current_epoch - 2).start_slot(MainnetEthSpec::slots_per_epoch());
-    let earlier_block = chain
+    let earlier_block = harness
+        .chain
         .block_at_slot(earlier_slot)
         .expect("should not error getting block at slot")
         .expect("should find block at slot");
 
-    let mut state = chain
+    let mut state = harness
+        .chain
         .get_state(&earlier_block.state_root(), Some(earlier_slot))
         .expect("should not error getting state")
         .expect("should find state");
@@ -830,6 +939,6 @@ fn attestation_that_skips_epochs() {
 
     harness
         .chain
-        .verify_unaggregated_attestation_for_gossip(attestation, subnet_id)
+        .verify_unaggregated_attestation_for_gossip(attestation, Some(subnet_id))
         .expect("should gossip verify attestation that skips slots");
 }

beacon_node/beacon_chain/tests/block_verification.rs

@@ -4,7 +4,7 @@
 extern crate lazy_static;
 
 use beacon_chain::{
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
     BeaconSnapshot, BlockError,
 };
 use store::config::StoreConfig;
@@ -18,8 +18,9 @@ use types::{
 type E = MainnetEthSpec;
 
 // Should ideally be divisible by 3.
-pub const VALIDATOR_COUNT: usize = 24;
-pub const CHAIN_SEGMENT_LENGTH: usize = 64 * 5;
+const VALIDATOR_COUNT: usize = 24;
+const CHAIN_SEGMENT_LENGTH: usize = 64 * 5;
+const BLOCK_INDICES: &[usize] = &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT_LENGTH - 1];
 
 lazy_static! {
     /// A cached set of keys.
@@ -47,8 +48,8 @@ fn get_chain_segment() -> Vec<BeaconSnapshot<E>> {
         .collect()
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<E>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -80,7 +81,7 @@ fn junk_aggregate_signature() -> AggregateSignature {
 
 fn update_proposal_signatures(
     snapshots: &mut [BeaconSnapshot<E>],
-    harness: &BeaconChainHarness<HarnessType<E>>,
+    harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
 ) {
     for snapshot in snapshots {
         let spec = &harness.chain.spec;
@@ -90,7 +91,7 @@ fn update_proposal_signatures(
             .get_beacon_proposer_index(slot, spec)
             .expect("should find proposer index");
         let keypair = harness
-            .keypairs
+            .validator_keypairs
             .get(proposer_index)
             .expect("proposer keypair should be available");
 
@@ -272,17 +273,73 @@ fn chain_segment_non_linear_slots() {
     );
 }
 
+fn assert_invalid_signature(
+    harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
+    block_index: usize,
+    snapshots: &[BeaconSnapshot<E>],
+    item: &str,
+) {
+    let blocks = snapshots
+        .iter()
+        .map(|snapshot| snapshot.beacon_block.clone())
+        .collect();
+
+    // Ensure the block will be rejected if imported in a chain segment.
+    assert!(
+        matches!(
+            harness
+                .chain
+                .process_chain_segment(blocks)
+                .into_block_error(),
+            Err(BlockError::InvalidSignature)
+        ),
+        "should not import chain segment with an invalid {} signature",
+        item
+    );
+
+    // Ensure the block will be rejected if imported on its own (without gossip checking).
+    let ancestor_blocks = CHAIN_SEGMENT
+        .iter()
+        .take(block_index)
+        .map(|snapshot| snapshot.beacon_block.clone())
+        .collect();
+    // We don't care if this fails, we just call this to ensure that all prior blocks have been
+    // imported prior to this test.
+    let _ = harness.chain.process_chain_segment(ancestor_blocks);
+    assert!(
+        matches!(
+            harness
+                .chain
+                .process_block(snapshots[block_index].beacon_block.clone()),
+            Err(BlockError::InvalidSignature)
+        ),
+        "should not import individual block with an invalid {} signature",
+        item
+    );
+
+    // NOTE: we choose not to check gossip verification here. It only checks one signature
+    // (proposal) and that is already tested elsewhere in this file.
+    //
+    // It's not trivial to just check gossip verification since it will start refusing
+    // blocks as soon as it has seen one valid proposal signature for a given (validator,
+    // slot) tuple.
+}
+
+fn get_invalid_sigs_harness() -> BeaconChainHarness<EphemeralHarnessType<E>> {
+    let harness = get_harness(VALIDATOR_COUNT);
+    harness
+        .chain
+        .slot_clock
+        .set_slot(CHAIN_SEGMENT.last().unwrap().beacon_block.slot().as_u64());
+    harness
+}
 #[test]
-fn invalid_signatures() {
-    let mut checked_attestation = false;
-
-    for &block_index in &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT.len() - 1] {
-        let harness = get_harness(VALIDATOR_COUNT);
-        harness
-            .chain
-            .slot_clock
-            .set_slot(CHAIN_SEGMENT.last().unwrap().beacon_block.slot().as_u64());
-
+fn invalid_signature_gossip_block() {
+    for &block_index in BLOCK_INDICES {
+        // Ensure the block will be rejected if imported on its own (without gossip checking).
+        let harness = get_invalid_sigs_harness();
+        let mut snapshots = CHAIN_SEGMENT.clone();
+        snapshots[block_index].beacon_block.signature = junk_signature();
         // Import all the ancestors before the `block_index` block.
         let ancestor_blocks = CHAIN_SEGMENT
             .iter()
@@ -294,75 +351,6 @@ fn invalid_signatures() {
             .process_chain_segment(ancestor_blocks)
             .into_block_error()
             .expect("should import all blocks prior to the one being tested");
-
-        // For the given snapshots, test the following:
-        //
-        // - The `process_chain_segment` function returns `InvalidSignature`.
-        // - The `process_block` function returns `InvalidSignature` when importing the
-        //    `SignedBeaconBlock` directly.
-        // - The `verify_block_for_gossip` function does _not_ return an error.
-        // - The `process_block` function returns `InvalidSignature` when verifying the
-        //    `GossipVerifiedBlock`.
-        let assert_invalid_signature = |snapshots: &[BeaconSnapshot<E>], item: &str| {
-            let blocks = snapshots
-                .iter()
-                .map(|snapshot| snapshot.beacon_block.clone())
-                .collect();
-
-            // Ensure the block will be rejected if imported in a chain segment.
-            assert!(
-                matches!(
-                    harness
-                        .chain
-                        .process_chain_segment(blocks)
-                        .into_block_error(),
-                    Err(BlockError::InvalidSignature)
-                ),
-                "should not import chain segment with an invalid {} signature",
-                item
-            );
-
-            // Ensure the block will be rejected if imported on its own (without gossip checking).
-            assert!(
-                matches!(
-                    harness
-                        .chain
-                        .process_block(snapshots[block_index].beacon_block.clone()),
-                    Err(BlockError::InvalidSignature)
-                ),
-                "should not import individual block with an invalid {} signature",
-                item
-            );
-
-            // NOTE: we choose not to check gossip verification here. It only checks one signature
-            // (proposal) and that is already tested elsewhere in this file.
-            //
-            // It's not trivial to just check gossip verification since it will start refusing
-            // blocks as soon as it has seen one valid proposal signature for a given (validator,
-            // slot) tuple.
-        };
-
-        /*
-         * Block proposal
-         */
-        let mut snapshots = CHAIN_SEGMENT.clone();
-        snapshots[block_index].beacon_block.signature = junk_signature();
-        let blocks = snapshots
-            .iter()
-            .map(|snapshot| snapshot.beacon_block.clone())
-            .collect();
-        // Ensure the block will be rejected if imported in a chain segment.
-        assert!(
-            matches!(
-                harness
-                    .chain
-                    .process_chain_segment(blocks)
-                    .into_block_error(),
-                Err(BlockError::InvalidSignature)
-            ),
-            "should not import chain segment with an invalid gossip signature",
-        );
-        // Ensure the block will be rejected if imported on its own (without gossip checking).
         assert!(
             matches!(
                 harness
@@ -372,10 +360,37 @@ fn invalid_signatures() {
             ),
             "should not import individual block with an invalid gossip signature",
         );
+    }
+}
 
-        /*
-         * Randao reveal
-         */
+#[test]
+fn invalid_signature_block_proposal() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
+        let mut snapshots = CHAIN_SEGMENT.clone();
+        snapshots[block_index].beacon_block.signature = junk_signature();
+        let blocks = snapshots
+            .iter()
+            .map(|snapshot| snapshot.beacon_block.clone())
+            .collect::<Vec<_>>();
+        // Ensure the block will be rejected if imported in a chain segment.
+        assert!(
+            matches!(
+                harness
+                    .chain
+                    .process_chain_segment(blocks)
+                    .into_block_error(),
+                Err(BlockError::InvalidSignature)
+            ),
+            "should not import chain segment with an invalid block signature",
+        );
+    }
+}
+
+#[test]
+fn invalid_signature_randao_reveal() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         snapshots[block_index]
             .beacon_block
@@ -384,11 +399,14 @@ fn invalid_signatures() {
             .randao_reveal = junk_signature();
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "randao");
+        assert_invalid_signature(&harness, block_index, &snapshots, "randao");
+    }
+}
 
-        /*
-         * Proposer slashing
-         */
+#[test]
+fn invalid_signature_proposer_slashing() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let proposer_slashing = ProposerSlashing {
             signed_header_1: SignedBeaconBlockHeader {
@@ -409,11 +427,14 @@ fn invalid_signatures() {
             .expect("should update proposer slashing");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "proposer slashing");
+        assert_invalid_signature(&harness, block_index, &snapshots, "proposer slashing");
+    }
+}
 
-        /*
-         * Attester slashing
-         */
+#[test]
+fn invalid_signature_attester_slashing() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let indexed_attestation = IndexedAttestation {
             attesting_indices: vec![0].into(),
@@ -445,11 +466,16 @@ fn invalid_signatures() {
             .expect("should update attester slashing");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "attester slashing");
+        assert_invalid_signature(&harness, block_index, &snapshots, "attester slashing");
+    }
+}
 
-        /*
-         * Attestation
-         */
+#[test]
+fn invalid_signature_attestation() {
+    let mut checked_attestation = false;
+
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         if let Some(attestation) = snapshots[block_index]
             .beacon_block
@@ -461,15 +487,22 @@ fn invalid_signatures() {
             attestation.signature = junk_aggregate_signature();
             update_parent_roots(&mut snapshots);
             update_proposal_signatures(&mut snapshots, &harness);
-            assert_invalid_signature(&snapshots, "attestation");
+            assert_invalid_signature(&harness, block_index, &snapshots, "attestation");
             checked_attestation = true;
         }
+    }
 
-        /*
-         * Deposit
-         *
-         * Note: an invalid deposit signature is permitted!
-         */
+    assert!(
+        checked_attestation,
+        "the test should check an attestation signature"
+    )
+}
+
+#[test]
+fn invalid_signature_deposit() {
+    for &block_index in BLOCK_INDICES {
+        // Note: an invalid deposit signature is permitted!
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let deposit = Deposit {
             proof: vec![Hash256::zero(); DEPOSIT_TREE_DEPTH + 1].into(),
@@ -503,10 +536,13 @@ fn invalid_signatures() {
             ),
             "should not throw an invalid signature error for a bad deposit signature"
         );
+    }
+}
 
-        /*
-         * Voluntary exit
-         */
+#[test]
+fn invalid_signature_exit() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let epoch = snapshots[block_index].beacon_state.current_epoch();
         snapshots[block_index]
@@ -524,13 +560,8 @@ fn invalid_signatures() {
             .expect("should update deposit");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "voluntary exit");
+        assert_invalid_signature(&harness, block_index, &snapshots, "voluntary exit");
     }
-
-    assert!(
-        checked_attestation,
-        "the test should check an attestation signature"
-    )
 }
 
 fn unwrap_err<T, E>(result: Result<T, E>) -> E {
@@ -641,6 +672,48 @@ fn block_gossip_verification() {
         "should not import a block with an invalid proposal signature"
     );
 
+    /*
+     * This test ensures that:
+     *
+     * Spec v0.12.2
+     *
+     * The block's parent (defined by block.parent_root) passes validation.
+     */
+
+    let mut block = CHAIN_SEGMENT[block_index].beacon_block.clone();
+    let parent_root = Hash256::from_low_u64_be(42);
+    block.message.parent_root = parent_root;
+    assert!(
+        matches!(
+            unwrap_err(harness.chain.verify_block_for_gossip(block)),
+            BlockError::ParentUnknown(block)
+            if block.parent_root() == parent_root
+        ),
+        "should not import a block for an unknown parent"
+    );
+
+    /*
+     * This test ensures that:
+     *
+     * Spec v0.12.2
+     *
+     * The current finalized_checkpoint is an ancestor of block -- i.e. get_ancestor(store,
+     * block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) ==
+     * store.finalized_checkpoint.root
+     */
+
+    let mut block = CHAIN_SEGMENT[block_index].beacon_block.clone();
+    let parent_root = CHAIN_SEGMENT[0].beacon_block_root;
+    block.message.parent_root = parent_root;
+    assert!(
+        matches!(
+            unwrap_err(harness.chain.verify_block_for_gossip(block)),
+            BlockError::NotFinalizedDescendant { block_parent_root }
+            if block_parent_root == parent_root
+        ),
+        "should not import a block that conflicts with finality"
+    );
+
     /*
      * This test ensures that:
      *

beacon_node/beacon_chain/tests/op_verification.rs

@@ -58,7 +58,7 @@ fn voluntary_exit() {
     let db_path = tempdir().unwrap();
     let store = get_store(&db_path);
     let harness = get_harness(store.clone(), VALIDATOR_COUNT);
-    let spec = &harness.chain.spec;
+    let spec = &harness.chain.spec.clone();
 
     harness.extend_chain(
         (E::slots_per_epoch() * (spec.shard_committee_period + 1)) as usize,

beacon_node/beacon_chain/tests/persistence_tests.rs

@@ -153,8 +153,11 @@ fn assert_chains_pretty_much_the_same<T: BeaconChainTypes>(a: &BeaconChain<T>, b
         a.genesis_block_root, b.genesis_block_root,
         "genesis_block_root should be equal"
     );
+
+    let slot = a.slot().unwrap();
     assert!(
-        *a.fork_choice.read() == *b.fork_choice.read(),
-        "fork_choice should be equal"
+        a.fork_choice.write().get_head(slot).unwrap()
+            == b.fork_choice.write().get_head(slot).unwrap(),
+        "fork_choice heads should be equal"
     );
 }

beacon_node/beacon_chain/tests/tests.rs

@@ -6,7 +6,8 @@ extern crate lazy_static;
 use beacon_chain::{
     attestation_verification::Error as AttnError,
     test_utils::{
-        AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType, OP_POOL_DB_KEY,
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType,
+        OP_POOL_DB_KEY,
     },
 };
 use operation_pool::PersistedOperationPool;
@@ -24,8 +25,8 @@ lazy_static! {
     static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<MinimalEthSpec>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<MinimalEthSpec>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MinimalEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -354,11 +355,10 @@ fn roundtrip_operation_pool() {
         .persist_op_pool()
         .expect("should persist op pool");
 
-    let key = Hash256::from_slice(&OP_POOL_DB_KEY);
     let restored_op_pool = harness
         .chain
         .store
-        .get_item::<PersistedOperationPool<MinimalEthSpec>>(&key)
+        .get_item::<PersistedOperationPool<MinimalEthSpec>>(&OP_POOL_DB_KEY)
         .expect("should read db")
         .expect("should find op pool")
         .into_operation_pool();
@@ -436,8 +436,8 @@ fn attestations_with_increasing_slots() {
             AttestationStrategy::SomeValidators(vec![]),
         );
 
-        attestations.append(
-            &mut harness.get_unaggregated_attestations(
+        attestations.extend(
+            harness.get_unaggregated_attestations(
                 &AttestationStrategy::AllValidators,
                 &harness.chain.head().expect("should get head").beacon_state,
                 harness
@@ -460,7 +460,7 @@ fn attestations_with_increasing_slots() {
     for (attestation, subnet_id) in attestations.into_iter().flatten() {
         let res = harness
             .chain
-            .verify_unaggregated_attestation_for_gossip(attestation.clone(), subnet_id);
+            .verify_unaggregated_attestation_for_gossip(attestation.clone(), Some(subnet_id));
 
         let current_slot = harness.chain.slot().expect("should get slot");
         let expected_attestation_slot = attestation.data.slot;

beacon_node/client/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "client"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
 [dev-dependencies]
-sloggers = "1.0.0"
+sloggers = "1.0.1"
 toml = "0.5.6"
 
 [dependencies]
@@ -14,30 +14,33 @@ store = { path = "../store" }
 network = { path = "../network" }
 timer = { path = "../timer" }
 eth2_libp2p = { path = "../eth2_libp2p" }
-rest_api = { path = "../rest_api" }
 parking_lot = "0.11.0"
 websocket_server = { path = "../websocket_server" }
-prometheus = "0.9.0"
+prometheus = "0.10.0"
 types = { path = "../../consensus/types" }
-tree_hash = "0.1.0"
+tree_hash = "0.1.1"
 eth2_config = { path = "../../common/eth2_config" }
 slot_clock = { path = "../../common/slot_clock" }
-serde = "1.0.110"
-serde_derive = "1.0.110"
-error-chain = "0.12.2"
-serde_yaml = "0.8.11"
+serde = "1.0.116"
+serde_derive = "1.0.116"
+error-chain = "0.12.4"
+serde_yaml = "0.8.13"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 slog-async = "2.5.0"
-tokio = "0.2.21"
-dirs = "2.0.2"
+tokio = "0.2.22"
+dirs = "3.0.1"
 futures = "0.3.5"
-reqwest = "0.10.4"
+reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
 url = "2.1.1"
 eth1 = { path = "../eth1" }
 genesis = { path = "../genesis" }
+task_executor = { path = "../../common/task_executor" }
 environment = { path = "../../lighthouse/environment" }
 eth2_ssz = "0.1.2"
 lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-time = "0.2.16"
+time = "0.2.22"
 bus = "2.2.3"
+directory = {path = "../../common/directory"}
+http_api = { path = "../http_api" }
+http_metrics = { path = "../http_metrics" }

beacon_node/client/src/builder.rs

@@ -5,7 +5,6 @@ use beacon_chain::events::TeeEventHandler;
 use beacon_chain::{
     builder::{BeaconChainBuilder, Witness},
     eth1_chain::{CachingEth1Backend, Eth1Chain},
-    migrate::{BackgroundMigrator, Migrate},
     slot_clock::{SlotClock, SystemTimeSlotClock},
     store::{HotColdDB, ItemStore, LevelDB, StoreConfig},
     BeaconChain, BeaconChainTypes, Eth1ChainBackend, EventHandler,
@@ -13,15 +12,14 @@ use beacon_chain::{
 use bus::Bus;
 use environment::RuntimeContext;
 use eth1::{Config as Eth1Config, Service as Eth1Service};
-use eth2_config::Eth2Config;
 use eth2_libp2p::NetworkGlobals;
 use genesis::{interop_genesis_state, Eth1GenesisService};
 use network::{NetworkConfig, NetworkMessage, NetworkService};
 use parking_lot::Mutex;
-use slog::info;
+use slog::{debug, info};
 use ssz::Decode;
 use std::net::SocketAddr;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 use timer::spawn_timer;
@@ -52,7 +50,6 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
     slot_clock: Option<T::SlotClock>,
     #[allow(clippy::type_complexity)]
     store: Option<Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>>,
-    store_migrator: Option<T::StoreMigrator>,
     runtime_context: Option<RuntimeContext<T::EthSpec>>,
     chain_spec: Option<ChainSpec>,
     beacon_chain_builder: Option<BeaconChainBuilder<T>>,
@@ -61,25 +58,17 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
     event_handler: Option<T::EventHandler>,
     network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
     network_send: Option<UnboundedSender<NetworkMessage<T::EthSpec>>>,
-    http_listen_addr: Option<SocketAddr>,
+    db_path: Option<PathBuf>,
+    freezer_db_path: Option<PathBuf>,
+    http_api_config: http_api::Config,
+    http_metrics_config: http_metrics::Config,
     websocket_listen_addr: Option<SocketAddr>,
     eth_spec_instance: T::EthSpec,
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+    ClientBuilder<Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>>
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + Clone + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -94,7 +83,6 @@ where
         Self {
             slot_clock: None,
             store: None,
-            store_migrator: None,
             runtime_context: None,
             chain_spec: None,
             beacon_chain_builder: None,
@@ -103,7 +91,10 @@ where
             event_handler: None,
             network_globals: None,
             network_send: None,
-            http_listen_addr: None,
+            db_path: None,
+            freezer_db_path: None,
+            http_api_config: <_>::default(),
+            http_metrics_config: <_>::default(),
             websocket_listen_addr: None,
             eth_spec_instance,
         }
@@ -129,18 +120,16 @@ where
         config: ClientConfig,
     ) -> Result<Self, String> {
         let store = self.store.clone();
-        let store_migrator = self.store_migrator.take();
         let chain_spec = self.chain_spec.clone();
         let runtime_context = self.runtime_context.clone();
         let eth_spec_instance = self.eth_spec_instance.clone();
         let data_dir = config.data_dir.clone();
         let disabled_forks = config.disabled_forks.clone();
+        let chain_config = config.chain.clone();
         let graffiti = config.graffiti;
 
         let store =
             store.ok_or_else(|| "beacon_chain_start_method requires a store".to_string())?;
-        let store_migrator = store_migrator
-            .ok_or_else(|| "beacon_chain_start_method requires a store migrator".to_string())?;
         let context = runtime_context
             .ok_or_else(|| "beacon_chain_start_method requires a runtime context".to_string())?
             .service_context("beacon".into());
@@ -150,9 +139,9 @@ where
         let builder = BeaconChainBuilder::new(eth_spec_instance)
             .logger(context.log().clone())
             .store(store)
-            .store_migrator(store_migrator)
             .data_dir(data_dir)
             .custom_spec(spec.clone())
+            .chain_config(chain_config)
             .disabled_forks(disabled_forks)
             .graffiti(graffiti);
 
@@ -232,8 +221,8 @@ where
         Ok(self)
     }
 
-    /// Immediately starts the networking stack.
-    pub fn network(mut self, config: &NetworkConfig) -> Result<Self, String> {
+    /// Starts the networking stack.
+    pub async fn network(mut self, config: &NetworkConfig) -> Result<Self, String> {
         let beacon_chain = self
             .beacon_chain
             .clone()
@@ -246,6 +235,7 @@ where
 
         let (network_globals, network_send) =
             NetworkService::start(beacon_chain, config, context.executor)
+                .await
                 .map_err(|e| format!("Failed to start network: {:?}", e))?;
 
         self.network_globals = Some(network_globals);
@@ -277,55 +267,16 @@ where
         Ok(self)
     }
 
-    /// Immediately starts the beacon node REST API http server.
-    pub fn http_server(
-        mut self,
-        client_config: &ClientConfig,
-        eth2_config: &Eth2Config,
-        events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
-    ) -> Result<Self, String> {
-        let beacon_chain = self
-            .beacon_chain
-            .clone()
-            .ok_or_else(|| "http_server requires a beacon chain")?;
-        let context = self
-            .runtime_context
-            .as_ref()
-            .ok_or_else(|| "http_server requires a runtime_context")?
-            .service_context("http".into());
-        let network_globals = self
-            .network_globals
-            .clone()
-            .ok_or_else(|| "http_server requires a libp2p network")?;
-        let network_send = self
-            .network_send
-            .clone()
-            .ok_or_else(|| "http_server requires a libp2p network sender")?;
+    /// Provides configuration for the HTTP API.
+    pub fn http_api_config(mut self, config: http_api::Config) -> Self {
+        self.http_api_config = config;
+        self
+    }
 
-        let network_info = rest_api::NetworkInfo {
-            network_globals,
-            network_chan: network_send,
-        };
-
-        let listening_addr = rest_api::start_server(
-            context.executor,
-            &client_config.rest_api,
-            beacon_chain,
-            network_info,
-            client_config
-                .create_db_path()
-                .map_err(|_| "unable to read data dir")?,
-            client_config
-                .create_freezer_db_path()
-                .map_err(|_| "unable to read freezer DB dir")?,
-            eth2_config.clone(),
-            events,
-        )
-        .map_err(|e| format!("Failed to start HTTP API: {:?}", e))?;
-
-        self.http_listen_addr = Some(listening_addr);
-
-        Ok(self)
+    /// Provides configuration for the HTTP server that serves Prometheus metrics.
+    pub fn http_metrics_config(mut self, config: http_metrics::Config) -> Self {
+        self.http_metrics_config = config;
+        self
     }
 
     /// Immediately starts the service that periodically logs information each slot.
@@ -364,42 +315,81 @@ where
     /// specified.
     ///
     /// If type inference errors are being raised, see the comment on the definition of `Self`.
+    #[allow(clippy::type_complexity)]
     pub fn build(
         self,
-    ) -> Client<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+    ) -> Result<
+        Client<Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>>,
+        String,
     > {
-        Client {
+        let runtime_context = self
+            .runtime_context
+            .as_ref()
+            .ok_or_else(|| "build requires a runtime context".to_string())?;
+        let log = runtime_context.log().clone();
+
+        let http_api_listen_addr = if self.http_api_config.enabled {
+            let ctx = Arc::new(http_api::Context {
+                config: self.http_api_config.clone(),
+                chain: self.beacon_chain.clone(),
+                network_tx: self.network_send.clone(),
+                network_globals: self.network_globals.clone(),
+                log: log.clone(),
+            });
+
+            let exit = runtime_context.executor.exit();
+
+            let (listen_addr, server) = http_api::serve(ctx, exit)
+                .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
+
+            runtime_context
+                .clone()
+                .executor
+                .spawn_without_exit(async move { server.await }, "http-api");
+
+            Some(listen_addr)
+        } else {
+            info!(log, "HTTP server is disabled");
+            None
+        };
+
+        let http_metrics_listen_addr = if self.http_metrics_config.enabled {
+            let ctx = Arc::new(http_metrics::Context {
+                config: self.http_metrics_config.clone(),
+                chain: self.beacon_chain.clone(),
+                db_path: self.db_path.clone(),
+                freezer_db_path: self.freezer_db_path.clone(),
+                log: log.clone(),
+            });
+
+            let exit = runtime_context.executor.exit();
+
+            let (listen_addr, server) = http_metrics::serve(ctx, exit)
+                .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
+
+            runtime_context
+                .executor
+                .spawn_without_exit(async move { server.await }, "http-api");
+
+            Some(listen_addr)
+        } else {
+            debug!(log, "Metrics server is disabled");
+            None
+        };
+
+        Ok(Client {
             beacon_chain: self.beacon_chain,
             network_globals: self.network_globals,
-            http_listen_addr: self.http_listen_addr,
+            http_api_listen_addr,
+            http_metrics_listen_addr,
             websocket_listen_addr: self.websocket_listen_addr,
-        }
+        })
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+    ClientBuilder<Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>>
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + Clone + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -409,6 +399,12 @@ where
 {
     /// Consumes the internal `BeaconChainBuilder`, attaching the resulting `BeaconChain` to self.
     pub fn build_beacon_chain(mut self) -> Result<Self, String> {
+        let context = self
+            .runtime_context
+            .as_ref()
+            .ok_or_else(|| "beacon_chain requires a runtime context")?
+            .clone();
+
         let chain = self
             .beacon_chain_builder
             .ok_or_else(|| "beacon_chain requires a beacon_chain_builder")?
@@ -421,6 +417,7 @@ where
                     .clone()
                     .ok_or_else(|| "beacon_chain requires a slot clock")?,
             )
+            .shutdown_sender(context.executor.shutdown_sender())
             .build()
             .map_err(|e| format!("Failed to build beacon chain: {}", e))?;
 
@@ -433,10 +430,9 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
+impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
     ClientBuilder<
         Witness<
-            TStoreMigrator,
             TSlotClock,
             TEth1Backend,
             TEthSpec,
@@ -446,7 +442,6 @@ impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
         >,
     >
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -481,10 +476,9 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
     ClientBuilder<
         Witness<
-            TStoreMigrator,
             TSlotClock,
             TEth1Backend,
             TEthSpec,
@@ -495,7 +489,6 @@ impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
     >
 where
     TSlotClock: SlotClock + 'static,
-    TStoreMigrator: Migrate<TEthSpec, LevelDB<TEthSpec>, LevelDB<TEthSpec>> + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -517,6 +510,9 @@ where
             .clone()
             .ok_or_else(|| "disk_store requires a chain spec".to_string())?;
 
+        self.db_path = Some(hot_path.into());
+        self.freezer_db_path = Some(cold_path.into());
+
         let store = HotColdDB::open(hot_path, cold_path, config, spec, context.log().clone())
             .map_err(|e| format!("Unable to open database: {:?}", e))?;
         self.store = Some(Arc::new(store));
@@ -524,44 +520,9 @@ where
     }
 }
 
-impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
     ClientBuilder<
         Witness<
-            BackgroundMigrator<TEthSpec, THotStore, TColdStore>,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
-where
-    TSlotClock: SlotClock + 'static,
-    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
-    TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
-    THotStore: ItemStore<TEthSpec> + 'static,
-    TColdStore: ItemStore<TEthSpec> + 'static,
-{
-    pub fn background_migrator(mut self) -> Result<Self, String> {
-        let context = self
-            .runtime_context
-            .as_ref()
-            .ok_or_else(|| "disk_store requires a log".to_string())?
-            .service_context("freezer_db".into());
-        let store = self.store.clone().ok_or_else(|| {
-            "background_migrator requires the store to be initialized".to_string()
-        })?;
-        self.store_migrator = Some(BackgroundMigrator::new(store, context.log().clone()));
-        Ok(self)
-    }
-}
-
-impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
             TSlotClock,
             CachingEth1Backend<TEthSpec>,
             TEthSpec,
@@ -571,7 +532,6 @@ impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
         >,
     >
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -581,7 +541,7 @@ where
     /// Specifies that the `BeaconChain` should cache eth1 blocks/logs from a remote eth1 node
     /// (e.g., Parity/Geth) and refer to that cache when collecting deposits or eth1 votes during
     /// block production.
-    pub fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
+    pub async fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
         let context = self
             .runtime_context
             .as_ref()
@@ -671,20 +631,11 @@ where
     }
 }
 
-impl<TStoreMigrator, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
     ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            SystemTimeSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+        Witness<SystemTimeSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>,
     >
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -699,11 +650,8 @@ where
             .ok_or_else(|| "system_time_slot_clock requires a beacon_chain_builder")?;
 
         let genesis_time = beacon_chain_builder
-            .finalized_snapshot
-            .as_ref()
-            .ok_or_else(|| "system_time_slot_clock requires an initialized beacon state")?
-            .beacon_state
-            .genesis_time;
+            .genesis_time
+            .ok_or_else(|| "system_time_slot_clock requires an initialized beacon state")?;
 
         let spec = self
             .chain_spec

beacon_node/client/src/config.rs

@@ -1,14 +1,10 @@
+use directory::DEFAULT_ROOT_DIR;
 use network::NetworkConfig;
 use serde_derive::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
 use types::Graffiti;
 
-pub const DEFAULT_DATADIR: &str = ".lighthouse";
-
-/// The number initial validators when starting the `Minimal`.
-const TESTNET_SPEC_CONSTANTS: &str = "minimal";
-
 /// Default directory name for the freezer database under the top-level data dir.
 const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db";
 
@@ -47,7 +43,6 @@ pub struct Config {
     /// Path where the freezer database will be located.
     pub freezer_db_path: Option<PathBuf>,
     pub log_file: PathBuf,
-    pub spec_constants: String,
     /// If true, the node will use co-ordinated junk for eth1 values.
     ///
     /// This is the method used for the 2019 client interop in Canada.
@@ -63,29 +58,32 @@ pub struct Config {
     pub genesis: ClientGenesis,
     pub store: store::StoreConfig,
     pub network: network::NetworkConfig,
-    pub rest_api: rest_api::Config,
+    pub chain: beacon_chain::ChainConfig,
     pub websocket_server: websocket_server::Config,
     pub eth1: eth1::Config,
+    pub http_api: http_api::Config,
+    pub http_metrics: http_metrics::Config,
 }
 
 impl Default for Config {
     fn default() -> Self {
         Self {
-            data_dir: PathBuf::from(DEFAULT_DATADIR),
+            data_dir: PathBuf::from(DEFAULT_ROOT_DIR),
             db_name: "chain_db".to_string(),
             freezer_db_path: None,
             log_file: PathBuf::from(""),
             genesis: <_>::default(),
             store: <_>::default(),
             network: NetworkConfig::default(),
-            rest_api: <_>::default(),
+            chain: <_>::default(),
             websocket_server: <_>::default(),
-            spec_constants: TESTNET_SPEC_CONSTANTS.into(),
             dummy_eth1_backend: false,
             sync_eth1_chain: false,
             eth1: <_>::default(),
             disabled_forks: Vec::new(),
             graffiti: Graffiti::default(),
+            http_api: <_>::default(),
+            http_metrics: <_>::default(),
         }
     }
 }

beacon_node/client/src/lib.rs

@@ -23,7 +23,10 @@ pub use eth2_config::Eth2Config;
 pub struct Client<T: BeaconChainTypes> {
     beacon_chain: Option<Arc<BeaconChain<T>>>,
     network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
-    http_listen_addr: Option<SocketAddr>,
+    /// Listen address for the standard eth2.0 API, if the service was started.
+    http_api_listen_addr: Option<SocketAddr>,
+    /// Listen address for the HTTP server which serves Prometheus metrics.
+    http_metrics_listen_addr: Option<SocketAddr>,
     websocket_listen_addr: Option<SocketAddr>,
 }
 
@@ -33,9 +36,14 @@ impl<T: BeaconChainTypes> Client<T> {
         self.beacon_chain.clone()
     }
 
-    /// Returns the address of the client's HTTP API server, if it was started.
-    pub fn http_listen_addr(&self) -> Option<SocketAddr> {
-        self.http_listen_addr
+    /// Returns the address of the client's standard eth2.0 API server, if it was started.
+    pub fn http_api_listen_addr(&self) -> Option<SocketAddr> {
+        self.http_api_listen_addr
+    }
+
+    /// Returns the address of the client's HTTP Prometheus metrics server, if it was started.
+    pub fn http_metrics_listen_addr(&self) -> Option<SocketAddr> {
+        self.http_metrics_listen_addr
     }
 
     /// Returns the address of the client's WebSocket API server, if it was started.

beacon_node/client/src/notifier.rs

@@ -22,7 +22,7 @@ const SPEEDO_OBSERVATIONS: usize = 4;
 
 /// Spawns a notifier service which periodically logs information about the node.
 pub fn spawn_notifier<T: BeaconChainTypes>(
-    executor: environment::TaskExecutor,
+    executor: task_executor::TaskExecutor,
     beacon_chain: Arc<BeaconChain<T>>,
     network: Arc<NetworkGlobals<T::EthSpec>>,
     milliseconds_per_slot: u64,
@@ -122,14 +122,28 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                     head_distance.as_u64(),
                     slot_distance_pretty(head_distance, slot_duration)
                 );
-                info!(
-                    log,
-                    "Syncing";
-                    "peers" => peer_count_pretty(connected_peer_count),
-                    "distance" => distance,
-                    "speed" => sync_speed_pretty(speedo.slots_per_second()),
-                    "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
-                );
+
+                let speed = speedo.slots_per_second();
+                let display_speed = speed.map_or(false, |speed| speed != 0.0);
+
+                if display_speed {
+                    info!(
+                        log,
+                        "Syncing";
+                        "peers" => peer_count_pretty(connected_peer_count),
+                        "distance" => distance,
+                        "speed" => sync_speed_pretty(speed),
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                    );
+                } else {
+                    info!(
+                        log,
+                        "Syncing";
+                        "peers" => peer_count_pretty(connected_peer_count),
+                        "distance" => distance,
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                    );
+                }
             } else if sync_state.is_synced() {
                 let block_info = if current_slot > head_slot {
                     "   …  empty".to_string()
@@ -220,14 +234,37 @@ fn seconds_pretty(secs: f64) -> String {
     let hours = d.whole_hours();
     let minutes = d.whole_minutes();
 
+    let week_string = if weeks == 1 { "week" } else { "weeks" };
+    let day_string = if days == 1 { "day" } else { "days" };
+    let hour_string = if hours == 1 { "hr" } else { "hrs" };
+    let min_string = if minutes == 1 { "min" } else { "mins" };
+
     if weeks > 0 {
-        format!("{:.0} weeks {:.0} days", weeks, days % DAYS_PER_WEEK)
+        format!(
+            "{:.0} {} {:.0} {}",
+            weeks,
+            week_string,
+            days % DAYS_PER_WEEK,
+            day_string
+        )
     } else if days > 0 {
-        format!("{:.0} days {:.0} hrs", days, hours % HOURS_PER_DAY)
+        format!(
+            "{:.0} {} {:.0} {}",
+            days,
+            day_string,
+            hours % HOURS_PER_DAY,
+            hour_string
+        )
     } else if hours > 0 {
-        format!("{:.0} hrs {:.0} mins", hours, minutes % MINUTES_PER_HOUR)
+        format!(
+            "{:.0} {} {:.0} {}",
+            hours,
+            hour_string,
+            minutes % MINUTES_PER_HOUR,
+            min_string
+        )
     } else {
-        format!("{:.0} mins", minutes)
+        format!("{:.0} {}", minutes, min_string)
     }
 }
 

beacon_node/eth1/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "eth1"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
@@ -8,25 +8,26 @@ edition = "2018"
 eth1_test_rig = { path = "../../testing/eth1_test_rig" }
 toml = "0.5.6"
 web3 = "0.11.0"
-sloggers = "1.0.0"
+sloggers = "1.0.1"
+environment = { path = "../../lighthouse/environment" }
 
 [dependencies]
-reqwest = "0.10.4"
+reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
 futures = { version = "0.3.5", features = ["compat"] }
-serde_json = "1.0.52"
-serde = { version = "1.0.110", features = ["derive"] }
+serde_json = "1.0.58"
+serde = { version = "1.0.116", features = ["derive"] }
 hex = "0.4.2"
 types = { path = "../../consensus/types"}
 merkle_proof = { path = "../../consensus/merkle_proof"}
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
-tree_hash = "0.1.0"
+tree_hash = "0.1.1"
 eth2_hashing = "0.1.0"
 parking_lot = "0.11.0"
 slog = "2.5.2"
-tokio = { version = "0.2.21", features = ["full"] }
+tokio = { version = "0.2.22", features = ["full"] }
 state_processing = { path = "../../consensus/state_processing" }
-libflate = "1.0.0"
+libflate = "1.0.2"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics"}
 lazy_static = "1.4.0"
-environment = { path = "../../lighthouse/environment" }
+task_executor = { path = "../../common/task_executor" }

beacon_node/eth1/src/http.rs

@@ -12,8 +12,10 @@
 
 use futures::future::TryFutureExt;
 use reqwest::{header::CONTENT_TYPE, ClientBuilder, StatusCode};
+use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
 use std::ops::Range;
+use std::str::FromStr;
 use std::time::Duration;
 use types::Hash256;
 
@@ -30,6 +32,55 @@ pub const DEPOSIT_COUNT_RESPONSE_BYTES: usize = 96;
 /// Number of bytes in deposit contract deposit root (value only).
 pub const DEPOSIT_ROOT_BYTES: usize = 32;
 
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub enum Eth1NetworkId {
+    Goerli,
+    Mainnet,
+    Custom(u64),
+}
+
+impl Into<u64> for Eth1NetworkId {
+    fn into(self) -> u64 {
+        match self {
+            Eth1NetworkId::Mainnet => 1,
+            Eth1NetworkId::Goerli => 5,
+            Eth1NetworkId::Custom(id) => id,
+        }
+    }
+}
+
+impl From<u64> for Eth1NetworkId {
+    fn from(id: u64) -> Self {
+        let into = |x: Eth1NetworkId| -> u64 { x.into() };
+        match id {
+            id if id == into(Eth1NetworkId::Mainnet) => Eth1NetworkId::Mainnet,
+            id if id == into(Eth1NetworkId::Goerli) => Eth1NetworkId::Goerli,
+            id => Eth1NetworkId::Custom(id),
+        }
+    }
+}
+
+impl FromStr for Eth1NetworkId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        u64::from_str_radix(s, 10)
+            .map(Into::into)
+            .map_err(|e| format!("Failed to parse eth1 network id {}", e))
+    }
+}
+
+/// Get the eth1 network id of the given endpoint.
+pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result<Eth1NetworkId, String> {
+    let response_body = send_rpc_request(endpoint, "net_version", json!([]), timeout).await?;
+    Eth1NetworkId::from_str(
+        response_result(&response_body)?
+            .ok_or_else(|| "No result was returned for block number".to_string())?
+            .as_str()
+            .ok_or_else(|| "Data was not string")?,
+    )
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct Block {
     pub hash: Hash256,

beacon_node/eth1/src/lib.rs

@@ -13,4 +13,6 @@ pub use block_cache::{BlockCache, Eth1Block};
 pub use deposit_cache::DepositCache;
 pub use deposit_log::DepositLog;
 pub use inner::SszEth1Cache;
-pub use service::{BlockCacheUpdateOutcome, Config, DepositCacheUpdateOutcome, Error, Service};
+pub use service::{
+    BlockCacheUpdateOutcome, Config, DepositCacheUpdateOutcome, Error, Service, DEFAULT_NETWORK_ID,
+};

beacon_node/eth1/src/service.rs

@@ -2,20 +2,25 @@ use crate::metrics;
 use crate::{
     block_cache::{BlockCache, Error as BlockCacheError, Eth1Block},
     deposit_cache::Error as DepositCacheError,
-    http::{get_block, get_block_number, get_deposit_logs_in_range, Log},
+    http::{
+        get_block, get_block_number, get_deposit_logs_in_range, get_network_id, Eth1NetworkId, Log,
+    },
     inner::{DepositUpdater, Inner},
     DepositLog,
 };
 use futures::{future::TryFutureExt, stream, stream::TryStreamExt, StreamExt};
 use parking_lot::{RwLock, RwLockReadGuard};
 use serde::{Deserialize, Serialize};
-use slog::{debug, error, info, trace, Logger};
+use slog::{crit, debug, error, info, trace, Logger};
 use std::ops::{Range, RangeInclusive};
 use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
 use tokio::time::{interval_at, Duration, Instant};
 use types::ChainSpec;
 
+/// Indicates the default eth1 network we use for the deposit contract.
+pub const DEFAULT_NETWORK_ID: Eth1NetworkId = Eth1NetworkId::Goerli;
+
 const STANDARD_TIMEOUT_MILLIS: u64 = 15_000;
 
 /// Timeout when doing a eth_blockNumber call.
@@ -25,6 +30,8 @@ const GET_BLOCK_TIMEOUT_MILLIS: u64 = STANDARD_TIMEOUT_MILLIS;
 /// Timeout when doing an eth_getLogs to read the deposit contract logs.
 const GET_DEPOSIT_LOG_TIMEOUT_MILLIS: u64 = STANDARD_TIMEOUT_MILLIS;
 
+const WARNING_MSG: &str = "BLOCK PROPOSALS WILL FAIL WITHOUT VALID ETH1 CONNECTION";
+
 #[derive(Debug, PartialEq)]
 pub enum Error {
     /// The remote node is less synced that we expect, it is not useful until has done more
@@ -76,6 +83,8 @@ pub struct Config {
     pub endpoint: String,
     /// The address the `BlockCache` and `DepositCache` should assume is the canonical deposit contract.
     pub deposit_contract_address: String,
+    /// The eth1 network id where the deposit contract is deployed (Goerli/Mainnet).
+    pub network_id: Eth1NetworkId,
     /// Defines the first block that the `DepositCache` will start searching for deposit logs.
     ///
     /// Setting too high can result in missed logs. Setting too low will result in unnecessary
@@ -105,6 +114,7 @@ impl Default for Config {
         Self {
             endpoint: "http://localhost:8545".into(),
             deposit_contract_address: "0x0000000000000000000000000000000000000000".into(),
+            network_id: DEFAULT_NETWORK_ID,
             deposit_contract_deploy_block: 1,
             lowest_cached_block_number: 1,
             follow_distance: 128,
@@ -335,7 +345,7 @@ impl Service {
     /// - Err(_) if there is an error.
     ///
     /// Emits logs for debugging and errors.
-    pub fn auto_update(self, handle: environment::TaskExecutor) {
+    pub fn auto_update(self, handle: task_executor::TaskExecutor) {
         let update_interval = Duration::from_millis(self.config().auto_update_interval_millis);
 
         let mut interval = interval_at(Instant::now(), update_interval);
@@ -350,6 +360,34 @@ impl Service {
     }
 
     async fn do_update(&self, update_interval: Duration) -> Result<(), ()> {
+        let endpoint = self.config().endpoint.clone();
+        let config_network = self.config().network_id.clone();
+        let result =
+            get_network_id(&endpoint, Duration::from_millis(STANDARD_TIMEOUT_MILLIS)).await;
+        match result {
+            Ok(network_id) => {
+                if network_id != config_network {
+                    crit!(
+                        self.log,
+                        "Invalid eth1 network. Please switch to correct network";
+                        "expected" => format!("{:?}",config_network),
+                        "received" => format!("{:?}",network_id),
+                        "warning" => WARNING_MSG,
+                    );
+                    return Ok(());
+                }
+            }
+            Err(_) => {
+                crit!(
+                    self.log,
+                    "Error connecting to eth1 node. Please ensure that you have an eth1 http server running locally on http://localhost:8545 or \
+                    pass an external endpoint using `--eth1-endpoint <SERVER-ADDRESS>`. Also ensure that `eth` and `net` apis are enabled on the eth1 http server";
+                    "warning" => WARNING_MSG,
+                );
+                return Ok(());
+            }
+        }
+
         let update_result = self.update().await;
         match update_result {
             Err(e) => error!(

beacon_node/eth2_libp2p/Cargo.toml

@@ -1,54 +1,54 @@
 [package]
 name = "eth2_libp2p"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
 [dependencies]
-hex = "0.4.2"
+discv5 = { git = "https://github.com/sigp/discv5", rev = "fba7ceb5cfebd219ebbad6ffdb5d8c31dc8e4bc0", features = ["libp2p"] }
 types = { path =  "../../consensus/types" }
 hashset_delay = { path = "../../common/hashset_delay" }
 eth2_ssz_types = { path =  "../../consensus/ssz_types" }
-serde = { version = "1.0.110", features = ["derive"] }
-serde_derive = "1.0.110"
+serde = { version = "1.0.116", features = ["derive"] }
+serde_derive = "1.0.116"
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 lighthouse_version = { path = "../../common/lighthouse_version" }
-tokio = { version = "0.2.21", features = ["time", "macros"] }
+tokio = { version = "0.2.22", features = ["time", "macros"] }
 futures = "0.3.5"
-error-chain = "0.12.2"
-dirs = "2.0.2"
+error-chain = "0.12.4"
+dirs = "3.0.1"
 fnv = "1.0.7"
 unsigned-varint = { git = "https://github.com/sigp/unsigned-varint", branch = "latest-codecs", features = ["codec"] }
 lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-smallvec = "1.4.1"
-lru = "0.5.1"
+smallvec = "1.4.2"
+lru = "0.6.0"
 parking_lot = "0.11.0"
 sha2 = "0.9.1"
-base64 = "0.12.1"
-snap = "1.0.0"
+base64 = "0.13.0"
+snap = "1.0.1"
 void = "1.0.2"
+hex = "0.4.2"
 tokio-io-timeout = "0.4.0"
 tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
-discv5 = { version = "0.1.0-alpha.7", features = ["libp2p"] }
 tiny-keccak = "2.0.2"
-environment = { path = "../../lighthouse/environment" }
-# TODO: Remove rand crate for mainnet
+task_executor = { path = "../../common/task_executor" }
 rand = "0.7.3"
+directory = { path = "../../common/directory" }
+regex = "1.3.9"
 
 [dependencies.libp2p]
-#version = "0.19.1"
+#version = "0.23.0"
 git = "https://github.com/sigp/rust-libp2p"
-rev = "f1b660a1a96c1b6198cd62062e75d357893faf16"
+rev = "8c6ce6eb1228de568568f6cd72fb134dea5f9669"
 default-features = false
-features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "secio", "tcp-tokio"]
+features = ["websocket", "identify", "mplex", "noise", "gossipsub", "dns", "tcp-tokio"]
 
 [dev-dependencies]
-tokio = { version = "0.2.21", features = ["full"] }
-slog-stdlog = "4.0.0"
-slog-term = "2.5.0"
+tokio = { version = "0.2.22", features = ["full"] }
+slog-term = "2.6.0"
 slog-async = "2.5.0"
 tempdir = "0.3.7"
 exit-future = "0.2.0"

beacon_node/eth2_libp2p/src/behaviour/handler/delegate.rs

@@ -1,8 +1,8 @@
+use crate::behaviour::Gossipsub;
 use crate::rpc::*;
 use libp2p::{
     core::either::{EitherError, EitherOutput},
     core::upgrade::{EitherUpgrade, InboundUpgrade, OutboundUpgrade, SelectUpgrade, UpgradeError},
-    gossipsub::Gossipsub,
     identify::Identify,
     swarm::{
         protocols_handler::{
@@ -49,13 +49,11 @@ impl<TSpec: EthSpec> DelegatingHandler<TSpec> {
     }
 
     /// Gives access to identify's handler.
-    pub fn identify(&self) -> &IdentifyHandler {
+    pub fn _identify(&self) -> &IdentifyHandler {
         &self.identify_handler
     }
 }
 
-// TODO: this can all be created with macros
-
 /// Wrapper around the `ProtocolsHandler::InEvent` types of the handlers.
 /// Simply delegated to the corresponding behaviour's handler.
 #[derive(Debug, Clone)]
@@ -115,7 +113,6 @@ pub type DelegateOutProto<TSpec> = EitherUpgrade<
     >,
 >;
 
-// TODO: prob make this an enum
 pub type DelegateOutInfo<TSpec> = EitherOutput<
     <GossipHandler as ProtocolsHandler>::OutboundOpenInfo,
     EitherOutput<
@@ -131,8 +128,9 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         let gossip_proto = self.gossip_handler.listen_protocol();
         let rpc_proto = self.rpc_handler.listen_protocol();
         let identify_proto = self.identify_handler.listen_protocol();
@@ -147,24 +145,27 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             SelectUpgrade::new(rpc_proto.into_upgrade().1, identify_proto.into_upgrade().1),
         );
 
-        SubstreamProtocol::new(select).with_timeout(timeout)
+        SubstreamProtocol::new(select, ()).with_timeout(timeout)
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         match out {
             // Gossipsub
-            EitherOutput::First(out) => self.gossip_handler.inject_fully_negotiated_inbound(out),
+            EitherOutput::First(out) => {
+                self.gossip_handler.inject_fully_negotiated_inbound(out, ())
+            }
             // RPC
             EitherOutput::Second(EitherOutput::First(out)) => {
-                self.rpc_handler.inject_fully_negotiated_inbound(out)
+                self.rpc_handler.inject_fully_negotiated_inbound(out, ())
             }
             // Identify
-            EitherOutput::Second(EitherOutput::Second(out)) => {
-                self.identify_handler.inject_fully_negotiated_inbound(out)
-            }
+            EitherOutput::Second(EitherOutput::Second(out)) => self
+                .identify_handler
+                .inject_fully_negotiated_inbound(out, ()),
         }
     }
 
@@ -212,7 +213,6 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             <Self::OutboundProtocol as OutboundUpgrade<NegotiatedSubstream>>::Error,
         >,
     ) {
-        // TODO: find how to clean up
         match info {
             // Gossipsub
             EitherOutput::First(info) => match error {
@@ -317,10 +317,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
                     event,
                 )));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(EitherUpgrade::A),
-                    info: EitherOutput::First(info),
+                    protocol: protocol
+                        .map_upgrade(EitherUpgrade::A)
+                        .map_info(EitherOutput::First),
                 });
             }
             Poll::Pending => (),
@@ -333,10 +334,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::RPC(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u))),
-                    info: EitherOutput::Second(EitherOutput::First(info)),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u)))
+                        .map_info(|info| EitherOutput::Second(EitherOutput::First(info))),
                 });
             }
             Poll::Pending => (),
@@ -351,10 +353,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::Identify(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u))),
-                    info: EitherOutput::Second(EitherOutput::Second(())),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u)))
+                        .map_info(|_| EitherOutput::Second(EitherOutput::Second(()))),
                 });
             }
             Poll::Pending => (),

beacon_node/eth2_libp2p/src/behaviour/handler/mod.rs

@@ -1,3 +1,4 @@
+use crate::behaviour::Gossipsub;
 use crate::rpc::*;
 use delegate::DelegatingHandler;
 pub(super) use delegate::{
@@ -5,7 +6,6 @@ pub(super) use delegate::{
 };
 use libp2p::{
     core::upgrade::{InboundUpgrade, OutboundUpgrade},
-    gossipsub::Gossipsub,
     identify::Identify,
     swarm::protocols_handler::{
         KeepAlive, ProtocolsHandlerEvent, ProtocolsHandlerUpgrErr, SubstreamProtocol,
@@ -41,29 +41,25 @@ pub enum BehaviourHandlerIn<TSpec: EthSpec> {
     Shutdown(Option<(RequestId, RPCRequest<TSpec>)>),
 }
 
-pub enum BehaviourHandlerOut<TSpec: EthSpec> {
-    Delegate(Box<DelegateOut<TSpec>>),
-    // TODO: replace custom with events to send
-    Custom,
-}
-
 impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
     type InEvent = BehaviourHandlerIn<TSpec>;
-    type OutEvent = BehaviourHandlerOut<TSpec>;
+    type OutEvent = DelegateOut<TSpec>;
     type Error = DelegateError<TSpec>;
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.delegate.listen_protocol()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
-        self.delegate.inject_fully_negotiated_inbound(out)
+        self.delegate.inject_fully_negotiated_inbound(out, ())
     }
 
     fn inject_fully_negotiated_outbound(
@@ -77,7 +73,7 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
     fn inject_event(&mut self, event: Self::InEvent) {
         match event {
             BehaviourHandlerIn::Delegate(delegated_ev) => self.delegate.inject_event(delegated_ev),
-            /* Events comming from the behaviour */
+            /* Events coming from the behaviour */
             BehaviourHandlerIn::Shutdown(last_message) => {
                 self.shutting_down = true;
                 self.delegate.rpc_mut().shutdown(last_message);
@@ -113,28 +109,20 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
         >,
     > {
         // Disconnect if the sub-handlers are ready.
-        if self.shutting_down {
-            let rpc_keep_alive = self.delegate.rpc().connection_keep_alive();
-            let identify_keep_alive = self.delegate.identify().connection_keep_alive();
-            if KeepAlive::No == rpc_keep_alive.max(identify_keep_alive) {
-                return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::Disconnected));
-            }
+        // Currently we only respect the RPC handler.
+        if self.shutting_down && KeepAlive::No == self.delegate.rpc().connection_keep_alive() {
+            return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::Disconnected));
         }
 
         match self.delegate.poll(cx) {
             Poll::Ready(ProtocolsHandlerEvent::Custom(event)) => {
-                return Poll::Ready(ProtocolsHandlerEvent::Custom(
-                    BehaviourHandlerOut::Delegate(Box::new(event)),
-                ))
+                return Poll::Ready(ProtocolsHandlerEvent::Custom(event))
             }
             Poll::Ready(ProtocolsHandlerEvent::Close(err)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(err))
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
-                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol,
-                    info,
-                });
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
+                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol });
             }
             Poll::Pending => (),
         }

beacon_node/eth2_libp2p/src/behaviour/mod.rs

@@ -1,17 +1,24 @@
 use crate::peer_manager::{score::PeerAction, PeerManager, PeerManagerEvent};
 use crate::rpc::*;
-use crate::types::{GossipEncoding, GossipKind, GossipTopic};
+use crate::service::METADATA_FILENAME;
+use crate::types::{GossipEncoding, GossipKind, GossipTopic, MessageData, SubnetDiscovery};
 use crate::Eth2Enr;
 use crate::{error, metrics, Enr, NetworkConfig, NetworkGlobals, PubsubMessage, TopicHash};
 use futures::prelude::*;
-use handler::{BehaviourHandler, BehaviourHandlerIn, BehaviourHandlerOut, DelegateIn, DelegateOut};
+use handler::{BehaviourHandler, BehaviourHandlerIn, DelegateIn, DelegateOut};
+use libp2p::gossipsub::subscription_filter::{
+    MaxCountSubscriptionFilter, WhitelistSubscriptionFilter,
+};
 use libp2p::{
     core::{
         connection::{ConnectedPoint, ConnectionId, ListenerId},
         identity::Keypair,
         Multiaddr,
     },
-    gossipsub::{Gossipsub, GossipsubEvent, MessageAuthenticity, MessageId},
+    gossipsub::{
+        GenericGossipsub, GenericGossipsubEvent, IdentTopic as Topic, MessageAcceptance,
+        MessageAuthenticity, MessageId,
+    },
     identify::{Identify, IdentifyEvent},
     swarm::{
         NetworkBehaviour, NetworkBehaviourAction as NBAction, NotifyHandler, PollParameters,
@@ -19,13 +26,17 @@ use libp2p::{
     },
     PeerId,
 };
-use slog::{crit, debug, o, trace};
+use slog::{crit, debug, o, trace, warn};
+use ssz::Encode;
+use std::collections::HashSet;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
 use std::{
     collections::VecDeque,
     marker::PhantomData,
     sync::Arc,
     task::{Context, Poll},
-    time::Instant,
 };
 use types::{EnrForkId, EthSpec, SignedBeaconBlock, SubnetId};
 
@@ -33,6 +44,63 @@ mod handler;
 
 const MAX_IDENTIFY_ADDRESSES: usize = 10;
 
+/// Identifier of requests sent by a peer.
+pub type PeerRequestId = (ConnectionId, SubstreamId);
+
+pub type SubscriptionFilter = MaxCountSubscriptionFilter<WhitelistSubscriptionFilter>;
+pub type Gossipsub = GenericGossipsub<MessageData, SubscriptionFilter>;
+pub type GossipsubEvent = GenericGossipsubEvent<MessageData>;
+
+/// The types of events than can be obtained from polling the behaviour.
+#[derive(Debug)]
+pub enum BehaviourEvent<TSpec: EthSpec> {
+    /// We have successfully dialed and connected to a peer.
+    PeerDialed(PeerId),
+    /// A peer has successfully dialed and connected to us.
+    PeerConnected(PeerId),
+    /// A peer has disconnected.
+    PeerDisconnected(PeerId),
+    /// An RPC Request that was sent failed.
+    RPCFailed {
+        /// The id of the failed request.
+        id: RequestId,
+        /// The peer to which this request was sent.
+        peer_id: PeerId,
+        /// The error that occurred.
+        error: RPCError,
+    },
+    RequestReceived {
+        /// The peer that sent the request.
+        peer_id: PeerId,
+        /// Identifier of the request. All responses to this request must use this id.
+        id: PeerRequestId,
+        /// Request the peer sent.
+        request: Request,
+    },
+    ResponseReceived {
+        /// Peer that sent the response.
+        peer_id: PeerId,
+        /// Id of the request to which the peer is responding.
+        id: RequestId,
+        /// Response the peer sent.
+        response: Response<TSpec>,
+    },
+    PubsubMessage {
+        /// The gossipsub message id. Used when propagating blocks after validation.
+        id: MessageId,
+        /// The peer from which we received this message, not the peer that published it.
+        source: PeerId,
+        /// The topic that this message was sent on.
+        topic: TopicHash,
+        /// The message itself.
+        message: PubsubMessage<TSpec>,
+    },
+    /// Subscribed to peer for given topic
+    PeerSubscribed(PeerId, TopicHash),
+    /// Inform the network to send a Status to this peer.
+    StatusPeer(PeerId),
+}
+
 /// Builds the network behaviour that manages the core protocols of eth2.
 /// This core behaviour is managed by `Behaviour` which adds peer management to all core
 /// behaviours.
@@ -42,7 +110,7 @@ pub struct Behaviour<TSpec: EthSpec> {
     /// The Eth2 RPC specified in the wire-0 protocol.
     eth2_rpc: RPC<TSpec>,
     /// Keep regular connection to peers and disconnect if absent.
-    // TODO: Using id for initial interop. This will be removed by mainnet.
+    // NOTE: The id protocol is used for initial interop. This will be removed by mainnet.
     /// Provides IP addresses and peer information.
     identify: Identify,
     /// The peer manager that keeps track of peer's reputation and status.
@@ -51,8 +119,6 @@ pub struct Behaviour<TSpec: EthSpec> {
     events: VecDeque<BehaviourEvent<TSpec>>,
     /// Queue of peers to disconnect and an optional reason for the disconnection.
     peers_to_dc: VecDeque<(PeerId, Option<GoodbyeReason>)>,
-    /// The current meta data of the node, so respond to pings and get metadata
-    meta_data: MetaData<TSpec>,
     /// A collections of variables accessible outside the network service.
     network_globals: Arc<NetworkGlobals<TSpec>>,
     /// Keeps track of the current EnrForkId for upgrading gossipsub topics.
@@ -61,13 +127,15 @@ pub struct Behaviour<TSpec: EthSpec> {
     enr_fork_id: EnrForkId,
     /// The waker for the current thread.
     waker: Option<std::task::Waker>,
+    /// Directory where metadata is stored
+    network_dir: PathBuf,
     /// Logger for behaviour actions.
     log: slog::Logger,
 }
 
 /// Implements the combined behaviour for the libp2p service.
 impl<TSpec: EthSpec> Behaviour<TSpec> {
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         net_conf: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
@@ -86,33 +154,39 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .eth2()
             .expect("Local ENR must have a fork id");
 
-        let attnets = network_globals
-            .local_enr()
-            .bitfield::<TSpec>()
-            .expect("Local ENR must have subnet bitfield");
-
-        let meta_data = MetaData {
-            seq_number: 1,
-            attnets,
+        let possible_fork_digests = vec![enr_fork_id.fork_digest];
+        let filter = MaxCountSubscriptionFilter {
+            filter: Self::create_whitelist_filter(possible_fork_digests, 64), //TODO change this to a constant
+            max_subscribed_topics: 200, //TODO change this to a constant
+            max_subscriptions_per_request: 100, //this is according to the current go implementation
         };
 
-        // TODO: Until other clients support no author, we will use a 0 peer_id as our author.
-        let message_author = PeerId::from_bytes(vec![0, 1, 0]).expect("Valid peer id");
+        let gossipsub = Gossipsub::new_with_subscription_filter(
+            MessageAuthenticity::Anonymous,
+            net_conf.gs_config.clone(),
+            filter,
+        )
+        .map_err(|e| format!("Could not construct gossipsub: {:?}", e))?;
+
+        // Temporarily disable scoring until parameters are tested.
+        /*
+        gossipsub
+            .with_peer_score(PeerScoreParams::default(), PeerScoreThresholds::default())
+            .expect("Valid score params and thresholds");
+        */
 
         Ok(Behaviour {
             eth2_rpc: RPC::new(log.clone()),
-            gossipsub: Gossipsub::new(
-                MessageAuthenticity::Author(message_author),
-                net_conf.gs_config.clone(),
-            ),
+            gossipsub,
             identify,
-            peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)?,
+            peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)
+                .await?,
             events: VecDeque::new(),
             peers_to_dc: VecDeque::new(),
-            meta_data,
             network_globals,
             enr_fork_id,
             waker: None,
+            network_dir: net_conf.network_dir.clone(),
             log: behaviour_log,
         })
     }
@@ -123,7 +197,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
     ///
     /// All external dials, dial a multiaddr. This is currently unused but kept here in case any
     /// part of lighthouse needs to connect to a peer_id in the future.
-    pub fn _dial(&mut self, peer_id: &PeerId) {
+    pub fn dial(&mut self, peer_id: &PeerId) {
         self.peer_manager.dial_peer(peer_id);
     }
 
@@ -147,6 +221,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             GossipEncoding::default(),
             self.enr_fork_id.fork_digest,
         );
+
         self.subscribe(gossip_topic)
     }
 
@@ -189,9 +264,18 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .write()
             .insert(topic.clone());
 
-        let topic_str: String = topic.clone().into();
-        debug!(self.log, "Subscribed to topic"; "topic" => topic_str);
-        self.gossipsub.subscribe(topic.into())
+        let topic: Topic = topic.into();
+
+        match self.gossipsub.subscribe(&topic) {
+            Err(_) => {
+                warn!(self.log, "Failed to subscribe to topic"; "topic" => topic.to_string());
+                false
+            }
+            Ok(v) => {
+                debug!(self.log, "Subscribed to topic"; "topic" => topic.to_string());
+                v
+            }
+        }
     }
 
     /// Unsubscribe from a gossipsub topic.
@@ -201,8 +285,20 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .gossipsub_subscriptions
             .write()
             .remove(&topic);
+
         // unsubscribe from the topic
-        self.gossipsub.unsubscribe(topic.into())
+        let topic: Topic = topic.into();
+
+        match self.gossipsub.unsubscribe(&topic) {
+            Err(_) => {
+                warn!(self.log, "Failed to unsubscribe from topic"; "topic" => topic.to_string());
+                false
+            }
+            Ok(v) => {
+                debug!(self.log, "Unsubscribed to topic"; "topic" => topic.to_string());
+                v
+            }
+        }
     }
 
     /// Publishes a list of messages on the pubsub (gossipsub) behaviour, choosing the encoding.
@@ -211,8 +307,28 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) {
                 match message.encode(GossipEncoding::default()) {
                     Ok(message_data) => {
-                        if let Err(e) = self.gossipsub.publish(&topic.into(), message_data) {
+                        if let Err(e) = self.gossipsub.publish(topic.clone().into(), message_data) {
                             slog::warn!(self.log, "Could not publish message"; "error" => format!("{:?}", e));
+
+                            // add to metrics
+                            match topic.kind() {
+                                GossipKind::Attestation(subnet_id) => {
+                                    if let Some(v) = metrics::get_int_gauge(
+                                        &metrics::FAILED_ATTESTATION_PUBLISHES_PER_SUBNET,
+                                        &[&subnet_id.to_string()],
+                                    ) {
+                                        v.inc()
+                                    };
+                                }
+                                kind => {
+                                    if let Some(v) = metrics::get_int_gauge(
+                                        &metrics::FAILED_PUBLISHES_PER_MAIN_TOPIC,
+                                        &[&format!("{:?}", kind)],
+                                    ) {
+                                        v.inc()
+                                    };
+                                }
+                            }
                         }
                     }
                     Err(e) => crit!(self.log, "Could not publish message"; "error" => e),
@@ -221,11 +337,21 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
         }
     }
 
-    /// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated
-    /// once validated by the beacon chain.
-    pub fn validate_message(&mut self, propagation_source: &PeerId, message_id: MessageId) {
-        self.gossipsub
-            .validate_message(&message_id, propagation_source);
+    /// Informs the gossipsub about the result of a message validation.
+    /// If the message is valid it will get propagated by gossipsub.
+    pub fn report_message_validation_result(
+        &mut self,
+        propagation_source: &PeerId,
+        message_id: MessageId,
+        validation_result: MessageAcceptance,
+    ) {
+        if let Err(e) = self.gossipsub.report_message_validation_result(
+            &message_id,
+            propagation_source,
+            validation_result,
+        ) {
+            warn!(self.log, "Failed to report message validation"; "message_id" => message_id.to_string(), "peer_id" => propagation_source.to_string(), "error" => format!("{:?}", e));
+        }
     }
 
     /* Eth2 RPC behaviour functions */
@@ -300,8 +426,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Attempts to discover new peers for a given subnet. The `min_ttl` gives the time at which we
     /// would like to retain the peers for.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
-        self.peer_manager.discover_subnet_peers(subnet_id, min_ttl)
+    pub fn discover_subnet_peers(&mut self, subnet_subscriptions: Vec<SubnetDiscovery>) {
+        self.peer_manager
+            .discover_subnet_peers(subnet_subscriptions)
     }
 
     /// Updates the local ENR's "eth2" field with the latest EnrForkId.
@@ -338,19 +465,31 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Updates the current meta data of the node to match the local ENR.
     fn update_metadata(&mut self) {
-        self.meta_data.seq_number += 1;
-        self.meta_data.attnets = self
+        let local_attnets = self
             .peer_manager
             .discovery()
             .local_enr()
             .bitfield::<TSpec>()
             .expect("Local discovery must have bitfield");
+
+        {
+            // write lock scope
+            let mut meta_data = self.network_globals.local_metadata.write();
+            meta_data.seq_number += 1;
+            meta_data.attnets = local_attnets;
+        }
+        // Save the updated metadata to disk
+        save_metadata_to_disk(
+            &self.network_dir,
+            self.network_globals.local_metadata.read().clone(),
+            &self.log,
+        );
     }
 
     /// Sends a Ping request to the peer.
     fn ping(&mut self, id: RequestId, peer_id: PeerId) {
         let ping = crate::rpc::Ping {
-            data: self.meta_data.seq_number,
+            data: self.network_globals.local_metadata.read().seq_number,
         };
         trace!(self.log, "Sending Ping"; "request_id" => id, "peer_id" => peer_id.to_string());
 
@@ -361,7 +500,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
     /// Sends a Pong response to the peer.
     fn pong(&mut self, id: PeerRequestId, peer_id: PeerId) {
         let ping = crate::rpc::Ping {
-            data: self.meta_data.seq_number,
+            data: self.network_globals.local_metadata.read().seq_number,
         };
         trace!(self.log, "Sending Pong"; "request_id" => id.1, "peer_id" => peer_id.to_string());
         let event = RPCCodedResponse::Success(RPCResponse::Pong(ping));
@@ -377,7 +516,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Sends a METADATA response to a peer.
     fn send_meta_data_response(&mut self, id: PeerRequestId, peer_id: PeerId) {
-        let event = RPCCodedResponse::Success(RPCResponse::MetaData(self.meta_data.clone()));
+        let event = RPCCodedResponse::Success(RPCResponse::MetaData(
+            self.network_globals.local_metadata.read().clone(),
+        ));
         self.eth2_rpc.send_response(peer_id, id, event);
     }
 
@@ -389,40 +530,40 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     fn on_gossip_event(&mut self, event: GossipsubEvent) {
         match event {
-            GossipsubEvent::Message(propagation_source, id, gs_msg) => {
+            GossipsubEvent::Message {
+                propagation_source,
+                message_id: id,
+                message: gs_msg,
+            } => {
                 // Note: We are keeping track here of the peer that sent us the message, not the
                 // peer that originally published the message.
-                match PubsubMessage::decode(&gs_msg.topics, &gs_msg.data) {
-                    Err(e) => debug!(self.log, "Could not decode gossipsub message"; "error" => e),
+                match PubsubMessage::decode(&gs_msg.topic, gs_msg.data()) {
+                    Err(e) => {
+                        debug!(self.log, "Could not decode gossipsub message"; "error" => e);
+                        //reject the message
+                        if let Err(e) = self.gossipsub.report_message_validation_result(
+                            &id,
+                            &propagation_source,
+                            MessageAcceptance::Reject,
+                        ) {
+                            warn!(self.log, "Failed to report message validation"; "message_id" => id.to_string(), "peer_id" => propagation_source.to_string(), "error" => format!("{:?}", e));
+                        }
+                    }
                     Ok(msg) => {
                         // Notify the network
                         self.add_event(BehaviourEvent::PubsubMessage {
                             id,
                             source: propagation_source,
-                            topics: gs_msg.topics,
+                            topic: gs_msg.topic,
                             message: msg,
                         });
                     }
                 }
             }
             GossipsubEvent::Subscribed { peer_id, topic } => {
-                if let Some(topic_metric) = metrics::get_int_gauge(
-                    &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_COUNT,
-                    &[topic.as_str()],
-                ) {
-                    topic_metric.inc()
-                }
-
                 self.add_event(BehaviourEvent::PeerSubscribed(peer_id, topic));
             }
-            GossipsubEvent::Unsubscribed { peer_id: _, topic } => {
-                if let Some(topic_metric) = metrics::get_int_gauge(
-                    &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_COUNT,
-                    &[topic.as_str()],
-                ) {
-                    topic_metric.dec()
-                }
-            }
+            GossipsubEvent::Unsubscribed { .. } => {}
         }
     }
 
@@ -460,7 +601,6 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                     } => {
                         if matches!(error, RPCError::HandlerRejected) {
                             // this peer's request got canceled
-                            // TODO: cancel processing for this request
                         }
                         // Inform the peer manager of the error.
                         // An inbound error here means we sent an error to the peer, or the stream
@@ -490,11 +630,8 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                     RPCRequest::MetaData(_) => {
                         // send the requested meta-data
                         self.send_meta_data_response((handler_id, id), peer_id);
-                        // TODO: inform the peer manager?
                     }
                     RPCRequest::Goodbye(reason) => {
-                        // let the peer manager know this peer is in the process of disconnecting
-                        self.peer_manager._disconnecting_peer(&peer_id);
                         // queue for disconnection without a goodbye message
                         debug!(
                             self.log, "Peer sent Goodbye";
@@ -664,6 +801,33 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             waker.wake_by_ref();
         }
     }
+
+    /// Creates a whitelist topic filter that covers all possible topics using the given set of
+    /// possible fork digests.
+    fn create_whitelist_filter(
+        possible_fork_digests: Vec<[u8; 4]>,
+        attestation_subnet_count: u64,
+    ) -> WhitelistSubscriptionFilter {
+        let mut possible_hashes = HashSet::new();
+        for fork_digest in possible_fork_digests {
+            let mut add = |kind| {
+                let topic: Topic =
+                    GossipTopic::new(kind, GossipEncoding::SSZSnappy, fork_digest).into();
+                possible_hashes.insert(topic.hash());
+            };
+
+            use GossipKind::*;
+            add(BeaconBlock);
+            add(BeaconAggregateAndProof);
+            add(VoluntaryExit);
+            add(ProposerSlashing);
+            add(AttesterSlashing);
+            for id in 0..attestation_subnet_count {
+                add(Attestation(SubnetId::new(id)));
+            }
+        }
+        WhitelistSubscriptionFilter(possible_hashes)
+    }
 }
 
 /// Calls the given function with the given args on all sub behaviours.
@@ -687,35 +851,10 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
         self.peer_manager.addresses_of_peer(peer_id)
     }
 
-    // This gets called every time a connection is closed.
-    fn inject_connection_closed(
-        &mut self,
-        peer_id: &PeerId,
-        conn_id: &ConnectionId,
-        endpoint: &ConnectedPoint,
-    ) {
-        delegate_to_behaviours!(self, inject_connection_closed, peer_id, conn_id, endpoint);
-    }
-
-    // This gets called once there are no more active connections.
-    fn inject_disconnected(&mut self, peer_id: &PeerId) {
-        // Inform the peer manager.
-        self.peer_manager.notify_disconnect(&peer_id);
-        // Inform the application.
-        self.add_event(BehaviourEvent::PeerDisconnected(peer_id.clone()));
-
-        // Update the prometheus metrics
-        metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT);
-        metrics::set_gauge(
-            &metrics::PEERS_CONNECTED,
-            self.network_globals.connected_peers() as i64,
-        );
-
-        // Inform the behaviour.
-        delegate_to_behaviours!(self, inject_disconnected, peer_id);
-    }
-
     // This gets called every time a connection is established.
+    // NOTE: The current logic implies that we would reject extra connections for already connected
+    // peers if we have reached our peer limit. This is fine for the time being as we currently
+    // only allow a single connection per peer.
     fn inject_connection_established(
         &mut self,
         peer_id: &PeerId,
@@ -724,6 +863,9 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
     ) {
         let goodbye_reason: Option<GoodbyeReason> = if self.peer_manager.is_banned(peer_id) {
             // If the peer is banned, send goodbye with reason banned.
+            // A peer that has recently transitioned to the banned state should be in the
+            // disconnecting state, but the `is_banned()` function is dependent on score so should
+            // be true here in this case.
             Some(GoodbyeReason::Banned)
         } else if self.peer_manager.peer_limit_reached()
             && self
@@ -733,28 +875,34 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
                 .peer_info(peer_id)
                 .map_or(true, |i| !i.has_future_duty())
         {
-            //If we are at our peer limit and we don't need the peer for a future validator
-            //duty, send goodbye with reason TooManyPeers
+            // If we are at our peer limit and we don't need the peer for a future validator
+            // duty, send goodbye with reason TooManyPeers
             Some(GoodbyeReason::TooManyPeers)
         } else {
             None
         };
 
-        if goodbye_reason.is_some() {
+        if let Some(goodbye_reason) = goodbye_reason {
+            debug!(self.log, "Disconnecting newly connected peer"; "peer_id" => peer_id.to_string(), "reason" => goodbye_reason.to_string());
             self.peers_to_dc
-                .push_back((peer_id.clone(), goodbye_reason));
+                .push_back((peer_id.clone(), Some(goodbye_reason)));
+            // NOTE: We don't inform the peer manager that this peer is disconnecting. It is simply
+            // rejected with a goodbye.
             return;
         }
 
-        // notify the peer manager of a successful connection
+        // All peers at this point will be registered as being connected.
+        // Notify the peer manager of a successful connection
         match endpoint {
-            ConnectedPoint::Listener { .. } => {
-                self.peer_manager.connect_ingoing(&peer_id);
+            ConnectedPoint::Listener { send_back_addr, .. } => {
+                self.peer_manager
+                    .connect_ingoing(&peer_id, send_back_addr.clone());
                 self.add_event(BehaviourEvent::PeerConnected(peer_id.clone()));
                 debug!(self.log, "Connection established"; "peer_id" => peer_id.to_string(), "connection" => "Incoming");
             }
-            ConnectedPoint::Dialer { .. } => {
-                self.peer_manager.connect_outgoing(&peer_id);
+            ConnectedPoint::Dialer { address } => {
+                self.peer_manager
+                    .connect_outgoing(&peer_id, address.clone());
                 self.add_event(BehaviourEvent::PeerDialed(peer_id.clone()));
                 debug!(self.log, "Connection established"; "peer_id" => peer_id.to_string(), "connection" => "Dialed");
             }
@@ -770,19 +918,11 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
     }
 
     // This gets called on the initial connection establishment.
+    // NOTE: This gets called after inject_connection_established. Therefore the logic in that
+    // function dictates the logic here.
     fn inject_connected(&mut self, peer_id: &PeerId) {
-        // Drop any connection from a banned peer. The goodbye and disconnects are handled in
-        // `inject_connection_established()`, which gets called first.
-        // The same holds if we reached the peer limit and the connected peer has no future duty.
-        if self.peer_manager.is_banned(peer_id)
-            || (self.peer_manager.peer_limit_reached()
-                && self
-                    .network_globals
-                    .peers
-                    .read()
-                    .peer_info(peer_id)
-                    .map_or(true, |i| !i.has_future_duty()))
-        {
+        // If the PeerManager has connected this peer, inform the behaviours
+        if !self.network_globals.peers.read().is_connected(&peer_id) {
             return;
         }
 
@@ -796,6 +936,79 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
         delegate_to_behaviours!(self, inject_connected, peer_id);
     }
 
+    // This gets called every time a connection is closed.
+    // NOTE: The peer manager state can be modified in the lifetime of the peer. Due to the scoring
+    // mechanism. Peers can become banned. In this case, we still want to inform the behaviours.
+    fn inject_connection_closed(
+        &mut self,
+        peer_id: &PeerId,
+        conn_id: &ConnectionId,
+        endpoint: &ConnectedPoint,
+    ) {
+        // If the peer manager (and therefore the behaviour's) believe this peer connected, inform
+        // about the disconnection.
+        // It could be the peer was in the process of being disconnected. In this case the
+        // sub-behaviours are expecting this peer to be connected and we inform them.
+        if self
+            .network_globals
+            .peers
+            .read()
+            .is_connected_or_disconnecting(peer_id)
+        {
+            // We are disconnecting the peer or the peer has already been connected.
+            // Both these cases, the peer has been previously registered in the sub protocols.
+            delegate_to_behaviours!(self, inject_connection_closed, peer_id, conn_id, endpoint);
+        }
+    }
+
+    // This gets called once there are no more active connections.
+    fn inject_disconnected(&mut self, peer_id: &PeerId) {
+        // If the application/behaviour layers thinks this peer has connected inform it of the disconnect.
+
+        if self
+            .network_globals
+            .peers
+            .read()
+            .is_connected_or_disconnecting(peer_id)
+        {
+            // We are disconnecting the peer or the peer has already been connected.
+            // Both these cases, the peer has been previously registered in the sub protocols and
+            // potentially the application layer.
+            // Inform the application.
+            self.add_event(BehaviourEvent::PeerDisconnected(peer_id.clone()));
+            // Inform the behaviour.
+            delegate_to_behaviours!(self, inject_disconnected, peer_id);
+
+            // Decrement the PEERS_PER_CLIENT metric
+            if let Some(kind) = self
+                .network_globals
+                .peers
+                .read()
+                .peer_info(peer_id)
+                .map(|info| info.client.kind.clone())
+            {
+                if let Some(v) =
+                    metrics::get_int_gauge(&metrics::PEERS_PER_CLIENT, &[&kind.to_string()])
+                {
+                    v.dec()
+                };
+            }
+        }
+
+        // Inform the peer manager.
+        // NOTE: It may be the case that a rejected node, due to too many peers is disconnected
+        // here and the peer manager has no knowledge of its connection. We insert it here for
+        // reference so that peer manager can track this peer.
+        self.peer_manager.notify_disconnect(&peer_id);
+
+        // Update the prometheus metrics
+        metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT);
+        metrics::set_gauge(
+            &metrics::PEERS_CONNECTED,
+            self.network_globals.connected_peers() as i64,
+        );
+    }
+
     fn inject_addr_reach_failure(
         &mut self,
         peer_id: Option<&PeerId>,
@@ -842,17 +1055,11 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
             return;
         }
 
+        // Events comming from the handler, redirected to each behaviour
         match event {
-            // Events comming from the handler, redirected to each behaviour
-            BehaviourHandlerOut::Delegate(delegate) => match *delegate {
-                DelegateOut::Gossipsub(ev) => self.gossipsub.inject_event(peer_id, conn_id, ev),
-                DelegateOut::RPC(ev) => self.eth2_rpc.inject_event(peer_id, conn_id, ev),
-                DelegateOut::Identify(ev) => self.identify.inject_event(peer_id, conn_id, *ev),
-            },
-            /* Custom events sent BY the handler */
-            BehaviourHandlerOut::Custom => {
-                // TODO: implement
-            }
+            DelegateOut::Gossipsub(ev) => self.gossipsub.inject_event(peer_id, conn_id, ev),
+            DelegateOut::RPC(ev) => self.eth2_rpc.inject_event(peer_id, conn_id, ev),
+            DelegateOut::Identify(ev) => self.identify.inject_event(peer_id, conn_id, *ev),
         }
     }
 
@@ -870,7 +1077,6 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
             self.waker = Some(cx.waker().clone());
         }
 
-        // TODO: move where it's less distracting
         macro_rules! poll_behaviour {
             /* $behaviour:  The sub-behaviour being polled.
              * $on_event_fn:  Function to call if we get an event from the sub-behaviour.
@@ -982,55 +1188,22 @@ impl<TSpec: EthSpec> std::convert::From<Response<TSpec>> for RPCCodedResponse<TS
     }
 }
 
-/// Identifier of requests sent by a peer.
-pub type PeerRequestId = (ConnectionId, SubstreamId);
-
-/// The types of events than can be obtained from polling the behaviour.
-#[derive(Debug)]
-pub enum BehaviourEvent<TSpec: EthSpec> {
-    /// We have successfully dialed and connected to a peer.
-    PeerDialed(PeerId),
-    /// A peer has successfully dialed and connected to us.
-    PeerConnected(PeerId),
-    /// A peer has disconnected.
-    PeerDisconnected(PeerId),
-    /// An RPC Request that was sent failed.
-    RPCFailed {
-        /// The id of the failed request.
-        id: RequestId,
-        /// The peer to which this request was sent.
-        peer_id: PeerId,
-        /// The error that occurred.
-        error: RPCError,
-    },
-    RequestReceived {
-        /// The peer that sent the request.
-        peer_id: PeerId,
-        /// Identifier of the request. All responses to this request must use this id.
-        id: PeerRequestId,
-        /// Request the peer sent.
-        request: Request,
-    },
-    ResponseReceived {
-        /// Peer that sent the response.
-        peer_id: PeerId,
-        /// Id of the request to which the peer is responding.
-        id: RequestId,
-        /// Response the peer sent.
-        response: Response<TSpec>,
-    },
-    PubsubMessage {
-        /// The gossipsub message id. Used when propagating blocks after validation.
-        id: MessageId,
-        /// The peer from which we received this message, not the peer that published it.
-        source: PeerId,
-        /// The topics that this message was sent on.
-        topics: Vec<TopicHash>,
-        /// The message itself.
-        message: PubsubMessage<TSpec>,
-    },
-    /// Subscribed to peer for given topic
-    PeerSubscribed(PeerId, TopicHash),
-    /// Inform the network to send a Status to this peer.
-    StatusPeer(PeerId),
+/// Persist metadata to disk
+pub fn save_metadata_to_disk<E: EthSpec>(dir: &PathBuf, metadata: MetaData<E>, log: &slog::Logger) {
+    let _ = std::fs::create_dir_all(&dir);
+    match File::create(dir.join(METADATA_FILENAME))
+        .and_then(|mut f| f.write_all(&metadata.as_ssz_bytes()))
+    {
+        Ok(_) => {
+            debug!(log, "Metadata written to disk");
+        }
+        Err(e) => {
+            warn!(
+                log,
+                "Could not write metadata to disk";
+                "file" => format!("{:?}{:?}",dir, METADATA_FILENAME),
+                "error" => format!("{}", e)
+            );
+        }
+    }
 }

beacon_node/eth2_libp2p/src/config.rs

@@ -1,8 +1,12 @@
-use crate::types::GossipKind;
-use crate::Enr;
+use crate::types::{GossipKind, MessageData};
+use crate::{Enr, PeerIdSerialized};
+use directory::{
+    DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_TESTNET, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR,
+};
 use discv5::{Discv5Config, Discv5ConfigBuilder};
 use libp2p::gossipsub::{
-    GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode,
+    FastMessageId, GenericGossipsubConfig, GenericGossipsubConfigBuilder, GenericGossipsubMessage,
+    MessageId, RawGossipsubMessage, ValidationMode,
 };
 use libp2p::Multiaddr;
 use serde_derive::{Deserialize, Serialize};
@@ -11,6 +15,12 @@ use std::path::PathBuf;
 use std::time::Duration;
 
 pub const GOSSIP_MAX_SIZE: usize = 1_048_576;
+const MESSAGE_DOMAIN_INVALID_SNAPPY: [u8; 4] = [0, 0, 0, 0];
+const MESSAGE_DOMAIN_VALID_SNAPPY: [u8; 4] = [1, 0, 0, 0];
+
+pub type GossipsubConfig = GenericGossipsubConfig<MessageData>;
+pub type GossipsubConfigBuilder = GenericGossipsubConfigBuilder<MessageData>;
+pub type GossipsubMessage = GenericGossipsubMessage<MessageData>;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
@@ -50,17 +60,26 @@ pub struct Config {
     pub discv5_config: Discv5Config,
 
     /// List of nodes to initially connect to.
-    pub boot_nodes: Vec<Enr>,
+    pub boot_nodes_enr: Vec<Enr>,
+
+    /// List of nodes to initially connect to, on Multiaddr format.
+    pub boot_nodes_multiaddr: Vec<Multiaddr>,
 
     /// List of libp2p nodes to initially connect to.
     pub libp2p_nodes: Vec<Multiaddr>,
 
+    /// List of trusted libp2p nodes which are not scored.
+    pub trusted_peers: Vec<PeerIdSerialized>,
+
     /// Client version
     pub client_version: String,
 
     /// Disables the discovery protocol from starting.
     pub disable_discovery: bool,
 
+    /// Attempt to construct external port mappings with UPnP.
+    pub upnp_enabled: bool,
+
     /// List of extra topics to initially subscribe to as strings.
     pub topics: Vec<GossipKind>,
 }
@@ -68,26 +87,40 @@ pub struct Config {
 impl Default for Config {
     /// Generate a default network configuration.
     fn default() -> Self {
-        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
-        network_dir.push(".lighthouse");
-        network_dir.push("network");
-
-        // The default topics that we will initially subscribe to
-        let topics = vec![
-            GossipKind::BeaconBlock,
-            GossipKind::BeaconAggregateAndProof,
-            GossipKind::VoluntaryExit,
-            GossipKind::ProposerSlashing,
-            GossipKind::AttesterSlashing,
-        ];
+        // WARNING: this directory default should be always overrided with parameters
+        // from cli for specific networks.
+        let network_dir = dirs::home_dir()
+            .unwrap_or_else(|| PathBuf::from("."))
+            .join(DEFAULT_ROOT_DIR)
+            .join(DEFAULT_HARDCODED_TESTNET)
+            .join(DEFAULT_BEACON_NODE_DIR)
+            .join(DEFAULT_NETWORK_DIR);
 
         // The function used to generate a gossipsub message id
-        // We use base64(SHA256(data)) for content addressing
+        // We use the first 8 bytes of SHA256(data) for content addressing
+        let fast_gossip_message_id =
+            |message: &RawGossipsubMessage| FastMessageId::from(&Sha256::digest(&message.data)[..]);
+
+        fn prefix(prefix: [u8; 4], data: &[u8]) -> Vec<u8> {
+            prefix
+                .to_vec()
+                .into_iter()
+                .chain(data.iter().cloned())
+                .collect()
+        }
+
         let gossip_message_id = |message: &GossipsubMessage| {
-            MessageId::from(base64::encode_config(
-                &Sha256::digest(&message.data),
-                base64::URL_SAFE_NO_PAD,
-            ))
+            MessageId::from(
+                &Sha256::digest(
+                    {
+                        match &message.data.decompressed {
+                            Ok(decompressed) => prefix(MESSAGE_DOMAIN_VALID_SNAPPY, decompressed),
+                            _ => prefix(MESSAGE_DOMAIN_INVALID_SNAPPY, &message.data.raw),
+                        }
+                    }
+                    .as_slice(),
+                )[..20],
+            )
         };
 
         // gossipsub configuration
@@ -95,13 +128,22 @@ impl Default for Config {
         // parameter.
         let gs_config = GossipsubConfigBuilder::new()
             .max_transmit_size(GOSSIP_MAX_SIZE)
-            .heartbeat_interval(Duration::from_secs(1))
+            .heartbeat_interval(Duration::from_millis(700))
+            .mesh_n(8)
+            .mesh_n_low(6)
+            .mesh_n_high(12)
+            .gossip_lazy(6)
+            .fanout_ttl(Duration::from_secs(60))
+            .history_length(6)
+            .history_gossip(3)
             .validate_messages() // require validation before propagation
-            .validation_mode(ValidationMode::Permissive)
-            // Prevent duplicates by caching messages from an epoch + 1 slot amount of time (33*12)
-            .duplicate_cache_time(Duration::from_secs(396))
+            .validation_mode(ValidationMode::Anonymous)
+            // prevent duplicates for 550 heartbeats(700millis * 550) = 385 secs
+            .duplicate_cache_time(Duration::from_secs(385))
             .message_id_fn(gossip_message_id)
-            .build();
+            .fast_message_id_fn(fast_gossip_message_id)
+            .build()
+            .expect("valid gossipsub configuration");
 
         // discv5 configuration
         let discv5_config = Discv5ConfigBuilder::new()
@@ -111,6 +153,7 @@ impl Default for Config {
             .request_retries(1)
             .enr_peer_update_min(10)
             .query_parallelism(5)
+            .disable_report_discovered_peers()
             .query_timeout(Duration::from_secs(30))
             .query_peer_timeout(Duration::from_secs(2))
             .ip_limit() // limits /24 IP's in buckets.
@@ -129,11 +172,14 @@ impl Default for Config {
             target_peers: 50,
             gs_config,
             discv5_config,
-            boot_nodes: vec![],
+            boot_nodes_enr: vec![],
+            boot_nodes_multiaddr: vec![],
             libp2p_nodes: vec![],
+            trusted_peers: vec![],
             client_version: lighthouse_version::version_with_platform(),
             disable_discovery: false,
-            topics,
+            upnp_enabled: true,
+            topics: Vec::new(),
         }
     }
 }

beacon_node/eth2_libp2p/src/discovery/enr.rs

@@ -6,6 +6,7 @@ use super::enr_ext::CombinedKeyExt;
 use super::ENR_FILENAME;
 use crate::types::{Enr, EnrBitfield};
 use crate::NetworkConfig;
+use discv5::enr::EnrKey;
 use libp2p::core::identity::Keypair;
 use slog::{debug, warn};
 use ssz::{Decode, Encode};
@@ -48,6 +49,56 @@ impl Eth2Enr for Enr {
     }
 }
 
+/// Either use the given ENR or load an ENR from file if it exists and matches the current NodeId
+/// and sequence number.
+/// If an ENR exists, with the same NodeId, this function checks to see if the loaded ENR from
+/// disk is suitable to use, otherwise we increment the given ENR's sequence number.
+pub fn use_or_load_enr(
+    enr_key: &CombinedKey,
+    local_enr: &mut Enr,
+    config: &NetworkConfig,
+    log: &slog::Logger,
+) -> Result<(), String> {
+    let enr_f = config.network_dir.join(ENR_FILENAME);
+    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
+        let mut enr_string = String::new();
+        match enr_file.read_to_string(&mut enr_string) {
+            Err(_) => debug!(log, "Could not read ENR from file"),
+            Ok(_) => {
+                match Enr::from_str(&enr_string) {
+                    Ok(disk_enr) => {
+                        // if the same node id, then we may need to update our sequence number
+                        if local_enr.node_id() == disk_enr.node_id() {
+                            if compare_enr(&local_enr, &disk_enr) {
+                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
+                                // the stored ENR has the same configuration, use it
+                                *local_enr = disk_enr;
+                                return Ok(());
+                            }
+
+                            // same node id, different configuration - update the sequence number
+                            // Note: local_enr is generated with default(0) attnets value,
+                            // so a non default value in persisted enr will also update sequence number.
+                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
+                            local_enr.set_seq(new_seq_no, enr_key).map_err(|e| {
+                                format!("Could not update ENR sequence number: {:?}", e)
+                            })?;
+                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
+                        }
+                    }
+                    Err(e) => {
+                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
+                    }
+                }
+            }
+        }
+    }
+
+    save_enr_to_disk(&config.network_dir, &local_enr, log);
+
+    Ok(())
+}
+
 /// Loads an ENR from file if it exists and matches the current NodeId and sequence number. If none
 /// exists, generates a new one.
 ///
@@ -65,49 +116,11 @@ pub fn build_or_load_enr<T: EthSpec>(
     let enr_key = CombinedKey::from_libp2p(&local_key)?;
     let mut local_enr = build_enr::<T>(&enr_key, config, enr_fork_id)?;
 
-    let enr_f = config.network_dir.join(ENR_FILENAME);
-    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
-        let mut enr_string = String::new();
-        match enr_file.read_to_string(&mut enr_string) {
-            Err(_) => debug!(log, "Could not read ENR from file"),
-            Ok(_) => {
-                match Enr::from_str(&enr_string) {
-                    Ok(disk_enr) => {
-                        // if the same node id, then we may need to update our sequence number
-                        if local_enr.node_id() == disk_enr.node_id() {
-                            if compare_enr(&local_enr, &disk_enr) {
-                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
-                                // the stored ENR has the same configuration, use it
-                                return Ok(disk_enr);
-                            }
-
-                            // same node id, different configuration - update the sequence number
-                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
-                            local_enr.set_seq(new_seq_no, &enr_key).map_err(|e| {
-                                format!("Could not update ENR sequence number: {:?}", e)
-                            })?;
-                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
-                        }
-                    }
-                    Err(e) => {
-                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
-                    }
-                }
-            }
-        }
-    }
-
-    save_enr_to_disk(&config.network_dir, &local_enr, log);
-
+    use_or_load_enr(&enr_key, &mut local_enr, config, log)?;
     Ok(local_enr)
 }
 
-/// Builds a lighthouse ENR given a `NetworkConfig`.
-pub fn build_enr<T: EthSpec>(
-    enr_key: &CombinedKey,
-    config: &NetworkConfig,
-    enr_fork_id: EnrForkId,
-) -> Result<Enr, String> {
+pub fn create_enr_builder_from_config<T: EnrKey>(config: &NetworkConfig) -> EnrBuilder<T> {
     let mut builder = EnrBuilder::new("v4");
     if let Some(enr_address) = config.enr_address {
         builder.ip(enr_address);
@@ -116,20 +129,28 @@ pub fn build_enr<T: EthSpec>(
         builder.udp(udp_port);
     }
     // we always give it our listening tcp port
-    // TODO: Add uPnP support to map udp and tcp ports
     let tcp_port = config.enr_tcp_port.unwrap_or_else(|| config.libp2p_port);
-    builder.tcp(tcp_port);
+    builder.tcp(tcp_port).tcp(config.libp2p_port);
+    builder
+}
+
+/// Builds a lighthouse ENR given a `NetworkConfig`.
+pub fn build_enr<T: EthSpec>(
+    enr_key: &CombinedKey,
+    config: &NetworkConfig,
+    enr_fork_id: EnrForkId,
+) -> Result<Enr, String> {
+    let mut builder = create_enr_builder_from_config(config);
 
     // set the `eth2` field on our ENR
-    builder.add_value(ETH2_ENR_KEY.into(), enr_fork_id.as_ssz_bytes());
+    builder.add_value(ETH2_ENR_KEY, &enr_fork_id.as_ssz_bytes());
 
     // set the "attnets" field on our ENR
     let bitfield = BitVector::<T::SubnetBitfieldLength>::new();
 
-    builder.add_value(BITFIELD_ENR_KEY.into(), bitfield.as_ssz_bytes());
+    builder.add_value(BITFIELD_ENR_KEY, &bitfield.as_ssz_bytes());
 
     builder
-        .tcp(config.libp2p_port)
         .build(enr_key)
         .map_err(|e| format!("Could not build Local ENR: {:?}", e))
 }

beacon_node/eth2_libp2p/src/discovery/enr_ext.rs

@@ -13,9 +13,15 @@ pub trait EnrExt {
     /// The vector remains empty if these fields are not defined.
     fn multiaddr(&self) -> Vec<Multiaddr>;
 
-    /// Returns the multiaddr with the `PeerId` prepended.
+    /// Returns a list of multiaddrs with the `PeerId` prepended.
     fn multiaddr_p2p(&self) -> Vec<Multiaddr>;
 
+    /// Returns any multiaddrs that contain the TCP protocol with the `PeerId` prepended.
+    fn multiaddr_p2p_tcp(&self) -> Vec<Multiaddr>;
+
+    /// Returns any multiaddrs that contain the UDP protocol with the `PeerId` prepended.
+    fn multiaddr_p2p_udp(&self) -> Vec<Multiaddr>;
+
     /// Returns any multiaddrs that contain the TCP protocol.
     fn multiaddr_tcp(&self) -> Vec<Multiaddr>;
 }
@@ -111,6 +117,58 @@ impl EnrExt for Enr {
         multiaddrs
     }
 
+    /// Returns a list of multiaddrs if the ENR has an `ip` and a `tcp` key **or** an `ip6` and a `tcp6`.
+    /// The vector remains empty if these fields are not defined.
+    ///
+    /// This also prepends the `PeerId` into each multiaddr with the `P2p` protocol.
+    fn multiaddr_p2p_tcp(&self) -> Vec<Multiaddr> {
+        let peer_id = self.peer_id();
+        let mut multiaddrs: Vec<Multiaddr> = Vec::new();
+        if let Some(ip) = self.ip() {
+            if let Some(tcp) = self.tcp() {
+                let mut multiaddr: Multiaddr = ip.into();
+                multiaddr.push(Protocol::Tcp(tcp));
+                multiaddr.push(Protocol::P2p(peer_id.clone().into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        if let Some(ip6) = self.ip6() {
+            if let Some(tcp6) = self.tcp6() {
+                let mut multiaddr: Multiaddr = ip6.into();
+                multiaddr.push(Protocol::Tcp(tcp6));
+                multiaddr.push(Protocol::P2p(peer_id.into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        multiaddrs
+    }
+
+    /// Returns a list of multiaddrs if the ENR has an `ip` and a `udp` key **or** an `ip6` and a `udp6`.
+    /// The vector remains empty if these fields are not defined.
+    ///
+    /// This also prepends the `PeerId` into each multiaddr with the `P2p` protocol.
+    fn multiaddr_p2p_udp(&self) -> Vec<Multiaddr> {
+        let peer_id = self.peer_id();
+        let mut multiaddrs: Vec<Multiaddr> = Vec::new();
+        if let Some(ip) = self.ip() {
+            if let Some(udp) = self.udp() {
+                let mut multiaddr: Multiaddr = ip.into();
+                multiaddr.push(Protocol::Udp(udp));
+                multiaddr.push(Protocol::P2p(peer_id.clone().into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        if let Some(ip6) = self.ip6() {
+            if let Some(udp6) = self.udp6() {
+                let mut multiaddr: Multiaddr = ip6.into();
+                multiaddr.push(Protocol::Udp(udp6));
+                multiaddr.push(Protocol::P2p(peer_id.into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        multiaddrs
+    }
+
     /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`.
     /// The vector remains empty if these fields are not defined.
     fn multiaddr_tcp(&self) -> Vec<Multiaddr> {
@@ -140,7 +198,7 @@ impl CombinedKeyPublicExt for CombinedPublicKey {
     fn into_peer_id(&self) -> PeerId {
         match self {
             Self::Secp256k1(pk) => {
-                let pk_bytes = pk.serialize_compressed();
+                let pk_bytes = pk.to_bytes();
                 let libp2p_pk = libp2p::core::PublicKey::Secp256k1(
                     libp2p::core::identity::secp256k1::PublicKey::decode(&pk_bytes)
                         .expect("valid public key"),
@@ -163,7 +221,7 @@ impl CombinedKeyExt for CombinedKey {
     fn from_libp2p(key: &libp2p::core::identity::Keypair) -> Result<CombinedKey, &'static str> {
         match key {
             Keypair::Secp256k1(key) => {
-                let secret = discv5::enr::secp256k1::SecretKey::parse(&key.secret().to_bytes())
+                let secret = discv5::enr::k256::ecdsa::SigningKey::new(&key.secret().to_bytes())
                     .expect("libp2p key must be valid");
                 Ok(CombinedKey::Secp256k1(secret))
             }
@@ -219,7 +277,7 @@ mod tests {
     fn test_secp256k1_peer_id_conversion() {
         let sk_hex = "df94a73d528434ce2309abb19c16aedb535322797dbd59c157b1e04095900f48";
         let sk_bytes = hex::decode(sk_hex).unwrap();
-        let secret_key = discv5::enr::secp256k1::SecretKey::parse_slice(&sk_bytes).unwrap();
+        let secret_key = discv5::enr::k256::ecdsa::SigningKey::new(&sk_bytes).unwrap();
 
         let libp2p_sk = libp2p::identity::secp256k1::SecretKey::from_bytes(sk_bytes).unwrap();
         let secp256k1_kp: libp2p::identity::secp256k1::Keypair = libp2p_sk.into();

beacon_node/eth2_libp2p/src/discovery/mod.rs

@@ -3,24 +3,24 @@ pub(crate) mod enr;
 pub mod enr_ext;
 
 // Allow external use of the lighthouse ENR builder
-pub use enr::{build_enr, CombinedKey, Eth2Enr};
-pub use enr_ext::{CombinedKeyExt, EnrExt};
-pub use libp2p::core::identity::Keypair;
+pub use enr::{build_enr, create_enr_builder_from_config, use_or_load_enr, CombinedKey, Eth2Enr};
+pub use enr_ext::{peer_id_to_node_id, CombinedKeyExt, EnrExt};
+pub use libp2p::core::identity::{Keypair, PublicKey};
 
 use crate::metrics;
-use crate::{error, Enr, NetworkConfig, NetworkGlobals};
+use crate::{error, Enr, NetworkConfig, NetworkGlobals, SubnetDiscovery};
 use discv5::{enr::NodeId, Discv5, Discv5Event};
 use enr::{BITFIELD_ENR_KEY, ETH2_ENR_KEY};
 use futures::prelude::*;
 use futures::stream::FuturesUnordered;
 use libp2p::core::PeerId;
 use lru::LruCache;
-use slog::{crit, debug, info, warn};
+use slog::{crit, debug, error, info, warn};
 use ssz::{Decode, Encode};
 use ssz_types::BitVector;
 use std::{
     collections::{HashMap, VecDeque},
-    net::SocketAddr,
+    net::{IpAddr, SocketAddr},
     path::Path,
     pin::Pin,
     sync::Arc,
@@ -31,12 +31,12 @@ use tokio::sync::mpsc;
 use types::{EnrForkId, EthSpec, SubnetId};
 
 mod subnet_predicate;
-use subnet_predicate::subnet_predicate;
+pub use subnet_predicate::subnet_predicate;
 
 /// Local ENR storage filename.
 pub const ENR_FILENAME: &str = "enr.dat";
 /// Target number of peers we'd like to have connected to a given long-lived subnet.
-const TARGET_SUBNET_PEERS: usize = 3;
+pub const TARGET_SUBNET_PEERS: usize = 3;
 /// Target number of peers to search for given a grouped subnet query.
 const TARGET_PEERS_FOR_GROUPED_QUERY: usize = 6;
 /// Number of times to attempt a discovery request.
@@ -155,7 +155,7 @@ pub struct Discovery<TSpec: EthSpec> {
 
     /// Indicates if the discovery service has been started. When the service is disabled, this is
     /// always false.
-    started: bool,
+    pub started: bool,
 
     /// Logger for the discovery behaviour.
     log: slog::Logger,
@@ -163,7 +163,7 @@ pub struct Discovery<TSpec: EthSpec> {
 
 impl<TSpec: EthSpec> Discovery<TSpec> {
     /// NOTE: Creating discovery requires running within a tokio execution environment.
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         config: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
@@ -189,35 +189,81 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             .map_err(|e| format!("Discv5 service failed. Error: {:?}", e))?;
 
         // Add bootnodes to routing table
-        for bootnode_enr in config.boot_nodes.clone() {
+        for bootnode_enr in config.boot_nodes_enr.clone() {
             debug!(
                 log,
                 "Adding node to routing table";
-                "node_id" => format!("{}", bootnode_enr.node_id()),
-                "peer_id" => format!("{}", bootnode_enr.peer_id()),
+                "node_id" => bootnode_enr.node_id().to_string(),
+                "peer_id" => bootnode_enr.peer_id().to_string(),
                 "ip" => format!("{:?}", bootnode_enr.ip()),
                 "udp" => format!("{:?}", bootnode_enr.udp()),
                 "tcp" => format!("{:?}", bootnode_enr.tcp())
             );
+            let repr = bootnode_enr.to_string();
             let _ = discv5.add_enr(bootnode_enr).map_err(|e| {
-                debug!(
+                error!(
                     log,
                     "Could not add peer to the local routing table";
-                    "error" => e.to_string()
+                    "addr" => repr,
+                    "error" => e.to_string(),
                 )
             });
         }
 
         // Start the discv5 service and obtain an event stream
         let event_stream = if !config.disable_discovery {
-            discv5.start(listen_socket);
+            discv5.start(listen_socket).map_err(|e| e.to_string())?;
             debug!(log, "Discovery service started");
             EventStream::Awaiting(Box::pin(discv5.event_stream()))
         } else {
             EventStream::InActive
         };
 
-        // Obtain the event stream
+        if !config.boot_nodes_multiaddr.is_empty() {
+            info!(log, "Contacting Multiaddr boot-nodes for their ENR");
+        }
+
+        // get futures for requesting the Enrs associated to these multiaddr and wait for their
+        // completion
+        let mut fut_coll = config
+            .boot_nodes_multiaddr
+            .iter()
+            .map(|addr| addr.to_string())
+            // request the ENR for this multiaddr and keep the original for logging
+            .map(|addr| {
+                futures::future::join(
+                    discv5.request_enr(addr.clone()),
+                    futures::future::ready(addr),
+                )
+            })
+            .collect::<FuturesUnordered<_>>();
+
+        while let Some((result, original_addr)) = fut_coll.next().await {
+            match result {
+                Ok(enr) => {
+                    debug!(
+                        log,
+                        "Adding node to routing table";
+                        "node_id" => enr.node_id().to_string(),
+                        "peer_id" => enr.peer_id().to_string(),
+                        "ip" => format!("{:?}", enr.ip()),
+                        "udp" => format!("{:?}", enr.udp()),
+                        "tcp" => format!("{:?}", enr.tcp())
+                    );
+                    let _ = discv5.add_enr(enr).map_err(|e| {
+                        error!(
+                            log,
+                            "Could not add peer to the local routing table";
+                            "addr" => original_addr.to_string(),
+                            "error" => e.to_string(),
+                        )
+                    });
+                }
+                Err(e) => {
+                    error!(log, "Error getting mapping to ENR"; "multiaddr" => original_addr.to_string(), "error" => e.to_string())
+                }
+            }
+        }
 
         Ok(Self {
             cached_enrs: LruCache::new(50),
@@ -238,6 +284,11 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         self.discv5.local_enr()
     }
 
+    /// Return the cached enrs.
+    pub fn cached_enrs(&self) -> impl Iterator<Item = (&PeerId, &Enr)> {
+        self.cached_enrs.iter()
+    }
+
     /// This adds a new `FindPeers` query to the queue if one doesn't already exist.
     pub fn discover_peers(&mut self) {
         // If the discv5 service isn't running or we are in the process of a query, don't bother queuing a new one.
@@ -256,12 +307,19 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
     }
 
     /// Processes a request to search for more peers on a subnet.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
+    pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
         // If the discv5 service isn't running, ignore queries
         if !self.started {
             return;
         }
-        self.add_subnet_query(subnet_id, min_ttl, 0);
+        debug!(
+            self.log,
+            "Making discovery query for subnets";
+            "subnets" => format!("{:?}", subnets_to_discover.iter().map(|s| s.subnet_id).collect::<Vec<_>>())
+        );
+        for subnet in subnets_to_discover {
+            self.add_subnet_query(subnet.subnet_id, subnet.min_ttl, 0);
+        }
     }
 
     /// Add an ENR to the routing table of the discovery mechanism.
@@ -297,6 +355,54 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         }
     }
 
+    /// Updates the local ENR TCP port.
+    /// There currently isn't a case to update the address here. We opt for discovery to
+    /// automatically update the external address.
+    ///
+    /// If the external address needs to be modified, use `update_enr_udp_socket.
+    pub fn update_enr_tcp_port(&mut self, port: u16) -> Result<(), String> {
+        self.discv5
+            .enr_insert("tcp", &port.to_be_bytes())
+            .map_err(|e| format!("{:?}", e))?;
+
+        // replace the global version
+        *self.network_globals.local_enr.write() = self.discv5.local_enr();
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
+        Ok(())
+    }
+
+    /// Updates the local ENR UDP socket.
+    ///
+    /// This is with caution. Discovery should automatically maintain this. This should only be
+    /// used when automatic discovery is disabled.
+    pub fn update_enr_udp_socket(&mut self, socket_addr: SocketAddr) -> Result<(), String> {
+        match socket_addr {
+            SocketAddr::V4(socket) => {
+                self.discv5
+                    .enr_insert("ip", &socket.ip().octets())
+                    .map_err(|e| format!("{:?}", e))?;
+                self.discv5
+                    .enr_insert("udp", &socket.port().to_be_bytes())
+                    .map_err(|e| format!("{:?}", e))?;
+            }
+            SocketAddr::V6(socket) => {
+                self.discv5
+                    .enr_insert("ip6", &socket.ip().octets())
+                    .map_err(|e| format!("{:?}", e))?;
+                self.discv5
+                    .enr_insert("udp6", &socket.port().to_be_bytes())
+                    .map_err(|e| format!("{:?}", e))?;
+            }
+        }
+
+        // replace the global version
+        *self.network_globals.local_enr.write() = self.discv5.local_enr();
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
+        Ok(())
+    }
+
     /// Adds/Removes a subnet from the ENR Bitfield
     pub fn update_enr_bitfield(&mut self, subnet_id: SubnetId, value: bool) -> Result<(), String> {
         let id = *subnet_id as usize;
@@ -329,12 +435,15 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             .map_err(|_| String::from("Subnet ID out of bounds, could not set subnet ID"))?;
 
         // insert the bitfield into the ENR record
-        let _ = self
-            .discv5
-            .enr_insert(BITFIELD_ENR_KEY, current_bitfield.as_ssz_bytes());
+        self.discv5
+            .enr_insert(BITFIELD_ENR_KEY, &current_bitfield.as_ssz_bytes())
+            .map_err(|e| format!("{:?}", e))?;
 
         // replace the global version
         *self.network_globals.local_enr.write() = self.discv5.local_enr();
+
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
         Ok(())
     }
 
@@ -356,7 +465,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         let _ = self
             .discv5
-            .enr_insert(ETH2_ENR_KEY, enr_fork_id.as_ssz_bytes())
+            .enr_insert(ETH2_ENR_KEY, &enr_fork_id.as_ssz_bytes())
             .map_err(|e| {
                 warn!(
                     self.log,
@@ -367,6 +476,36 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         // replace the global version with discovery version
         *self.network_globals.local_enr.write() = self.discv5.local_enr();
+
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
+    }
+
+    // Bans a peer and it's associated seen IP addresses.
+    pub fn ban_peer(&mut self, peer_id: &PeerId, ip_addresses: Vec<IpAddr>) {
+        // first try and convert the peer_id to a node_id.
+        if let Ok(node_id) = peer_id_to_node_id(peer_id) {
+            // If we could convert this peer id, remove it from the DHT and ban it from discovery.
+            self.discv5.ban_node(&node_id);
+            // Remove the node from the routing table.
+            self.discv5.remove_node(&node_id);
+        }
+
+        for ip_address in ip_addresses {
+            self.discv5.ban_ip(ip_address);
+        }
+    }
+
+    pub fn unban_peer(&mut self, peer_id: &PeerId, ip_addresses: Vec<IpAddr>) {
+        // first try and convert the peer_id to a node_id.
+        if let Ok(node_id) = peer_id_to_node_id(peer_id) {
+            // If we could convert this peer id, remove it from the DHT and ban it from discovery.
+            self.discv5.permit_node(&node_id);
+        }
+
+        for ip_address in ip_addresses {
+            self.discv5.permit_ip(ip_address);
+        }
     }
 
     /* Internal Functions */
@@ -459,6 +598,11 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                         // This query is for searching for peers of a particular subnet
                         // Drain subnet_queries so we can re-use it as we continue to process the queue
                         let grouped_queries: Vec<SubnetQuery> = subnet_queries.drain(..).collect();
+                        debug!(
+                            self.log,
+                            "Starting grouped subnet query";
+                            "subnets" => format!("{:?}", grouped_queries.iter().map(|q| q.subnet_id).collect::<Vec<_>>()),
+                        );
                         self.start_subnet_query(grouped_queries);
                     }
                 }
@@ -491,7 +635,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                     .peers_on_subnet(subnet_query.subnet_id)
                     .count();
 
-                if peers_on_subnet > TARGET_SUBNET_PEERS {
+                if peers_on_subnet >= TARGET_SUBNET_PEERS {
                     debug!(self.log, "Discovery ignored";
                         "reason" => "Already connected to desired peers",
                         "connected_peers_on_subnet" => peers_on_subnet,
@@ -711,7 +855,10 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                 // Still awaiting the event stream, poll it
                 if let Poll::Ready(event_stream) = fut.poll_unpin(cx) {
                     match event_stream {
-                        Ok(stream) => self.event_stream = EventStream::Present(stream),
+                        Ok(stream) => {
+                            debug!(self.log, "Discv5 event stream ready");
+                            self.event_stream = EventStream::Present(stream);
+                        }
                         Err(e) => {
                             slog::crit!(self.log, "Discv5 event stream failed"; "error" => e.to_string());
                             self.event_stream = EventStream::InActive;
@@ -733,8 +880,8 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                             if enr.eth2() == self.local_enr().eth2() {
                                 trace!(self.log, "Peer found in process of query"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
                             } else {
-                                // this is temporary warning for debugging the DHT
-                                warn!(self.log, "Found peer during discovery not on correct fork"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
+                            // this is temporary warning for debugging the DHT
+                            warn!(self.log, "Found peer during discovery not on correct fork"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
                             }
                             */
                         }

beacon_node/eth2_libp2p/src/discovery/subnet_predicate.rs

@@ -1,5 +1,6 @@
 ///! The subnet predicate used for searching for a particular subnet.
 use super::*;
+use slog::trace;
 use std::ops::Deref;
 
 /// Returns the predicate for a given subnet.
@@ -30,14 +31,14 @@ where
                 .collect();
 
             if matches.is_empty() {
-                debug!(
+                trace!(
                     log_clone,
                     "Peer found but not on any of the desired subnets";
                     "peer_id" => format!("{}", enr.peer_id())
                 );
                 return false;
             } else {
-                debug!(
+                trace!(
                    log_clone,
                    "Peer found on desired subnet(s)";
                    "peer_id" => format!("{}", enr.peer_id()),

beacon_node/eth2_libp2p/src/lib.rs

@@ -14,16 +14,62 @@ pub mod rpc;
 mod service;
 pub mod types;
 
-pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage};
-pub use behaviour::{BehaviourEvent, PeerRequestId, Request, Response};
+use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
+use std::str::FromStr;
+
+/// Wrapper over a libp2p `PeerId` which implements `Serialize` and `Deserialize`
+#[derive(Clone, Debug)]
+pub struct PeerIdSerialized(libp2p::PeerId);
+
+impl From<PeerIdSerialized> for PeerId {
+    fn from(peer_id: PeerIdSerialized) -> Self {
+        peer_id.0
+    }
+}
+
+impl FromStr for PeerIdSerialized {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Self(
+            PeerId::from_str(s).map_err(|e| format!("Invalid peer id: {}", e))?,
+        ))
+    }
+}
+
+impl Serialize for PeerIdSerialized {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.0.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for PeerIdSerialized {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s: String = Deserialize::deserialize(deserializer)?;
+        Ok(Self(PeerId::from_str(&s).map_err(|e| {
+            de::Error::custom(format!("Failed to deserialise peer id: {:?}", e))
+        })?))
+    }
+}
+
+pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage, SubnetDiscovery};
+pub use behaviour::{BehaviourEvent, Gossipsub, PeerRequestId, Request, Response};
 pub use config::Config as NetworkConfig;
+pub use config::{GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage};
 pub use discovery::{CombinedKeyExt, EnrExt, Eth2Enr};
 pub use discv5;
-pub use libp2p::gossipsub::{MessageId, Topic, TopicHash};
+pub use libp2p::gossipsub::{MessageAcceptance, MessageId, Topic, TopicHash};
 pub use libp2p::{core::ConnectedPoint, PeerId, Swarm};
 pub use libp2p::{multiaddr, Multiaddr};
 pub use metrics::scrape_discovery_metrics;
 pub use peer_manager::{
-    client::Client, score::PeerAction, PeerDB, PeerInfo, PeerSyncStatus, SyncInfo,
+    client::Client, score::PeerAction, ConnectionDirection, PeerConnectionStatus, PeerDB, PeerInfo,
+    PeerSyncStatus, SyncInfo,
 };
-pub use service::{Libp2pEvent, Service, NETWORK_KEY_FILENAME};
+pub use service::{load_private_key, Libp2pEvent, Service, NETWORK_KEY_FILENAME};

beacon_node/eth2_libp2p/src/metrics.rs

@@ -34,9 +34,20 @@ lazy_static! {
         "Unsolicited discovery requests per ip per second",
         &["Addresses"]
     );
-    pub static ref GOSSIPSUB_SUBSCRIBED_PEERS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
-        "gossipsub_peers_per_topic_count",
-        "Peers subscribed per topic",
+    pub static ref PEERS_PER_CLIENT: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "libp2p_peers_per_client",
+        "The connected peers via client implementation",
+        &["Client"]
+    );
+    pub static ref FAILED_ATTESTATION_PUBLISHES_PER_SUBNET: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "gossipsub_failed_attestation_publishes_per_subnet",
+            "Failed attestation publishes per subnet",
+            &["subnet"]
+        );
+    pub static ref FAILED_PUBLISHES_PER_MAIN_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_failed_publishes_per_main_topic",
+        "Failed gossip publishes",
         &["topic_hash"]
     );
 }

beacon_node/eth2_libp2p/src/peer_manager/client.rs

@@ -20,7 +20,7 @@ pub struct Client {
     pub agent_string: Option<String>,
 }
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, PartialEq)]
 pub enum ClientKind {
     /// A lighthouse node (the best kind).
     Lighthouse,
@@ -30,6 +30,8 @@ pub enum ClientKind {
     Teku,
     /// A Prysm node.
     Prysm,
+    /// A lodestar node.
+    Lodestar,
     /// An unknown client.
     Unknown,
 }
@@ -84,6 +86,7 @@ impl std::fmt::Display for Client {
                 "Prysm: version: {}, os_version: {}",
                 self.version, self.os_version
             ),
+            ClientKind::Lodestar => write!(f, "Lodestar: version: {}", self.version),
             ClientKind::Unknown => {
                 if let Some(agent_string) = &self.agent_string {
                     write!(f, "Unknown: {}", agent_string)
@@ -95,6 +98,12 @@ impl std::fmt::Display for Client {
     }
 }
 
+impl std::fmt::Display for ClientKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
 // helper function to identify clients from their agent_version. Returns the client
 // kind and it's associated version and the OS kind.
 fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String) {
@@ -157,6 +166,18 @@ fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String
             }
             (kind, version, os_version)
         }
+        Some("js-libp2p") => {
+            let kind = ClientKind::Lodestar;
+            let mut version = String::from("unknown");
+            let mut os_version = version.clone();
+            if let Some(agent_version) = agent_split.next() {
+                version = agent_version.into();
+                if let Some(agent_os_version) = agent_split.next() {
+                    os_version = agent_os_version.into();
+                }
+            }
+            (kind, version, os_version)
+        }
         _ => {
             let unknown = String::from("unknown");
             (ClientKind::Unknown, unknown.clone(), unknown)

beacon_node/eth2_libp2p/src/peer_manager/mod.rs

@@ -1,10 +1,10 @@
 //! Implementation of a Lighthouse's peer management system.
 
 pub use self::peerdb::*;
-use crate::discovery::{Discovery, DiscoveryEvent};
+use crate::discovery::{subnet_predicate, Discovery, DiscoveryEvent, TARGET_SUBNET_PEERS};
 use crate::rpc::{GoodbyeReason, MetaData, Protocol, RPCError, RPCResponseErrorCode};
 use crate::{error, metrics};
-use crate::{EnrExt, NetworkConfig, NetworkGlobals, PeerId};
+use crate::{EnrExt, NetworkConfig, NetworkGlobals, PeerId, SubnetDiscovery};
 use futures::prelude::*;
 use futures::Stream;
 use hashset_delay::HashSetDelay;
@@ -29,14 +29,14 @@ mod peer_sync_status;
 mod peerdb;
 pub(crate) mod score;
 
-pub use peer_info::{PeerConnectionStatus::*, PeerInfo};
+pub use peer_info::{ConnectionDirection, PeerConnectionStatus, PeerConnectionStatus::*, PeerInfo};
 pub use peer_sync_status::{PeerSyncStatus, SyncInfo};
 use score::{PeerAction, ScoreState};
 use std::collections::HashMap;
 /// The time in seconds between re-status's peers.
 const STATUS_INTERVAL: u64 = 300;
-/// The time in seconds between PING events. We do not send a ping if the other peer as PING'd us within
-/// this time frame (Seconds)
+/// The time in seconds between PING events. We do not send a ping if the other peer has PING'd us
+/// within this time frame (Seconds)
 const PING_INTERVAL: u64 = 30;
 
 /// The heartbeat performs regular updates such as updating reputations and performing discovery
@@ -88,14 +88,14 @@ pub enum PeerManagerEvent {
 
 impl<TSpec: EthSpec> PeerManager<TSpec> {
     // NOTE: Must be run inside a tokio executor.
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         config: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
         log: &slog::Logger,
     ) -> error::Result<Self> {
         // start the discovery service
-        let mut discovery = Discovery::new(local_key, config, network_globals.clone(), log)?;
+        let mut discovery = Discovery::new(local_key, config, network_globals.clone(), log).await?;
 
         // start searching for peers
         discovery.discover_peers();
@@ -133,13 +133,21 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     pub fn goodbye_peer(&mut self, peer_id: &PeerId, reason: GoodbyeReason) {
         // get the peer info
         if let Some(info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
-            debug!(self.log, "Sending goodbye to peer"; "peer_id" => peer_id.to_string(), "reason" => reason.to_string(), "score" => info.score.to_string());
+            debug!(self.log, "Sending goodbye to peer"; "peer_id" => peer_id.to_string(), "reason" => reason.to_string(), "score" => info.score().to_string());
             // Goodbye's are fatal
-            info.score.apply_peer_action(PeerAction::Fatal);
-            if info.connection_status.is_connected_or_dialing() {
-                self.events
-                    .push(PeerManagerEvent::DisconnectPeer(peer_id.clone(), reason));
-            }
+            info.apply_peer_action_to_score(PeerAction::Fatal);
+        }
+
+        // Update the peerdb and peer state accordingly
+        if self
+            .network_globals
+            .peers
+            .write()
+            .disconnect_and_ban(peer_id)
+        {
+            // update the state of the peer.
+            self.events
+                .push(PeerManagerEvent::DisconnectPeer(peer_id.clone(), reason));
         }
     }
 
@@ -147,56 +155,55 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// If the peer doesn't exist, log a warning and insert defaults.
     pub fn report_peer(&mut self, peer_id: &PeerId, action: PeerAction) {
-        // TODO: Remove duplicate code  - This is duplicated in the update_peer_scores()
-        // function.
-
-        // Variables to update the PeerDb if required.
+        // Helper function to avoid any potential deadlocks.
         let mut ban_peer = None;
         let mut unban_peer = None;
 
-        if let Some(info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
-            let previous_state = info.score.state();
-            info.score.apply_peer_action(action);
-            if previous_state != info.score.state() {
-                match info.score.state() {
-                    ScoreState::Banned => {
-                        debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string());
-                        ban_peer = Some(peer_id.clone());
-                        if info.connection_status.is_connected_or_dialing() {
-                            self.events.push(PeerManagerEvent::DisconnectPeer(
-                                peer_id.clone(),
-                                GoodbyeReason::BadScore,
-                            ));
+        {
+            let mut peer_db = self.network_globals.peers.write();
+            if let Some(info) = peer_db.peer_info_mut(peer_id) {
+                let previous_state = info.score_state();
+                info.apply_peer_action_to_score(action);
+                if previous_state != info.score_state() {
+                    match info.score_state() {
+                        ScoreState::Banned => {
+                            debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string());
+                            ban_peer = Some(peer_id);
+                        }
+                        ScoreState::Disconnected => {
+                            debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
+                            // disconnect the peer if it's currently connected or dialing
+                            if info.is_connected_or_dialing() {
+                                self.events.push(PeerManagerEvent::DisconnectPeer(
+                                    peer_id.clone(),
+                                    GoodbyeReason::BadScore,
+                                ));
+                                peer_db.notify_disconnecting(peer_id);
+                            } else if info.is_banned() {
+                                unban_peer = Some(peer_id);
+                            }
+                        }
+                        ScoreState::Healthy => {
+                            debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
+                            // unban the peer if it was previously banned.
+                            if info.is_banned() {
+                                unban_peer = Some(peer_id);
+                            }
                         }
                     }
-                    ScoreState::Disconnected => {
-                        debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
-                        // disconnect the peer if it's currently connected or dialing
-                        unban_peer = Some(peer_id.clone());
-                        if info.connection_status.is_connected_or_dialing() {
-                            self.events.push(PeerManagerEvent::DisconnectPeer(
-                                peer_id.clone(),
-                                GoodbyeReason::BadScore,
-                            ));
-                        }
-                        // TODO: Update the peer manager to inform that the peer is disconnecting.
-                    }
-                    ScoreState::Healthy => {
-                        debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
-                        // unban the peer if it was previously banned.
-                        unban_peer = Some(peer_id.clone());
-                    }
+                } else {
+                    debug!(self.log, "Peer score adjusted"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string());
                 }
-            } else {
-                debug!(self.log, "Peer score adjusted"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string());
             }
-        }
+        } // end write lock
 
-        // Update the PeerDB state.
-        if let Some(peer_id) = ban_peer.take() {
-            self.network_globals.peers.write().ban(&peer_id);
-        } else if let Some(peer_id) = unban_peer.take() {
-            self.network_globals.peers.write().unban(&peer_id);
+        if let Some(peer_id) = ban_peer {
+            self.ban_peer(peer_id);
+        }
+        if let Some(peer_id) = unban_peer {
+            if let Err(e) = self.unban_peer(peer_id) {
+                error!(self.log, "{}", e; "peer_id" => %peer_id);
+            }
         }
     }
 
@@ -213,17 +220,48 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     }
 
     /// A request to find peers on a given subnet.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
-        // Extend the time to maintain peers if required.
-        if let Some(min_ttl) = min_ttl {
-            self.network_globals
-                .peers
-                .write()
-                .extend_peers_on_subnet(subnet_id, min_ttl);
-        }
+    pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
+        let filtered: Vec<SubnetDiscovery> = subnets_to_discover
+            .into_iter()
+            .filter(|s| {
+                // Extend min_ttl of connected peers on required subnets
+                if let Some(min_ttl) = s.min_ttl {
+                    self.network_globals
+                        .peers
+                        .write()
+                        .extend_peers_on_subnet(s.subnet_id, min_ttl);
+                }
+                // Already have target number of peers, no need for subnet discovery
+                let peers_on_subnet = self
+                    .network_globals
+                    .peers
+                    .read()
+                    .peers_on_subnet(s.subnet_id)
+                    .count();
+                if peers_on_subnet >= TARGET_SUBNET_PEERS {
+                    debug!(
+                        self.log,
+                        "Discovery query ignored";
+                        "subnet_id" => format!("{:?}",s.subnet_id),
+                        "reason" => "Already connected to desired peers",
+                        "connected_peers_on_subnet" => peers_on_subnet,
+                        "target_subnet_peers" => TARGET_SUBNET_PEERS,
+                    );
+                    false
+                // Queue an outgoing connection request to the cached peers that are on `s.subnet_id`.
+                // If we connect to the cached peers before the discovery query starts, then we potentially
+                // save a costly discovery query.
+                } else {
+                    self.dial_cached_enrs_in_subnet(s.subnet_id);
+                    true
+                }
+            })
+            .collect();
 
         // request the subnet query from discovery
-        self.discovery.discover_subnet_peers(subnet_id, min_ttl);
+        if !filtered.is_empty() {
+            self.discovery.discover_subnet_peers(filtered);
+        }
     }
 
     /// A STATUS message has been received from a peer. This resets the status timer.
@@ -237,16 +275,14 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// This is also called when dialing a peer fails.
     pub fn notify_disconnect(&mut self, peer_id: &PeerId) {
-        self.network_globals.peers.write().disconnect(peer_id);
+        self.network_globals
+            .peers
+            .write()
+            .notify_disconnect(peer_id);
 
         // remove the ping and status timer for the peer
         self.ping_peers.remove(peer_id);
         self.status_peers.remove(peer_id);
-        metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT);
-        metrics::set_gauge(
-            &metrics::PEERS_CONNECTED,
-            self.network_globals.connected_peers() as i64,
-        );
     }
 
     /// A dial attempt has failed.
@@ -262,20 +298,14 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
 
     /// Sets a peer as connected as long as their reputation allows it
     /// Informs if the peer was accepted
-    pub fn connect_ingoing(&mut self, peer_id: &PeerId) -> bool {
-        self.connect_peer(peer_id, ConnectingType::IngoingConnected)
+    pub fn connect_ingoing(&mut self, peer_id: &PeerId, multiaddr: Multiaddr) -> bool {
+        self.connect_peer(peer_id, ConnectingType::IngoingConnected { multiaddr })
     }
 
     /// Sets a peer as connected as long as their reputation allows it
     /// Informs if the peer was accepted
-    pub fn connect_outgoing(&mut self, peer_id: &PeerId) -> bool {
-        self.connect_peer(peer_id, ConnectingType::OutgoingConnected)
-    }
-
-    /// Updates the database informing that a peer is being disconnected.
-    pub fn _disconnecting_peer(&mut self, _peer_id: &PeerId) -> bool {
-        // TODO: implement
-        true
+    pub fn connect_outgoing(&mut self, peer_id: &PeerId, multiaddr: Multiaddr) -> bool {
+        self.connect_peer(peer_id, ConnectingType::OutgoingConnected { multiaddr })
     }
 
     /// Reports if a peer is banned or not.
@@ -294,8 +324,25 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// Updates `PeerInfo` with `identify` information.
     pub fn identify(&mut self, peer_id: &PeerId, info: &IdentifyInfo) {
         if let Some(peer_info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
+            let previous_kind = peer_info.client.kind.clone();
             peer_info.client = client::Client::from_identify_info(info);
             peer_info.listening_addresses = info.listen_addrs.clone();
+
+            if previous_kind != peer_info.client.kind {
+                // update the peer client kind metric
+                if let Some(v) = metrics::get_int_gauge(
+                    &metrics::PEERS_PER_CLIENT,
+                    &[&peer_info.client.kind.to_string()],
+                ) {
+                    v.inc()
+                };
+                if let Some(v) = metrics::get_int_gauge(
+                    &metrics::PEERS_PER_CLIENT,
+                    &[&previous_kind.to_string()],
+                ) {
+                    v.dec()
+                };
+            }
         } else {
             crit!(self.log, "Received an Identify response from an unknown peer"; "peer_id" => peer_id.to_string());
         }
@@ -338,10 +385,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                 // Not supporting a protocol shouldn't be considered a malicious action, but
                 // it is an action that in some cases will make the peer unfit to continue
                 // communicating.
-                // TODO: To avoid punishing a peer repeatedly for not supporting a protocol, this
-                // information could be stored and used to prevent sending requests for the given
-                // protocol to this peer. Similarly, to avoid blacklisting a peer for a protocol
-                // forever, if stored this information should expire.
+
                 match protocol {
                     Protocol::Ping => PeerAction::Fatal,
                     Protocol::BlocksByRange => return,
@@ -375,7 +419,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
 
     /// A ping request has been received.
     // NOTE: The behaviour responds with a PONG automatically
-    // TODO: Update last seen
     pub fn ping_request(&mut self, peer_id: &PeerId, seq: u64) {
         if let Some(peer_info) = self.network_globals.peers.read().peer_info(peer_id) {
             // received a ping
@@ -405,7 +448,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     }
 
     /// A PONG has been returned from a peer.
-    // TODO: Update last seen
     pub fn pong_response(&mut self, peer_id: &PeerId, seq: u64) {
         if let Some(peer_info) = self.network_globals.peers.read().peer_info(peer_id) {
             // received a pong
@@ -431,7 +473,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     }
 
     /// Received a metadata response from a peer.
-    // TODO: Update last seen
     pub fn meta_data_response(&mut self, peer_id: &PeerId, meta_data: MetaData<TSpec>) {
         if let Some(peer_info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
             if let Some(known_meta_data) = &peer_info.meta_data {
@@ -464,10 +505,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             let mut out_list = enr.multiaddr();
             out_list.retain(|addr| {
                 addr.iter()
-                    .find(|v| match v {
-                        MProtocol::Udp(_) => true,
-                        _ => false,
-                    })
+                    .find(|v| matches!(v, MProtocol::Udp(_)))
                     .is_none()
             });
 
@@ -491,6 +529,30 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         self.events.push(PeerManagerEvent::SocketUpdated(multiaddr));
     }
 
+    /// Dial cached enrs in discovery service that are in the given `subnet_id` and aren't
+    /// in Connected, Dialing or Banned state.
+    fn dial_cached_enrs_in_subnet(&mut self, subnet_id: SubnetId) {
+        let predicate = subnet_predicate::<TSpec>(vec![subnet_id], &self.log);
+        let peers_to_dial: Vec<PeerId> = self
+            .discovery()
+            .cached_enrs()
+            .filter_map(|(peer_id, enr)| {
+                let peers = self.network_globals.peers.read();
+                if predicate(enr)
+                    && !peers.is_connected_or_dialing(peer_id)
+                    && !peers.is_banned(peer_id)
+                {
+                    Some(peer_id.clone())
+                } else {
+                    None
+                }
+            })
+            .collect();
+        for peer in &peers_to_dial {
+            self.dial_peer(peer);
+        }
+    }
+
     /// Peers that have been returned by discovery requests are dialed here if they are suitable.
     ///
     /// NOTE: By dialing `PeerId`s and not multiaddrs, libp2p requests the multiaddr associated
@@ -503,7 +565,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         let connected_or_dialing = self.network_globals.connected_or_dialing_peers();
         for (peer_id, min_ttl) in results {
             // we attempt a connection if this peer is a subnet peer or if the max peer count
-            // is not yet filled (including dialling peers)
+            // is not yet filled (including dialing peers)
             if (min_ttl.is_some() || connected_or_dialing + to_dial_peers.len() < self.max_peers)
                 && !self
                     .network_globals
@@ -516,7 +578,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                     .read()
                     .is_banned_or_disconnected(&peer_id)
             {
-                // TODO: Update output
                 // This should be updated with the peer dialing. In fact created once the peer is
                 // dialed
                 if let Some(min_ttl) = min_ttl {
@@ -539,22 +600,28 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// This is called by `connect_ingoing` and `connect_outgoing`.
     ///
-    /// This informs if the peer was accepted in to the db or not.
+    /// Informs if the peer was accepted in to the db or not.
     fn connect_peer(&mut self, peer_id: &PeerId, connection: ConnectingType) -> bool {
-        // TODO: remove after timed updates
-        //self.update_reputations();
-
         {
             let mut peerdb = self.network_globals.peers.write();
-            if peerdb.connection_status(peer_id).map(|c| c.is_banned()) == Some(true) {
+            if peerdb.is_banned(&peer_id) {
                 // don't connect if the peer is banned
                 slog::crit!(self.log, "Connection has been allowed to a banned peer"; "peer_id" => peer_id.to_string());
             }
 
+            let enr = self.discovery.enr_of_peer(peer_id);
+
             match connection {
-                ConnectingType::Dialing => peerdb.dialing_peer(peer_id),
-                ConnectingType::IngoingConnected => peerdb.connect_outgoing(peer_id),
-                ConnectingType::OutgoingConnected => peerdb.connect_ingoing(peer_id),
+                ConnectingType::Dialing => {
+                    peerdb.dialing_peer(peer_id, enr);
+                    return true;
+                }
+                ConnectingType::IngoingConnected { multiaddr } => {
+                    peerdb.connect_outgoing(peer_id, multiaddr, enr)
+                }
+                ConnectingType::OutgoingConnected { multiaddr } => {
+                    peerdb.connect_ingoing(peer_id, multiaddr, enr)
+                }
             }
         }
 
@@ -569,6 +636,21 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             self.network_globals.connected_peers() as i64,
         );
 
+        // Increment the PEERS_PER_CLIENT metric
+        if let Some(kind) = self
+            .network_globals
+            .peers
+            .read()
+            .peer_info(peer_id)
+            .map(|peer_info| peer_info.client.kind.clone())
+        {
+            if let Some(v) =
+                metrics::get_int_gauge(&metrics::PEERS_PER_CLIENT, &[&kind.to_string()])
+            {
+                v.inc()
+            };
+        }
+
         true
     }
 
@@ -577,111 +659,110 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// NOTE: This is experimental and will likely be adjusted
     fn update_peer_scores(&mut self) {
         /* Check how long have peers been in this state and update their reputations if needed */
-        let mut pdb = self.network_globals.peers.write();
 
         let mut to_ban_peers = Vec::new();
         let mut to_unban_peers = Vec::new();
 
-        for (peer_id, info) in pdb.peers_mut() {
-            let previous_state = info.score.state();
+        for (peer_id, info) in self.network_globals.peers.write().peers_mut() {
+            let previous_state = info.score_state();
             // Update scores
-            info.score.update();
-
-            /* TODO: Implement logic about connection lifetimes
-            match info.connection_status {
-                Connected { .. } => {
-                    // Connected peers gain reputation by sending useful messages
-                }
-                Disconnected { since } | Banned { since } => {
-                    // For disconnected peers, lower their reputation by 1 for every hour they
-                    // stay disconnected. This helps us slowly forget disconnected peers.
-                    // In the same way, slowly allow banned peers back again.
-                    let dc_hours = now
-                        .checked_duration_since(since)
-                        .unwrap_or_else(|| Duration::from_secs(0))
-                        .as_secs()
-                        / 3600;
-                    let last_dc_hours = self
-                        ._last_updated
-                        .checked_duration_since(since)
-                        .unwrap_or_else(|| Duration::from_secs(0))
-                        .as_secs()
-                        / 3600;
-                    if dc_hours > last_dc_hours {
-                        // this should be 1 most of the time
-                        let rep_dif = (dc_hours - last_dc_hours)
-                            .try_into()
-                            .unwrap_or(Rep::max_value());
-
-                        info.reputation = if info.connection_status.is_banned() {
-                            info.reputation.saturating_add(rep_dif)
-                        } else {
-                            info.reputation.saturating_sub(rep_dif)
-                        };
-                    }
-                }
-                Dialing { since } => {
-                    // A peer shouldn't be dialing for more than 2 minutes
-                    if since.elapsed().as_secs() > 120 {
-                        warn!(self.log,"Peer has been dialing for too long"; "peer_id" => id.to_string());
-                        // TODO: decide how to handle this
-                    }
-                }
-                Unknown => {} //TODO: Handle this case
-            }
-            // Check if the peer gets banned or unbanned and if it should be disconnected
-            if info.reputation < _MIN_REP_BEFORE_BAN && !info.connection_status.is_banned() {
-                // This peer gets banned. Check if we should request disconnection
-                ban_queue.push(id.clone());
-            } else if info.reputation >= _MIN_REP_BEFORE_BAN && info.connection_status.is_banned() {
-                // This peer gets unbanned
-                unban_queue.push(id.clone());
-            }
-            */
+            info.score_update();
 
             // handle score transitions
-            if previous_state != info.score.state() {
-                match info.score.state() {
+            if previous_state != info.score_state() {
+                match info.score_state() {
                     ScoreState::Banned => {
-                        debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string());
+                        debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string());
                         to_ban_peers.push(peer_id.clone());
-                        if info.connection_status.is_connected_or_dialing() {
-                            self.events.push(PeerManagerEvent::DisconnectPeer(
-                                peer_id.clone(),
-                                GoodbyeReason::BadScore,
-                            ));
-                        }
                     }
                     ScoreState::Disconnected => {
-                        debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
+                        debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
                         // disconnect the peer if it's currently connected or dialing
-                        to_unban_peers.push(peer_id.clone());
-                        if info.connection_status.is_connected_or_dialing() {
+                        if info.is_connected_or_dialing() {
+                            // Change the state to inform that we are disconnecting the peer.
+                            info.disconnecting(false);
                             self.events.push(PeerManagerEvent::DisconnectPeer(
                                 peer_id.clone(),
                                 GoodbyeReason::BadScore,
                             ));
+                        } else if info.is_banned() {
+                            to_unban_peers.push(peer_id.clone());
                         }
-                        // TODO: Update peer manager to report that it's disconnecting.
                     }
                     ScoreState::Healthy => {
-                        debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
+                        debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
                         // unban the peer if it was previously banned.
-                        to_unban_peers.push(peer_id.clone());
+                        if info.is_banned() {
+                            to_unban_peers.push(peer_id.clone());
+                        }
                     }
                 }
             }
         }
         // process banning peers
         for peer_id in to_ban_peers {
-            pdb.ban(&peer_id);
+            self.ban_peer(&peer_id);
         }
         // process unbanning peers
         for peer_id in to_unban_peers {
-            pdb.unban(&peer_id);
+            if let Err(e) = self.unban_peer(&peer_id) {
+                error!(self.log, "{}", e; "peer_id" => %peer_id);
+            }
         }
     }
 
+    /// Bans a peer.
+    ///
+    /// Records updates the peers connection status and updates the peer db as well as blocks the
+    /// peer from participating in discovery and removes them from the routing table.
+    fn ban_peer(&mut self, peer_id: &PeerId) {
+        {
+            // write lock scope
+            let mut peer_db = self.network_globals.peers.write();
+
+            if peer_db.disconnect_and_ban(peer_id) {
+                // The peer was currently connected, so we start a disconnection.
+                self.events.push(PeerManagerEvent::DisconnectPeer(
+                    peer_id.clone(),
+                    GoodbyeReason::BadScore,
+                ));
+            }
+        } // end write lock
+
+        // take a read lock
+        let peer_db = self.network_globals.peers.read();
+
+        let banned_ip_addresses = peer_db
+            .peer_info(peer_id)
+            .map(|info| {
+                info.seen_addresses()
+                    .iter()
+                    .filter(|ip| peer_db.is_ip_banned(ip))
+                    .cloned()
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default();
+
+        self.discovery.ban_peer(&peer_id, banned_ip_addresses);
+    }
+
+    /// Unbans a peer.
+    ///
+    /// Records updates the peers connection status and updates the peer db as well as removes
+    /// previous bans from discovery.
+    fn unban_peer(&mut self, peer_id: &PeerId) -> Result<(), &'static str> {
+        let mut peer_db = self.network_globals.peers.write();
+        peer_db.unban(&peer_id)?;
+
+        let seen_ip_addresses = peer_db
+            .peer_info(peer_id)
+            .map(|info| info.seen_addresses().iter().cloned().collect::<Vec<_>>())
+            .unwrap_or_default();
+
+        self.discovery.unban_peer(&peer_id, seen_ip_addresses);
+        Ok(())
+    }
+
     /// The Peer manager's heartbeat maintains the peer count and maintains peer reputations.
     ///
     /// It will request discovery queries if the peer count has not reached the desired number of
@@ -689,8 +770,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// NOTE: Discovery will only add a new query if one isn't already queued.
     fn heartbeat(&mut self) {
-        // TODO: Provide a back-off time for discovery queries. I.e Queue many initially, then only
-        // perform discoveries over a larger fixed interval. Perhaps one every 6 heartbeats
         let peer_count = self.network_globals.connected_or_dialing_peers();
         if peer_count < self.target_peers {
             // If we need more peers, queue a discovery lookup.
@@ -701,6 +780,9 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         // Updates peer's scores.
         self.update_peer_scores();
 
+        // Keep a list of peers we are disconnecting
+        let mut disconnecting_peers = Vec::new();
+
         let connected_peer_count = self.network_globals.connected_peers();
         if connected_peer_count > self.target_peers {
             //remove excess peers with the worst scores, but keep subnet peers
@@ -714,14 +796,20 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                 .take(connected_peer_count - self.target_peers)
                 //we only need to disconnect peers with healthy scores, since the others got already
                 //disconnected in update_peer_scores
-                .filter(|(_, info)| info.score.state() == ScoreState::Healthy)
+                .filter(|(_, info)| info.score_state() == ScoreState::Healthy)
             {
-                self.events.push(PeerManagerEvent::DisconnectPeer(
-                    (*peer_id).clone(),
-                    GoodbyeReason::TooManyPeers,
-                ));
+                disconnecting_peers.push((*peer_id).clone());
             }
         }
+
+        let mut peer_db = self.network_globals.peers.write();
+        for peer_id in disconnecting_peers {
+            peer_db.notify_disconnecting(&peer_id);
+            self.events.push(PeerManagerEvent::DisconnectPeer(
+                peer_id,
+                GoodbyeReason::TooManyPeers,
+            ));
+        }
     }
 }
 
@@ -756,20 +844,16 @@ impl<TSpec: EthSpec> Stream for PeerManager<TSpec> {
             }
         }
 
-        // We don't want to update peers during syncing, since this may result in a new chain being
-        // synced which leads to inefficient re-downloads of blocks.
-        if !self.network_globals.is_syncing() {
-            loop {
-                match self.status_peers.poll_next_unpin(cx) {
-                    Poll::Ready(Some(Ok(peer_id))) => {
-                        self.status_peers.insert(peer_id.clone());
-                        self.events.push(PeerManagerEvent::Status(peer_id))
-                    }
-                    Poll::Ready(Some(Err(e))) => {
-                        error!(self.log, "Failed to check for peers to ping"; "error" => e.to_string())
-                    }
-                    Poll::Ready(None) | Poll::Pending => break,
+        loop {
+            match self.status_peers.poll_next_unpin(cx) {
+                Poll::Ready(Some(Ok(peer_id))) => {
+                    self.status_peers.insert(peer_id.clone());
+                    self.events.push(PeerManagerEvent::Status(peer_id))
                 }
+                Poll::Ready(Some(Err(e))) => {
+                    error!(self.log, "Failed to check for peers to ping"; "error" => e.to_string())
+                }
+                Poll::Ready(None) | Poll::Pending => break,
             }
         }
 
@@ -787,7 +871,13 @@ enum ConnectingType {
     /// We are in the process of dialing this peer.
     Dialing,
     /// A peer has dialed us.
-    IngoingConnected,
+    IngoingConnected {
+        // The multiaddr the peer connected to us on.
+        multiaddr: Multiaddr,
+    },
     /// We have successfully dialed a peer.
-    OutgoingConnected,
+    OutgoingConnected {
+        /// The multiaddr we dialed to reach the peer.
+        multiaddr: Multiaddr,
+    },
 }

beacon_node/eth2_libp2p/src/peer_manager/peer_info.rs

@@ -1,12 +1,15 @@
 use super::client::Client;
-use super::score::Score;
+use super::score::{PeerAction, Score, ScoreState};
 use super::PeerSyncStatus;
 use crate::rpc::MetaData;
 use crate::Multiaddr;
+use discv5::Enr;
 use serde::{
-    ser::{SerializeStructVariant, Serializer},
+    ser::{SerializeStruct, Serializer},
     Serialize,
 };
+use std::collections::HashSet;
+use std::net::IpAddr;
 use std::time::Instant;
 use types::{EthSpec, SubnetId};
 use PeerConnectionStatus::*;
@@ -18,13 +21,17 @@ pub struct PeerInfo<T: EthSpec> {
     /// The connection status of the peer
     _status: PeerStatus,
     /// The peers reputation
-    pub score: Score,
+    score: Score,
     /// Client managing this peer
     pub client: Client,
     /// Connection status of this peer
-    pub connection_status: PeerConnectionStatus,
-    /// The known listening addresses of this peer.
+    connection_status: PeerConnectionStatus,
+    /// The known listening addresses of this peer. This is given by identify and can be arbitrary
+    /// (including local IPs).
     pub listening_addresses: Vec<Multiaddr>,
+    /// This is addresses we have physically seen and this is what we use for banning/un-banning
+    /// peers.
+    seen_addresses: HashSet<IpAddr>,
     /// The current syncing state of the peer. The state may be determined after it's initial
     /// connection.
     pub sync_status: PeerSyncStatus,
@@ -35,6 +42,13 @@ pub struct PeerInfo<T: EthSpec> {
     /// necessary.
     #[serde(skip)]
     pub min_ttl: Option<Instant>,
+    /// Is the peer a trusted peer.
+    pub is_trusted: bool,
+    /// Direction of the first connection of the last (or current) connected session with this peer.
+    /// None if this peer was never connected.
+    pub connection_direction: Option<ConnectionDirection>,
+    /// The enr of the peer, if known.
+    pub enr: Option<Enr>,
 }
 
 impl<TSpec: EthSpec> Default for PeerInfo<TSpec> {
@@ -44,15 +58,28 @@ impl<TSpec: EthSpec> Default for PeerInfo<TSpec> {
             score: Score::default(),
             client: Client::default(),
             connection_status: Default::default(),
-            listening_addresses: vec![],
+            listening_addresses: Vec::new(),
+            seen_addresses: HashSet::new(),
             sync_status: PeerSyncStatus::Unknown,
             meta_data: None,
             min_ttl: None,
+            is_trusted: false,
+            connection_direction: None,
+            enr: None,
         }
     }
 }
 
 impl<T: EthSpec> PeerInfo<T> {
+    /// Return a PeerInfo struct for a trusted peer.
+    pub fn trusted_peer_info() -> Self {
+        PeerInfo {
+            score: Score::max_score(),
+            is_trusted: true,
+            ..Default::default()
+        }
+    }
+
     /// Returns if the peer is subscribed to a given `SubnetId`
     pub fn on_subnet(&self, subnet_id: SubnetId) -> bool {
         if let Some(meta_data) = &self.meta_data {
@@ -64,10 +91,202 @@ impl<T: EthSpec> PeerInfo<T> {
         false
     }
 
+    /// Returns the seen addresses of the peer.
+    pub fn seen_addresses(&self) -> &HashSet<IpAddr> {
+        &self.seen_addresses
+    }
+
+    /// Returns the connection status of the peer.
+    pub fn connection_status(&self) -> &PeerConnectionStatus {
+        &self.connection_status
+    }
+
     /// Reports if this peer has some future validator duty in which case it is valuable to keep it.
     pub fn has_future_duty(&self) -> bool {
         self.min_ttl.map_or(false, |i| i >= Instant::now())
     }
+
+    /// Returns score of the peer.
+    pub fn score(&self) -> Score {
+        self.score
+    }
+
+    /// Returns the state of the peer based on the score.
+    pub(crate) fn score_state(&self) -> ScoreState {
+        self.score.state()
+    }
+
+    /// Applies decay rates to a non-trusted peer's score.
+    pub fn score_update(&mut self) {
+        if !self.is_trusted {
+            self.score.update()
+        }
+    }
+
+    /// Apply peer action to a non-trusted peer's score.
+    pub fn apply_peer_action_to_score(&mut self, peer_action: PeerAction) {
+        if !self.is_trusted {
+            self.score.apply_peer_action(peer_action)
+        }
+    }
+
+    #[cfg(test)]
+    /// Resets the peers score.
+    pub fn reset_score(&mut self) {
+        self.score.test_reset();
+    }
+
+    /* Peer connection status API */
+
+    /// Checks if the status is connected.
+    pub fn is_connected(&self) -> bool {
+        matches!(self.connection_status, PeerConnectionStatus::Connected { .. })
+    }
+
+    /// Checks if the status is connected.
+    pub fn is_dialing(&self) -> bool {
+        matches!(self.connection_status, PeerConnectionStatus::Dialing { .. })
+    }
+
+    /// The peer is either connected or in the process of being dialed.
+    pub fn is_connected_or_dialing(&self) -> bool {
+        self.is_connected() || self.is_dialing()
+    }
+
+    /// Checks if the status is banned.
+    pub fn is_banned(&self) -> bool {
+        matches!(self.connection_status, PeerConnectionStatus::Banned { .. })
+    }
+
+    /// Checks if the status is disconnected.
+    pub fn is_disconnected(&self) -> bool {
+        matches!(self.connection_status, Disconnected { .. })
+    }
+
+    /// Returns the number of connections with this peer.
+    pub fn connections(&self) -> (u8, u8) {
+        match self.connection_status {
+            Connected { n_in, n_out } => (n_in, n_out),
+            _ => (0, 0),
+        }
+    }
+
+    // Setters
+
+    /// Modifies the status to Disconnected and sets the last seen instant to now. Returns None if
+    /// no changes were made. Returns Some(bool) where the bool represents if peer became banned or
+    /// simply just disconnected.
+    pub fn notify_disconnect(&mut self) -> Option<bool> {
+        match self.connection_status {
+            Banned { .. } | Disconnected { .. } => None,
+            Disconnecting { to_ban } => {
+                // If we are disconnecting this peer in the process of banning, we now ban the
+                // peer.
+                if to_ban {
+                    self.connection_status = Banned {
+                        since: Instant::now(),
+                    };
+                    Some(true)
+                } else {
+                    self.connection_status = Disconnected {
+                        since: Instant::now(),
+                    };
+                    Some(false)
+                }
+            }
+            Connected { .. } | Dialing { .. } | Unknown => {
+                self.connection_status = Disconnected {
+                    since: Instant::now(),
+                };
+                Some(false)
+            }
+        }
+    }
+
+    /// Notify the we are currently disconnecting this peer, after which the peer will be
+    /// considered banned.
+    // This intermediate state is required to inform the network behaviours that the sub-protocols
+    // are aware this peer exists and it is in the process of being banned. Compared to nodes that
+    // try to connect to us and are already banned (sub protocols do not know of these peers).
+    pub fn disconnecting(&mut self, to_ban: bool) {
+        self.connection_status = Disconnecting { to_ban }
+    }
+
+    /// Modifies the status to Banned
+    pub fn ban(&mut self) {
+        self.connection_status = Banned {
+            since: Instant::now(),
+        };
+    }
+
+    /// The score system has unbanned the peer. Update the connection status
+    pub fn unban(&mut self) {
+        if let PeerConnectionStatus::Banned { since, .. } = self.connection_status {
+            self.connection_status = PeerConnectionStatus::Disconnected { since };
+        }
+    }
+
+    /// Modifies the status to Dialing
+    /// Returns an error if the current state is unexpected.
+    pub(crate) fn dialing_peer(&mut self) -> Result<(), &'static str> {
+        match &mut self.connection_status {
+            Connected { .. } => return Err("Dialing connected peer"),
+            Dialing { .. } => return Err("Dialing an already dialing peer"),
+            Disconnecting { .. } => return Err("Dialing a disconnecting peer"),
+            Disconnected { .. } | Banned { .. } | Unknown => {}
+        }
+        self.connection_status = Dialing {
+            since: Instant::now(),
+        };
+        Ok(())
+    }
+
+    /// Modifies the status to Connected and increases the number of ingoing
+    /// connections by one
+    pub(crate) fn connect_ingoing(&mut self, seen_address: Option<IpAddr>) {
+        match &mut self.connection_status {
+            Connected { n_in, .. } => *n_in += 1,
+            Disconnected { .. }
+            | Banned { .. }
+            | Dialing { .. }
+            | Disconnecting { .. }
+            | Unknown => {
+                self.connection_status = Connected { n_in: 1, n_out: 0 };
+                self.connection_direction = Some(ConnectionDirection::Incoming);
+            }
+        }
+
+        if let Some(ip_addr) = seen_address {
+            self.seen_addresses.insert(ip_addr);
+        }
+    }
+
+    /// Modifies the status to Connected and increases the number of outgoing
+    /// connections by one
+    pub(crate) fn connect_outgoing(&mut self, seen_address: Option<IpAddr>) {
+        match &mut self.connection_status {
+            Connected { n_out, .. } => *n_out += 1,
+            Disconnected { .. }
+            | Banned { .. }
+            | Dialing { .. }
+            | Disconnecting { .. }
+            | Unknown => {
+                self.connection_status = Connected { n_in: 0, n_out: 1 };
+                self.connection_direction = Some(ConnectionDirection::Outgoing);
+            }
+        }
+        if let Some(ip_addr) = seen_address {
+            self.seen_addresses.insert(ip_addr);
+        }
+    }
+
+    #[cfg(test)]
+    /// Add an f64 to a non-trusted peer's score abiding by the limits.
+    pub fn add_to_score(&mut self, score: f64) {
+        if !self.is_trusted {
+            self.score.test_add(score)
+        }
+    }
 }
 
 #[derive(Clone, Debug, Serialize)]
@@ -85,6 +304,13 @@ impl Default for PeerStatus {
     }
 }
 
+/// Connection Direction of connection.
+#[derive(Debug, Clone, Serialize)]
+pub enum ConnectionDirection {
+    Incoming,
+    Outgoing,
+}
+
 /// Connection Status of the peer.
 #[derive(Debug, Clone)]
 pub enum PeerConnectionStatus {
@@ -95,11 +321,17 @@ pub enum PeerConnectionStatus {
         /// number of outgoing connections.
         n_out: u8,
     },
+    /// The peer is being disconnected.
+    Disconnecting {
+        // After the disconnection the peer will be considered banned.
+        to_ban: bool,
+    },
     /// The peer has disconnected.
     Disconnected {
         /// last time the peer was connected or discovered.
         since: Instant,
     },
+
     /// The peer has been banned and is disconnected.
     Banned {
         /// moment when the peer was banned.
@@ -117,29 +349,51 @@ pub enum PeerConnectionStatus {
 /// Serialization for http requests.
 impl Serialize for PeerConnectionStatus {
     fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let mut s = serializer.serialize_struct("connection_status", 6)?;
         match self {
             Connected { n_in, n_out } => {
-                let mut s = serializer.serialize_struct_variant("", 0, "Connected", 2)?;
-                s.serialize_field("in", n_in)?;
-                s.serialize_field("out", n_out)?;
+                s.serialize_field("status", "connected")?;
+                s.serialize_field("connections_in", n_in)?;
+                s.serialize_field("connections_out", n_out)?;
+                s.serialize_field("last_seen", &0)?;
+                s.end()
+            }
+            Disconnecting { .. } => {
+                s.serialize_field("status", "disconnecting")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &0)?;
                 s.end()
             }
             Disconnected { since } => {
-                let mut s = serializer.serialize_struct_variant("", 1, "Disconnected", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "disconnected")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
                 s.end()
             }
             Banned { since } => {
-                let mut s = serializer.serialize_struct_variant("", 2, "Banned", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "banned")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
                 s.end()
             }
             Dialing { since } => {
-                let mut s = serializer.serialize_struct_variant("", 3, "Dialing", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "dialing")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.end()
+            }
+            Unknown => {
+                s.serialize_field("status", "unknown")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &0)?;
                 s.end()
             }
-            Unknown => serializer.serialize_unit_variant("", 4, "Unknown"),
         }
     }
 }
@@ -149,92 +403,3 @@ impl Default for PeerConnectionStatus {
         PeerConnectionStatus::Unknown
     }
 }
-
-impl PeerConnectionStatus {
-    /// Checks if the status is connected.
-    pub fn is_connected(&self) -> bool {
-        match self {
-            PeerConnectionStatus::Connected { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Checks if the status is connected.
-    pub fn is_dialing(&self) -> bool {
-        match self {
-            PeerConnectionStatus::Dialing { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// The peer is either connected or in the process of being dialed.
-    pub fn is_connected_or_dialing(&self) -> bool {
-        self.is_connected() || self.is_dialing()
-    }
-
-    /// Checks if the status is banned.
-    pub fn is_banned(&self) -> bool {
-        match self {
-            PeerConnectionStatus::Banned { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Checks if the status is disconnected.
-    pub fn is_disconnected(&self) -> bool {
-        match self {
-            Disconnected { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Modifies the status to Connected and increases the number of ingoing
-    /// connections by one
-    pub fn connect_ingoing(&mut self) {
-        match self {
-            Connected { n_in, .. } => *n_in += 1,
-            Disconnected { .. } | Banned { .. } | Dialing { .. } | Unknown => {
-                *self = Connected { n_in: 1, n_out: 0 }
-            }
-        }
-    }
-
-    /// Modifies the status to Connected and increases the number of outgoing
-    /// connections by one
-    pub fn connect_outgoing(&mut self) {
-        match self {
-            Connected { n_out, .. } => *n_out += 1,
-            Disconnected { .. } | Banned { .. } | Dialing { .. } | Unknown => {
-                *self = Connected { n_in: 0, n_out: 1 }
-            }
-        }
-    }
-
-    /// Modifies the status to Disconnected and sets the last seen instant to now
-    pub fn disconnect(&mut self) {
-        *self = Disconnected {
-            since: Instant::now(),
-        };
-    }
-
-    /// Modifies the status to Banned
-    pub fn ban(&mut self) {
-        *self = Banned {
-            since: Instant::now(),
-        };
-    }
-
-    /// The score system has unbanned the peer. Update the connection status
-    pub fn unban(&mut self) {
-        if let PeerConnectionStatus::Banned { since } = self {
-            *self = PeerConnectionStatus::Disconnected { since: *since }
-        }
-    }
-
-    pub fn connections(&self) -> (u8, u8) {
-        match self {
-            Connected { n_in, n_out } => (*n_in, *n_out),
-            _ => (0, 0),
-        }
-    }
-}

beacon_node/eth2_libp2p/src/peer_manager/peer_sync_status.rs

@@ -29,26 +29,17 @@ pub struct SyncInfo {
 impl PeerSyncStatus {
     /// Returns true if the peer has advanced knowledge of the chain.
     pub fn is_advanced(&self) -> bool {
-        match self {
-            PeerSyncStatus::Advanced { .. } => true,
-            _ => false,
-        }
+        matches!(self, PeerSyncStatus::Advanced { .. })
     }
 
     /// Returns true if the peer is up to date with the current chain.
     pub fn is_synced(&self) -> bool {
-        match self {
-            PeerSyncStatus::Synced { .. } => true,
-            _ => false,
-        }
+        matches!(self, PeerSyncStatus::Synced { .. })
     }
 
     /// Returns true if the peer is behind the current chain.
     pub fn is_behind(&self) -> bool {
-        match self {
-            PeerSyncStatus::Behind { .. } => true,
-            _ => false,
-        }
+        matches!(self, PeerSyncStatus::Behind { .. })
     }
 
     /// Updates the sync state given a fully synced peer.

beacon_node/eth2_libp2p/src/peer_manager/score.rs

@@ -48,7 +48,7 @@ pub enum PeerAction {
     /// An error occurred with this peer but it is not necessarily malicious.
     /// We have high tolerance for this actions: several occurrences are needed for a peer to get
     /// kicked.
-    /// NOTE: ~15 occurrences will get the peer banned
+    /// NOTE: ~50 occurrences will get the peer banned
     HighToleranceError,
     /// Received an expected message.
     _ValidMessage,
@@ -145,6 +145,13 @@ impl std::fmt::Display for ScoreState {
 }
 
 impl Score {
+    /// Return max possible score.
+    pub fn max_score() -> Self {
+        Score {
+            score: MAX_SCORE,
+            last_updated: Instant::now(),
+        }
+    }
     /// Access to the underlying score.
     pub fn score(&self) -> f64 {
         self.score
@@ -171,7 +178,7 @@ impl Score {
     }
 
     /// Add an f64 to the score abiding by the limits.
-    pub fn add(&mut self, score: f64) {
+    fn add(&mut self, score: f64) {
         let mut new_score = self.score + score;
         if new_score > MAX_SCORE {
             new_score = MAX_SCORE;
@@ -183,6 +190,26 @@ impl Score {
         self.score = new_score;
     }
 
+    /// Add an f64 to the score abiding by the limits.
+    #[cfg(test)]
+    pub fn test_add(&mut self, score: f64) {
+        let mut new_score = self.score + score;
+        if new_score > MAX_SCORE {
+            new_score = MAX_SCORE;
+        }
+        if new_score < MIN_SCORE {
+            new_score = MIN_SCORE;
+        }
+
+        self.score = new_score;
+    }
+
+    #[cfg(test)]
+    // reset the score
+    pub fn test_reset(&mut self) {
+        self.score = 0f64;
+    }
+
     /// Applies time-based logic such as decay rates to the score.
     /// This function should be called periodically.
     pub fn update(&mut self) {

beacon_node/eth2_libp2p/src/rpc/codec/base.rs

@@ -1,5 +1,6 @@
 //! This handles the various supported encoding mechanism for the Eth 2.0 RPC.
 
+use crate::rpc::methods::ErrorType;
 use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
 use libp2p::bytes::BufMut;
 use libp2p::bytes::BytesMut;
@@ -8,12 +9,12 @@ use tokio_util::codec::{Decoder, Encoder};
 use types::EthSpec;
 
 pub trait OutboundCodec<TItem>: Encoder<TItem> + Decoder {
-    type ErrorType;
+    type CodecErrorType;
 
     fn decode_error(
         &mut self,
         src: &mut BytesMut,
-    ) -> Result<Option<Self::ErrorType>, <Self as Decoder>::Error>;
+    ) -> Result<Option<Self::CodecErrorType>, <Self as Decoder>::Error>;
 }
 
 /* Global Inbound Codec */
@@ -130,8 +131,8 @@ where
 impl<TCodec, TSpec> Decoder for BaseOutboundCodec<TCodec, TSpec>
 where
     TSpec: EthSpec,
-    TCodec:
-        OutboundCodec<RPCRequest<TSpec>, ErrorType = String> + Decoder<Item = RPCResponse<TSpec>>,
+    TCodec: OutboundCodec<RPCRequest<TSpec>, CodecErrorType = ErrorType>
+        + Decoder<Item = RPCResponse<TSpec>>,
 {
     type Item = RPCCodedResponse<TSpec>;
     type Error = <TCodec as Decoder>::Error;
@@ -174,10 +175,17 @@ where
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::*;
     use super::super::ssz_snappy::*;
     use super::*;
+    use crate::rpc::methods::StatusMessage;
     use crate::rpc::protocol::*;
+    use snap::write::FrameEncoder;
+    use ssz::Encode;
+    use std::io::Write;
+    use types::{Epoch, Hash256, Slot};
+    use unsigned_varint::codec::Uvi;
+
+    type Spec = types::MainnetEthSpec;
 
     #[test]
     fn test_decode_status_message() {
@@ -185,33 +193,77 @@ mod tests {
         let mut buf = BytesMut::new();
         buf.extend_from_slice(&message);
 
-        type Spec = types::MainnetEthSpec;
-
         let snappy_protocol_id =
             ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy);
-        let ssz_protocol_id = ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZ);
 
         let mut snappy_outbound_codec =
             SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, 1_048_576);
-        let mut ssz_outbound_codec = SSZOutboundCodec::<Spec>::new(ssz_protocol_id, 1_048_576);
 
         // decode message just as snappy message
         let snappy_decoded_message = snappy_outbound_codec.decode(&mut buf.clone());
         // decode message just a ssz message
-        let ssz_decoded_message = ssz_outbound_codec.decode(&mut buf.clone());
 
         // build codecs for entire chunk
         let mut snappy_base_outbound_codec = BaseOutboundCodec::new(snappy_outbound_codec);
-        let mut ssz_base_outbound_codec = BaseOutboundCodec::new(ssz_outbound_codec);
 
         // decode message as ssz snappy chunk
         let snappy_decoded_chunk = snappy_base_outbound_codec.decode(&mut buf.clone());
         // decode message just a ssz chunk
-        let ssz_decoded_chunk = ssz_base_outbound_codec.decode(&mut buf.clone());
 
         let _ = dbg!(snappy_decoded_message);
-        let _ = dbg!(ssz_decoded_message);
         let _ = dbg!(snappy_decoded_chunk);
-        let _ = dbg!(ssz_decoded_chunk);
+    }
+
+    #[test]
+    fn test_decode_malicious_status_message() {
+        // Snappy stream identifier
+        let stream_identifier: &'static [u8] = b"\xFF\x06\x00\x00sNaPpY";
+
+        // byte 0(0xFE) is padding chunk type identifier for snappy messages
+        // byte 1,2,3 are chunk length (little endian)
+        let malicious_padding: &'static [u8] = b"\xFE\x00\x00\x00";
+
+        // Status message is 84 bytes uncompressed. `max_compressed_len` is 130.
+        let status_message_bytes = StatusMessage {
+            fork_digest: [0; 4],
+            finalized_root: Hash256::from_low_u64_be(0),
+            finalized_epoch: Epoch::new(1),
+            head_root: Hash256::from_low_u64_be(0),
+            head_slot: Slot::new(1),
+        }
+        .as_ssz_bytes();
+
+        let mut uvi_codec: Uvi<usize> = Uvi::default();
+        let mut dst = BytesMut::with_capacity(1024);
+
+        // Insert length-prefix
+        uvi_codec
+            .encode(status_message_bytes.len(), &mut dst)
+            .unwrap();
+
+        // Insert snappy stream identifier
+        dst.extend_from_slice(stream_identifier);
+
+        // Insert malicious padding of 80 bytes.
+        for _ in 0..20 {
+            dst.extend_from_slice(malicious_padding);
+        }
+
+        // Insert payload (42 bytes compressed)
+        let mut writer = FrameEncoder::new(Vec::new());
+        writer.write_all(&status_message_bytes).unwrap();
+        writer.flush().unwrap();
+        dst.extend_from_slice(writer.get_ref());
+
+        // 42 + 80 = 132 > max_compressed_len. Hence, decoding should fail with `InvalidData`.
+
+        let snappy_protocol_id =
+            ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy);
+
+        let mut snappy_outbound_codec =
+            SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, 1_048_576);
+
+        let snappy_decoded_message = snappy_outbound_codec.decode(&mut dst.clone()).unwrap_err();
+        assert_eq!(snappy_decoded_message, RPCError::InvalidData);
     }
 }

beacon_node/eth2_libp2p/src/rpc/codec/mod.rs

@@ -1,9 +1,7 @@
 pub(crate) mod base;
-pub(crate) mod ssz;
 pub(crate) mod ssz_snappy;
 
 use self::base::{BaseInboundCodec, BaseOutboundCodec};
-use self::ssz::{SSZInboundCodec, SSZOutboundCodec};
 use self::ssz_snappy::{SSZSnappyInboundCodec, SSZSnappyOutboundCodec};
 use crate::rpc::protocol::RPCError;
 use crate::rpc::{RPCCodedResponse, RPCRequest};
@@ -14,12 +12,10 @@ use types::EthSpec;
 // Known types of codecs
 pub enum InboundCodec<TSpec: EthSpec> {
     SSZSnappy(BaseInboundCodec<SSZSnappyInboundCodec<TSpec>, TSpec>),
-    SSZ(BaseInboundCodec<SSZInboundCodec<TSpec>, TSpec>),
 }
 
 pub enum OutboundCodec<TSpec: EthSpec> {
     SSZSnappy(BaseOutboundCodec<SSZSnappyOutboundCodec<TSpec>, TSpec>),
-    SSZ(BaseOutboundCodec<SSZOutboundCodec<TSpec>, TSpec>),
 }
 
 impl<T: EthSpec> Encoder<RPCCodedResponse<T>> for InboundCodec<T> {
@@ -27,7 +23,6 @@ impl<T: EthSpec> Encoder<RPCCodedResponse<T>> for InboundCodec<T> {
 
     fn encode(&mut self, item: RPCCodedResponse<T>, dst: &mut BytesMut) -> Result<(), Self::Error> {
         match self {
-            InboundCodec::SSZ(codec) => codec.encode(item, dst),
             InboundCodec::SSZSnappy(codec) => codec.encode(item, dst),
         }
     }
@@ -39,7 +34,6 @@ impl<TSpec: EthSpec> Decoder for InboundCodec<TSpec> {
 
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         match self {
-            InboundCodec::SSZ(codec) => codec.decode(src),
             InboundCodec::SSZSnappy(codec) => codec.decode(src),
         }
     }
@@ -50,7 +44,6 @@ impl<TSpec: EthSpec> Encoder<RPCRequest<TSpec>> for OutboundCodec<TSpec> {
 
     fn encode(&mut self, item: RPCRequest<TSpec>, dst: &mut BytesMut) -> Result<(), Self::Error> {
         match self {
-            OutboundCodec::SSZ(codec) => codec.encode(item, dst),
             OutboundCodec::SSZSnappy(codec) => codec.encode(item, dst),
         }
     }
@@ -62,7 +55,6 @@ impl<T: EthSpec> Decoder for OutboundCodec<T> {
 
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         match self {
-            OutboundCodec::SSZ(codec) => codec.decode(src),
             OutboundCodec::SSZSnappy(codec) => codec.decode(src),
         }
     }

beacon_node/eth2_libp2p/src/rpc/codec/ssz.rs

@@ -1,324 +0,0 @@
-use crate::rpc::methods::*;
-use crate::rpc::{
-    codec::base::OutboundCodec,
-    protocol::{
-        Encoding, Protocol, ProtocolId, RPCError, Version, BLOCKS_BY_ROOT_REQUEST_MAX,
-        BLOCKS_BY_ROOT_REQUEST_MIN, SIGNED_BEACON_BLOCK_MAX, SIGNED_BEACON_BLOCK_MIN,
-    },
-};
-use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
-use libp2p::bytes::{BufMut, Bytes, BytesMut};
-use ssz::{Decode, Encode};
-use ssz_types::VariableList;
-use std::marker::PhantomData;
-use tokio_util::codec::{Decoder, Encoder};
-use types::{EthSpec, SignedBeaconBlock};
-use unsigned_varint::codec::UviBytes;
-
-/* Inbound Codec */
-
-pub struct SSZInboundCodec<TSpec: EthSpec> {
-    inner: UviBytes,
-    protocol: ProtocolId,
-    phantom: PhantomData<TSpec>,
-}
-
-impl<TSpec: EthSpec> SSZInboundCodec<TSpec> {
-    pub fn new(protocol: ProtocolId, max_packet_size: usize) -> Self {
-        let mut uvi_codec = UviBytes::default();
-        uvi_codec.set_max_len(max_packet_size);
-
-        // this encoding only applies to ssz.
-        debug_assert_eq!(protocol.encoding, Encoding::SSZ);
-
-        SSZInboundCodec {
-            inner: uvi_codec,
-            protocol,
-            phantom: PhantomData,
-        }
-    }
-}
-
-// Encoder for inbound streams: Encodes RPC Responses sent to peers.
-impl<TSpec: EthSpec> Encoder<RPCCodedResponse<TSpec>> for SSZInboundCodec<TSpec> {
-    type Error = RPCError;
-
-    fn encode(
-        &mut self,
-        item: RPCCodedResponse<TSpec>,
-        dst: &mut BytesMut,
-    ) -> Result<(), Self::Error> {
-        let bytes = match item {
-            RPCCodedResponse::Success(resp) => match resp {
-                RPCResponse::Status(res) => res.as_ssz_bytes(),
-                RPCResponse::BlocksByRange(res) => res.as_ssz_bytes(),
-                RPCResponse::BlocksByRoot(res) => res.as_ssz_bytes(),
-                RPCResponse::Pong(res) => res.data.as_ssz_bytes(),
-                RPCResponse::MetaData(res) => res.as_ssz_bytes(),
-            },
-            RPCCodedResponse::Error(_, err) => err.as_ssz_bytes(),
-            RPCCodedResponse::StreamTermination(_) => {
-                unreachable!("Code error - attempting to encode a stream termination")
-            }
-        };
-        if !bytes.is_empty() {
-            // length-prefix and return
-            return self
-                .inner
-                .encode(Bytes::from(bytes), dst)
-                .map_err(RPCError::from);
-        } else {
-            // payload is empty, add a 0-byte length prefix
-            dst.reserve(1);
-            dst.put_u8(0);
-        }
-        Ok(())
-    }
-}
-
-// Decoder for inbound streams: Decodes RPC requests from peers
-impl<TSpec: EthSpec> Decoder for SSZInboundCodec<TSpec> {
-    type Item = RPCRequest<TSpec>;
-    type Error = RPCError;
-
-    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        match self.inner.decode(src).map_err(RPCError::from) {
-            Ok(Some(packet)) => match self.protocol.message_name {
-                Protocol::Status => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                            Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
-                                &packet,
-                            )?)))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::Goodbye => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() == <GoodbyeReason as Encode>::ssz_fixed_len() {
-                            Ok(Some(RPCRequest::Goodbye(GoodbyeReason::from_ssz_bytes(
-                                &packet,
-                            )?)))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::BlocksByRange => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() == <BlocksByRangeRequest as Encode>::ssz_fixed_len() {
-                            Ok(Some(RPCRequest::BlocksByRange(
-                                BlocksByRangeRequest::from_ssz_bytes(&packet)?,
-                            )))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::BlocksByRoot => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() >= *BLOCKS_BY_ROOT_REQUEST_MIN
-                            && packet.len() <= *BLOCKS_BY_ROOT_REQUEST_MAX
-                        {
-                            Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
-                                block_roots: VariableList::from_ssz_bytes(&packet)?,
-                            })))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::Ping => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() == <Ping as Encode>::ssz_fixed_len() {
-                            Ok(Some(RPCRequest::Ping(Ping {
-                                data: u64::from_ssz_bytes(&packet)?,
-                            })))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::MetaData => match self.protocol.version {
-                    Version::V1 => {
-                        if !packet.is_empty() {
-                            Err(RPCError::InvalidData)
-                        } else {
-                            Ok(Some(RPCRequest::MetaData(PhantomData)))
-                        }
-                    }
-                },
-            },
-            Ok(None) => Ok(None),
-            Err(e) => Err(e),
-        }
-    }
-}
-
-/* Outbound Codec: Codec for initiating RPC requests */
-
-pub struct SSZOutboundCodec<TSpec: EthSpec> {
-    inner: UviBytes,
-    protocol: ProtocolId,
-    phantom: PhantomData<TSpec>,
-}
-
-impl<TSpec: EthSpec> SSZOutboundCodec<TSpec> {
-    pub fn new(protocol: ProtocolId, max_packet_size: usize) -> Self {
-        let mut uvi_codec = UviBytes::default();
-        uvi_codec.set_max_len(max_packet_size);
-
-        // this encoding only applies to ssz.
-        debug_assert_eq!(protocol.encoding, Encoding::SSZ);
-
-        SSZOutboundCodec {
-            inner: uvi_codec,
-            protocol,
-            phantom: PhantomData,
-        }
-    }
-}
-
-// Encoder for outbound streams: Encodes RPC Requests to peers
-impl<TSpec: EthSpec> Encoder<RPCRequest<TSpec>> for SSZOutboundCodec<TSpec> {
-    type Error = RPCError;
-
-    fn encode(&mut self, item: RPCRequest<TSpec>, dst: &mut BytesMut) -> Result<(), Self::Error> {
-        let bytes = match item {
-            RPCRequest::Status(req) => req.as_ssz_bytes(),
-            RPCRequest::Goodbye(req) => req.as_ssz_bytes(),
-            RPCRequest::BlocksByRange(req) => req.as_ssz_bytes(),
-            RPCRequest::BlocksByRoot(req) => req.block_roots.as_ssz_bytes(),
-            RPCRequest::Ping(req) => req.as_ssz_bytes(),
-            RPCRequest::MetaData(_) => return Ok(()), // no metadata to encode
-        };
-        // length-prefix
-        self.inner
-            .encode(libp2p::bytes::Bytes::from(bytes), dst)
-            .map_err(RPCError::from)
-    }
-}
-
-// Decoder for outbound streams: Decodes RPC responses from peers.
-//
-// The majority of the decoding has now been pushed upstream due to the changing specification.
-// We prefer to decode blocks and attestations with extra knowledge about the chain to perform
-// faster verification checks before decoding entire blocks/attestations.
-impl<TSpec: EthSpec> Decoder for SSZOutboundCodec<TSpec> {
-    type Item = RPCResponse<TSpec>;
-    type Error = RPCError;
-
-    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        if src.len() == 1 && src[0] == 0_u8 {
-            // the object is empty. We return the empty object if this is the case
-            // clear the buffer and return an empty object
-            src.clear();
-            match self.protocol.message_name {
-                Protocol::Status => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty HELLO message. The stream has terminated unexpectedly
-                },
-                Protocol::Goodbye => Err(RPCError::InvalidData),
-                Protocol::BlocksByRange => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty block message.
-                },
-                Protocol::BlocksByRoot => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty block message.
-                },
-                Protocol::Ping => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty block message.
-                },
-                Protocol::MetaData => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty block message.
-                },
-            }
-        } else {
-            match self.inner.decode(src).map_err(RPCError::from) {
-                Ok(Some(mut packet)) => {
-                    // take the bytes from the buffer
-                    let raw_bytes = packet.split();
-
-                    match self.protocol.message_name {
-                        Protocol::Status => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                                    Ok(Some(RPCResponse::Status(StatusMessage::from_ssz_bytes(
-                                        &raw_bytes,
-                                    )?)))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                        Protocol::Goodbye => Err(RPCError::InvalidData),
-                        Protocol::BlocksByRange => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                    && raw_bytes.len() <= *SIGNED_BEACON_BLOCK_MAX
-                                {
-                                    Ok(Some(RPCResponse::BlocksByRange(Box::new(
-                                        SignedBeaconBlock::from_ssz_bytes(&raw_bytes)?,
-                                    ))))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                        Protocol::BlocksByRoot => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                    && raw_bytes.len() <= *SIGNED_BEACON_BLOCK_MAX
-                                {
-                                    Ok(Some(RPCResponse::BlocksByRoot(Box::new(
-                                        SignedBeaconBlock::from_ssz_bytes(&raw_bytes)?,
-                                    ))))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                        Protocol::Ping => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() == <Ping as Encode>::ssz_fixed_len() {
-                                    Ok(Some(RPCResponse::Pong(Ping {
-                                        data: u64::from_ssz_bytes(&raw_bytes)?,
-                                    })))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                        Protocol::MetaData => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() == <MetaData<TSpec> as Encode>::ssz_fixed_len() {
-                                    Ok(Some(RPCResponse::MetaData(MetaData::from_ssz_bytes(
-                                        &raw_bytes,
-                                    )?)))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                    }
-                }
-                Ok(None) => Ok(None), // waiting for more bytes
-                Err(e) => Err(e),
-            }
-        }
-    }
-}
-
-impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZOutboundCodec<TSpec> {
-    type ErrorType = String;
-
-    fn decode_error(&mut self, src: &mut BytesMut) -> Result<Option<Self::ErrorType>, RPCError> {
-        match self.inner.decode(src).map_err(RPCError::from) {
-            Ok(Some(packet)) => Ok(Some(
-                String::from_utf8_lossy(&<Vec<u8>>::from_ssz_bytes(&packet)?).into(),
-            )),
-            Ok(None) => Ok(None),
-            Err(e) => Err(e),
-        }
-    }
-}

beacon_node/eth2_libp2p/src/rpc/codec/ssz_snappy.rs

@@ -1,10 +1,7 @@
 use crate::rpc::methods::*;
 use crate::rpc::{
     codec::base::OutboundCodec,
-    protocol::{
-        Encoding, Protocol, ProtocolId, RPCError, Version, BLOCKS_BY_ROOT_REQUEST_MAX,
-        BLOCKS_BY_ROOT_REQUEST_MIN, SIGNED_BEACON_BLOCK_MAX, SIGNED_BEACON_BLOCK_MIN,
-    },
+    protocol::{Encoding, Protocol, ProtocolId, RPCError, Version, ERROR_TYPE_MAX, ERROR_TYPE_MIN},
 };
 use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
 use libp2p::bytes::BytesMut;
@@ -99,6 +96,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
+            // Note: length-prefix of > 10 bytes(uint64) would be a decoding error
             match self.inner.decode(src).map_err(RPCError::from)? {
                 Some(length) => {
                     self.len = Some(length);
@@ -109,79 +107,56 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
 
         let length = self.len.expect("length should be Some");
 
-        // Should not attempt to decode rpc chunks with length > max_packet_size
-        if length > self.max_packet_size {
+        // Should not attempt to decode rpc chunks with `length > max_packet_size` or not within bounds of
+        // packet size for ssz container corresponding to `self.protocol`.
+        let ssz_limits = self.protocol.rpc_request_limits();
+        if length > self.max_packet_size || ssz_limits.is_out_of_bounds(length) {
             return Err(RPCError::InvalidData);
         }
-        let mut reader = FrameDecoder::new(Cursor::new(&src));
+        // Calculate worst case compression length for given uncompressed length
+        let max_compressed_len = snap::raw::max_compress_len(length) as u64;
+
+        // Create a limit reader as a wrapper that reads only upto `max_compressed_len` from `src`.
+        let limit_reader = Cursor::new(src.as_ref()).take(max_compressed_len);
+        let mut reader = FrameDecoder::new(limit_reader);
         let mut decoded_buffer = vec![0; length];
 
         match reader.read_exact(&mut decoded_buffer) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
-                let n = reader.get_ref().position();
+                let n = reader.get_ref().get_ref().position();
                 self.len = None;
                 let _read_bytes = src.split_to(n as usize);
+
+                // We need not check that decoded_buffer.len() is within bounds here
+                // since we have already checked `length` above.
                 match self.protocol.message_name {
                     Protocol::Status => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
+                            &decoded_buffer,
+                        )?))),
                     },
                     Protocol::Goodbye => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <GoodbyeReason as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCRequest::Goodbye(GoodbyeReason::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::Goodbye(
+                            GoodbyeReason::from_ssz_bytes(&decoded_buffer)?,
+                        ))),
                     },
                     Protocol::BlocksByRange => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len()
-                                == <BlocksByRangeRequest as Encode>::ssz_fixed_len()
-                            {
-                                Ok(Some(RPCRequest::BlocksByRange(
-                                    BlocksByRangeRequest::from_ssz_bytes(&decoded_buffer)?,
-                                )))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::BlocksByRange(
+                            BlocksByRangeRequest::from_ssz_bytes(&decoded_buffer)?,
+                        ))),
                     },
                     Protocol::BlocksByRoot => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() >= *BLOCKS_BY_ROOT_REQUEST_MIN
-                                && decoded_buffer.len() <= *BLOCKS_BY_ROOT_REQUEST_MAX
-                            {
-                                Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
-                                    block_roots: VariableList::from_ssz_bytes(&decoded_buffer)?,
-                                })))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
+                            block_roots: VariableList::from_ssz_bytes(&decoded_buffer)?,
+                        }))),
                     },
                     Protocol::Ping => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <Ping as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCRequest::Ping(Ping {
-                                    data: u64::from_ssz_bytes(&decoded_buffer)?,
-                                })))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::Ping(Ping {
+                            data: u64::from_ssz_bytes(&decoded_buffer)?,
+                        }))),
                     },
+                    // This case should be unreachable as `MetaData` requests are handled separately in the `InboundUpgrade`
                     Protocol::MetaData => match self.protocol.version {
                         Version::V1 => {
                             if !decoded_buffer.is_empty() {
@@ -193,12 +168,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
                     },
                 }
             }
-            Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
-                ErrorKind::UnexpectedEof => Ok(None),
-                _ => Err(e).map_err(RPCError::from),
-            },
+            Err(e) => handle_error(e, reader.get_ref().get_ref().position(), max_compressed_len),
         }
     }
 }
@@ -277,6 +247,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
+            // Note: length-prefix of > 10 bytes(uint64) would be a decoding error
             match self.inner.decode(src).map_err(RPCError::from)? {
                 Some(length) => {
                     self.len = Some(length as usize);
@@ -287,96 +258,71 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
 
         let length = self.len.expect("length should be Some");
 
-        // Should not attempt to decode rpc chunks with length > max_packet_size
-        if length > self.max_packet_size {
+        // Should not attempt to decode rpc chunks with `length > max_packet_size` or not within bounds of
+        // packet size for ssz container corresponding to `self.protocol`.
+        let ssz_limits = self.protocol.rpc_response_limits::<TSpec>();
+        if length > self.max_packet_size || ssz_limits.is_out_of_bounds(length) {
             return Err(RPCError::InvalidData);
         }
-        let mut reader = FrameDecoder::new(Cursor::new(&src));
+        // Calculate worst case compression length for given uncompressed length
+        let max_compressed_len = snap::raw::max_compress_len(length) as u64;
+        // Create a limit reader as a wrapper that reads only upto `max_compressed_len` from `src`.
+        let limit_reader = Cursor::new(src.as_ref()).take(max_compressed_len);
+        let mut reader = FrameDecoder::new(limit_reader);
+
         let mut decoded_buffer = vec![0; length];
+
         match reader.read_exact(&mut decoded_buffer) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
-                let n = reader.get_ref().position();
+                let n = reader.get_ref().get_ref().position();
                 self.len = None;
-                let _read_byts = src.split_to(n as usize);
+                let _read_bytes = src.split_to(n as usize);
+
+                // We need not check that decoded_buffer.len() is within bounds here
+                // since we have already checked `length` above.
                 match self.protocol.message_name {
                     Protocol::Status => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCResponse::Status(StatusMessage::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::Status(
+                            StatusMessage::from_ssz_bytes(&decoded_buffer)?,
+                        ))),
                     },
+                    // This case should be unreachable as `Goodbye` has no response.
                     Protocol::Goodbye => Err(RPCError::InvalidData),
                     Protocol::BlocksByRange => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                && decoded_buffer.len() <= *SIGNED_BEACON_BLOCK_MAX
-                            {
-                                Ok(Some(RPCResponse::BlocksByRange(Box::new(
-                                    SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
-                                ))))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::BlocksByRange(Box::new(
+                            SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
+                        )))),
                     },
                     Protocol::BlocksByRoot => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                && decoded_buffer.len() <= *SIGNED_BEACON_BLOCK_MAX
-                            {
-                                Ok(Some(RPCResponse::BlocksByRoot(Box::new(
-                                    SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
-                                ))))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::BlocksByRoot(Box::new(
+                            SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
+                        )))),
                     },
                     Protocol::Ping => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <Ping as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCResponse::Pong(Ping {
-                                    data: u64::from_ssz_bytes(&decoded_buffer)?,
-                                })))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::Pong(Ping {
+                            data: u64::from_ssz_bytes(&decoded_buffer)?,
+                        }))),
                     },
                     Protocol::MetaData => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <MetaData<TSpec> as Encode>::ssz_fixed_len()
-                            {
-                                Ok(Some(RPCResponse::MetaData(MetaData::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::MetaData(MetaData::from_ssz_bytes(
+                            &decoded_buffer,
+                        )?))),
                     },
                 }
             }
-            Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
-                ErrorKind::UnexpectedEof => Ok(None),
-                _ => Err(e).map_err(RPCError::from),
-            },
+            Err(e) => handle_error(e, reader.get_ref().get_ref().position(), max_compressed_len),
         }
     }
 }
 
 impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZSnappyOutboundCodec<TSpec> {
-    type ErrorType = String;
+    type CodecErrorType = ErrorType;
 
-    fn decode_error(&mut self, src: &mut BytesMut) -> Result<Option<Self::ErrorType>, RPCError> {
+    fn decode_error(
+        &mut self,
+        src: &mut BytesMut,
+    ) -> Result<Option<Self::CodecErrorType>, RPCError> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
             match self.inner.decode(src).map_err(RPCError::from)? {
@@ -389,28 +335,52 @@ impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZSnappyOutboundCodec
 
         let length = self.len.expect("length should be Some");
 
-        // Should not attempt to decode rpc chunks with length > max_packet_size
-        if length > self.max_packet_size {
+        // Should not attempt to decode rpc chunks with `length > max_packet_size` or not within bounds of
+        // packet size for ssz container corresponding to `ErrorType`.
+        if length > self.max_packet_size || length > *ERROR_TYPE_MAX || length < *ERROR_TYPE_MIN {
             return Err(RPCError::InvalidData);
         }
-        let mut reader = FrameDecoder::new(Cursor::new(&src));
+
+        // Calculate worst case compression length for given uncompressed length
+        let max_compressed_len = snap::raw::max_compress_len(length) as u64;
+        // // Create a limit reader as a wrapper that reads only upto `max_compressed_len` from `src`.
+        let limit_reader = Cursor::new(src.as_ref()).take(max_compressed_len);
+        let mut reader = FrameDecoder::new(limit_reader);
         let mut decoded_buffer = vec![0; length];
         match reader.read_exact(&mut decoded_buffer) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
-                let n = reader.get_ref().position();
+                let n = reader.get_ref().get_ref().position();
                 self.len = None;
                 let _read_bytes = src.split_to(n as usize);
-                Ok(Some(
-                    String::from_utf8_lossy(&<Vec<u8>>::from_ssz_bytes(&decoded_buffer)?).into(),
-                ))
+                Ok(Some(ErrorType(VariableList::from_ssz_bytes(
+                    &decoded_buffer,
+                )?)))
             }
-            Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
-                ErrorKind::UnexpectedEof => Ok(None),
-                _ => Err(e).map_err(RPCError::from),
-            },
+            Err(e) => handle_error(e, reader.get_ref().get_ref().position(), max_compressed_len),
         }
     }
 }
+
+/// Handle errors that we get from decoding an RPC message from the stream.
+/// `num_bytes_read` is the number of bytes the snappy decoder has read from the underlying stream.
+/// `max_compressed_len` is the maximum compressed size for a given uncompressed size.
+fn handle_error<T>(
+    err: std::io::Error,
+    num_bytes: u64,
+    max_compressed_len: u64,
+) -> Result<Option<T>, RPCError> {
+    match err.kind() {
+        ErrorKind::UnexpectedEof => {
+            // If snappy has read `max_compressed_len` from underlying stream and still can't fill buffer, we have a malicious message.
+            // Report as `InvalidData` so that malicious peer gets banned.
+            if num_bytes >= max_compressed_len {
+                Err(RPCError::InvalidData)
+            } else {
+                // Haven't received enough bytes to decode yet, wait for more
+                Ok(None)
+            }
+        }
+        _ => Err(err).map_err(RPCError::from),
+    }
+}

beacon_node/eth2_libp2p/src/rpc/handler.rs

@@ -25,8 +25,6 @@ use std::{
 use tokio::time::{delay_queue, delay_until, Delay, DelayQueue, Instant as TInstant};
 use types::EthSpec;
 
-//TODO: Implement check_timeout() on the substream types
-
 /// The time (in seconds) before a substream that is awaiting a response from the user times out.
 pub const RESPONSE_TIMEOUT: u64 = 10;
 
@@ -81,7 +79,7 @@ where
     TSpec: EthSpec,
 {
     /// The upgrade for inbound substreams.
-    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>,
+    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
 
     /// Errors occurring on outbound and inbound connections queued for reporting back.
     pending_errors: Vec<HandlerErr>,
@@ -116,9 +114,6 @@ where
     /// Maximum number of concurrent outbound substreams being opened. Value is never modified.
     max_dial_negotiated: u32,
 
-    /// Value to return from `connection_keep_alive`.
-    keep_alive: KeepAlive,
-
     /// State of the handler.
     state: HandlerState,
 
@@ -166,8 +161,6 @@ struct OutboundInfo<TSpec: EthSpec> {
     /// Info over the protocol this substream is handling.
     proto: Protocol,
     /// Number of chunks to be seen from the peer's response.
-    // TODO: removing the option could allow clossing the streams after the number of
-    // expected responses is met for all protocols.
     remaining_chunks: Option<u64>,
     /// `RequestId` as given by the application that sent the request.
     req_id: RequestId,
@@ -228,7 +221,10 @@ impl<TSpec> RPCHandler<TSpec>
 where
     TSpec: EthSpec,
 {
-    pub fn new(listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>, log: &slog::Logger) -> Self {
+    pub fn new(
+        listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
+        log: &slog::Logger,
+    ) -> Self {
         RPCHandler {
             listen_protocol,
             pending_errors: Vec::new(),
@@ -243,7 +239,6 @@ where
             current_outbound_substream_id: SubstreamId(0),
             state: HandlerState::Active,
             max_dial_negotiated: 8,
-            keep_alive: KeepAlive::Yes,
             outbound_io_error_retries: 0,
             log: log.clone(),
         }
@@ -253,7 +248,7 @@ where
     ///
     /// > **Note**: If you modify the protocol, modifications will only applies to future inbound
     /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_ref(&self) -> &SubstreamProtocol<RPCProtocol<TSpec>> {
+    pub fn listen_protocol_ref(&self) -> &SubstreamProtocol<RPCProtocol<TSpec>, ()> {
         &self.listen_protocol
     }
 
@@ -261,7 +256,7 @@ where
     ///
     /// > **Note**: If you modify the protocol, modifications will only apply to future inbound
     /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_mut(&mut self) -> &mut SubstreamProtocol<RPCProtocol<TSpec>> {
+    pub fn listen_protocol_mut(&mut self) -> &mut SubstreamProtocol<RPCProtocol<TSpec>, ()> {
         &mut self.listen_protocol
     }
 
@@ -287,7 +282,6 @@ where
                 TInstant::now() + Duration::from_secs(SHUTDOWN_TIMEOUT_SECS as u64),
             ));
         }
-        self.update_keep_alive();
     }
 
     /// Opens an outbound substream with a request.
@@ -295,7 +289,6 @@ where
         match self.state {
             HandlerState::Active => {
                 self.dial_queue.push((id, req));
-                self.update_keep_alive();
             }
             _ => {
                 self.pending_errors.push(HandlerErr::Outbound {
@@ -338,38 +331,6 @@ where
         }
         inbound_info.pending_items.push(response);
     }
-
-    /// Updates the `KeepAlive` returned by `connection_keep_alive`.
-    ///
-    /// The handler stays alive as long as there are inbound/outbound substreams established and no
-    /// items dialing/to be dialed. Otherwise it is given a grace period of inactivity of
-    /// `self.inactive_timeout`.
-    fn update_keep_alive(&mut self) {
-        // Check that we don't have outbound items pending for dialing, nor dialing, nor
-        // established. Also check that there are no established inbound substreams.
-        // Errors and events need to be reported back, so check those too.
-        let should_shutdown = if let HandlerState::ShuttingDown(_) = self.state {
-            self.dial_queue.is_empty()
-                && self.outbound_substreams.is_empty()
-                && self.inbound_substreams.is_empty()
-                && self.pending_errors.is_empty()
-                && self.events_out.is_empty()
-                && self.dial_negotiated == 0
-        } else {
-            false
-        };
-
-        match self.keep_alive {
-            KeepAlive::Yes if should_shutdown => self.keep_alive = KeepAlive::No,
-            KeepAlive::Yes => {} // We continue being active
-            KeepAlive::Until(_) if should_shutdown => self.keep_alive = KeepAlive::No, // Already deemed inactive
-            KeepAlive::Until(_) => {
-                // No longer idle
-                self.keep_alive = KeepAlive::Yes;
-            }
-            KeepAlive::No => {} // currently not used
-        }
-    }
 }
 
 impl<TSpec> ProtocolsHandler for RPCHandler<TSpec>
@@ -382,14 +343,16 @@ where
     type InboundProtocol = RPCProtocol<TSpec>;
     type OutboundProtocol = RPCRequest<TSpec>;
     type OutboundOpenInfo = (RequestId, RPCRequest<TSpec>); // Keep track of the id and the request
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.listen_protocol.clone()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         substream: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         // only accept new peer requests when active
         if !matches!(self.state, HandlerState::Active) {
@@ -422,8 +385,6 @@ where
         self.events_out
             .push(RPCReceived::Request(self.current_inbound_substream_id, req));
         self.current_inbound_substream_id.0 += 1;
-
-        self.update_keep_alive();
     }
 
     fn inject_fully_negotiated_outbound(
@@ -481,8 +442,6 @@ where
             }
             self.current_outbound_substream_id.0 += 1;
         }
-
-        self.update_keep_alive();
     }
 
     fn inject_event(&mut self, rpc_event: Self::InEvent) {
@@ -510,7 +469,6 @@ where
 
         // This dialing is now considered failed
         self.dial_negotiated -= 1;
-        self.update_keep_alive();
 
         self.outbound_io_error_retries = 0;
         // map the error
@@ -543,7 +501,29 @@ where
     }
 
     fn connection_keep_alive(&self) -> KeepAlive {
-        self.keep_alive
+        // Check that we don't have outbound items pending for dialing, nor dialing, nor
+        // established. Also check that there are no established inbound substreams.
+        // Errors and events need to be reported back, so check those too.
+        let should_shutdown = match self.state {
+            HandlerState::ShuttingDown(_) => {
+                self.dial_queue.is_empty()
+                    && self.outbound_substreams.is_empty()
+                    && self.inbound_substreams.is_empty()
+                    && self.pending_errors.is_empty()
+                    && self.events_out.is_empty()
+                    && self.dial_negotiated == 0
+            }
+            HandlerState::Deactivated => {
+                // Regardless of events, the timeout has expired. Force the disconnect.
+                true
+            }
+            _ => false,
+        };
+        if should_shutdown {
+            KeepAlive::No
+        } else {
+            KeepAlive::Yes
+        }
     }
 
     fn poll(
@@ -619,8 +599,6 @@ where
                     if let Some(OutboundInfo { proto, req_id, .. }) =
                         self.outbound_substreams.remove(outbound_id.get_ref())
                     {
-                        self.update_keep_alive();
-
                         let outbound_err = HandlerErr::Outbound {
                             id: req_id,
                             proto,
@@ -719,7 +697,6 @@ where
             self.inbound_substreams.remove(&inbound_id);
         }
 
-        self.update_keep_alive();
         // drive outbound streams that need to be processed
         for outbound_id in self.outbound_substreams.keys().copied().collect::<Vec<_>>() {
             // get the state and mark it as poisoned
@@ -808,7 +785,6 @@ where
                         let request_id = entry.get().req_id;
                         self.outbound_substreams_delay.remove(delay_key);
                         entry.remove_entry();
-                        self.update_keep_alive();
                         // notify the application error
                         if request.expected_responses() > 1 {
                             // return an end of stream result
@@ -839,7 +815,6 @@ where
                             error: e,
                         };
                         entry.remove_entry();
-                        self.update_keep_alive();
                         return Poll::Ready(ProtocolsHandlerEvent::Custom(Err(outbound_err)));
                     }
                 },
@@ -852,7 +827,6 @@ where
                             let request_id = entry.get().req_id;
                             self.outbound_substreams_delay.remove(delay_key);
                             entry.remove_entry();
-                            self.update_keep_alive();
 
                             // report the stream termination to the user
                             //
@@ -889,10 +863,8 @@ where
             self.dial_negotiated += 1;
             let (id, req) = self.dial_queue.remove(0);
             self.dial_queue.shrink_to_fit();
-            self.update_keep_alive();
             return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                protocol: SubstreamProtocol::new(req.clone()),
-                info: (id, req),
+                protocol: SubstreamProtocol::new(req.clone(), ()).map_info(|()| (id, req)),
             });
         }
         Poll::Pending

beacon_node/eth2_libp2p/src/rpc/methods.rs

@@ -1,6 +1,7 @@
 //! Available RPC methods types and ids.
 
 use crate::types::EnrBitfield;
+use regex::bytes::Regex;
 use serde::Serialize;
 use ssz_derive::{Decode, Encode};
 use ssz_types::{
@@ -15,11 +16,12 @@ pub type MaxRequestBlocks = U1024;
 pub const MAX_REQUEST_BLOCKS: u64 = 1024;
 
 /// Maximum length of error message.
-type MaxErrorLen = U256;
+pub type MaxErrorLen = U256;
+pub const MAX_ERROR_LEN: u64 = 256;
 
 /// Wrapper over SSZ List to represent error message in rpc responses.
 #[derive(Debug, Clone)]
-pub struct ErrorType(VariableList<u8, MaxErrorLen>);
+pub struct ErrorType(pub VariableList<u8, MaxErrorLen>);
 
 impl From<String> for ErrorType {
     fn from(s: String) -> Self {
@@ -42,10 +44,9 @@ impl Deref for ErrorType {
 
 impl ToString for ErrorType {
     fn to_string(&self) -> String {
-        match std::str::from_utf8(self.0.deref()) {
-            Ok(s) => s.to_string(),
-            Err(_) => format!("{:?}", self.0.deref()), // Display raw bytes if not a UTF-8 string
-        }
+        #[allow(clippy::invalid_regex)]
+        let re = Regex::new("\\p{C}").expect("Regex is valid");
+        String::from_utf8_lossy(&re.replace_all(self.0.deref(), &b""[..])).to_string()
     }
 }
 
@@ -256,7 +257,7 @@ pub enum RPCCodedResponse<T: EthSpec> {
 }
 
 /// The code assigned to an erroneous `RPCResponse`.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 pub enum RPCResponseErrorCode {
     RateLimited,
     InvalidRequest,
@@ -276,20 +277,17 @@ impl<T: EthSpec> RPCCodedResponse<T> {
 
     /// Tells the codec whether to decode as an RPCResponse or an error.
     pub fn is_response(response_code: u8) -> bool {
-        match response_code {
-            0 => true,
-            _ => false,
-        }
+        matches!(response_code, 0)
     }
 
     /// Builds an RPCCodedResponse from a response code and an ErrorMessage
-    pub fn from_error(response_code: u8, err: String) -> Self {
+    pub fn from_error(response_code: u8, err: ErrorType) -> Self {
         let code = match response_code {
             1 => RPCResponseErrorCode::InvalidRequest,
             2 => RPCResponseErrorCode::ServerError,
             _ => RPCResponseErrorCode::Unknown,
         };
-        RPCCodedResponse::Error(code, err.into())
+        RPCCodedResponse::Error(code, err)
     }
 
     /// Specifies which response allows for multiple chunks for the stream handler.
@@ -310,10 +308,7 @@ impl<T: EthSpec> RPCCodedResponse<T> {
 
     /// Returns true if this response always terminates the stream.
     pub fn close_after(&self) -> bool {
-        match self {
-            RPCCodedResponse::Success(_) => false,
-            _ => true,
-        }
+        !matches!(self, RPCCodedResponse::Success(_))
     }
 }
 
@@ -366,7 +361,7 @@ impl<T: EthSpec> std::fmt::Display for RPCCodedResponse<T> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             RPCCodedResponse::Success(res) => write!(f, "{}", res),
-            RPCCodedResponse::Error(code, err) => write!(f, "{}: {:?}", code, err),
+            RPCCodedResponse::Error(code, err) => write!(f, "{}: {}", code, err.to_string()),
             RPCCodedResponse::StreamTermination(_) => write!(f, "Stream Termination"),
         }
     }

beacon_node/eth2_libp2p/src/rpc/mod.rs

@@ -169,9 +169,12 @@ where
 
     fn new_handler(&mut self) -> Self::ProtocolsHandler {
         RPCHandler::new(
-            SubstreamProtocol::new(RPCProtocol {
-                phantom: PhantomData,
-            }),
+            SubstreamProtocol::new(
+                RPCProtocol {
+                    phantom: PhantomData,
+                },
+                (),
+            ),
             &self.log,
         )
     }
@@ -230,9 +233,23 @@ where
                         }))
                 }
                 Err(RateLimitedErr::TooLarge) => {
-                    // we set the batch sizes, so this is a coding/config err
-                    crit!(self.log, "Batch too large to ever be processed";
-                        "protocol" => format!("{}", req.protocol()));
+                    // we set the batch sizes, so this is a coding/config err for most protocols
+                    let protocol = req.protocol();
+                    if matches!(protocol, Protocol::BlocksByRange) {
+                        debug!(self.log, "Blocks by range request will never be processed"; "request" => %req);
+                    } else {
+                        crit!(self.log, "Request size too large to ever be processed"; "protocol" => %protocol);
+                    }
+                    // send an error code to the peer.
+                    // the handler upon receiving the error code will send it back to the behaviour
+                    self.send_response(
+                        peer_id,
+                        (conn_id, *id),
+                        RPCCodedResponse::Error(
+                            RPCResponseErrorCode::RateLimited,
+                            "Rate limited. Request too large".into(),
+                        ),
+                    );
                 }
                 Err(RateLimitedErr::TooSoon(wait_time)) => {
                     debug!(self.log, "Request exceeds the rate limit";

beacon_node/eth2_libp2p/src/rpc/protocol.rs

@@ -2,11 +2,10 @@ use super::methods::*;
 use crate::rpc::{
     codec::{
         base::{BaseInboundCodec, BaseOutboundCodec},
-        ssz::{SSZInboundCodec, SSZOutboundCodec},
         ssz_snappy::{SSZSnappyInboundCodec, SSZSnappyOutboundCodec},
         InboundCodec, OutboundCodec,
     },
-    methods::ResponseTermination,
+    methods::{MaxErrorLen, ResponseTermination, MAX_ERROR_LEN},
     MaxRequestBlocks, MAX_REQUEST_BLOCKS,
 };
 use futures::future::BoxFuture;
@@ -52,6 +51,19 @@ lazy_static! {
         ])
     .as_ssz_bytes()
     .len();
+    pub static ref ERROR_TYPE_MIN: usize =
+        VariableList::<u8, MaxErrorLen>::from(Vec::<u8>::new())
+    .as_ssz_bytes()
+    .len();
+    pub static ref ERROR_TYPE_MAX: usize =
+        VariableList::<u8, MaxErrorLen>::from(vec![
+            0u8;
+            MAX_ERROR_LEN
+                as usize
+        ])
+    .as_ssz_bytes()
+    .len();
+
 }
 
 /// The maximum bytes that can be sent across the RPC.
@@ -91,7 +103,6 @@ pub enum Version {
 /// RPC Encondings supported.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Encoding {
-    SSZ,
     SSZSnappy,
 }
 
@@ -112,7 +123,6 @@ impl std::fmt::Display for Protocol {
 impl std::fmt::Display for Encoding {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let repr = match self {
-            Encoding::SSZ => "ssz",
             Encoding::SSZSnappy => "ssz_snappy",
         };
         f.write_str(repr)
@@ -141,21 +151,33 @@ impl<TSpec: EthSpec> UpgradeInfo for RPCProtocol<TSpec> {
     fn protocol_info(&self) -> Self::InfoIter {
         vec![
             ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::Goodbye, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::Goodbye, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::Ping, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::Ping, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::MetaData, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::MetaData, Version::V1, Encoding::SSZ),
         ]
     }
 }
 
+/// Represents the ssz length bounds for RPC messages.
+#[derive(Debug, PartialEq)]
+pub struct RpcLimits {
+    min: usize,
+    max: usize,
+}
+
+impl RpcLimits {
+    pub fn new(min: usize, max: usize) -> Self {
+        Self { min, max }
+    }
+
+    /// Returns true if the given length is out of bounds, false otherwise.
+    pub fn is_out_of_bounds(&self, length: usize) -> bool {
+        length > self.max || length < self.min
+    }
+}
+
 /// Tracks the types in a protocol id.
 #[derive(Clone, Debug)]
 pub struct ProtocolId {
@@ -172,6 +194,59 @@ pub struct ProtocolId {
     protocol_id: String,
 }
 
+impl ProtocolId {
+    /// Returns min and max size for messages of given protocol id requests.
+    pub fn rpc_request_limits(&self) -> RpcLimits {
+        match self.message_name {
+            Protocol::Status => RpcLimits::new(
+                <StatusMessage as Encode>::ssz_fixed_len(),
+                <StatusMessage as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::Goodbye => RpcLimits::new(
+                <GoodbyeReason as Encode>::ssz_fixed_len(),
+                <GoodbyeReason as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::BlocksByRange => RpcLimits::new(
+                <BlocksByRangeRequest as Encode>::ssz_fixed_len(),
+                <BlocksByRangeRequest as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::BlocksByRoot => {
+                RpcLimits::new(*BLOCKS_BY_ROOT_REQUEST_MIN, *BLOCKS_BY_ROOT_REQUEST_MAX)
+            }
+            Protocol::Ping => RpcLimits::new(
+                <Ping as Encode>::ssz_fixed_len(),
+                <Ping as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::MetaData => RpcLimits::new(0, 0), // Metadata requests are empty
+        }
+    }
+
+    /// Returns min and max size for messages of given protocol id responses.
+    pub fn rpc_response_limits<T: EthSpec>(&self) -> RpcLimits {
+        match self.message_name {
+            Protocol::Status => RpcLimits::new(
+                <StatusMessage as Encode>::ssz_fixed_len(),
+                <StatusMessage as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::Goodbye => RpcLimits::new(0, 0), // Goodbye request has no response
+            Protocol::BlocksByRange => {
+                RpcLimits::new(*SIGNED_BEACON_BLOCK_MIN, *SIGNED_BEACON_BLOCK_MAX)
+            }
+            Protocol::BlocksByRoot => {
+                RpcLimits::new(*SIGNED_BEACON_BLOCK_MIN, *SIGNED_BEACON_BLOCK_MAX)
+            }
+            Protocol::Ping => RpcLimits::new(
+                <Ping as Encode>::ssz_fixed_len(),
+                <Ping as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::MetaData => RpcLimits::new(
+                <MetaData<T> as Encode>::ssz_fixed_len(),
+                <MetaData<T> as Encode>::ssz_fixed_len(),
+            ),
+        }
+    }
+}
+
 /// An RPC protocol ID.
 impl ProtocolId {
     pub fn new(message_name: Protocol, version: Version, encoding: Encoding) -> Self {
@@ -224,11 +299,6 @@ where
                         BaseInboundCodec::new(SSZSnappyInboundCodec::new(protocol, MAX_RPC_SIZE));
                     InboundCodec::SSZSnappy(ssz_snappy_codec)
                 }
-                Encoding::SSZ => {
-                    let ssz_codec =
-                        BaseInboundCodec::new(SSZInboundCodec::new(protocol, MAX_RPC_SIZE));
-                    InboundCodec::SSZ(ssz_codec)
-                }
             };
             let mut timed_socket = TimeoutStream::new(socket);
             timed_socket.set_read_timeout(Some(Duration::from_secs(TTFB_TIMEOUT)));
@@ -247,7 +317,8 @@ where
                     {
                         Err(e) => Err(RPCError::from(e)),
                         Ok((Some(Ok(request)), stream)) => Ok((request, stream)),
-                        Ok((Some(Err(_)), _)) | Ok((None, _)) => Err(RPCError::IncompleteStream),
+                        Ok((Some(Err(e)), _)) => Err(e),
+                        Ok((None, _)) => Err(RPCError::IncompleteStream),
                     }
                 }
             }
@@ -286,30 +357,36 @@ impl<TSpec: EthSpec> RPCRequest<TSpec> {
     pub fn supported_protocols(&self) -> Vec<ProtocolId> {
         match self {
             // add more protocols when versions/encodings are supported
-            RPCRequest::Status(_) => vec![
-                ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::Goodbye(_) => vec![
-                ProtocolId::new(Protocol::Goodbye, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::Goodbye, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::BlocksByRange(_) => vec![
-                ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::BlocksByRoot(_) => vec![
-                ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::Ping(_) => vec![
-                ProtocolId::new(Protocol::Ping, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::Ping, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::MetaData(_) => vec![
-                ProtocolId::new(Protocol::MetaData, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::MetaData, Version::V1, Encoding::SSZ),
-            ],
+            RPCRequest::Status(_) => vec![ProtocolId::new(
+                Protocol::Status,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::Goodbye(_) => vec![ProtocolId::new(
+                Protocol::Goodbye,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::BlocksByRange(_) => vec![ProtocolId::new(
+                Protocol::BlocksByRange,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::BlocksByRoot(_) => vec![ProtocolId::new(
+                Protocol::BlocksByRoot,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::Ping(_) => vec![ProtocolId::new(
+                Protocol::Ping,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::MetaData(_) => vec![ProtocolId::new(
+                Protocol::MetaData,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
         }
     }
 
@@ -379,11 +456,6 @@ where
                     BaseOutboundCodec::new(SSZSnappyOutboundCodec::new(protocol, MAX_RPC_SIZE));
                 OutboundCodec::SSZSnappy(ssz_snappy_codec)
             }
-            Encoding::SSZ => {
-                let ssz_codec =
-                    BaseOutboundCodec::new(SSZOutboundCodec::new(protocol, MAX_RPC_SIZE));
-                OutboundCodec::SSZ(ssz_codec)
-            }
         };
 
         let mut socket = Framed::new(socket, codec);
@@ -398,7 +470,7 @@ where
 }
 
 /// Error in RPC Encoding/Decoding.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub enum RPCError {
     /// Error when decoding the raw buffer from ssz.
     // NOTE: in the future a ssz::DecodeError should map to an InvalidData error

beacon_node/eth2_libp2p/src/rpc/rate_limiter.rs

@@ -189,7 +189,30 @@ impl RPCRateLimiter {
         request: &RPCRequest<T>,
     ) -> Result<(), RateLimitedErr> {
         let time_since_start = self.init_time.elapsed();
-        let tokens = request.expected_responses().max(1);
+        let mut tokens = request.expected_responses().max(1);
+
+        // Increase the rate limit for blocks by range requests with large step counts.
+        // We count to tokens as a quadratic increase with step size.
+        // Using (step_size/5)^2 + 1 as penalty factor allows step sizes of 1-4 to have no penalty
+        // but step sizes higher than this add a quadratic penalty.
+        // Penalty's go:
+        // Step size | Penalty Factor
+        //     1     |   1
+        //     2     |   1
+        //     3     |   1
+        //     4     |   1
+        //     5     |   2
+        //     6     |   2
+        //     7     |   2
+        //     8     |   3
+        //     9     |   4
+        //     10    |   5
+
+        if let RPCRequest::BlocksByRange(bbr_req) = request {
+            let penalty_factor = (bbr_req.step as f64 / 5.0).powi(2) as u64 + 1;
+            tokens *= penalty_factor;
+        }
+
         let check =
             |limiter: &mut Limiter<PeerId>| limiter.allows(time_since_start, peer_id, tokens);
         let limiter = match request.protocol() {

beacon_node/eth2_libp2p/src/service.rs

@@ -1,24 +1,23 @@
-use crate::behaviour::{Behaviour, BehaviourEvent, PeerRequestId, Request, Response};
+use crate::behaviour::{
+    save_metadata_to_disk, Behaviour, BehaviourEvent, PeerRequestId, Request, Response,
+};
 use crate::discovery::enr;
 use crate::multiaddr::Protocol;
-use crate::rpc::{GoodbyeReason, RPCResponseErrorCode, RequestId};
-use crate::types::{error, GossipKind};
+use crate::rpc::{GoodbyeReason, MetaData, RPCResponseErrorCode, RequestId};
+use crate::types::{error, EnrBitfield, GossipKind};
 use crate::EnrExt;
 use crate::{NetworkConfig, NetworkGlobals, PeerAction};
 use futures::prelude::*;
 use libp2p::core::{
-    identity::Keypair,
-    multiaddr::Multiaddr,
-    muxing::StreamMuxerBox,
-    transport::boxed::Boxed,
-    upgrade::{InboundUpgradeExt, OutboundUpgradeExt},
+    identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::boxed::Boxed,
 };
 use libp2p::{
-    core, noise, secio,
+    core, noise,
     swarm::{SwarmBuilder, SwarmEvent},
     PeerId, Swarm, Transport,
 };
 use slog::{crit, debug, info, o, trace, warn};
+use ssz::Decode;
 use std::fs::File;
 use std::io::prelude::*;
 use std::io::{Error, ErrorKind};
@@ -30,6 +29,8 @@ use types::{EnrForkId, EthSpec};
 pub const NETWORK_KEY_FILENAME: &str = "key";
 /// The maximum simultaneous libp2p connections per peer.
 const MAX_CONNECTIONS_PER_PEER: usize = 1;
+/// The filename to store our local metadata.
+pub const METADATA_FILENAME: &str = "metadata";
 
 /// The types of events than can be obtained from polling the libp2p service.
 ///
@@ -40,6 +41,8 @@ pub enum Libp2pEvent<TSpec: EthSpec> {
     Behaviour(BehaviourEvent<TSpec>),
     /// A new listening address has been established.
     NewListenAddr(Multiaddr),
+    /// We reached zero listening addresses.
+    ZeroListeners,
 }
 
 /// The configuration and state of the libp2p components for the beacon node.
@@ -55,8 +58,8 @@ pub struct Service<TSpec: EthSpec> {
 }
 
 impl<TSpec: EthSpec> Service<TSpec> {
-    pub fn new(
-        executor: environment::TaskExecutor,
+    pub async fn new(
+        executor: task_executor::TaskExecutor,
         config: &NetworkConfig,
         enr_fork_id: EnrForkId,
         log: &slog::Logger,
@@ -72,15 +75,24 @@ impl<TSpec: EthSpec> Service<TSpec> {
             enr::build_or_load_enr::<TSpec>(local_keypair.clone(), config, enr_fork_id, &log)?;
 
         let local_peer_id = enr.peer_id();
+
+        let meta_data = load_or_build_metadata(&config.network_dir, &log);
+
         // set up a collection of variables accessible outside of the network crate
         let network_globals = Arc::new(NetworkGlobals::new(
             enr.clone(),
             config.libp2p_port,
             config.discovery_port,
+            meta_data,
+            config
+                .trusted_peers
+                .iter()
+                .map(|x| PeerId::from(x.clone()))
+                .collect(),
             &log,
         ));
 
-        info!(log, "Libp2p Service"; "peer_id" => format!("{:?}", enr.peer_id()));
+        info!(log, "Libp2p Service"; "peer_id" => enr.peer_id().to_string());
         let discovery_string = if config.disable_discovery {
             "None".into()
         } else {
@@ -89,14 +101,15 @@ impl<TSpec: EthSpec> Service<TSpec> {
         debug!(log, "Attempting to open listening ports"; "address" => format!("{}", config.listen_address), "tcp_port" => config.libp2p_port, "udp_port" => discovery_string);
 
         let mut swarm = {
-            // Set up the transport - tcp/ws with noise and yamux/mplex
+            // Set up the transport - tcp/ws with noise and mplex
             let transport = build_transport(local_keypair.clone())
                 .map_err(|e| format!("Failed to build transport: {:?}", e))?;
             // Lighthouse network behaviour
-            let behaviour = Behaviour::new(&local_keypair, config, network_globals.clone(), &log)?;
+            let behaviour =
+                Behaviour::new(&local_keypair, config, network_globals.clone(), &log).await?;
 
             // use the executor for libp2p
-            struct Executor(environment::TaskExecutor);
+            struct Executor(task_executor::TaskExecutor);
             impl libp2p::core::Executor for Executor {
                 fn exec(&self, f: Pin<Box<dyn Future<Output = ()> + Send>>) {
                     self.0.spawn(f, "libp2p");
@@ -106,6 +119,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
                 .notify_handler_buffer_size(std::num::NonZeroUsize::new(32).expect("Not zero"))
                 .connection_event_buffer_size(64)
                 .incoming_connection_limit(10)
+                .outgoing_connection_limit(config.target_peers * 2)
                 .peer_connection_limit(MAX_CONNECTIONS_PER_PEER)
                 .executor(Box::new(Executor(executor)))
                 .build()
@@ -154,7 +168,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
         }
 
         // attempt to connect to any specified boot-nodes
-        let mut boot_nodes = config.boot_nodes.clone();
+        let mut boot_nodes = config.boot_nodes_enr.clone();
         boot_nodes.dedup();
 
         for bootnode_enr in boot_nodes {
@@ -175,6 +189,16 @@ impl<TSpec: EthSpec> Service<TSpec> {
             }
         }
 
+        for multiaddr in &config.boot_nodes_multiaddr {
+            // check TCP support for dialing
+            if multiaddr
+                .iter()
+                .any(|proto| matches!(proto, Protocol::Tcp(_)))
+            {
+                dial_addr(multiaddr.clone());
+            }
+        }
+
         let mut subscribed_topics: Vec<GossipKind> = vec![];
         for topic_kind in &config.topics {
             if swarm.subscribe_kind(topic_kind.clone()) {
@@ -183,7 +207,9 @@ impl<TSpec: EthSpec> Service<TSpec> {
                 warn!(log, "Could not subscribe to topic"; "topic" => format!("{}",topic_kind));
             }
         }
-        info!(log, "Subscribed to topics"; "topics" => format!("{:?}", subscribed_topics));
+        if !subscribed_topics.is_empty() {
+            info!(log, "Subscribed to topics"; "topics" => format!("{:?}", subscribed_topics));
+        }
 
         let service = Service {
             local_peer_id,
@@ -239,7 +265,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
                     endpoint: _,
                     num_established,
                 } => {
-                    debug!(self.log, "Connection closed"; "peer_id"=> peer_id.to_string(), "cause" => cause.to_string(), "connections" => num_established);
+                    debug!(self.log, "Connection closed"; "peer_id"=> peer_id.to_string(), "cause" => format!("{:?}", cause), "connections" => num_established);
                 }
                 SwarmEvent::NewListenAddr(multiaddr) => {
                     return Libp2pEvent::NewListenAddr(multiaddr)
@@ -275,10 +301,17 @@ impl<TSpec: EthSpec> Service<TSpec> {
                     debug!(self.log, "Listen address expired"; "multiaddr" => multiaddr.to_string())
                 }
                 SwarmEvent::ListenerClosed { addresses, reason } => {
-                    debug!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason))
+                    crit!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason));
+                    if Swarm::listeners(&self.swarm).count() == 0 {
+                        return Libp2pEvent::ZeroListeners;
+                    }
                 }
                 SwarmEvent::ListenerError { error } => {
-                    debug!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string()))
+                    // this is non fatal, but we still check
+                    warn!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string()));
+                    if Swarm::listeners(&self.swarm).count() == 0 {
+                        return Libp2pEvent::ZeroListeners;
+                    }
                 }
                 SwarmEvent::Dialing(peer_id) => {
                     debug!(self.log, "Dialing peer"; "peer_id" => peer_id.to_string());
@@ -289,8 +322,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
 }
 
 /// The implementation supports TCP/IP, WebSockets over TCP/IP, noise as the encryption layer, and
-/// yamux or mplex as the multiplexing layer.
-
+/// mplex as the multiplexing layer.
 fn build_transport(
     local_private_key: Keypair,
 ) -> Result<Boxed<(PeerId, StreamMuxerBox), Error>, Error> {
@@ -302,47 +334,15 @@ fn build_transport(
         transport.or_transport(libp2p::websocket::WsConfig::new(trans_clone))
     };
     // Authentication
-    let transport = transport
-        .and_then(move |stream, endpoint| {
-            let upgrade = core::upgrade::SelectUpgrade::new(
-                secio::SecioConfig::new(local_private_key.clone()),
-                generate_noise_config(&local_private_key),
-            );
-            core::upgrade::apply(stream, upgrade, endpoint, core::upgrade::Version::V1).and_then(
-                |out| async move {
-                    match out {
-                        // Secio was negotiated
-                        core::either::EitherOutput::First((remote_id, out)) => {
-                            Ok((core::either::EitherOutput::First(out), remote_id))
-                        }
-                        // Noise was negotiated
-                        core::either::EitherOutput::Second((remote_id, out)) => {
-                            Ok((core::either::EitherOutput::Second(out), remote_id))
-                        }
-                    }
-                },
-            )
-        })
-        .timeout(Duration::from_secs(20));
-
-    // Multiplexing
-    let transport = transport
-        .and_then(move |(stream, peer_id), endpoint| {
-            let peer_id2 = peer_id.clone();
-            let upgrade = core::upgrade::SelectUpgrade::new(
-                libp2p::mplex::MplexConfig::new(),
-                libp2p::yamux::Config::default(),
-            )
-            .map_inbound(move |muxer| (peer_id, muxer))
-            .map_outbound(move |muxer| (peer_id2, muxer));
-
-            core::upgrade::apply(stream, upgrade, endpoint, core::upgrade::Version::V1)
-                .map_ok(|(id, muxer)| (id, core::muxing::StreamMuxerBox::new(muxer)))
-        })
-        .timeout(Duration::from_secs(20))
+    Ok(transport
+        .upgrade(core::upgrade::Version::V1)
+        .authenticate(generate_noise_config(&local_private_key))
+        .multiplex(libp2p::mplex::MplexConfig::new())
+        .map(|(peer, muxer), _| (peer, core::muxing::StreamMuxerBox::new(muxer)))
+        .timeout(Duration::from_secs(10))
+        .timeout(Duration::from_secs(10))
         .map_err(|err| Error::new(ErrorKind::Other, err))
-        .boxed();
-    Ok(transport)
+        .boxed())
 }
 
 // Useful helper functions for debugging. Currently not used in the client.
@@ -373,7 +373,7 @@ fn keypair_from_bytes(mut bytes: Vec<u8>) -> error::Result<Keypair> {
 /// generated and is then saved to disk.
 ///
 /// Currently only secp256k1 keys are allowed, as these are the only keys supported by discv5.
-fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
+pub fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
     // check for key from disk
     let network_key_f = config.network_dir.join(NETWORK_KEY_FILENAME);
     if let Ok(mut network_key_file) = File::open(network_key_f.clone()) {
@@ -436,3 +436,43 @@ fn strip_peer_id(addr: &mut Multiaddr) {
         _ => {}
     }
 }
+
+/// Load metadata from persisted file. Return default metadata if loading fails.
+fn load_or_build_metadata<E: EthSpec>(
+    network_dir: &std::path::PathBuf,
+    log: &slog::Logger,
+) -> MetaData<E> {
+    // Default metadata
+    let mut meta_data = MetaData {
+        seq_number: 0,
+        attnets: EnrBitfield::<E>::default(),
+    };
+    // Read metadata from persisted file if available
+    let metadata_path = network_dir.join(METADATA_FILENAME);
+    if let Ok(mut metadata_file) = File::open(metadata_path) {
+        let mut metadata_ssz = Vec::new();
+        if metadata_file.read_to_end(&mut metadata_ssz).is_ok() {
+            match MetaData::<E>::from_ssz_bytes(&metadata_ssz) {
+                Ok(persisted_metadata) => {
+                    meta_data.seq_number = persisted_metadata.seq_number;
+                    // Increment seq number if persisted attnet is not default
+                    if persisted_metadata.attnets != meta_data.attnets {
+                        meta_data.seq_number += 1;
+                    }
+                    debug!(log, "Loaded metadata from disk");
+                }
+                Err(e) => {
+                    debug!(
+                        log,
+                        "Metadata from file could not be decoded";
+                        "error" => format!("{:?}", e),
+                    );
+                }
+            }
+        }
+    };
+
+    debug!(log, "Metadata sequence number"; "seq_num" => meta_data.seq_number);
+    save_metadata_to_disk(network_dir, meta_data.clone(), &log);
+    meta_data
+}

beacon_node/eth2_libp2p/src/types/globals.rs

@@ -1,10 +1,10 @@
 //! A collection of variables that are accessible outside of the network thread itself.
 use crate::peer_manager::PeerDB;
-use crate::rpc::methods::MetaData;
+use crate::rpc::MetaData;
 use crate::types::SyncState;
 use crate::Client;
 use crate::EnrExt;
-use crate::{Enr, Eth2Enr, GossipTopic, Multiaddr, PeerId};
+use crate::{Enr, GossipTopic, Multiaddr, PeerId};
 use parking_lot::RwLock;
 use std::collections::HashSet;
 use std::sync::atomic::{AtomicU16, Ordering};
@@ -13,8 +13,6 @@ use types::EthSpec;
 pub struct NetworkGlobals<TSpec: EthSpec> {
     /// The current local ENR.
     pub local_enr: RwLock<Enr>,
-    /// The current node's meta-data.
-    pub meta_data: RwLock<MetaData<TSpec>>,
     /// The local peer_id.
     pub peer_id: RwLock<PeerId>,
     /// Listening multiaddrs.
@@ -25,6 +23,8 @@ pub struct NetworkGlobals<TSpec: EthSpec> {
     pub listen_port_udp: AtomicU16,
     /// The collection of known peers.
     pub peers: RwLock<PeerDB<TSpec>>,
+    // The local meta data of our node.
+    pub local_metadata: RwLock<MetaData<TSpec>>,
     /// The current gossipsub topic subscriptions.
     pub gossipsub_subscriptions: RwLock<HashSet<GossipTopic>>,
     /// The current sync status of the node.
@@ -32,23 +32,22 @@ pub struct NetworkGlobals<TSpec: EthSpec> {
 }
 
 impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
-    pub fn new(enr: Enr, tcp_port: u16, udp_port: u16, log: &slog::Logger) -> Self {
-        // set up the local meta data of the node
-        let meta_data = RwLock::new(MetaData {
-            seq_number: 0,
-            attnets: enr
-                .bitfield::<TSpec>()
-                .expect("Local ENR must have a bitfield specified"),
-        });
-
+    pub fn new(
+        enr: Enr,
+        tcp_port: u16,
+        udp_port: u16,
+        local_metadata: MetaData<TSpec>,
+        trusted_peers: Vec<PeerId>,
+        log: &slog::Logger,
+    ) -> Self {
         NetworkGlobals {
             local_enr: RwLock::new(enr.clone()),
-            meta_data,
             peer_id: RwLock::new(enr.peer_id()),
             listen_multiaddrs: RwLock::new(Vec::new()),
             listen_port_tcp: AtomicU16::new(tcp_port),
             listen_port_udp: AtomicU16::new(udp_port),
-            peers: RwLock::new(PeerDB::new(log)),
+            local_metadata: RwLock::new(local_metadata),
+            peers: RwLock::new(PeerDB::new(trusted_peers, log)),
             gossipsub_subscriptions: RwLock::new(HashSet::new()),
             sync_state: RwLock::new(SyncState::Stalled),
         }
@@ -111,25 +110,8 @@ impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
 
     /// Updates the syncing state of the node.
     ///
-    /// If there is a new state, the old state and the new states are returned.
-    pub fn update_sync_state(&self) -> Option<(SyncState, SyncState)> {
-        let mut result = None;
-        // if we are in a range sync, nothing changes. Range sync will update this.
-        if !self.is_syncing() {
-            let new_state = self
-                .peers
-                .read()
-                .synced_peers()
-                .next()
-                .map(|_| SyncState::Synced)
-                .unwrap_or_else(|| SyncState::Stalled);
-
-            let mut peer_state = self.sync_state.write();
-            if new_state != *peer_state {
-                result = Some((peer_state.clone(), new_state.clone()));
-            }
-            *peer_state = new_state;
-        }
-        result
+    /// The old state is returned
+    pub fn set_sync_state(&self, new_state: SyncState) -> SyncState {
+        std::mem::replace(&mut *self.sync_state.write(), new_state)
     }
 }

beacon_node/eth2_libp2p/src/types/mod.rs

@@ -1,6 +1,7 @@
 pub mod error;
 mod globals;
 mod pubsub;
+mod subnet;
 mod sync_state;
 mod topics;
 
@@ -12,6 +13,7 @@ pub type EnrBitfield<T: EthSpec> = BitVector<T::SubnetBitfieldLength>;
 pub type Enr = discv5::enr::Enr<discv5::enr::CombinedKey>;
 
 pub use globals::NetworkGlobals;
-pub use pubsub::PubsubMessage;
+pub use pubsub::{MessageData, PubsubMessage};
+pub use subnet::SubnetDiscovery;
 pub use sync_state::SyncState;
-pub use topics::{GossipEncoding, GossipKind, GossipTopic};
+pub use topics::{GossipEncoding, GossipKind, GossipTopic, CORE_TOPICS};

beacon_node/eth2_libp2p/src/types/pubsub.rs

@@ -12,6 +12,33 @@ use types::{
     SignedBeaconBlock, SignedVoluntaryExit,
 };
 
+#[derive(Clone)]
+pub struct MessageData {
+    pub raw: Vec<u8>,
+    pub decompressed: Result<Vec<u8>, String>,
+}
+
+impl AsRef<[u8]> for MessageData {
+    fn as_ref(&self) -> &[u8] {
+        self.raw.as_ref()
+    }
+}
+
+impl Into<Vec<u8>> for MessageData {
+    fn into(self) -> Vec<u8> {
+        self.raw
+    }
+}
+
+impl From<Vec<u8>> for MessageData {
+    fn from(raw: Vec<u8>) -> Self {
+        Self {
+            decompressed: decompress_snappy(raw.as_ref()),
+            raw,
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub enum PubsubMessage<T: EthSpec> {
     /// Gossipsub message providing notification of a new block.
@@ -28,6 +55,24 @@ pub enum PubsubMessage<T: EthSpec> {
     AttesterSlashing(Box<AttesterSlashing<T>>),
 }
 
+fn decompress_snappy(data: &[u8]) -> Result<Vec<u8>, String> {
+    // Exit early if uncompressed data is > GOSSIP_MAX_SIZE
+    match decompress_len(data) {
+        Ok(n) if n > GOSSIP_MAX_SIZE => {
+            return Err("ssz_snappy decoded data > GOSSIP_MAX_SIZE".into());
+        }
+        Ok(_) => {}
+        Err(e) => {
+            return Err(format!("{}", e));
+        }
+    };
+    let mut decoder = Decoder::new();
+    match decoder.decompress_vec(data) {
+        Ok(decompressed_data) => Ok(decompressed_data),
+        Err(e) => Err(format!("{}", e)),
+    }
+}
+
 impl<T: EthSpec> PubsubMessage<T> {
     /// Returns the topics that each pubsub message will be sent across, given a supported
     /// gossipsub encoding and fork version.
@@ -49,94 +94,58 @@ impl<T: EthSpec> PubsubMessage<T> {
         }
     }
 
-    /// This decodes `data` into a `PubsubMessage` given a list of topics.
-    ///
-    /// The topics are checked
-    /// in order and as soon as one topic matches the decoded data, we return the data.
+    /// This decodes `data` into a `PubsubMessage` given a topic.
     /* Note: This is assuming we are not hashing topics. If we choose to hash topics, these will
      * need to be modified.
-     *
-     * Also note that a message can be associated with many topics. As soon as one of the topics is
-     * known we match. If none of the topics are known we return an unknown state.
      */
-    pub fn decode(topics: &[TopicHash], data: &[u8]) -> Result<Self, String> {
-        let mut unknown_topics = Vec::new();
-        for topic in topics {
-            match GossipTopic::decode(topic.as_str()) {
-                Err(_) => {
-                    unknown_topics.push(topic);
-                    continue;
-                }
-                Ok(gossip_topic) => {
-                    let decompressed_data = &(match gossip_topic.encoding() {
-                        GossipEncoding::SSZSnappy => {
-                            // Exit early if uncompressed data is > GOSSIP_MAX_SIZE
-                            match decompress_len(data) {
-                                Ok(n) if n > GOSSIP_MAX_SIZE => {
-                                    return Err("ssz_snappy decoded data > GOSSIP_MAX_SIZE".into());
-                                }
-                                Ok(_) => {}
-                                Err(e) => {
-                                    return Err(format!("{}", e));
-                                }
-                            };
-                            let mut decoder = Decoder::new();
-                            match decoder.decompress_vec(data) {
-                                Ok(decompressed_data) => decompressed_data,
-                                Err(e) => return Err(format!("{}", e)),
-                            }
-                        }
-                    });
-                    // the ssz decoders
-                    match gossip_topic.kind() {
-                        GossipKind::BeaconAggregateAndProof => {
-                            let agg_and_proof =
-                                SignedAggregateAndProof::from_ssz_bytes(decompressed_data)
-                                    .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::AggregateAndProofAttestation(Box::new(
-                                agg_and_proof,
-                            )));
-                        }
-                        GossipKind::Attestation(subnet_id) => {
-                            let attestation = Attestation::from_ssz_bytes(decompressed_data)
+    pub fn decode(topic: &TopicHash, data: &MessageData) -> Result<Self, String> {
+        match GossipTopic::decode(topic.as_str()) {
+            Err(_) => Err(format!("Unknown gossipsub topic: {:?}", topic)),
+            Ok(gossip_topic) => {
+                let decompressed_data = match gossip_topic.encoding() {
+                    GossipEncoding::SSZSnappy => data.decompressed.as_ref()?.as_slice(),
+                };
+                // the ssz decoders
+                match gossip_topic.kind() {
+                    GossipKind::BeaconAggregateAndProof => {
+                        let agg_and_proof =
+                            SignedAggregateAndProof::from_ssz_bytes(decompressed_data)
                                 .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::Attestation(Box::new((
-                                *subnet_id,
-                                attestation,
-                            ))));
-                        }
-                        GossipKind::BeaconBlock => {
-                            let beacon_block = SignedBeaconBlock::from_ssz_bytes(decompressed_data)
-                                .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::BeaconBlock(Box::new(beacon_block)));
-                        }
-                        GossipKind::VoluntaryExit => {
-                            let voluntary_exit =
-                                SignedVoluntaryExit::from_ssz_bytes(decompressed_data)
-                                    .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::VoluntaryExit(Box::new(voluntary_exit)));
-                        }
-                        GossipKind::ProposerSlashing => {
-                            let proposer_slashing =
-                                ProposerSlashing::from_ssz_bytes(decompressed_data)
-                                    .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::ProposerSlashing(Box::new(
-                                proposer_slashing,
-                            )));
-                        }
-                        GossipKind::AttesterSlashing => {
-                            let attester_slashing =
-                                AttesterSlashing::from_ssz_bytes(decompressed_data)
-                                    .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::AttesterSlashing(Box::new(
-                                attester_slashing,
-                            )));
-                        }
+                        Ok(PubsubMessage::AggregateAndProofAttestation(Box::new(
+                            agg_and_proof,
+                        )))
+                    }
+                    GossipKind::Attestation(subnet_id) => {
+                        let attestation = Attestation::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::Attestation(Box::new((
+                            *subnet_id,
+                            attestation,
+                        ))))
+                    }
+                    GossipKind::BeaconBlock => {
+                        let beacon_block = SignedBeaconBlock::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::BeaconBlock(Box::new(beacon_block)))
+                    }
+                    GossipKind::VoluntaryExit => {
+                        let voluntary_exit = SignedVoluntaryExit::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::VoluntaryExit(Box::new(voluntary_exit)))
+                    }
+                    GossipKind::ProposerSlashing => {
+                        let proposer_slashing = ProposerSlashing::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::ProposerSlashing(Box::new(proposer_slashing)))
+                    }
+                    GossipKind::AttesterSlashing => {
+                        let attester_slashing = AttesterSlashing::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::AttesterSlashing(Box::new(attester_slashing)))
                     }
                 }
             }
         }
-        Err(format!("Unknown gossipsub topics: {:?}", unknown_topics))
     }
 
     /// Encodes a `PubsubMessage` based on the topic encodings. The first known encoding is used. If

beacon_node/eth2_libp2p/src/types/subnet.rs

@@ -0,0 +1,28 @@
+use std::time::{Duration, Instant};
+use types::SubnetId;
+
+const DURATION_DIFFERENCE: Duration = Duration::from_millis(1);
+
+/// A subnet to discover peers on along with the instant after which it's no longer useful.
+#[derive(Debug, Clone)]
+pub struct SubnetDiscovery {
+    pub subnet_id: SubnetId,
+    pub min_ttl: Option<Instant>,
+}
+
+impl PartialEq for SubnetDiscovery {
+    fn eq(&self, other: &SubnetDiscovery) -> bool {
+        self.subnet_id == other.subnet_id
+            && match (self.min_ttl, other.min_ttl) {
+                (Some(min_ttl_instant), Some(other_min_ttl_instant)) => {
+                    min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
+                        < DURATION_DIFFERENCE
+                        && other_min_ttl_instant.saturating_duration_since(min_ttl_instant)
+                            < DURATION_DIFFERENCE
+                }
+                (None, None) => true,
+                (None, Some(_)) => true,
+                (Some(_), None) => true,
+            }
+    }
+}

beacon_node/eth2_libp2p/src/types/sync_state.rs

@@ -1,19 +1,15 @@
 use serde::{Deserialize, Serialize};
-use types::{Hash256, Slot};
+use types::Slot;
 
 /// The current state of the node.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum SyncState {
     /// The node is performing a long-range (batch) sync over a finalized chain.
     /// In this state, parent lookups are disabled.
-    SyncingFinalized {
-        start_slot: Slot,
-        head_slot: Slot,
-        head_root: Hash256,
-    },
+    SyncingFinalized { start_slot: Slot, target_slot: Slot },
     /// The node is performing a long-range (batch) sync over one or many head chains.
     /// In this state parent lookups are disabled.
-    SyncingHead { start_slot: Slot, head_slot: Slot },
+    SyncingHead { start_slot: Slot, target_slot: Slot },
     /// The node is up to date with all known peers and is connected to at least one
     /// fully synced peer. In this state, parent lookups are enabled.
     Synced,
@@ -47,10 +43,7 @@ impl SyncState {
 
     /// Returns true if the node is synced.
     pub fn is_synced(&self) -> bool {
-        match self {
-            SyncState::Synced => true,
-            _ => false,
-        }
+        matches!(self, SyncState::Synced)
     }
 }
 

beacon_node/eth2_libp2p/src/types/topics.rs

@@ -1,4 +1,4 @@
-use libp2p::gossipsub::Topic;
+use libp2p::gossipsub::IdentTopic as Topic;
 use serde_derive::{Deserialize, Serialize};
 use types::SubnetId;
 
@@ -14,6 +14,14 @@ pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit";
 pub const PROPOSER_SLASHING_TOPIC: &str = "proposer_slashing";
 pub const ATTESTER_SLASHING_TOPIC: &str = "attester_slashing";
 
+pub const CORE_TOPICS: [GossipKind; 5] = [
+    GossipKind::BeaconBlock,
+    GossipKind::BeaconAggregateAndProof,
+    GossipKind::VoluntaryExit,
+    GossipKind::ProposerSlashing,
+    GossipKind::AttesterSlashing,
+];
+
 /// A gossipsub topic which encapsulates the type of messages that should be sent and received over
 /// the pubsub protocol and the way the messages should be encoded.
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
@@ -139,7 +147,7 @@ impl GossipTopic {
 
 impl Into<Topic> for GossipTopic {
     fn into(self) -> Topic {
-        Topic::new(self.into())
+        Topic::new(self)
     }
 }
 

beacon_node/eth2_libp2p/tests/common/mod.rs

@@ -3,7 +3,7 @@ use eth2_libp2p::Enr;
 use eth2_libp2p::EnrExt;
 use eth2_libp2p::Multiaddr;
 use eth2_libp2p::Service as LibP2PService;
-use eth2_libp2p::{Libp2pEvent, NetworkConfig};
+use eth2_libp2p::{GossipsubConfigBuilder, Libp2pEvent, NetworkConfig};
 use slog::{debug, error, o, Drain};
 use std::net::{TcpListener, UdpSocket};
 use std::time::Duration;
@@ -80,24 +80,33 @@ pub fn build_config(port: u16, mut boot_nodes: Vec<Enr>) -> NetworkConfig {
     config.enr_tcp_port = Some(port);
     config.enr_udp_port = Some(port);
     config.enr_address = Some("127.0.0.1".parse().unwrap());
-    config.boot_nodes.append(&mut boot_nodes);
+    config.boot_nodes_enr.append(&mut boot_nodes);
     config.network_dir = path.into_path();
     // Reduce gossipsub heartbeat parameters
-    config.gs_config.heartbeat_initial_delay = Duration::from_millis(500);
-    config.gs_config.heartbeat_interval = Duration::from_millis(500);
+    config.gs_config = GossipsubConfigBuilder::from(config.gs_config)
+        .heartbeat_initial_delay(Duration::from_millis(500))
+        .heartbeat_interval(Duration::from_millis(500))
+        .build()
+        .unwrap();
     config
 }
 
-pub fn build_libp2p_instance(boot_nodes: Vec<Enr>, log: slog::Logger) -> Libp2pInstance {
+pub async fn build_libp2p_instance(boot_nodes: Vec<Enr>, log: slog::Logger) -> Libp2pInstance {
     let port = unused_port("tcp").unwrap();
     let config = build_config(port, boot_nodes);
     // launch libp2p service
 
     let (signal, exit) = exit_future::signal();
-    let executor =
-        environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone());
+    let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+    let executor = task_executor::TaskExecutor::new(
+        tokio::runtime::Handle::current(),
+        exit,
+        log.clone(),
+        shutdown_tx,
+    );
     Libp2pInstance(
         LibP2PService::new(executor, &config, EnrForkId::default(), &log)
+            .await
             .expect("should build libp2p instance")
             .1,
         signal,
@@ -112,10 +121,11 @@ pub fn get_enr(node: &LibP2PService<E>) -> Enr {
 
 // Returns `n` libp2p peers in fully connected topology.
 #[allow(dead_code)]
-pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
-    let mut nodes: Vec<_> = (0..n)
-        .map(|_| build_libp2p_instance(vec![], log.clone()))
-        .collect();
+pub async fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+    let mut nodes = Vec::with_capacity(n);
+    for _ in 0..n {
+        nodes.push(build_libp2p_instance(vec![], log.clone()).await);
+    }
     let multiaddrs: Vec<Multiaddr> = nodes
         .iter()
         .map(|x| get_enr(&x).multiaddr()[1].clone())
@@ -141,8 +151,8 @@ pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInsta
     let sender_log = log.new(o!("who" => "sender"));
     let receiver_log = log.new(o!("who" => "receiver"));
 
-    let mut sender = build_libp2p_instance(vec![], sender_log);
-    let mut receiver = build_libp2p_instance(vec![], receiver_log);
+    let mut sender = build_libp2p_instance(vec![], sender_log).await;
+    let mut receiver = build_libp2p_instance(vec![], receiver_log).await;
 
     let receiver_multiaddr = receiver.swarm.local_enr().multiaddr()[1].clone();
 
@@ -181,10 +191,12 @@ pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInsta
 
 // Returns `n` peers in a linear topology
 #[allow(dead_code)]
-pub fn build_linear(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
-    let mut nodes: Vec<_> = (0..n)
-        .map(|_| build_libp2p_instance(vec![], log.clone()))
-        .collect();
+pub async fn build_linear(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+    let mut nodes = Vec::with_capacity(n);
+    for _ in 0..n {
+        nodes.push(build_libp2p_instance(vec![], log.clone()).await);
+    }
+
     let multiaddrs: Vec<Multiaddr> = nodes
         .iter()
         .map(|x| get_enr(&x).multiaddr()[1].clone())

beacon_node/genesis/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "genesis"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
@@ -12,16 +12,16 @@ futures = "0.3.5"
 types = { path = "../../consensus/types"}
 environment = { path = "../../lighthouse/environment"}
 eth1 = { path = "../eth1"}
-rayon = "1.3.0"
+rayon = "1.4.1"
 state_processing = { path = "../../consensus/state_processing" }
 merkle_proof = { path = "../../consensus/merkle_proof" }
 eth2_ssz = "0.1.2"
 eth2_hashing = "0.1.0"
-tree_hash = "0.1.0"
-tokio = { version = "0.2.21", features = ["full"] }
+tree_hash = "0.1.1"
+tokio = { version = "0.2.22", features = ["full"] }
 parking_lot = "0.11.0"
 slog = "2.5.2"
 exit-future = "0.2.0"
-serde = "1.0.110"
-serde_derive = "1.0.110"
+serde = "1.0.116"
+serde_derive = "1.0.116"
 int_to_bytes = { path = "../../consensus/int_to_bytes" }

beacon_node/http_api/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "http_api"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+warp = { git = "https://github.com/paulhauner/warp", branch = "cors-wildcard" }
+serde = { version = "1.0.116", features = ["derive"] }
+tokio = { version = "0.2.22", features = ["macros"] }
+parking_lot = "0.11.0"
+types = { path = "../../consensus/types" }
+hex = "0.4.2"
+beacon_chain = { path = "../beacon_chain" }
+eth2 = { path = "../../common/eth2", features = ["lighthouse"] }
+slog = "2.5.2"
+network = { path = "../network" }
+eth2_libp2p = { path = "../eth2_libp2p" }
+eth1 = { path = "../eth1" }
+fork_choice = { path = "../../consensus/fork_choice" }
+state_processing = { path = "../../consensus/state_processing" }
+lighthouse_version = { path = "../../common/lighthouse_version" }
+lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
+lazy_static = "1.4.0"
+warp_utils = { path = "../../common/warp_utils" }
+slot_clock = { path = "../../common/slot_clock" }
+eth2_ssz = { path = "../../consensus/ssz" }
+bs58 = "0.3.1"
+
+[dev-dependencies]
+store = { path = "../store" }
+environment = { path = "../../lighthouse/environment" }
+tree_hash = "0.1.1"
+discv5 = { version = "0.1.0-alpha.13", features = ["libp2p"] }

beacon_node/http_api/src/beacon_proposer_cache.rs

@@ -0,0 +1,185 @@
+use crate::metrics;
+use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use eth2::types::ProposerData;
+use fork_choice::ProtoBlock;
+use slot_clock::SlotClock;
+use state_processing::per_slot_processing;
+use types::{BeaconState, Epoch, EthSpec, Hash256, PublicKeyBytes};
+
+/// This sets a maximum bound on the number of epochs to skip whilst instantiating the cache for
+/// the first time.
+const EPOCHS_TO_SKIP: u64 = 2;
+
+/// Caches the beacon block proposers for a given `epoch` and `epoch_boundary_root`.
+///
+/// This cache is only able to contain a single set of proposers and is only
+/// intended to cache the proposers for the current epoch according to the head
+/// of the chain. A change in epoch or re-org to a different chain may cause a
+/// cache miss and rebuild.
+pub struct BeaconProposerCache {
+    epoch: Epoch,
+    decision_block_root: Hash256,
+    proposers: Vec<ProposerData>,
+}
+
+impl BeaconProposerCache {
+    /// Create a new cache for the current epoch of the `chain`.
+    pub fn new<T: BeaconChainTypes>(chain: &BeaconChain<T>) -> Result<Self, BeaconChainError> {
+        let head_root = chain.head_beacon_block_root()?;
+        let head_block = chain
+            .fork_choice
+            .read()
+            .get_block(&head_root)
+            .ok_or_else(|| BeaconChainError::MissingBeaconBlock(head_root))?;
+
+        // If the head epoch is more than `EPOCHS_TO_SKIP` in the future, just build the cache at
+        // the epoch of the head. This prevents doing a massive amount of skip slots when starting
+        // a new database from genesis.
+        let epoch = {
+            let epoch_now = chain
+                .epoch()
+                .unwrap_or_else(|_| chain.spec.genesis_slot.epoch(T::EthSpec::slots_per_epoch()));
+            let head_epoch = head_block.slot.epoch(T::EthSpec::slots_per_epoch());
+            if epoch_now > head_epoch + EPOCHS_TO_SKIP {
+                head_epoch
+            } else {
+                epoch_now
+            }
+        };
+
+        Self::for_head_block(chain, epoch, head_root, head_block)
+    }
+
+    /// Create a new cache that contains the shuffling for `current_epoch`,
+    /// assuming that `head_root` and `head_block` represents the most recent
+    /// canonical block.
+    fn for_head_block<T: BeaconChainTypes>(
+        chain: &BeaconChain<T>,
+        current_epoch: Epoch,
+        head_root: Hash256,
+        head_block: ProtoBlock,
+    ) -> Result<Self, BeaconChainError> {
+        let _timer = metrics::start_timer(&metrics::HTTP_API_BEACON_PROPOSER_CACHE_TIMES);
+
+        let mut head_state = chain
+            .get_state(&head_block.state_root, Some(head_block.slot))?
+            .ok_or_else(|| BeaconChainError::MissingBeaconState(head_block.state_root))?;
+
+        let decision_block_root = Self::decision_block_root(current_epoch, head_root, &head_state)?;
+
+        // We *must* skip forward to the current epoch to obtain valid proposer
+        // duties. We cannot skip to the previous epoch, like we do with
+        // attester duties.
+        while head_state.current_epoch() < current_epoch {
+            // Skip slots until the current epoch, providing `Hash256::zero()` as the state root
+            // since we don't require it to be valid to identify producers.
+            per_slot_processing(&mut head_state, Some(Hash256::zero()), &chain.spec)?;
+        }
+
+        let proposers = current_epoch
+            .slot_iter(T::EthSpec::slots_per_epoch())
+            .map(|slot| {
+                head_state
+                    .get_beacon_proposer_index(slot, &chain.spec)
+                    .map_err(BeaconChainError::from)
+                    .and_then(|i| {
+                        let pubkey = chain
+                            .validator_pubkey(i)?
+                            .ok_or_else(|| BeaconChainError::ValidatorPubkeyCacheIncomplete(i))?;
+
+                        Ok(ProposerData {
+                            pubkey: PublicKeyBytes::from(pubkey),
+                            slot,
+                        })
+                    })
+            })
+            .collect::<Result<_, _>>()?;
+
+        Ok(Self {
+            epoch: current_epoch,
+            decision_block_root,
+            proposers,
+        })
+    }
+
+    /// Returns a block root which can be used to key the shuffling obtained from the following
+    /// parameters:
+    ///
+    /// - `shuffling_epoch`: the epoch for which the shuffling pertains.
+    /// - `head_block_root`: the block root at the head of the chain.
+    /// - `head_block_state`: the state of `head_block_root`.
+    pub fn decision_block_root<E: EthSpec>(
+        shuffling_epoch: Epoch,
+        head_block_root: Hash256,
+        head_block_state: &BeaconState<E>,
+    ) -> Result<Hash256, BeaconChainError> {
+        let decision_slot = shuffling_epoch
+            .start_slot(E::slots_per_epoch())
+            .saturating_sub(1_u64);
+
+        // If decision slot is equal to or ahead of the head, the block root is the head block root
+        if decision_slot >= head_block_state.slot {
+            Ok(head_block_root)
+        } else {
+            head_block_state
+                .get_block_root(decision_slot)
+                .map(|root| *root)
+                .map_err(Into::into)
+        }
+    }
+
+    /// Return the proposers for the given `Epoch`.
+    ///
+    /// The cache may be rebuilt if:
+    ///
+    /// - The epoch has changed since the last cache build.
+    /// - There has been a re-org that crosses an epoch boundary.
+    pub fn get_proposers<T: BeaconChainTypes>(
+        &mut self,
+        chain: &BeaconChain<T>,
+        epoch: Epoch,
+    ) -> Result<Vec<ProposerData>, warp::Rejection> {
+        let current_epoch = chain
+            .slot_clock
+            .now_or_genesis()
+            .ok_or_else(|| {
+                warp_utils::reject::custom_server_error("unable to read slot clock".to_string())
+            })?
+            .epoch(T::EthSpec::slots_per_epoch());
+
+        // Disallow requests that are outside the current epoch. This ensures the cache doesn't get
+        // washed-out with old values.
+        if current_epoch != epoch {
+            return Err(warp_utils::reject::custom_bad_request(format!(
+                "requested epoch is {} but only current epoch {} is allowed",
+                epoch, current_epoch
+            )));
+        }
+
+        let (head_block_root, head_decision_block_root) = chain
+            .with_head(|head| {
+                Self::decision_block_root(current_epoch, head.beacon_block_root, &head.beacon_state)
+                    .map(|decision_root| (head.beacon_block_root, decision_root))
+            })
+            .map_err(warp_utils::reject::beacon_chain_error)?;
+
+        let head_block = chain
+            .fork_choice
+            .read()
+            .get_block(&head_block_root)
+            .ok_or_else(|| BeaconChainError::MissingBeaconBlock(head_block_root))
+            .map_err(warp_utils::reject::beacon_chain_error)?;
+
+        // Rebuild the cache if this call causes a cache-miss.
+        if self.epoch != current_epoch || self.decision_block_root != head_decision_block_root {
+            metrics::inc_counter(&metrics::HTTP_API_BEACON_PROPOSER_CACHE_MISSES_TOTAL);
+
+            *self = Self::for_head_block(chain, current_epoch, head_block_root, head_block)
+                .map_err(warp_utils::reject::beacon_chain_error)?;
+        } else {
+            metrics::inc_counter(&metrics::HTTP_API_BEACON_PROPOSER_CACHE_HITS_TOTAL);
+        }
+
+        Ok(self.proposers.clone())
+    }
+}

beacon_node/http_api/src/block_id.rs

@@ -0,0 +1,87 @@
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use eth2::types::BlockId as CoreBlockId;
+use std::str::FromStr;
+use types::{Hash256, SignedBeaconBlock, Slot};
+
+/// Wraps `eth2::types::BlockId` and provides a simple way to obtain a block or root for a given
+/// `BlockId`.
+#[derive(Debug)]
+pub struct BlockId(pub CoreBlockId);
+
+impl BlockId {
+    pub fn from_slot(slot: Slot) -> Self {
+        Self(CoreBlockId::Slot(slot))
+    }
+
+    pub fn from_root(root: Hash256) -> Self {
+        Self(CoreBlockId::Root(root))
+    }
+
+    /// Return the block root identified by `self`.
+    pub fn root<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<Hash256, warp::Rejection> {
+        match &self.0 {
+            CoreBlockId::Head => chain
+                .head_info()
+                .map(|head| head.block_root)
+                .map_err(warp_utils::reject::beacon_chain_error),
+            CoreBlockId::Genesis => Ok(chain.genesis_block_root),
+            CoreBlockId::Finalized => chain
+                .head_info()
+                .map(|head| head.finalized_checkpoint.root)
+                .map_err(warp_utils::reject::beacon_chain_error),
+            CoreBlockId::Justified => chain
+                .head_info()
+                .map(|head| head.current_justified_checkpoint.root)
+                .map_err(warp_utils::reject::beacon_chain_error),
+            CoreBlockId::Slot(slot) => chain
+                .block_root_at_slot(*slot)
+                .map_err(warp_utils::reject::beacon_chain_error)
+                .and_then(|root_opt| {
+                    root_opt.ok_or_else(|| {
+                        warp_utils::reject::custom_not_found(format!(
+                            "beacon block at slot {}",
+                            slot
+                        ))
+                    })
+                }),
+            CoreBlockId::Root(root) => Ok(*root),
+        }
+    }
+
+    /// Return the `SignedBeaconBlock` identified by `self`.
+    pub fn block<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<SignedBeaconBlock<T::EthSpec>, warp::Rejection> {
+        match &self.0 {
+            CoreBlockId::Head => chain
+                .head_beacon_block()
+                .map_err(warp_utils::reject::beacon_chain_error),
+            _ => {
+                let root = self.root(chain)?;
+                chain
+                    .get_block(&root)
+                    .map_err(warp_utils::reject::beacon_chain_error)
+                    .and_then(|root_opt| {
+                        root_opt.ok_or_else(|| {
+                            warp_utils::reject::custom_not_found(format!(
+                                "beacon block with root {}",
+                                root
+                            ))
+                        })
+                    })
+            }
+        }
+    }
+}
+
+impl FromStr for BlockId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        CoreBlockId::from_str(s).map(Self)
+    }
+}

beacon_node/http_api/src/metrics.rs

@@ -0,0 +1,32 @@
+pub use lighthouse_metrics::*;
+
+lazy_static::lazy_static! {
+    pub static ref HTTP_API_PATHS_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "http_api_paths_total",
+        "Count of HTTP requests received",
+        &["path"]
+    );
+    pub static ref HTTP_API_STATUS_CODES_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "http_api_status_codes_total",
+        "Count of HTTP status codes returned",
+        &["status"]
+    );
+    pub static ref HTTP_API_PATHS_TIMES: Result<HistogramVec> = try_create_histogram_vec(
+        "http_api_paths_times",
+        "Duration to process HTTP requests per path",
+        &["path"]
+    );
+
+    pub static ref HTTP_API_BEACON_PROPOSER_CACHE_TIMES: Result<Histogram> = try_create_histogram(
+        "http_api_beacon_proposer_cache_build_times",
+        "Duration to process HTTP requests per path",
+    );
+    pub static ref HTTP_API_BEACON_PROPOSER_CACHE_HITS_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "http_api_beacon_proposer_cache_hits_total",
+        "Count of times the proposer cache has been hit",
+    );
+    pub static ref HTTP_API_BEACON_PROPOSER_CACHE_MISSES_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "http_api_beacon_proposer_cache_misses_total",
+        "Count of times the proposer cache has been missed",
+    );
+}

beacon_node/http_api/src/state_id.rs

@@ -0,0 +1,118 @@
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use eth2::types::StateId as CoreStateId;
+use std::str::FromStr;
+use types::{BeaconState, EthSpec, Fork, Hash256, Slot};
+
+/// Wraps `eth2::types::StateId` and provides common state-access functionality. E.g., reading
+/// states or parts of states from the database.
+pub struct StateId(CoreStateId);
+
+impl StateId {
+    pub fn head() -> Self {
+        Self(CoreStateId::Head)
+    }
+
+    pub fn slot(slot: Slot) -> Self {
+        Self(CoreStateId::Slot(slot))
+    }
+
+    /// Return the state root identified by `self`.
+    pub fn root<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<Hash256, warp::Rejection> {
+        let slot = match &self.0 {
+            CoreStateId::Head => {
+                return chain
+                    .head_info()
+                    .map(|head| head.state_root)
+                    .map_err(warp_utils::reject::beacon_chain_error)
+            }
+            CoreStateId::Genesis => return Ok(chain.genesis_state_root),
+            CoreStateId::Finalized => chain.head_info().map(|head| {
+                head.finalized_checkpoint
+                    .epoch
+                    .start_slot(T::EthSpec::slots_per_epoch())
+            }),
+            CoreStateId::Justified => chain.head_info().map(|head| {
+                head.current_justified_checkpoint
+                    .epoch
+                    .start_slot(T::EthSpec::slots_per_epoch())
+            }),
+            CoreStateId::Slot(slot) => Ok(*slot),
+            CoreStateId::Root(root) => return Ok(*root),
+        }
+        .map_err(warp_utils::reject::beacon_chain_error)?;
+
+        chain
+            .state_root_at_slot(slot)
+            .map_err(warp_utils::reject::beacon_chain_error)?
+            .ok_or_else(|| {
+                warp_utils::reject::custom_not_found(format!("beacon state at slot {}", slot))
+            })
+    }
+
+    /// Return the `fork` field of the state identified by `self`.
+    pub fn fork<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<Fork, warp::Rejection> {
+        self.map_state(chain, |state| Ok(state.fork))
+    }
+
+    /// Return the `BeaconState` identified by `self`.
+    pub fn state<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<BeaconState<T::EthSpec>, warp::Rejection> {
+        let (state_root, slot_opt) = match &self.0 {
+            CoreStateId::Head => {
+                return chain
+                    .head_beacon_state()
+                    .map_err(warp_utils::reject::beacon_chain_error)
+            }
+            CoreStateId::Slot(slot) => (self.root(chain)?, Some(*slot)),
+            _ => (self.root(chain)?, None),
+        };
+
+        chain
+            .get_state(&state_root, slot_opt)
+            .map_err(warp_utils::reject::beacon_chain_error)
+            .and_then(|opt| {
+                opt.ok_or_else(|| {
+                    warp_utils::reject::custom_not_found(format!(
+                        "beacon state at root {}",
+                        state_root
+                    ))
+                })
+            })
+    }
+
+    /// Map a function across the `BeaconState` identified by `self`.
+    ///
+    /// This function will avoid instantiating/copying a new state when `self` points to the head
+    /// of the chain.
+    pub fn map_state<T: BeaconChainTypes, F, U>(
+        &self,
+        chain: &BeaconChain<T>,
+        func: F,
+    ) -> Result<U, warp::Rejection>
+    where
+        F: Fn(&BeaconState<T::EthSpec>) -> Result<U, warp::Rejection>,
+    {
+        match &self.0 {
+            CoreStateId::Head => chain
+                .with_head(|snapshot| Ok(func(&snapshot.beacon_state)))
+                .map_err(warp_utils::reject::beacon_chain_error)?,
+            _ => func(&self.state(chain)?),
+        }
+    }
+}
+
+impl FromStr for StateId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        CoreStateId::from_str(s).map(Self)
+    }
+}

beacon_node/http_api/src/validator_inclusion.rs

@@ -0,0 +1,88 @@
+use crate::state_id::StateId;
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use eth2::{
+    lighthouse::{GlobalValidatorInclusionData, ValidatorInclusionData},
+    types::ValidatorId,
+};
+use state_processing::per_epoch_processing::ValidatorStatuses;
+use types::{Epoch, EthSpec};
+
+/// Returns information about *all validators* (i.e., global) and how they performed during a given
+/// epoch.
+pub fn global_validator_inclusion_data<T: BeaconChainTypes>(
+    epoch: Epoch,
+    chain: &BeaconChain<T>,
+) -> Result<GlobalValidatorInclusionData, warp::Rejection> {
+    let target_slot = epoch.end_slot(T::EthSpec::slots_per_epoch());
+
+    let state = StateId::slot(target_slot).state(chain)?;
+
+    let mut validator_statuses = ValidatorStatuses::new(&state, &chain.spec)
+        .map_err(warp_utils::reject::beacon_state_error)?;
+    validator_statuses
+        .process_attestations(&state, &chain.spec)
+        .map_err(warp_utils::reject::beacon_state_error)?;
+
+    let totals = validator_statuses.total_balances;
+
+    Ok(GlobalValidatorInclusionData {
+        current_epoch_active_gwei: totals.current_epoch(),
+        previous_epoch_active_gwei: totals.previous_epoch(),
+        current_epoch_attesting_gwei: totals.current_epoch_attesters(),
+        current_epoch_target_attesting_gwei: totals.current_epoch_target_attesters(),
+        previous_epoch_attesting_gwei: totals.previous_epoch_attesters(),
+        previous_epoch_target_attesting_gwei: totals.previous_epoch_target_attesters(),
+        previous_epoch_head_attesting_gwei: totals.previous_epoch_head_attesters(),
+    })
+}
+
+/// Returns information about a single validator and how it performed during a given epoch.
+pub fn validator_inclusion_data<T: BeaconChainTypes>(
+    epoch: Epoch,
+    validator_id: &ValidatorId,
+    chain: &BeaconChain<T>,
+) -> Result<Option<ValidatorInclusionData>, warp::Rejection> {
+    let target_slot = epoch.end_slot(T::EthSpec::slots_per_epoch());
+
+    let mut state = StateId::slot(target_slot).state(chain)?;
+
+    let mut validator_statuses = ValidatorStatuses::new(&state, &chain.spec)
+        .map_err(warp_utils::reject::beacon_state_error)?;
+    validator_statuses
+        .process_attestations(&state, &chain.spec)
+        .map_err(warp_utils::reject::beacon_state_error)?;
+
+    state
+        .update_pubkey_cache()
+        .map_err(warp_utils::reject::beacon_state_error)?;
+
+    let validator_index = match validator_id {
+        ValidatorId::Index(index) => *index as usize,
+        ValidatorId::PublicKey(pubkey) => {
+            if let Some(index) = state
+                .get_validator_index(pubkey)
+                .map_err(warp_utils::reject::beacon_state_error)?
+            {
+                index
+            } else {
+                return Ok(None);
+            }
+        }
+    };
+
+    Ok(validator_statuses
+        .statuses
+        .get(validator_index)
+        .map(|vote| ValidatorInclusionData {
+            is_slashed: vote.is_slashed,
+            is_withdrawable_in_current_epoch: vote.is_withdrawable_in_current_epoch,
+            is_active_in_current_epoch: vote.is_active_in_current_epoch,
+            is_active_in_previous_epoch: vote.is_active_in_previous_epoch,
+            current_epoch_effective_balance_gwei: vote.current_epoch_effective_balance,
+            is_current_epoch_attester: vote.is_current_epoch_attester,
+            is_current_epoch_target_attester: vote.is_current_epoch_target_attester,
+            is_previous_epoch_attester: vote.is_previous_epoch_attester,
+            is_previous_epoch_target_attester: vote.is_previous_epoch_target_attester,
+            is_previous_epoch_head_attester: vote.is_previous_epoch_head_attester,
+        }))
+}

beacon_node/http_metrics/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "http_metrics"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+prometheus = "0.10.0"
+warp = { git = "https://github.com/paulhauner/warp", branch = "cors-wildcard" }
+serde = { version = "1.0.116", features = ["derive"] }
+slog = "2.5.2"
+beacon_chain = { path = "../beacon_chain" }
+store = { path = "../store" }
+eth2_libp2p = { path = "../eth2_libp2p" }
+slot_clock = { path = "../../common/slot_clock" }
+lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
+lazy_static = "1.4.0"
+eth2 = { path = "../../common/eth2" }
+lighthouse_version = { path = "../../common/lighthouse_version" }
+warp_utils = { path = "../../common/warp_utils" }
+
+[dev-dependencies]
+tokio = { version = "0.2.22", features = ["sync"] }
+reqwest = { version = "0.10.8", features = ["json"] }
+environment = { path = "../../lighthouse/environment" }
+types = { path = "../../consensus/types" }