Update to v1.0.1, run cargo update

## Issue Addressed

part of  #1883

## Proposed Changes

Adds a new cli argument `--eth1-endpoints` that can be used instead of `--eth1-endpoint` to specify a comma-separated list of endpoints. If the first endpoint returns an error for some request the other endpoints are tried in the given order.

## Additional Info

Currently if the first endpoint fails the fallbacks are used silently (except for `try_fallback_test_endpoint` that is used in `do_update` which logs a `WARN` for each endpoint that is not reachable). A question is if we should add more logs so that the user gets warned if his main endpoint is for example just slow and sometimes hits timeouts.

.dockerignore

@@ -1,4 +1,4 @@
-tests/ef_tests/eth2.0-spec-tests
+testing/ef_tests/eth2.0-spec-tests
 target/
 *.data
 *.tar.gz

.github/workflows/book.yml

@@ -3,7 +3,7 @@ name: mdbook
 on:
   push:
     branches:
-      - master
+      - unstable
 
 jobs:
   build-and-upload-to-s3:

.github/workflows/test-suite.yml

@@ -33,20 +33,6 @@ jobs:
       run: sudo npm install -g ganache-cli
     - name: Run tests in release
       run: make test-release
-  release-tests-and-install-macos:
-    name: release-tests-and-install-macos
-    runs-on: macos-latest
-    needs: cargo-fmt
-    steps:
-    - uses: actions/checkout@v1
-    - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
-    - name: Run tests in release
-      run: make test-release
-    - name: Install Lighthouse
-      run: make
   debug-tests-ubuntu:
     name: debug-tests-ubuntu
     runs-on: ubuntu-latest
@@ -115,6 +101,14 @@ jobs:
     - uses: actions/checkout@v1
     - name: Typecheck benchmark code without running it
       run: make check-benches
+  check-consensus:
+    name: check-consensus
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Typecheck consensus code in strict mode
+      run: make check-consensus
   clippy:
     name: clippy
     runs-on: ubuntu-latest
@@ -123,6 +117,8 @@ jobs:
     - uses: actions/checkout@v1
     - name: Lint code for quality and style with Clippy
       run: make lint
+    - name: Certify Cargo.lock freshness
+      run: git diff --exit-code Cargo.lock
   arbitrary-check:
     name: arbitrary-check
     runs-on: ubuntu-latest

.gitignore

@@ -6,4 +6,4 @@ target/
 flamegraph.svg
 perf.data*
 *.tar.gz
-bin/
+/bin

Cargo.toml

@@ -7,8 +7,9 @@ members = [
     "beacon_node/client",
     "beacon_node/eth1",
     "beacon_node/eth2_libp2p",
+    "beacon_node/http_api",
+    "beacon_node/http_metrics",
     "beacon_node/network",
-    "beacon_node/rest_api",
     "beacon_node/store",
     "beacon_node/timer",
     "beacon_node/websocket_server",
@@ -20,6 +21,8 @@ members = [
     "common/compare_fields",
     "common/compare_fields_derive",
     "common/deposit_contract",
+    "common/directory",
+    "common/eth2",
     "common/eth2_config",
     "common/eth2_interop_keypairs",
     "common/eth2_testnet_config",
@@ -27,13 +30,16 @@ members = [
     "common/hashset_delay",
     "common/lighthouse_metrics",
     "common/lighthouse_version",
+    "common/lockfile",
     "common/logging",
     "common/lru_cache",
-    "common/remote_beacon_node",
-    "common/rest_types",
+    "common/remote_signer_consumer",
     "common/slot_clock",
+    "common/task_executor",
     "common/test_random_derive",
     "common/validator_dir",
+    "common/warp_utils",
+    "common/fallback",
 
     "consensus/cached_tree_hash",
     "consensus/int_to_bytes",
@@ -43,7 +49,7 @@ members = [
     "consensus/ssz",
     "consensus/ssz_derive",
     "consensus/ssz_types",
-    "consensus/serde_hex",
+    "consensus/serde_utils",
     "consensus/state_processing",
     "consensus/swap_or_not_shuffle",
     "consensus/tree_hash",
@@ -60,10 +66,17 @@ members = [
     "lighthouse",
     "lighthouse/environment",
 
-    "testing/simulator",
+    "remote_signer",
+    "remote_signer/backend",
+    "remote_signer/client",
+
+    "slasher",
+
     "testing/ef_tests",
     "testing/eth1_test_rig",
     "testing/node_test_rig",
+    "testing/remote_signer_test",
+    "testing/simulator",
     "testing/state_transition_vectors",
 
     "validator_client",
@@ -78,4 +91,3 @@ eth2_ssz = { path = "consensus/ssz" }
 eth2_ssz_derive = { path = "consensus/ssz_derive" }
 eth2_ssz_types = { path = "consensus/ssz_types" }
 eth2_hashing = { path = "crypto/eth2_hashing" }
-leveldb-sys = { git = "https://github.com/michaelsproul/leveldb-sys", branch = "v2.0.6-cmake" }

Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.45.1 AS builder
+FROM rust:1.47.0 AS builder
 RUN apt-get update && apt-get install -y cmake
 COPY . lighthouse
 ARG PORTABLE

Makefile

@@ -42,7 +42,7 @@ endif
 # optimized CPU functions that may not be available on some systems. This
 # results in a more portable binary with ~20% slower BLS verification.
 build-x86_64:
-	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features modern
 build-x86_64-portable:
 	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features portable
 build-aarch64:
@@ -74,7 +74,6 @@ build-release-tarballs:
 	$(MAKE) build-aarch64-portable
 	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"-portable")
 
-
 # Runs the full workspace tests in **release**, without downloading any additional
 # test vectors.
 test-release:
@@ -93,6 +92,10 @@ cargo-fmt:
 check-benches:
 	cargo check --all --benches
 
+# Typechecks consensus code *without* allowing deprecated legacy arithmetic
+check-consensus:
+	cargo check --manifest-path=consensus/state_processing/Cargo.toml --no-default-features
+
 # Runs only the ef-test vectors.
 run-ef-tests:
 	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests"
@@ -133,7 +136,11 @@ arbitrary-fuzz:
 # Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database)
 audit:
 	cargo install --force cargo-audit
-	cargo audit
+	# TODO: we should address this --ignore.
+	#
+	# Tracking issue:
+	# https://github.com/sigp/lighthouse/issues/1669
+	cargo audit --ignore RUSTSEC-2020-0043
 
 # Runs `cargo udeps` to check for unused dependencies
 udeps:

README.md

@@ -2,76 +2,65 @@
 
 An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prime.
 
-[![Build Status]][Build Link] [![Book Status]][Book Link] [![RustDoc Status]][RustDoc Link] [![Chat Badge]][Chat Link]
+[![Build Status]][Build Link] [![Book Status]][Book Link] [![Chat Badge]][Chat Link]
 
 [Build Status]: https://github.com/sigp/lighthouse/workflows/test-suite/badge.svg?branch=master
 [Build Link]: https://github.com/sigp/lighthouse/actions
 [Chat Badge]: https://img.shields.io/badge/chat-discord-%237289da
 [Chat Link]: https://discord.gg/cyAszAh
 [Book Status]:https://img.shields.io/badge/user--docs-master-informational
-[Book Link]: http://lighthouse-book.sigmaprime.io/
-[RustDoc Status]:https://img.shields.io/badge/code--docs-master-orange
-[RustDoc Link]: http://lighthouse-docs.sigmaprime.io/
+[Book Link]: https://lighthouse-book.sigmaprime.io
+[stable]: https://github.com/sigp/lighthouse/tree/stable
+[unstable]: https://github.com/sigp/lighthouse/tree/unstable
+[blog]: https://lighthouse.sigmaprime.io
 
-[Documentation](http://lighthouse-book.sigmaprime.io/)
+[Documentation](https://lighthouse-book.sigmaprime.io)
 
-![terminalize](https://i.postimg.cc/kG11dpCW/lighthouse-cli-png.gif)
+![Banner](https://i.postimg.cc/hjdTGKPd/photo-2020-10-23-09-52-16.jpg)
 
 ## Overview
 
 Lighthouse is:
 
+- Ready for use on Eth2 mainnet.
 - Fully open-source, licensed under Apache 2.0.
-- Security-focused. Fuzzing has begun and security reviews are underway.
-- Built in [Rust](https://www.rust-lang.org/), a modern language providing unique safety guarantees and
+- Security-focused. Fuzzing techniques have been continuously applied and several external security reviews have been performed.
+- Built in [Rust](https://www.rust-lang.org), a modern language providing unique safety guarantees and
 	excellent performance (comparable to C++).
 - Funded by various organisations, including Sigma Prime, the
-	Ethereum Foundation, ConsenSys and private individuals.
-- Actively involved in the specification and security analysis of the emerging
-    Ethereum 2.0 specification.
+	Ethereum Foundation, ConsenSys, the Decentralization Foundation and private individuals.
+- Actively involved in the specification and security analysis of the
+	Ethereum 2.0 specification.
 
-Like all Ethereum 2.0 clients, Lighthouse is a work-in-progress.
-
-## Development Status
-
-Current development overview:
-
-- Specification `v0.12.1` implemented, optimized and passing test vectors.
-- Rust-native libp2p with Gossipsub and Discv5.
-- RESTful JSON API via HTTP server.
-- Events via WebSocket.
-- Metrics via Prometheus.
-
-### Roadmap
-
-- ~~**April 2019**: Inital single-client testnets.~~
-- ~~**September 2019**: Inter-operability with other Ethereum 2.0 clients.~~
-- ~~**Q1 2020**: `lighthouse-0.1.0` release: All major phase 0 features implemented.~~
-- ~~**Q2 2020**: Public, multi-client testnet with user-facing functionality.~~
-- ~~**Q2 2020**: Third-party security review.~~
-- **Q3 2020**: Additional third-party security reviews.
-- **Q3 2020**: Long-lived, multi-client Beacon Chain testnet
-- **Q4 2020**: Production Beacon Chain (tentative).
+## Eth2 Deposit Contract
 
+The Lighthouse team acknowledges
+[`0x00000000219ab540356cBB839Cbe05303d7705Fa`](https://etherscan.io/address/0x00000000219ab540356cbb839cbe05303d7705fa)
+as the canonical Eth2 deposit contract address.
 
 ## Documentation
 
-The [Lighthouse Book](http://lighthouse-book.sigmaprime.io/) contains information
-for testnet users and developers.
+The [Lighthouse Book](https://lighthouse-book.sigmaprime.io) contains information for users and
+developers.
 
-Code documentation is generated via `cargo doc` and hosted at
-[lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/).
+The Lighthouse team maintains a blog at [lighthouse.sigmaprime.io][blog] which contains periodical
+progress updates, roadmap insights and interesting findings.
 
-If you'd like some background on Sigma Prime, please see the [Lighthouse Update
-\#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or
-[sigmaprime.io](https://sigmaprime.io).
+## Branches
+
+Lighthouse maintains two permanent branches:
+
+- [`stable`][stable]: Always points to the latest stable release.
+  - This is ideal for most users.
+- [`unstable`][unstable]: Used for development, contains the latest PRs.
+  - Developers should base thier PRs on this branch.
 
 ## Contributing
 
 Lighthouse welcomes contributors.
 
 If you are looking to contribute, please head to the
-[Contributing](http://lighthouse-book.sigmaprime.io/contributing.html) section
+[Contributing](https://lighthouse-book.sigmaprime.io/contributing.html) section
 of the Lighthouse book.
 
 ## Contact
@@ -80,6 +69,9 @@ The best place for discussion is the [Lighthouse Discord
 server](https://discord.gg/cyAszAh). Alternatively, you may use the
 [sigp/lighthouse gitter](https://gitter.im/sigp/lighthouse).
 
+Sign up to the [Lighthouse Development Updates](https://mailchi.mp/3d9df0417779/lighthouse-dev-updates)
+mailing list for email notifications about releases, network status and other important information.
+
 Encrypt sensitive messages using our [PGP
 key](https://keybase.io/sigp/pgp_keys.asc?fingerprint=dcf37e025d6c9d42ea795b119e7c6cf9988604be).
 

account_manager/Cargo.toml

@@ -1,33 +1,39 @@
 [package]
 name = "account_manager"
-version = "0.2.6"
+version = "0.3.5"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
 edition = "2018"
 
 [dependencies]
 bls = { path = "../crypto/bls" }
-clap = "2.33.0"
+clap = "2.33.3"
 slog = "2.5.2"
-slog-term = "2.5.0"
+slog-term = "2.6.0"
 slog-async = "2.5.0"
 types = { path = "../consensus/types" }
-state_processing = { path = "../consensus/state_processing" }
-dirs = "2.0.2"
+dirs = "3.0.1"
 environment = { path = "../lighthouse/environment" }
 deposit_contract = { path = "../common/deposit_contract" }
-libc = "0.2.65"
+libc = "0.2.79"
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
 hex = "0.4.2"
-rayon = "1.3.0"
+rayon = "1.4.1"
 eth2_testnet_config = { path = "../common/eth2_testnet_config" }
-web3 = "0.11.0"
 futures = { version = "0.3.5", features = ["compat"] }
 clap_utils = { path = "../common/clap_utils" }
+directory = { path = "../common/directory" }
 eth2_wallet = { path = "../crypto/eth2_wallet" }
 eth2_wallet_manager = { path = "../common/eth2_wallet_manager" }
-rand = "0.7.2"
+rand = "0.7.3"
 validator_dir = { path = "../common/validator_dir" }
-tokio = { version = "0.2.21", features = ["full"] }
+tokio = { version = "0.2.22", features = ["full"] }
 eth2_keystore = { path = "../crypto/eth2_keystore" }
 account_utils = { path = "../common/account_utils" }
+slashing_protection = { path = "../validator_client/slashing_protection" }
+eth2 = {path = "../common/eth2"}
+safe_arith = {path = "../consensus/safe_arith"}
+slot_clock = { path = "../common/slot_clock" }
+
+[dev-dependencies]
+tempfile = "3.1.0"

account_manager/src/common.rs

@@ -1,21 +1,67 @@
-use clap::ArgMatches;
-use std::fs::create_dir_all;
-use std::path::{Path, PathBuf};
+use account_utils::PlainText;
+use account_utils::{read_input_from_user, strip_off_newlines};
+use eth2_wallet::bip39::{Language, Mnemonic};
+use std::fs;
+use std::path::PathBuf;
+use std::str::from_utf8;
+use std::thread::sleep;
+use std::time::Duration;
 
-pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> {
-    let path = path.as_ref();
+pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
+pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:";
 
-    if !path.exists() {
-        create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?;
+pub fn read_mnemonic_from_cli(
+    mnemonic_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<Mnemonic, String> {
+    let mnemonic = match mnemonic_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .and_then(|bytes| {
+                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
+                let phrase = from_utf8(&bytes_no_newlines.as_ref())
+                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
+                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
+                    format!(
+                        "Unable to derive mnemonic from string {:?}: {:?}",
+                        phrase, e
+                    )
+                })
+            })?,
+        None => loop {
+            eprintln!("");
+            eprintln!("{}", MNEMONIC_PROMPT);
+
+            let mnemonic = read_input_from_user(stdin_inputs)?;
+
+            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
+                Ok(mnemonic_m) => {
+                    eprintln!("Valid mnemonic provided.");
+                    eprintln!("");
+                    sleep(Duration::from_secs(1));
+                    break mnemonic_m;
+                }
+                Err(_) => {
+                    eprintln!("Invalid mnemonic");
+                }
+            }
+        },
+    };
+    Ok(mnemonic)
+}
+
+/// Reads in a wallet name from the user. If the `--wallet-name` flag is provided, use it. Otherwise
+/// read from an interactive prompt using tty unless the `--stdin-inputs` flag is provided.
+pub fn read_wallet_name_from_cli(
+    wallet_name: Option<String>,
+    stdin_inputs: bool,
+) -> Result<String, String> {
+    match wallet_name {
+        Some(name) => Ok(name),
+        None => {
+            eprintln!("{}", WALLET_NAME_PROMPT);
+
+            read_input_from_user(stdin_inputs)
+        }
     }
-
-    Ok(())
-}
-
-pub fn base_wallet_dir(matches: &ArgMatches, arg: &'static str) -> Result<PathBuf, String> {
-    clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        arg,
-        PathBuf::new().join(".lighthouse").join("wallets"),
-    )
 }

account_manager/src/lib.rs

@@ -10,7 +10,7 @@ use types::EthSpec;
 pub const CMD: &str = "account_manager";
 pub const SECRETS_DIR_FLAG: &str = "secrets-dir";
 pub const VALIDATOR_DIR_FLAG: &str = "validator-dir";
-pub const BASE_DIR_FLAG: &str = "base-dir";
+pub const WALLETS_DIR_FLAG: &str = "wallets-dir";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)

account_manager/src/validator/create.rs

@@ -1,9 +1,16 @@
-use crate::{common::ensure_dir_exists, SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG};
-use account_utils::{random_password, strip_off_newlines, validator_definitions};
+use crate::common::read_wallet_name_from_cli;
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use crate::{SECRETS_DIR_FLAG, WALLETS_DIR_FLAG};
+use account_utils::{
+    random_password, read_password_from_user, strip_off_newlines, validator_definitions, PlainText,
+};
 use clap::{App, Arg, ArgMatches};
+use directory::{
+    ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_SECRET_DIR, DEFAULT_WALLET_DIR,
+};
 use environment::Environment;
-use eth2_wallet::PlainText;
 use eth2_wallet_manager::WalletManager;
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
 use std::ffi::OsStr;
 use std::fs;
 use std::path::{Path, PathBuf};
@@ -11,13 +18,13 @@ use types::EthSpec;
 use validator_dir::Builder as ValidatorDirBuilder;
 
 pub const CMD: &str = "create";
-pub const BASE_DIR_FLAG: &str = "base-dir";
 pub const WALLET_NAME_FLAG: &str = "wallet-name";
-pub const WALLET_PASSPHRASE_FLAG: &str = "wallet-passphrase";
+pub const WALLET_PASSWORD_FLAG: &str = "wallet-password";
 pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
 pub const STORE_WITHDRAW_FLAG: &str = "store-withdrawal-keystore";
 pub const COUNT_FLAG: &str = "count";
 pub const AT_MOST_FLAG: &str = "at-most";
+pub const WALLET_PASSWORD_PROMPT: &str = "Enter your wallet's password:";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
@@ -30,26 +37,22 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .long(WALLET_NAME_FLAG)
                 .value_name("WALLET_NAME")
                 .help("Use the wallet identified by this name")
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
-            Arg::with_name(WALLET_PASSPHRASE_FLAG)
-                .long(WALLET_PASSPHRASE_FLAG)
+            Arg::with_name(WALLET_PASSWORD_FLAG)
+                .long(WALLET_PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help("A path to a file containing the password which will unlock the wallet.")
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
+            Arg::with_name(WALLETS_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
+                .value_name(WALLETS_DIR_FLAG)
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{network}/wallets")
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .arg(
             Arg::with_name(SECRETS_DIR_FLAG)
@@ -57,8 +60,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .value_name("SECRETS_DIR")
                 .help(
                     "The path where the validator keystore passwords will be stored. \
-                    Defaults to ~/.lighthouse/secrets",
+                    Defaults to ~/.lighthouse/{network}/secrets",
                 )
+                .conflicts_with("datadir")
                 .takes_value(true),
         )
         .arg(
@@ -99,28 +103,35 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .conflicts_with("count")
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
 }
 
 pub fn cli_run<T: EthSpec>(
     matches: &ArgMatches,
     mut env: Environment<T>,
-    wallet_base_dir: PathBuf,
+    validator_dir: PathBuf,
 ) -> Result<(), String> {
     let spec = env.core_context().eth2_config.spec;
 
-    let name: String = clap_utils::parse_required(matches, WALLET_NAME_FLAG)?;
-    let wallet_password_path: PathBuf =
-        clap_utils::parse_required(matches, WALLET_PASSPHRASE_FLAG)?;
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let secrets_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        SECRETS_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("secrets"),
-    )?;
+    let name: Option<String> = clap_utils::parse_optional(matches, WALLET_NAME_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
+
     let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
         .unwrap_or_else(|| spec.max_effective_balance);
     let count: Option<usize> = clap_utils::parse_optional(matches, COUNT_FLAG)?;
@@ -129,6 +140,10 @@ pub fn cli_run<T: EthSpec>(
     ensure_dir_exists(&validator_dir)?;
     ensure_dir_exists(&secrets_dir)?;
 
+    eprintln!("validator-dir path: {:?}", validator_dir);
+    eprintln!("secrets-dir path {:?}", secrets_dir);
+    eprintln!("wallets-dir path {:?}", wallet_base_dir);
+
     let starting_validator_count = existing_validator_count(&validator_dir)?;
 
     let n = match (count, at_most) {
@@ -152,17 +167,37 @@ pub fn cli_run<T: EthSpec>(
         return Ok(());
     }
 
-    let wallet_password = fs::read(&wallet_password_path)
-        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
-        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
+    let wallet_password_path: Option<PathBuf> =
+        clap_utils::parse_optional(matches, WALLET_PASSWORD_FLAG)?;
+
+    let wallet_name = read_wallet_name_from_cli(name, stdin_inputs)?;
+    let wallet_password = read_wallet_password_from_cli(wallet_password_path, stdin_inputs)?;
 
     let mgr = WalletManager::open(&wallet_base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     let mut wallet = mgr
-        .wallet_by_name(&name)
+        .wallet_by_name(&wallet_name)
         .map_err(|e| format!("Unable to open wallet: {:?}", e))?;
 
+    let slashing_protection_path = validator_dir.join(SLASHING_PROTECTION_FILENAME);
+    let slashing_protection =
+        SlashingDatabase::open_or_create(&slashing_protection_path).map_err(|e| {
+            format!(
+                "Unable to open or create slashing protection database at {}: {:?}",
+                slashing_protection_path.display(),
+                e
+            )
+        })?;
+
+    // Create an empty transaction and drops it. Used to test if the database is locked.
+    slashing_protection.test_transaction().map_err(|e| {
+        format!(
+            "Cannot create keys while the validator client is running: {:?}",
+            e
+        )
+    })?;
+
     for i in 0..n {
         let voting_password = random_password();
         let withdrawal_password = random_password();
@@ -175,9 +210,25 @@ pub fn cli_run<T: EthSpec>(
             )
             .map_err(|e| format!("Unable to create validator keys: {:?}", e))?;
 
-        let voting_pubkey = keystores.voting.pubkey().to_string();
+        let voting_pubkey = keystores.voting.public_key().ok_or_else(|| {
+            format!(
+                "Keystore public key is invalid: {}",
+                keystores.voting.pubkey()
+            )
+        })?;
 
-        ValidatorDirBuilder::new(validator_dir.clone(), secrets_dir.clone())
+        slashing_protection
+            .register_validator(&voting_pubkey)
+            .map_err(|e| {
+                format!(
+                    "Error registering validator {}: {:?}",
+                    voting_pubkey.to_hex_string(),
+                    e
+                )
+            })?;
+
+        ValidatorDirBuilder::new(validator_dir.clone())
+            .password_dir(secrets_dir.clone())
             .voting_keystore(keystores.voting, voting_password.as_bytes())
             .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
             .create_eth1_tx_data(deposit_gwei, &spec)
@@ -185,7 +236,7 @@ pub fn cli_run<T: EthSpec>(
             .build()
             .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
 
-        println!("{}/{}\t0x{}", i + 1, n, voting_pubkey);
+        println!("{}/{}\t{}", i + 1, n, voting_pubkey.to_hex_string());
     }
 
     Ok(())
@@ -193,15 +244,40 @@ pub fn cli_run<T: EthSpec>(
 
 /// Returns the number of validators that exist in the given `validator_dir`.
 ///
-/// This function just assumes all files and directories, excluding the validator definitions YAML,
-/// are validator directories, making it likely to return a higher number than accurate
-/// but never a lower one.
+/// This function just assumes all files and directories, excluding the validator definitions YAML
+/// and slashing protection database are validator directories, making it likely to return a higher
+/// number than accurate but never a lower one.
 fn existing_validator_count<P: AsRef<Path>>(validator_dir: P) -> Result<usize, String> {
     fs::read_dir(validator_dir.as_ref())
         .map(|iter| {
             iter.filter_map(|e| e.ok())
-                .filter(|e| e.file_name() != OsStr::new(validator_definitions::CONFIG_FILENAME))
+                .filter(|e| {
+                    e.file_name() != OsStr::new(validator_definitions::CONFIG_FILENAME)
+                        && e.file_name()
+                            != OsStr::new(slashing_protection::SLASHING_PROTECTION_FILENAME)
+                })
                 .count()
         })
         .map_err(|e| format!("Unable to read {:?}: {}", validator_dir.as_ref(), e))
 }
+
+/// Used when a user is accessing an existing wallet. Read in a wallet password from a file if the password file
+/// path is provided. Otherwise, read from an interactive prompt using tty unless the `--stdin-inputs`
+/// flag is provided.
+pub fn read_wallet_password_from_cli(
+    password_file_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<PlainText, String> {
+    match password_file_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .map(|bytes| strip_off_newlines(bytes).into()),
+        None => {
+            eprintln!("");
+            eprintln!("{}", WALLET_PASSWORD_PROMPT);
+            let password =
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+            Ok(password)
+        }
+    }
+}

account_manager/src/validator/deposit.rs

@@ -1,405 +0,0 @@
-use crate::VALIDATOR_DIR_FLAG;
-use clap::{App, Arg, ArgMatches};
-use deposit_contract::DEPOSIT_GAS;
-use environment::Environment;
-use futures::{
-    compat::Future01CompatExt,
-    stream::{FuturesUnordered, StreamExt},
-};
-use slog::{info, Logger};
-use state_processing::per_block_processing::verify_deposit_signature;
-use std::path::PathBuf;
-use tokio::time::{delay_until, Duration, Instant};
-use types::EthSpec;
-use validator_dir::{Eth1DepositData, Manager as ValidatorManager, ValidatorDir};
-use web3::{
-    transports::Http,
-    transports::Ipc,
-    types::{Address, SyncInfo, SyncState, TransactionRequest, U256},
-    Transport, Web3,
-};
-
-pub const CMD: &str = "deposit";
-pub const VALIDATOR_FLAG: &str = "validator";
-pub const ETH1_IPC_FLAG: &str = "eth1-ipc";
-pub const ETH1_HTTP_FLAG: &str = "eth1-http";
-pub const FROM_ADDRESS_FLAG: &str = "from-address";
-pub const CONFIRMATION_COUNT_FLAG: &str = "confirmation-count";
-pub const CONFIRMATION_BATCH_SIZE_FLAG: &str = "confirmation-batch-size";
-
-const GWEI: u64 = 1_000_000_000;
-
-const SYNCING_STATE_RETRY_DELAY: Duration = Duration::from_secs(2);
-
-const CONFIRMATIONS_POLL_TIME: Duration = Duration::from_secs(2);
-
-pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
-    App::new("deposit")
-        .about(
-            "Submits a deposit to an Eth1 validator registration contract via an IPC endpoint \
-            of an Eth1 client (e.g., Geth, OpenEthereum, etc.). The validators must already \
-            have been created and exist on the file-system. The process will exit immediately \
-            with an error if any error occurs. After each deposit is submitted to the Eth1 \
-            node, a file will be saved in the validator directory with the transaction hash. \
-            If confirmations are set to non-zero then the application will wait for confirmations \
-            before saving the transaction hash and moving onto the next batch of deposits. \
-            The deposit contract address will be determined by the --testnet-dir flag on the \
-            primary Lighthouse binary.",
-        )
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to the validator client data directory. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name(VALIDATOR_FLAG)
-                .long(VALIDATOR_FLAG)
-                .value_name("VALIDATOR_NAME")
-                .help(
-                    "The name of the directory in --data-dir for which to deposit. \
-                    Set to 'all' to deposit all validators in the --data-dir.",
-                )
-                .takes_value(true)
-                .required(true),
-        )
-        .arg(
-            Arg::with_name(ETH1_IPC_FLAG)
-                .long(ETH1_IPC_FLAG)
-                .value_name("ETH1_IPC_PATH")
-                .help("Path to an Eth1 JSON-RPC IPC endpoint")
-                .takes_value(true)
-                .required(false),
-        )
-        .arg(
-            Arg::with_name(ETH1_HTTP_FLAG)
-                .long(ETH1_HTTP_FLAG)
-                .value_name("ETH1_HTTP_URL")
-                .help("URL to an Eth1 JSON-RPC endpoint")
-                .takes_value(true)
-                .required(false),
-        )
-        .arg(
-            Arg::with_name(FROM_ADDRESS_FLAG)
-                .long(FROM_ADDRESS_FLAG)
-                .value_name("FROM_ETH1_ADDRESS")
-                .help(
-                    "The address that will submit the eth1 deposit. \
-                    Must be unlocked on the node at --eth1-ipc.",
-                )
-                .takes_value(true)
-                .required(true),
-        )
-        .arg(
-            Arg::with_name(CONFIRMATION_COUNT_FLAG)
-                .long(CONFIRMATION_COUNT_FLAG)
-                .value_name("CONFIRMATION_COUNT")
-                .help(
-                    "The number of Eth1 block confirmations required \
-                    before a transaction is considered complete. Set to \
-                    0 for no confirmations.",
-                )
-                .takes_value(true)
-                .default_value("1"),
-        )
-        .arg(
-            Arg::with_name(CONFIRMATION_BATCH_SIZE_FLAG)
-                .long(CONFIRMATION_BATCH_SIZE_FLAG)
-                .value_name("BATCH_SIZE")
-                .help(
-                    "Perform BATCH_SIZE deposits and wait for confirmations \
-                    in parallel. Useful for achieving faster bulk deposits.",
-                )
-                .takes_value(true)
-                .default_value("10"),
-        )
-}
-
-#[allow(clippy::too_many_arguments)]
-fn send_deposit_transactions<T1, T2: 'static>(
-    mut env: Environment<T1>,
-    log: Logger,
-    mut eth1_deposit_datas: Vec<(ValidatorDir, Eth1DepositData)>,
-    from_address: Address,
-    deposit_contract: Address,
-    transport: T2,
-    confirmation_count: usize,
-    confirmation_batch_size: usize,
-) -> Result<(), String>
-where
-    T1: EthSpec,
-    T2: Transport + std::marker::Send,
-    <T2 as web3::Transport>::Out: std::marker::Send,
-{
-    let web3 = Web3::new(transport);
-    let spec = env.eth2_config.spec.clone();
-
-    let deposits_fut = async {
-        poll_until_synced(web3.clone(), log.clone()).await?;
-
-        for chunk in eth1_deposit_datas.chunks_mut(confirmation_batch_size) {
-            let futures = FuturesUnordered::default();
-
-            for (ref mut validator_dir, eth1_deposit_data) in chunk.iter_mut() {
-                verify_deposit_signature(&eth1_deposit_data.deposit_data, &spec).map_err(|e| {
-                    format!(
-                        "Deposit for {:?} fails verification, \
-                         are you using the correct testnet configuration?\nError: {:?}",
-                        eth1_deposit_data.deposit_data.pubkey, e
-                    )
-                })?;
-
-                let web3 = web3.clone();
-                let log = log.clone();
-                futures.push(async move {
-                    let tx_hash = web3
-                        .send_transaction_with_confirmation(
-                            TransactionRequest {
-                                from: from_address,
-                                to: Some(deposit_contract),
-                                gas: Some(DEPOSIT_GAS.into()),
-                                gas_price: None,
-                                value: Some(from_gwei(eth1_deposit_data.deposit_data.amount)),
-                                data: Some(eth1_deposit_data.rlp.clone().into()),
-                                nonce: None,
-                                condition: None,
-                            },
-                            CONFIRMATIONS_POLL_TIME,
-                            confirmation_count,
-                        )
-                        .compat()
-                        .await
-                        .map_err(|e| format!("Failed to send transaction: {:?}", e))?;
-
-                    info!(
-                        log,
-                        "Submitted deposit";
-                        "tx_hash" => format!("{:?}", tx_hash),
-                    );
-
-                    validator_dir
-                        .save_eth1_deposit_tx_hash(&format!("{:?}", tx_hash))
-                        .map_err(|e| {
-                            format!("Failed to save tx hash {:?} to disk: {:?}", tx_hash, e)
-                        })?;
-
-                    Ok::<(), String>(())
-                });
-            }
-
-            futures
-                .collect::<Vec<_>>()
-                .await
-                .into_iter()
-                .collect::<Result<_, _>>()?;
-        }
-
-        Ok::<(), String>(())
-    };
-
-    env.runtime().block_on(deposits_fut)?;
-
-    Ok(())
-}
-
-pub fn cli_run<T: EthSpec>(
-    matches: &ArgMatches<'_>,
-    mut env: Environment<T>,
-) -> Result<(), String> {
-    let log = env.core_context().log().clone();
-
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let validator: String = clap_utils::parse_required(matches, VALIDATOR_FLAG)?;
-    let eth1_ipc_path: Option<PathBuf> = clap_utils::parse_optional(matches, ETH1_IPC_FLAG)?;
-    let eth1_http_url: Option<String> = clap_utils::parse_optional(matches, ETH1_HTTP_FLAG)?;
-    let from_address: Address = clap_utils::parse_required(matches, FROM_ADDRESS_FLAG)?;
-    let confirmation_count: usize = clap_utils::parse_required(matches, CONFIRMATION_COUNT_FLAG)?;
-    let confirmation_batch_size: usize =
-        clap_utils::parse_required(matches, CONFIRMATION_BATCH_SIZE_FLAG)?;
-
-    let manager = ValidatorManager::open(&data_dir)
-        .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
-
-    let validators = match validator.as_ref() {
-        "all" => manager
-            .open_all_validators()
-            .map_err(|e| format!("Unable to read all validators: {:?}", e)),
-        name => {
-            let path = manager
-                .directory_names()
-                .map_err(|e| {
-                    format!(
-                        "Unable to read --{} directory names: {:?}",
-                        VALIDATOR_DIR_FLAG, e
-                    )
-                })?
-                .get(name)
-                .ok_or_else(|| format!("Unknown validator:  {}", name))?
-                .clone();
-
-            manager
-                .open_validator(&path)
-                .map_err(|e| format!("Unable to open {}: {:?}", name, e))
-                .map(|v| vec![v])
-        }
-    }?;
-
-    let eth1_deposit_datas = validators
-        .into_iter()
-        .filter(|v| !v.eth1_deposit_tx_hash_exists())
-        .map(|v| match v.eth1_deposit_data() {
-            Ok(Some(data)) => Ok((v, data)),
-            Ok(None) => Err(format!(
-                "Validator is missing deposit data file: {:?}",
-                v.dir()
-            )),
-            Err(e) => Err(format!(
-                "Unable to read deposit data for {:?}: {:?}",
-                v.dir(),
-                e
-            )),
-        })
-        .collect::<Result<Vec<_>, _>>()?;
-
-    let total_gwei: u64 = eth1_deposit_datas
-        .iter()
-        .map(|(_, d)| d.deposit_data.amount)
-        .sum();
-
-    if eth1_deposit_datas.is_empty() {
-        info!(log, "No validators to deposit");
-
-        return Ok(());
-    }
-
-    info!(
-        log,
-        "Starting deposits";
-        "deposit_count" => eth1_deposit_datas.len(),
-        "total_eth" => total_gwei / GWEI,
-    );
-
-    let deposit_contract = env
-        .testnet
-        .as_ref()
-        .ok_or_else(|| "Unable to run account manager without a testnet dir".to_string())?
-        .deposit_contract_address()
-        .map_err(|e| format!("Unable to parse deposit contract address: {}", e))?;
-
-    if deposit_contract == Address::zero() {
-        return Err("Refusing to deposit to the zero address. Check testnet configuration.".into());
-    }
-
-    match (eth1_ipc_path, eth1_http_url) {
-        (Some(_), Some(_)) => Err(format!(
-            "error: Cannot supply both --{} and --{}",
-            ETH1_IPC_FLAG, ETH1_HTTP_FLAG
-        )),
-        (None, None) => Err(format!(
-            "error: Must supply one of --{} or --{}",
-            ETH1_IPC_FLAG, ETH1_HTTP_FLAG
-        )),
-        (Some(ipc_path), None) => {
-            let (_event_loop_handle, ipc_transport) = Ipc::new(ipc_path)
-                .map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?;
-            send_deposit_transactions(
-                env,
-                log,
-                eth1_deposit_datas,
-                from_address,
-                deposit_contract,
-                ipc_transport,
-                confirmation_count,
-                confirmation_batch_size,
-            )
-        }
-        (None, Some(http_url)) => {
-            let (_event_loop_handle, http_transport) = Http::new(http_url.as_str())
-                .map_err(|e| format!("Unable to connect to eth1 http RPC: {:?}", e))?;
-            send_deposit_transactions(
-                env,
-                log,
-                eth1_deposit_datas,
-                from_address,
-                deposit_contract,
-                http_transport,
-                confirmation_count,
-                confirmation_batch_size,
-            )
-        }
-    }
-}
-
-/// Converts gwei to wei.
-fn from_gwei(gwei: u64) -> U256 {
-    U256::from(gwei) * U256::exp10(9)
-}
-
-/// Run a poll on the `eth_syncing` endpoint, blocking until the node is synced.
-async fn poll_until_synced<T>(web3: Web3<T>, log: Logger) -> Result<(), String>
-where
-    T: Transport + Send + 'static,
-    <T as Transport>::Out: Send,
-{
-    loop {
-        let sync_state = web3
-            .clone()
-            .eth()
-            .syncing()
-            .compat()
-            .await
-            .map_err(|e| format!("Unable to read syncing state from eth1 node: {:?}", e))?;
-
-        match sync_state {
-            SyncState::Syncing(SyncInfo {
-                current_block,
-                highest_block,
-                ..
-            }) => {
-                info!(
-                    log,
-                    "Waiting for eth1 node to sync";
-                    "est_highest_block" => format!("{}", highest_block),
-                    "current_block" => format!("{}", current_block),
-                );
-
-                delay_until(Instant::now() + SYNCING_STATE_RETRY_DELAY).await;
-            }
-            SyncState::NotSyncing => {
-                let block_number = web3
-                    .clone()
-                    .eth()
-                    .block_number()
-                    .compat()
-                    .await
-                    .map_err(|e| format!("Unable to read block number from eth1 node: {:?}", e))?;
-
-                if block_number > 0.into() {
-                    info!(
-                        log,
-                        "Eth1 node is synced";
-                        "head_block" => format!("{}", block_number),
-                    );
-                    break;
-                } else {
-                    delay_until(Instant::now() + SYNCING_STATE_RETRY_DELAY).await;
-                    info!(
-                        log,
-                        "Waiting for eth1 node to sync";
-                        "current_block" => 0,
-                    );
-                }
-            }
-        }
-    }
-
-    Ok(())
-}

account_manager/src/validator/exit.rs

@@ -0,0 +1,348 @@
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use bls::{Keypair, PublicKey};
+use clap::{App, Arg, ArgMatches};
+use environment::Environment;
+use eth2::{
+    types::{GenesisData, StateId, ValidatorId, ValidatorStatus},
+    BeaconNodeHttpClient, Url,
+};
+use eth2_keystore::Keystore;
+use eth2_testnet_config::Eth2TestnetConfig;
+use safe_arith::SafeArith;
+use slot_clock::{SlotClock, SystemTimeSlotClock};
+use std::path::PathBuf;
+use std::time::Duration;
+use types::{ChainSpec, Epoch, EthSpec, Fork, VoluntaryExit};
+
+pub const CMD: &str = "exit";
+pub const KEYSTORE_FLAG: &str = "keystore";
+pub const PASSWORD_FILE_FLAG: &str = "password-file";
+pub const BEACON_SERVER_FLAG: &str = "beacon-node";
+pub const PASSWORD_PROMPT: &str = "Enter the keystore password";
+
+pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/";
+pub const CONFIRMATION_PHRASE: &str = "Exit my validator";
+pub const WEBSITE_URL: &str = "https://lighthouse-book.sigmaprime.io/voluntary-exit.html";
+pub const PROMPT: &str = "WARNING: WITHDRAWING STAKED ETH IS NOT CURRENTLY POSSIBLE";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new("exit")
+        .about("Submits a VoluntaryExit to the beacon chain for a given validator keystore.")
+        .arg(
+            Arg::with_name(KEYSTORE_FLAG)
+                .long(KEYSTORE_FLAG)
+                .value_name("KEYSTORE_PATH")
+                .help("The path to the EIP-2335 voting keystore for the validator")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FILE_FLAG)
+                .long(PASSWORD_FILE_FLAG)
+                .value_name("PASSWORD_FILE_PATH")
+                .help("The path to the password file which unlocks the validator voting keystore")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(BEACON_SERVER_FLAG)
+                .long(BEACON_SERVER_FLAG)
+                .value_name("NETWORK_ADDRESS")
+                .help("Address to a beacon node HTTP API")
+                .default_value(&DEFAULT_BEACON_NODE)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run<E: EthSpec>(matches: &ArgMatches, mut env: Environment<E>) -> Result<(), String> {
+    let keystore_path: PathBuf = clap_utils::parse_required(matches, KEYSTORE_FLAG)?;
+    let password_file_path: Option<PathBuf> =
+        clap_utils::parse_optional(matches, PASSWORD_FILE_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    let spec = env.eth2_config().spec.clone();
+    let server_url: String = clap_utils::parse_required(matches, BEACON_SERVER_FLAG)?;
+    let client = BeaconNodeHttpClient::new(
+        Url::parse(&server_url)
+            .map_err(|e| format!("Failed to parse beacon http server: {:?}", e))?,
+    );
+
+    let testnet_config = env
+        .testnet
+        .clone()
+        .expect("network should have a valid config");
+
+    env.runtime().block_on(publish_voluntary_exit::<E>(
+        &keystore_path,
+        password_file_path.as_ref(),
+        &client,
+        &spec,
+        stdin_inputs,
+        &testnet_config,
+    ))?;
+
+    Ok(())
+}
+
+/// Gets the keypair and validator_index for every validator and calls `publish_voluntary_exit` on it.
+async fn publish_voluntary_exit<E: EthSpec>(
+    keystore_path: &PathBuf,
+    password_file_path: Option<&PathBuf>,
+    client: &BeaconNodeHttpClient,
+    spec: &ChainSpec,
+    stdin_inputs: bool,
+    testnet_config: &Eth2TestnetConfig,
+) -> Result<(), String> {
+    let genesis_data = get_geneisis_data(client).await?;
+    let testnet_genesis_root = testnet_config
+        .beacon_state::<E>()
+        .as_ref()
+        .expect("network should have valid genesis state")
+        .genesis_validators_root;
+
+    // Verify that the beacon node and validator being exited are on the same network.
+    if genesis_data.genesis_validators_root != testnet_genesis_root {
+        return Err(
+            "Invalid genesis state. Please ensure that your beacon node is on the same network \
+                 as the validator you are publishing an exit for"
+                .to_string(),
+        );
+    }
+
+    // Return immediately if beacon node is not synced
+    if is_syncing(client).await? {
+        return Err("Beacon node is still syncing".to_string());
+    }
+
+    let keypair = load_voting_keypair(keystore_path, password_file_path, stdin_inputs)?;
+
+    let epoch = get_current_epoch::<E>(genesis_data.genesis_time, spec)
+        .ok_or_else(|| "Failed to get current epoch. Please check your system time".to_string())?;
+    let validator_index = get_validator_index_for_exit(client, &keypair.pk, epoch, spec).await?;
+
+    let fork = get_beacon_state_fork(client).await?;
+    let voluntary_exit = VoluntaryExit {
+        epoch,
+        validator_index,
+    };
+
+    eprintln!(
+        "Publishing a voluntary exit for validator: {} \n",
+        keypair.pk
+    );
+    eprintln!("WARNING: THIS IS AN IRREVERSIBLE OPERATION\n");
+    eprintln!("{}\n", PROMPT);
+    eprintln!(
+        "PLEASE VISIT {} TO MAKE SURE YOU UNDERSTAND THE IMPLICATIONS OF A VOLUNTARY EXIT.",
+        WEBSITE_URL
+    );
+    eprintln!("Enter the exit phrase from the above URL to confirm the voluntary exit: ");
+
+    let confirmation = account_utils::read_input_from_user(stdin_inputs)?;
+    if confirmation == CONFIRMATION_PHRASE {
+        // Sign and publish the voluntary exit to network
+        let signed_voluntary_exit = voluntary_exit.sign(
+            &keypair.sk,
+            &fork,
+            genesis_data.genesis_validators_root,
+            spec,
+        );
+        client
+            .post_beacon_pool_voluntary_exits(&signed_voluntary_exit)
+            .await
+            .map_err(|e| format!("Failed to publish voluntary exit: {}", e))?;
+        tokio::time::delay_for(std::time::Duration::from_secs(1)).await; // Provides nicer UX.
+        eprintln!(
+            "Successfully validated and published voluntary exit for validator {}",
+            keypair.pk
+        );
+    } else {
+        eprintln!(
+            "Did not publish voluntary exit for validator {}. Please check that you entered the correct exit phrase.",
+            keypair.pk
+        );
+    }
+
+    Ok(())
+}
+
+/// Get the validator index of a given the validator public key by querying the beacon node endpoint.
+///
+/// Returns an error if the beacon endpoint returns an error or given validator is not eligible for an exit.
+async fn get_validator_index_for_exit(
+    client: &BeaconNodeHttpClient,
+    validator_pubkey: &PublicKey,
+    epoch: Epoch,
+    spec: &ChainSpec,
+) -> Result<u64, String> {
+    let validator_data = client
+        .get_beacon_states_validator_id(
+            StateId::Head,
+            &ValidatorId::PublicKey(validator_pubkey.into()),
+        )
+        .await
+        .map_err(|e| format!("Failed to get validator details: {:?}", e))?
+        .ok_or_else(|| {
+            format!(
+                "Validator {} is not present in the beacon state. \
+                Please ensure that your beacon node is synced and the validator has been deposited.",
+                validator_pubkey
+            )
+        })?
+        .data;
+
+    match validator_data.status {
+        ValidatorStatus::Active => {
+            let eligible_epoch = validator_data
+                .validator
+                .activation_epoch
+                .safe_add(spec.shard_committee_period)
+                .map_err(|e| format!("Failed to calculate eligible epoch, validator activation epoch too high: {:?}", e))?;
+
+            if epoch >= eligible_epoch {
+                Ok(validator_data.index)
+            } else {
+                Err(format!(
+                    "Validator {:?} is not eligible for exit. It will become eligible on epoch {}",
+                    validator_pubkey, eligible_epoch
+                ))
+            }
+        }
+        status => Err(format!(
+            "Validator {:?} is not eligible for voluntary exit. Validator status: {:?}",
+            validator_pubkey, status
+        )),
+    }
+}
+
+/// Get genesis data by querying the beacon node client.
+async fn get_geneisis_data(client: &BeaconNodeHttpClient) -> Result<GenesisData, String> {
+    Ok(client
+        .get_beacon_genesis()
+        .await
+        .map_err(|e| format!("Failed to get beacon genesis: {}", e))?
+        .data)
+}
+
+/// Gets syncing status from beacon node client and returns true if syncing and false otherwise.
+async fn is_syncing(client: &BeaconNodeHttpClient) -> Result<bool, String> {
+    Ok(client
+        .get_node_syncing()
+        .await
+        .map_err(|e| format!("Failed to get sync status: {:?}", e))?
+        .data
+        .is_syncing)
+}
+
+/// Get fork object for the current state by querying the beacon node client.
+async fn get_beacon_state_fork(client: &BeaconNodeHttpClient) -> Result<Fork, String> {
+    Ok(client
+        .get_beacon_states_fork(StateId::Head)
+        .await
+        .map_err(|e| format!("Failed to get get fork: {:?}", e))?
+        .ok_or_else(|| "Failed to get fork, state not found".to_string())?
+        .data)
+}
+
+/// Calculates the current epoch from the genesis time and current time.
+fn get_current_epoch<E: EthSpec>(genesis_time: u64, spec: &ChainSpec) -> Option<Epoch> {
+    let slot_clock = SystemTimeSlotClock::new(
+        spec.genesis_slot,
+        Duration::from_secs(genesis_time),
+        Duration::from_millis(spec.milliseconds_per_slot),
+    );
+    slot_clock.now().map(|s| s.epoch(E::slots_per_epoch()))
+}
+
+/// Load the voting keypair by loading and decrypting the keystore.
+///
+/// If the `password_file_path` is Some, unlock keystore using password in given file
+/// otherwise, prompts user for a password to unlock the keystore.
+fn load_voting_keypair(
+    voting_keystore_path: &PathBuf,
+    password_file_path: Option<&PathBuf>,
+    stdin_inputs: bool,
+) -> Result<Keypair, String> {
+    let keystore = Keystore::from_json_file(&voting_keystore_path).map_err(|e| {
+        format!(
+            "Unable to read keystore JSON {:?}: {:?}",
+            voting_keystore_path, e
+        )
+    })?;
+
+    // Get password from password file.
+    if let Some(password_file) = password_file_path {
+        validator_dir::unlock_keypair_from_password_path(voting_keystore_path, password_file)
+            .map_err(|e| format!("Error while decrypting keypair: {:?}", e))
+    } else {
+        // Prompt password from user.
+        eprintln!("");
+        eprintln!(
+            "{} for validator in {:?}: ",
+            PASSWORD_PROMPT, voting_keystore_path
+        );
+        let password = account_utils::read_password_from_user(stdin_inputs)?;
+        match keystore.decrypt_keypair(password.as_ref()) {
+            Ok(keypair) => {
+                eprintln!("Password is correct.");
+                eprintln!("");
+                std::thread::sleep(std::time::Duration::from_secs(1)); // Provides nicer UX.
+                Ok(keypair)
+            }
+            Err(eth2_keystore::Error::InvalidPassword) => Err("Invalid password".to_string()),
+            Err(e) => Err(format!("Error while decrypting keypair: {:?}", e)),
+        }
+    }
+}
+
+#[cfg(test)]
+#[cfg(not(debug_assertions))]
+mod tests {
+    use super::*;
+    use eth2_keystore::KeystoreBuilder;
+    use std::fs::File;
+    use std::io::Write;
+    use tempfile::{tempdir, TempDir};
+
+    const PASSWORD: &str = "cats";
+    const KEYSTORE_NAME: &str = "keystore-m_12381_3600_0_0_0-1595406747.json";
+    const PASSWORD_FILE: &str = "password.pass";
+
+    fn create_and_save_keystore(dir: &TempDir, save_password: bool) -> PublicKey {
+        let keypair = Keypair::random();
+        let keystore = KeystoreBuilder::new(&keypair, PASSWORD.as_bytes(), "".into())
+            .unwrap()
+            .build()
+            .unwrap();
+
+        // Create a keystore.
+        File::create(dir.path().join(KEYSTORE_NAME))
+            .map(|mut file| keystore.to_json_writer(&mut file).unwrap())
+            .unwrap();
+        if save_password {
+            File::create(dir.path().join(PASSWORD_FILE))
+                .map(|mut file| file.write_all(PASSWORD.as_bytes()).unwrap())
+                .unwrap();
+        }
+        keystore.public_key().unwrap()
+    }
+
+    #[test]
+    fn test_load_keypair_password_file() {
+        let dir = tempdir().unwrap();
+        let expected_pk = create_and_save_keystore(&dir, true);
+
+        let kp = load_voting_keypair(
+            &dir.path().join(KEYSTORE_NAME),
+            Some(&dir.path().join(PASSWORD_FILE)),
+            false,
+        )
+        .unwrap();
+
+        assert_eq!(expected_pk, kp.pk.into());
+    }
+}

account_manager/src/validator/import.rs

@@ -1,4 +1,4 @@
-use crate::{common::ensure_dir_exists, VALIDATOR_DIR_FLAG};
+use crate::wallet::create::STDIN_INPUTS_FLAG;
 use account_utils::{
     eth2_keystore::Keystore,
     read_password_from_user,
@@ -6,8 +6,10 @@ use account_utils::{
         recursively_find_voting_keystores, ValidatorDefinition, ValidatorDefinitions,
         CONFIG_FILENAME,
     },
+    ZeroizeString,
 };
 use clap::{App, Arg, ArgMatches};
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
 use std::fs;
 use std::path::PathBuf;
 use std::thread::sleep;
@@ -16,7 +18,7 @@ use std::time::Duration;
 pub const CMD: &str = "import";
 pub const KEYSTORE_FLAG: &str = "keystore";
 pub const DIR_FLAG: &str = "directory";
-pub const STDIN_PASSWORD_FLAG: &str = "stdin-passwords";
+pub const REUSE_PASSWORD_FLAG: &str = "reuse-password";
 
 pub const PASSWORD_PROMPT: &str = "Enter the keystore password, or press enter to omit it:";
 pub const KEYSTORE_REUSE_WARNING: &str = "DO NOT USE THE ORIGINAL KEYSTORES TO VALIDATE WITH \
@@ -54,37 +56,45 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
         )
         .arg(
-            Arg::with_name(STDIN_PASSWORD_FLAG)
-                .long(STDIN_PASSWORD_FLAG)
-                .help("If present, read passwords from stdin instead of tty."),
+            Arg::with_name(REUSE_PASSWORD_FLAG)
+                .long(REUSE_PASSWORD_FLAG)
+                .help("If present, the same password will be used for all imported keystores."),
         )
 }
 
-pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
     let keystore: Option<PathBuf> = clap_utils::parse_optional(matches, KEYSTORE_FLAG)?;
     let keystores_dir: Option<PathBuf> = clap_utils::parse_optional(matches, DIR_FLAG)?;
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let stdin_password = matches.is_present(STDIN_PASSWORD_FLAG);
-
-    ensure_dir_exists(&validator_dir)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+    let reuse_password = matches.is_present(REUSE_PASSWORD_FLAG);
 
     let mut defs = ValidatorDefinitions::open_or_create(&validator_dir)
         .map_err(|e| format!("Unable to open {}: {:?}", CONFIG_FILENAME, e))?;
 
+    let slashing_protection_path = validator_dir.join(SLASHING_PROTECTION_FILENAME);
+    let slashing_protection =
+        SlashingDatabase::open_or_create(&slashing_protection_path).map_err(|e| {
+            format!(
+                "Unable to open or create slashing protection database at {}: {:?}",
+                slashing_protection_path.display(),
+                e
+            )
+        })?;
+
+    // Create an empty transaction and drop it. Used to test if the database is locked.
+    slashing_protection.test_transaction().map_err(|e| {
+        format!(
+            "Cannot import keys while the validator client is running: {:?}",
+            e
+        )
+    })?;
+
+    eprintln!("validator-dir path: {:?}", validator_dir);
     // Collect the paths for the keystores that should be imported.
     let keystore_paths = match (keystore, keystores_dir) {
         (Some(keystore), None) => vec![keystore],
@@ -115,10 +125,13 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
     //
     // - Obtain the keystore password, if the user desires.
     // - Copy the keystore into the `validator_dir`.
+    // - Register the voting key with the slashing protection database.
     // - Add the keystore to the validator definitions file.
     //
     // Skip keystores that already exist, but exit early if any operation fails.
+    // Reuses the same password for all keystores if the `REUSE_PASSWORD_FLAG` flag is set.
     let mut num_imported_keystores = 0;
+    let mut previous_password: Option<ZeroizeString> = None;
     for src_keystore in &keystore_paths {
         let keystore = Keystore::from_json_file(src_keystore)
             .map_err(|e| format!("Unable to read keystore JSON {:?}: {:?}", src_keystore, e))?;
@@ -136,10 +149,14 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         );
 
         let password_opt = loop {
+            if let Some(password) = previous_password.clone() {
+                eprintln!("Reuse previous password.");
+                break Some(password);
+            }
             eprintln!("");
             eprintln!("{}", PASSWORD_PROMPT);
 
-            let password = read_password_from_user(stdin_password)?;
+            let password = read_password_from_user(stdin_inputs)?;
 
             if password.as_ref().is_empty() {
                 eprintln!("Continuing without password.");
@@ -152,6 +169,9 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
                     eprintln!("Password is correct.");
                     eprintln!("");
                     sleep(Duration::from_secs(1)); // Provides nicer UX.
+                    if reuse_password {
+                        previous_password = Some(password.clone());
+                    }
                     break Some(password);
                 }
                 Err(eth2_keystore::Error::InvalidPassword) => {
@@ -186,6 +206,20 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         fs::copy(&src_keystore, &dest_keystore)
             .map_err(|e| format!("Unable to copy keystore: {:?}", e))?;
 
+        // Register with slashing protection.
+        let voting_pubkey = keystore
+            .public_key()
+            .ok_or_else(|| format!("Keystore public key is invalid: {}", keystore.pubkey()))?;
+        slashing_protection
+            .register_validator(&voting_pubkey)
+            .map_err(|e| {
+                format!(
+                    "Error registering validator {}: {:?}",
+                    voting_pubkey.to_hex_string(),
+                    e
+                )
+            })?;
+
         eprintln!("Successfully imported keystore.");
         num_imported_keystores += 1;
 

account_manager/src/validator/list.rs

@@ -1,40 +1,28 @@
-use crate::VALIDATOR_DIR_FLAG;
-use clap::{App, Arg, ArgMatches};
+use account_utils::validator_definitions::ValidatorDefinitions;
+use clap::App;
 use std::path::PathBuf;
-use validator_dir::Manager as ValidatorManager;
 
 pub const CMD: &str = "list";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
-    App::new(CMD)
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to search for validator directories. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
-        .about("Lists the names of all validators.")
+    App::new(CMD).about("Lists the public keys of all validators.")
 }
 
-pub fn cli_run(matches: &ArgMatches<'_>) -> Result<(), String> {
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
+pub fn cli_run(validator_dir: PathBuf) -> Result<(), String> {
+    eprintln!("validator-dir path: {:?}", validator_dir);
+    let validator_definitions = ValidatorDefinitions::open(&validator_dir).map_err(|e| {
+        format!(
+            "No validator definitions found in {:?}: {:?}",
+            validator_dir, e
+        )
+    })?;
 
-    let mgr = ValidatorManager::open(&data_dir)
-        .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
-
-    for (name, _path) in mgr
-        .directory_names()
-        .map_err(|e| format!("Unable to list wallets: {:?}", e))?
-    {
-        println!("{}", name)
+    for def in validator_definitions.as_slice() {
+        println!(
+            "{} ({})",
+            def.voting_public_key,
+            if def.enabled { "enabled" } else { "disabled" }
+        );
     }
 
     Ok(())

account_manager/src/validator/mod.rs

@@ -1,11 +1,15 @@
 pub mod create;
-pub mod deposit;
+pub mod exit;
 pub mod import;
 pub mod list;
+pub mod recover;
+pub mod slashing_protection;
 
-use crate::common::base_wallet_dir;
+use crate::VALIDATOR_DIR_FLAG;
 use clap::{App, Arg, ArgMatches};
+use directory::{parse_path_or_default_with_flag, DEFAULT_VALIDATOR_DIR};
 use environment::Environment;
+use std::path::PathBuf;
 use types::EthSpec;
 
 pub const CMD: &str = "validator";
@@ -14,26 +18,42 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Provides commands for managing Eth2 validators.")
         .arg(
-            Arg::with_name("base-dir")
-                .long("base-dir")
-                .value_name("BASE_DIRECTORY")
-                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets")
-                .takes_value(true),
+            Arg::with_name(VALIDATOR_DIR_FLAG)
+                .long(VALIDATOR_DIR_FLAG)
+                .value_name("VALIDATOR_DIRECTORY")
+                .help(
+                    "The path to search for validator directories. \
+                    Defaults to ~/.lighthouse/{network}/validators",
+                )
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .subcommand(create::cli_app())
-        .subcommand(deposit::cli_app())
         .subcommand(import::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
+        .subcommand(slashing_protection::cli_app())
+        .subcommand(exit::cli_app())
 }
 
 pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
-    let base_wallet_dir = base_wallet_dir(matches, "base-dir")?;
+    let validator_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_VALIDATOR_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, VALIDATOR_DIR_FLAG, DEFAULT_VALIDATOR_DIR)?
+    };
+    eprintln!("validator-dir path: {:?}", validator_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, base_wallet_dir),
-        (deposit::CMD, Some(matches)) => deposit::cli_run::<T>(matches, env),
-        (import::CMD, Some(matches)) => import::cli_run(matches),
-        (list::CMD, Some(matches)) => list::cli_run(matches),
+        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, validator_base_dir),
+        (import::CMD, Some(matches)) => import::cli_run(matches, validator_base_dir),
+        (list::CMD, Some(_)) => list::cli_run(validator_base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, validator_base_dir),
+        (slashing_protection::CMD, Some(matches)) => {
+            slashing_protection::cli_run(matches, env, validator_base_dir)
+        }
+        (exit::CMD, Some(matches)) => exit::cli_run(matches, env),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/validator/recover.rs

@@ -0,0 +1,148 @@
+use super::create::STORE_WITHDRAW_FLAG;
+use crate::common::read_mnemonic_from_cli;
+use crate::validator::create::COUNT_FLAG;
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use crate::SECRETS_DIR_FLAG;
+use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
+use account_utils::random_password;
+use clap::{App, Arg, ArgMatches};
+use directory::ensure_dir_exists;
+use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR};
+use eth2_wallet::bip39::Seed;
+use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType, ValidatorKeystores};
+use std::path::PathBuf;
+use validator_dir::Builder as ValidatorDirBuilder;
+pub const CMD: &str = "recover";
+pub const FIRST_INDEX_FLAG: &str = "first-index";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Recovers validator private keys given a BIP-39 mnemonic phrase. \
+            If you did not specify a `--first-index` or count `--count`, by default this will \
+            only recover the keys associated with the validator at index 0 for an HD wallet \
+            in accordance with the EIP-2333 spec.")
+        .arg(
+            Arg::with_name(FIRST_INDEX_FLAG)
+                .long(FIRST_INDEX_FLAG)
+                .value_name("FIRST_INDEX")
+                .help("The first of consecutive key indexes you wish to recover.")
+                .takes_value(true)
+                .required(false)
+                .default_value("0"),
+        )
+        .arg(
+            Arg::with_name(COUNT_FLAG)
+                .long(COUNT_FLAG)
+                .value_name("COUNT")
+                .help("The number of validator keys you wish to recover. Counted consecutively from the provided `--first_index`.")
+                .takes_value(true)
+                .required(false)
+                .default_value("1"),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help(
+                    "If present, the mnemonic will be read in from this file.",
+                )
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name(SECRETS_DIR_FLAG)
+                .long(SECRETS_DIR_FLAG)
+                .value_name("SECRETS_DIR")
+                .help(
+                    "The path where the validator keystore passwords will be stored. \
+                    Defaults to ~/.lighthouse/{network}/secrets",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STORE_WITHDRAW_FLAG)
+                .long(STORE_WITHDRAW_FLAG)
+                .help(
+                    "If present, the withdrawal keystore will be stored alongside the voting \
+                    keypair. It is generally recommended to *not* store the withdrawal key and \
+                    instead generate them from the wallet seed when required.",
+                ),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
+    let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
+    let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?;
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    eprintln!("validator-dir path: {:?}", validator_dir);
+    eprintln!("secrets-dir path: {:?}", secrets_dir);
+
+    ensure_dir_exists(&validator_dir)?;
+    ensure_dir_exists(&secrets_dir)?;
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+
+    let seed = Seed::new(&mnemonic, "");
+
+    for index in first_index..first_index + count {
+        let voting_password = random_password();
+        let withdrawal_password = random_password();
+
+        let derive = |key_type: KeyType, password: &[u8]| -> Result<Keystore, String> {
+            let (secret, path) =
+                recover_validator_secret_from_mnemonic(seed.as_bytes(), index, key_type)
+                    .map_err(|e| format!("Unable to recover validator keys: {:?}", e))?;
+
+            let keypair = keypair_from_secret(secret.as_bytes())
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?;
+
+            KeystoreBuilder::new(&keypair, password, format!("{}", path))
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?
+                .build()
+                .map_err(|e| format!("Unable build keystore: {:?}", e))
+        };
+
+        let keystores = ValidatorKeystores {
+            voting: derive(KeyType::Voting, voting_password.as_bytes())?,
+            withdrawal: derive(KeyType::Withdrawal, withdrawal_password.as_bytes())?,
+        };
+
+        let voting_pubkey = keystores.voting.pubkey().to_string();
+
+        ValidatorDirBuilder::new(validator_dir.clone())
+            .password_dir(secrets_dir.clone())
+            .voting_keystore(keystores.voting, voting_password.as_bytes())
+            .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
+            .store_withdrawal_keystore(matches.is_present(STORE_WITHDRAW_FLAG))
+            .build()
+            .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
+
+        println!(
+            "{}/{}\tIndex: {}\t0x{}",
+            index - first_index,
+            count - first_index,
+            index,
+            voting_pubkey
+        );
+    }
+
+    Ok(())
+}

account_manager/src/validator/slashing_protection.rs

@@ -0,0 +1,186 @@
+use clap::{App, Arg, ArgMatches};
+use environment::Environment;
+use slashing_protection::{
+    interchange::Interchange, InterchangeImportOutcome, SlashingDatabase,
+    SLASHING_PROTECTION_FILENAME,
+};
+use std::fs::File;
+use std::path::PathBuf;
+use types::{BeaconState, Epoch, EthSpec, Slot};
+
+pub const CMD: &str = "slashing-protection";
+pub const IMPORT_CMD: &str = "import";
+pub const EXPORT_CMD: &str = "export";
+
+pub const IMPORT_FILE_ARG: &str = "IMPORT-FILE";
+pub const EXPORT_FILE_ARG: &str = "EXPORT-FILE";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Import or export slashing protection data to or from another client")
+        .subcommand(
+            App::new(IMPORT_CMD)
+                .about("Import an interchange file")
+                .arg(
+                    Arg::with_name(IMPORT_FILE_ARG)
+                        .takes_value(true)
+                        .value_name("FILE")
+                        .help("The slashing protection interchange file to import (.json)"),
+                ),
+        )
+        .subcommand(
+            App::new(EXPORT_CMD)
+                .about("Export an interchange file")
+                .arg(
+                    Arg::with_name(EXPORT_FILE_ARG)
+                        .takes_value(true)
+                        .value_name("FILE")
+                        .help("The filename to export the interchange file to"),
+                ),
+        )
+}
+
+pub fn cli_run<T: EthSpec>(
+    matches: &ArgMatches<'_>,
+    env: Environment<T>,
+    validator_base_dir: PathBuf,
+) -> Result<(), String> {
+    eprintln!("validator-dir path: {:?}", validator_base_dir);
+    let slashing_protection_db_path = validator_base_dir.join(SLASHING_PROTECTION_FILENAME);
+
+    let testnet_config = env
+        .testnet
+        .ok_or_else(|| "Unable to get testnet configuration from the environment".to_string())?;
+
+    let genesis_validators_root = testnet_config
+        .beacon_state::<T>()
+        .map(|state: BeaconState<T>| state.genesis_validators_root)
+        .map_err(|e| {
+            format!(
+                "Unable to get genesis state, has genesis occurred? Detail: {:?}",
+                e
+            )
+        })?;
+
+    match matches.subcommand() {
+        (IMPORT_CMD, Some(matches)) => {
+            let import_filename: PathBuf = clap_utils::parse_required(&matches, IMPORT_FILE_ARG)?;
+            let import_file = File::open(&import_filename).map_err(|e| {
+                format!(
+                    "Unable to open import file at {}: {:?}",
+                    import_filename.display(),
+                    e
+                )
+            })?;
+
+            let interchange = Interchange::from_json_reader(&import_file)
+                .map_err(|e| format!("Error parsing file for import: {:?}", e))?;
+
+            let slashing_protection_database =
+                SlashingDatabase::open_or_create(&slashing_protection_db_path).map_err(|e| {
+                    format!(
+                        "Unable to open database at {}: {:?}",
+                        slashing_protection_db_path.display(),
+                        e
+                    )
+                })?;
+
+            let outcomes = slashing_protection_database
+                .import_interchange_info(interchange, genesis_validators_root)
+                .map_err(|e| {
+                    format!(
+                        "Error during import: {:?}\n\
+                         IT IS NOT SAFE TO START VALIDATING",
+                        e
+                    )
+                })?;
+
+            let display_slot = |slot: Option<Slot>| {
+                slot.map_or("none".to_string(), |slot| format!("{}", slot.as_u64()))
+            };
+            let display_epoch = |epoch: Option<Epoch>| {
+                epoch.map_or("?".to_string(), |epoch| format!("{}", epoch.as_u64()))
+            };
+            let display_attestation = |source, target| match (source, target) {
+                (None, None) => "none".to_string(),
+                (source, target) => format!("{}=>{}", display_epoch(source), display_epoch(target)),
+            };
+
+            let mut num_failed = 0;
+
+            for outcome in &outcomes {
+                match outcome {
+                    InterchangeImportOutcome::Success { pubkey, summary } => {
+                        eprintln!("- {:?} SUCCESS min block: {}, max block: {}, min attestation: {}, max attestation: {}",
+                            pubkey,
+                            display_slot(summary.min_block_slot),
+                            display_slot(summary.max_block_slot),
+                            display_attestation(summary.min_attestation_source, summary.min_attestation_target),
+                            display_attestation(summary.max_attestation_source,
+                            summary.max_attestation_target),
+                        );
+                    }
+                    InterchangeImportOutcome::Failure { pubkey, error } => {
+                        eprintln!("- {:?} ERROR: {:?}", pubkey, error);
+                        num_failed += 1;
+                    }
+                }
+            }
+
+            if num_failed == 0 {
+                eprintln!("Import completed successfully.");
+                eprintln!(
+                    "Please double-check that the minimum and maximum blocks and slots above \
+                     match your expectations."
+                );
+            } else {
+                eprintln!(
+                    "WARNING: history was NOT imported for {} of {} records",
+                    num_failed,
+                    outcomes.len()
+                );
+                eprintln!("IT IS NOT SAFE TO START VALIDATING");
+                eprintln!("Please see https://lighthouse-book.sigmaprime.io/slashing-protection.html#slashable-data-in-import");
+                return Err("Partial import".to_string());
+            }
+
+            Ok(())
+        }
+        (EXPORT_CMD, Some(matches)) => {
+            let export_filename: PathBuf = clap_utils::parse_required(&matches, EXPORT_FILE_ARG)?;
+
+            if !slashing_protection_db_path.exists() {
+                return Err(format!(
+                    "No slashing protection database exists at: {}",
+                    slashing_protection_db_path.display()
+                ));
+            }
+
+            let slashing_protection_database = SlashingDatabase::open(&slashing_protection_db_path)
+                .map_err(|e| {
+                    format!(
+                        "Unable to open database at {}: {:?}",
+                        slashing_protection_db_path.display(),
+                        e
+                    )
+                })?;
+
+            let interchange = slashing_protection_database
+                .export_interchange_info(genesis_validators_root)
+                .map_err(|e| format!("Error during export: {:?}", e))?;
+
+            let output_file = File::create(export_filename)
+                .map_err(|e| format!("Error creating output file: {:?}", e))?;
+
+            interchange
+                .write_to(&output_file)
+                .map_err(|e| format!("Error writing output file: {:?}", e))?;
+
+            eprintln!("Export completed successfully");
+
+            Ok(())
+        }
+        ("", _) => Err("No subcommand provided, see --help for options".to_string()),
+        (command, _) => Err(format!("No such subcommand `{}`", command)),
+    }
+}

account_manager/src/wallet/create.rs

@@ -1,13 +1,17 @@
-use crate::BASE_DIR_FLAG;
-use account_utils::{random_password, strip_off_newlines};
+use crate::common::read_wallet_name_from_cli;
+use crate::WALLETS_DIR_FLAG;
+use account_utils::{
+    is_password_sufficiently_complex, random_password, read_password_from_user, strip_off_newlines,
+};
 use clap::{App, Arg, ArgMatches};
 use eth2_wallet::{
     bip39::{Language, Mnemonic, MnemonicType},
     PlainText,
 };
-use eth2_wallet_manager::{WalletManager, WalletType};
+use eth2_wallet_manager::{LockedWallet, WalletManager, WalletType};
 use std::ffi::OsStr;
-use std::fs::{self, File};
+use std::fs;
+use std::fs::File;
 use std::io::prelude::*;
 use std::os::unix::fs::PermissionsExt;
 use std::path::{Path, PathBuf};
@@ -15,9 +19,21 @@ use std::path::{Path, PathBuf};
 pub const CMD: &str = "create";
 pub const HD_TYPE: &str = "hd";
 pub const NAME_FLAG: &str = "name";
-pub const PASSPHRASE_FLAG: &str = "passphrase-file";
+pub const PASSWORD_FLAG: &str = "password-file";
 pub const TYPE_FLAG: &str = "type";
 pub const MNEMONIC_FLAG: &str = "mnemonic-output-path";
+pub const STDIN_INPUTS_FLAG: &str = "stdin-inputs";
+pub const MNEMONIC_LENGTH_FLAG: &str = "mnemonic-length";
+pub const MNEMONIC_TYPES: &[MnemonicType] = &[
+    MnemonicType::Words12,
+    MnemonicType::Words15,
+    MnemonicType::Words18,
+    MnemonicType::Words21,
+    MnemonicType::Words24,
+];
+pub const NEW_WALLET_PASSWORD_PROMPT: &str =
+    "Enter a password for your new wallet that is at least 12 characters long:";
+pub const RETYPE_PASSWORD_PROMPT: &str = "Please re-enter your wallet's new password:";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
@@ -30,12 +46,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     "The wallet will be created with this name. It is not allowed to \
                             create two wallets with the same name for the same --base-dir.",
                 )
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
-            Arg::with_name(PASSPHRASE_FLAG)
-                .long(PASSPHRASE_FLAG)
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help(
                     "A path to a file containing the password which will unlock the wallet. \
@@ -43,8 +58,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     saved at that path. To avoid confusion, if the file does not already \
                     exist it must include a '.pass' suffix.",
                 )
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
             Arg::with_name(TYPE_FLAG)
@@ -67,56 +81,47 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 )
                 .takes_value(true)
         )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_LENGTH_FLAG)
+                .long(MNEMONIC_LENGTH_FLAG)
+                .value_name("MNEMONIC_LENGTH")
+                .help("The number of words to use for the mnemonic phrase.")
+                .takes_value(true)
+                .validator(|len| {
+                    match len.parse::<usize>().ok().and_then(|words| MnemonicType::for_word_count(words).ok()) {
+                        Some(_) => Ok(()),
+                        None => Err(format!("Mnemonic length must be one of {}", MNEMONIC_TYPES.iter().map(|t| t.word_count().to_string()).collect::<Vec<_>>().join(", "))),
+                    }
+                })
+                .default_value("24"),
+        )
 }
 
-pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
-    let name: String = clap_utils::parse_required(matches, NAME_FLAG)?;
-    let wallet_password_path: PathBuf = clap_utils::parse_required(matches, PASSPHRASE_FLAG)?;
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
     let mnemonic_output_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
-    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
-
-    let wallet_type = match type_field.as_ref() {
-        HD_TYPE => WalletType::Hd,
-        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
-    };
-
-    let mgr = WalletManager::open(&base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
 
     // Create a new random mnemonic.
     //
     // The `tiny-bip39` crate uses `thread_rng()` for this entropy.
-    let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English);
+    let mnemonic_length = clap_utils::parse_required(matches, MNEMONIC_LENGTH_FLAG)?;
+    let mnemonic = Mnemonic::new(
+        MnemonicType::for_word_count(mnemonic_length).expect("Mnemonic length already validated"),
+        Language::English,
+    );
 
-    // Create a random password if the file does not exist.
-    if !wallet_password_path.exists() {
-        // To prevent users from accidentally supplying their password to the PASSPHRASE_FLAG and
-        // create a file with that name, we require that the password has a .pass suffix.
-        if wallet_password_path.extension() != Some(&OsStr::new("pass")) {
-            return Err(format!(
-                "Only creates a password file if that file ends in .pass: {:?}",
-                wallet_password_path
-            ));
-        }
-
-        create_with_600_perms(&wallet_password_path, random_password().as_bytes())
-            .map_err(|e| format!("Unable to write to {:?}: {:?}", wallet_password_path, e))?;
-    }
-
-    let wallet_password = fs::read(&wallet_password_path)
-        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
-        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
-
-    let wallet = mgr
-        .create_wallet(name, wallet_type, &mnemonic, wallet_password.as_bytes())
-        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)?;
 
     if let Some(path) = mnemonic_output_path {
         create_with_600_perms(&path, mnemonic.phrase().as_bytes())
             .map_err(|e| format!("Unable to write mnemonic to {:?}: {:?}", path, e))?;
     }
 
-    println!("Your wallet's 12-word BIP-39 mnemonic is:");
+    println!("Your wallet's {}-word BIP-39 mnemonic is:", mnemonic_length);
     println!();
     println!("\t{}", mnemonic.phrase());
     println!();
@@ -140,6 +145,99 @@ pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
     Ok(())
 }
 
+pub fn create_wallet_from_mnemonic(
+    matches: &ArgMatches,
+    wallet_base_dir: &Path,
+    mnemonic: &Mnemonic,
+) -> Result<LockedWallet, String> {
+    let name: Option<String> = clap_utils::parse_optional(matches, NAME_FLAG)?;
+    let wallet_password_path: Option<PathBuf> = clap_utils::parse_optional(matches, PASSWORD_FLAG)?;
+    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    let wallet_type = match type_field.as_ref() {
+        HD_TYPE => WalletType::Hd,
+        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
+    };
+
+    let mgr = WalletManager::open(&wallet_base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
+
+    let wallet_password: PlainText = match wallet_password_path {
+        Some(path) => {
+            // Create a random password if the file does not exist.
+            if !path.exists() {
+                // To prevent users from accidentally supplying their password to the PASSWORD_FLAG and
+                // create a file with that name, we require that the password has a .pass suffix.
+                if path.extension() != Some(&OsStr::new("pass")) {
+                    return Err(format!(
+                        "Only creates a password file if that file ends in .pass: {:?}",
+                        path
+                    ));
+                }
+
+                create_with_600_perms(&path, random_password().as_bytes())
+                    .map_err(|e| format!("Unable to write to {:?}: {:?}", path, e))?;
+            }
+            read_new_wallet_password_from_cli(Some(path), stdin_inputs)?
+        }
+        None => read_new_wallet_password_from_cli(None, stdin_inputs)?,
+    };
+
+    let wallet_name = read_wallet_name_from_cli(name, stdin_inputs)?;
+
+    let wallet = mgr
+        .create_wallet(
+            wallet_name,
+            wallet_type,
+            &mnemonic,
+            wallet_password.as_bytes(),
+        )
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    Ok(wallet)
+}
+
+/// Used when a user is creating a new wallet. Read in a wallet password from a file if the password file
+/// path is provided. Otherwise, read from an interactive prompt using tty unless the `--stdin-inputs`
+/// flag is provided. This verifies the password complexity and verifies the password is correctly re-entered.
+pub fn read_new_wallet_password_from_cli(
+    password_file_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<PlainText, String> {
+    match password_file_path {
+        Some(path) => {
+            let password: PlainText = fs::read(&path)
+                .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+                .map(|bytes| strip_off_newlines(bytes).into())?;
+
+            // Ensure the password meets the minimum requirements.
+            is_password_sufficiently_complex(password.as_bytes())?;
+            Ok(password)
+        }
+        None => loop {
+            eprintln!("");
+            eprintln!("{}", NEW_WALLET_PASSWORD_PROMPT);
+            let password =
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+
+            // Ensure the password meets the minimum requirements.
+            match is_password_sufficiently_complex(password.as_bytes()) {
+                Ok(_) => {
+                    eprintln!("{}", RETYPE_PASSWORD_PROMPT);
+                    let retyped_password =
+                        PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+                    if retyped_password == password {
+                        break Ok(password);
+                    } else {
+                        eprintln!("Passwords do not match.");
+                    }
+                }
+                Err(message) => eprintln!("{}", message),
+            }
+        },
+    }
+}
+
 /// Creates a file with `600 (-rw-------)` permissions.
 pub fn create_with_600_perms<P: AsRef<Path>>(path: P, bytes: &[u8]) -> Result<(), String> {
     let path = path.as_ref();

account_manager/src/wallet/list.rs

@@ -1,4 +1,4 @@
-use crate::BASE_DIR_FLAG;
+use crate::WALLETS_DIR_FLAG;
 use clap::App;
 use eth2_wallet_manager::WalletManager;
 use std::path::PathBuf;
@@ -9,9 +9,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD).about("Lists the names of all wallets.")
 }
 
-pub fn cli_run(base_dir: PathBuf) -> Result<(), String> {
-    let mgr = WalletManager::open(&base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+pub fn cli_run(wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mgr = WalletManager::open(&wallet_base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     for (name, _uuid) in mgr
         .wallets()

account_manager/src/wallet/mod.rs

@@ -1,11 +1,11 @@
 pub mod create;
 pub mod list;
+pub mod recover;
 
-use crate::{
-    common::{base_wallet_dir, ensure_dir_exists},
-    BASE_DIR_FLAG,
-};
+use crate::WALLETS_DIR_FLAG;
 use clap::{App, Arg, ArgMatches};
+use directory::{ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_WALLET_DIR};
+use std::path::PathBuf;
 
 pub const CMD: &str = "wallet";
 
@@ -13,23 +13,33 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Manage wallets, from which validator keys can be derived.")
         .arg(
-            Arg::with_name(BASE_DIR_FLAG)
-                .long(BASE_DIR_FLAG)
-                .value_name("BASE_DIRECTORY")
-                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets")
-                .takes_value(true),
+            Arg::with_name(WALLETS_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
+                .value_name("WALLETS_DIRECTORY")
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{network}/wallets")
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .subcommand(create::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
 }
 
 pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
-    let base_dir = base_wallet_dir(matches, BASE_DIR_FLAG)?;
-    ensure_dir_exists(&base_dir)?;
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    ensure_dir_exists(&wallet_base_dir)?;
+
+    eprintln!("wallet-dir path: {:?}", wallet_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run(matches, base_dir),
-        (list::CMD, Some(_)) => list::cli_run(base_dir),
+        (create::CMD, Some(matches)) => create::cli_run(matches, wallet_base_dir),
+        (list::CMD, Some(_)) => list::cli_run(wallet_base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, wallet_base_dir),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/wallet/recover.rs

@@ -0,0 +1,84 @@
+use crate::common::read_mnemonic_from_cli;
+use crate::wallet::create::{create_wallet_from_mnemonic, STDIN_INPUTS_FLAG};
+use crate::wallet::create::{HD_TYPE, NAME_FLAG, PASSWORD_FLAG, TYPE_FLAG};
+use clap::{App, Arg, ArgMatches};
+use std::path::PathBuf;
+
+pub const CMD: &str = "recover";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Recovers an EIP-2386 wallet from a given a BIP-39 mnemonic phrase.")
+        .arg(
+            Arg::with_name(NAME_FLAG)
+                .long(NAME_FLAG)
+                .value_name("WALLET_NAME")
+                .help(
+                    "The wallet will be created with this name. It is not allowed to \
+                            create two wallets with the same name for the same --base-dir.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
+                .value_name("PASSWORD_FILE_PATH")
+                .help(
+                    "This will be the new password for your recovered wallet. \
+                    A path to a file containing the password which will unlock the wallet. \
+                    If the file does not exist, a random password will be generated and \
+                    saved at that path. To avoid confusion, if the file does not already \
+                    exist it must include a '.pass' suffix.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help("If present, the mnemonic will be read in from this file.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(TYPE_FLAG)
+                .long(TYPE_FLAG)
+                .value_name("WALLET_TYPE")
+                .help(
+                    "The type of wallet to create. Only HD (hierarchical-deterministic) \
+                            wallets are supported presently..",
+                )
+                .takes_value(true)
+                .possible_values(&[HD_TYPE])
+                .default_value(HD_TYPE),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+
+    let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+    println!("Your wallet has been successfully recovered.");
+    println!();
+    println!("Your wallet's UUID is:");
+    println!();
+    println!("\t{}", wallet.wallet().uuid());
+    println!();
+    println!("You do not need to backup your UUID or keep it secret.");
+
+    Ok(())
+}

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "0.2.6"
+version = "1.0.1"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
 edition = "2018"
 
@@ -20,23 +20,27 @@ beacon_chain = { path = "beacon_chain" }
 types = { path = "../consensus/types" }
 store = { path = "./store" }
 client = { path = "client" }
-clap = "2.33.0"
+clap = "2.33.3"
 rand = "0.7.3"
 slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_trace"] }
-slog-term = "2.5.0"
+slog-term = "2.6.0"
 slog-async = "2.5.0"
-ctrlc = { version = "3.1.4", features = ["termination"] }
-tokio = { version = "0.2.21", features = ["time"] }
+ctrlc = { version = "3.1.6", features = ["termination"] }
+tokio = { version = "0.2.22", features = ["time"] }
 exit-future = "0.2.0"
-dirs = "2.0.2"
+dirs = "3.0.1"
 logging = { path = "../common/logging" }
+directory = {path = "../common/directory"}
 futures = "0.3.5"
 environment = { path = "../lighthouse/environment" }
+task_executor = { path = "../common/task_executor" }
 genesis = { path = "genesis" }
 eth2_testnet_config = { path = "../common/eth2_testnet_config" }
 eth2_libp2p = { path = "./eth2_libp2p" }
 eth2_ssz = "0.1.2"
-serde = "1.0.110"
+serde = "1.0.116"
 clap_utils = { path = "../common/clap_utils" }
-hyper = "0.13.5"
+hyper = "0.13.8"
 lighthouse_version = { path = "../common/lighthouse_version" }
+hex = "0.4.2"
+slasher = { path = "../slasher" }

beacon_node/beacon_chain/Cargo.toml

@@ -8,9 +8,12 @@ edition = "2018"
 default = ["participation_metrics"]
 write_ssz_files = []  # Writes debugging .ssz files to /tmp during block processing.
 participation_metrics = []  # Exposes validator participation metrics to Prometheus.
+test_logger = [] # Print log output to stderr when running tests instead of dropping it
 
 [dev-dependencies]
 int_to_bytes = { path = "../../consensus/int_to_bytes" }
+maplit = "1.0.2"
+environment = { path = "../../lighthouse/environment" }
 
 [dependencies]
 eth2_config = { path = "../../common/eth2_config" }
@@ -18,41 +21,45 @@ merkle_proof = { path = "../../consensus/merkle_proof" }
 store = { path = "../store" }
 parking_lot = "0.11.0"
 lazy_static = "1.4.0"
-smallvec = "1.4.1"
+smallvec = "1.4.2"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-log = "0.4.8"
+log = "0.4.11"
 operation_pool = { path = "../operation_pool" }
-rayon = "1.3.0"
-serde = "1.0.110"
-serde_derive = "1.0.110"
-serde_yaml = "0.8.11"
-serde_json = "1.0.52"
+rayon = "1.4.1"
+serde = "1.0.116"
+serde_derive = "1.0.116"
+serde_yaml = "0.8.13"
+serde_json = "1.0.58"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
-slog-term = "2.6.0"
-sloggers = "1.0.0"
+sloggers = "1.0.1"
 slot_clock = { path = "../../common/slot_clock" }
 eth2_hashing = "0.1.0"
 eth2_ssz = "0.1.2"
 eth2_ssz_types = { path = "../../consensus/ssz_types" }
 eth2_ssz_derive = "0.1.0"
 state_processing = { path = "../../consensus/state_processing" }
-tree_hash = "0.1.0"
+tree_hash = "0.1.1"
 types = { path = "../../consensus/types" }
-tokio = "0.2.21"
+tokio = "0.2.22"
 eth1 = { path = "../eth1" }
 websocket_server = { path = "../websocket_server" }
 futures = "0.3.5"
 genesis = { path = "../genesis" }
-integer-sqrt = "0.1.3"
+integer-sqrt = "0.1.5"
 rand = "0.7.3"
+rand_core = "0.5.1"
 proto_array = { path = "../../consensus/proto_array" }
-lru = "0.5.1"
+lru = "0.6.0"
 tempfile = "3.1.0"
-bitvec = "0.17.4"
+bitvec = "0.19.3"
 bls = { path = "../../crypto/bls" }
 safe_arith = { path = "../../consensus/safe_arith" }
 fork_choice = { path = "../../consensus/fork_choice" }
-environment = { path = "../../lighthouse/environment" }
+task_executor = { path = "../../common/task_executor" }
 bus = "2.2.3"
 derivative = "2.1.1"
 itertools = "0.9.0"
+regex = "1.3.9"
+exit-future = "0.2.0"
+slasher = { path = "../../slasher" }
+eth2 = { path = "../../common/eth2" }

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -28,8 +28,7 @@
 
 use crate::{
     beacon_chain::{
-        ATTESTATION_CACHE_LOCK_TIMEOUT, HEAD_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY,
-        VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT,
+        HEAD_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY, VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT,
     },
     metrics,
     observed_attestations::ObserveOutcome,
@@ -37,12 +36,12 @@ use crate::{
     BeaconChain, BeaconChainError, BeaconChainTypes,
 };
 use bls::verify_signature_sets;
+use proto_array::Block as ProtoBlock;
 use slog::debug;
 use slot_clock::SlotClock;
 use state_processing::{
     common::get_indexed_attestation,
     per_block_processing::errors::AttestationValidationError,
-    per_slot_processing,
     signature_sets::{
         indexed_attestation_signature_set_from_pubkeys,
         signed_aggregate_selection_proof_signature_set, signed_aggregate_signature_set,
@@ -52,7 +51,7 @@ use std::borrow::Cow;
 use tree_hash::TreeHash;
 use types::{
     Attestation, BeaconCommittee, CommitteeIndex, Epoch, EthSpec, Hash256, IndexedAttestation,
-    RelativeEpoch, SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
+    SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
 };
 
 /// Returned when an attestation was not successfully verified. It might not have been verified for
@@ -226,6 +225,21 @@ pub enum Error {
         head_block_slot: Slot,
         attestation_slot: Slot,
     },
+    /// The attestation has an invalid target epoch.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    InvalidTargetEpoch { slot: Slot, epoch: Epoch },
+    /// The attestation references an invalid target block.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    InvalidTargetRoot {
+        attestation: Hash256,
+        expected: Option<Hash256>,
+    },
     /// There was an error whilst processing the attestation. It is not known if it is valid or invalid.
     ///
     /// ## Peer scoring
@@ -251,6 +265,7 @@ pub struct VerifiedAggregatedAttestation<T: BeaconChainTypes> {
 pub struct VerifiedUnaggregatedAttestation<T: BeaconChainTypes> {
     attestation: Attestation<T::EthSpec>,
     indexed_attestation: IndexedAttestation<T::EthSpec>,
+    subnet_id: SubnetId,
 }
 
 /// Custom `Clone` implementation is to avoid the restrictive trait bounds applied by the usual derive
@@ -260,6 +275,7 @@ impl<T: BeaconChainTypes> Clone for VerifiedUnaggregatedAttestation<T> {
         Self {
             attestation: self.attestation.clone(),
             indexed_attestation: self.indexed_attestation.clone(),
+            subnet_id: self.subnet_id,
         }
     }
 }
@@ -282,6 +298,76 @@ impl<T: BeaconChainTypes> SignatureVerifiedAttestation<T> for VerifiedUnaggregat
     }
 }
 
+/// Information about invalid attestations which might still be slashable despite being invalid.
+pub enum AttestationSlashInfo<T: BeaconChainTypes, TErr> {
+    /// The attestation is invalid, but its signature wasn't checked.
+    SignatureNotChecked(Attestation<T::EthSpec>, TErr),
+    /// As for `SignatureNotChecked`, but we know the `IndexedAttestation`.
+    SignatureNotCheckedIndexed(IndexedAttestation<T::EthSpec>, TErr),
+    /// The attestation's signature is invalid, so it will never be slashable.
+    SignatureInvalid(TErr),
+    /// The signature is valid but the attestation is invalid in some other way.
+    SignatureValid(IndexedAttestation<T::EthSpec>, TErr),
+}
+
+/// After processing an attestation normally, optionally process it further for the slasher.
+///
+/// This maps an `AttestationSlashInfo` error back into a regular `Error`, performing signature
+/// checks on attestations that failed verification for other reasons.
+///
+/// No substantial extra work will be done if there is no slasher configured.
+fn process_slash_info<T: BeaconChainTypes>(
+    slash_info: AttestationSlashInfo<T, Error>,
+    chain: &BeaconChain<T>,
+) -> Error {
+    use AttestationSlashInfo::*;
+
+    if let Some(slasher) = chain.slasher.as_ref() {
+        let (indexed_attestation, check_signature, err) = match slash_info {
+            SignatureNotChecked(attestation, err) => {
+                match obtain_indexed_attestation_and_committees_per_slot(chain, &attestation) {
+                    Ok((indexed, _)) => (indexed, true, err),
+                    Err(e) => {
+                        debug!(
+                            chain.log,
+                            "Unable to obtain indexed form of attestation for slasher";
+                            "attestation_root" => format!("{:?}", attestation.tree_hash_root()),
+                            "error" => format!("{:?}", e)
+                        );
+                        return err;
+                    }
+                }
+            }
+            SignatureNotCheckedIndexed(indexed, err) => (indexed, true, err),
+            SignatureInvalid(e) => return e,
+            SignatureValid(indexed, err) => (indexed, false, err),
+        };
+
+        if check_signature {
+            if let Err(e) = verify_attestation_signature(chain, &indexed_attestation) {
+                debug!(
+                    chain.log,
+                    "Signature verification for slasher failed";
+                    "error" => format!("{:?}", e),
+                );
+                return err;
+            }
+        }
+
+        // Supply to slasher.
+        slasher.accept_attestation(indexed_attestation);
+
+        err
+    } else {
+        match slash_info {
+            SignatureNotChecked(_, e)
+            | SignatureNotCheckedIndexed(_, e)
+            | SignatureInvalid(e)
+            | SignatureValid(_, e) => e,
+        }
+    }
+}
+
 impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
     /// Returns `Ok(Self)` if the `signed_aggregate` is valid to be (re)published on the gossip
     /// network.
@@ -289,6 +375,21 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         signed_aggregate: SignedAggregateAndProof<T::EthSpec>,
         chain: &BeaconChain<T>,
     ) -> Result<Self, Error> {
+        Self::verify_slashable(signed_aggregate, chain)
+            .map(|verified_aggregate| {
+                if let Some(slasher) = chain.slasher.as_ref() {
+                    slasher.accept_attestation(verified_aggregate.indexed_attestation.clone());
+                }
+                verified_aggregate
+            })
+            .map_err(|slash_info| process_slash_info(slash_info, chain))
+    }
+
+    /// Run the checks that happen before an indexed attestation is constructed.
+    fn verify_early_checks(
+        signed_aggregate: &SignedAggregateAndProof<T::EthSpec>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Hash256, Error> {
         let attestation = &signed_aggregate.message.aggregate;
 
         // Ensure attestation is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (within a
@@ -301,6 +402,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         let attestation_root = attestation.tree_hash_root();
         if chain
             .observed_attestations
+            .write()
             .is_known(attestation, attestation_root)
             .map_err(|e| Error::BeaconChainError(e.into()))?
         {
@@ -314,6 +416,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         // Note: do not observe yet, only observe once the attestation has been verfied.
         match chain
             .observed_aggregators
+            .read()
             .validator_has_been_observed(attestation, aggregator_index as usize)
         {
             Ok(true) => Err(Error::AggregatorAlreadyKnown(aggregator_index)),
@@ -334,15 +437,82 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         //
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
-        verify_head_block_is_known(chain, &attestation, None)?;
+        let head_block = verify_head_block_is_known(chain, &attestation, None)?;
+
+        // Check the attestation target root is consistent with the head root.
+        //
+        // This check is not in the specification, however we guard against it since it opens us up
+        // to weird edge cases during verification.
+        //
+        // Whilst this attestation *technically* could be used to add value to a block, it is
+        // invalid in the spirit of the protocol. Here we choose safety over profit.
+        verify_attestation_target_root::<T::EthSpec>(&head_block, &attestation)?;
 
         // Ensure that the attestation has participants.
         if attestation.aggregation_bits.is_zero() {
-            return Err(Error::EmptyAggregationBitfield);
+            Err(Error::EmptyAggregationBitfield)
+        } else {
+            Ok(attestation_root)
+        }
+    }
+
+    /// Run the checks that happen after the indexed attestation and signature have been checked.
+    fn verify_late_checks(
+        signed_aggregate: &SignedAggregateAndProof<T::EthSpec>,
+        attestation_root: Hash256,
+        chain: &BeaconChain<T>,
+    ) -> Result<(), Error> {
+        let attestation = &signed_aggregate.message.aggregate;
+        let aggregator_index = signed_aggregate.message.aggregator_index;
+
+        // Observe the valid attestation so we do not re-process it.
+        //
+        // It's important to double check that the attestation is not already known, otherwise two
+        // attestations processed at the same time could be published.
+        if let ObserveOutcome::AlreadyKnown = chain
+            .observed_attestations
+            .write()
+            .observe_attestation(attestation, Some(attestation_root))
+            .map_err(|e| Error::BeaconChainError(e.into()))?
+        {
+            return Err(Error::AttestationAlreadyKnown(attestation_root));
         }
 
+        // Observe the aggregator so we don't process another aggregate from them.
+        //
+        // It's important to double check that the attestation is not already known, otherwise two
+        // attestations processed at the same time could be published.
+        if chain
+            .observed_aggregators
+            .write()
+            .observe_validator(&attestation, aggregator_index as usize)
+            .map_err(BeaconChainError::from)?
+        {
+            return Err(Error::PriorAttestationKnown {
+                validator_index: aggregator_index,
+                epoch: attestation.data.target.epoch,
+            });
+        }
+
+        Ok(())
+    }
+
+    /// Verify the attestation, producing extra information about whether it might be slashable.
+    pub fn verify_slashable(
+        signed_aggregate: SignedAggregateAndProof<T::EthSpec>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, AttestationSlashInfo<T, Error>> {
+        use AttestationSlashInfo::*;
+
+        let attestation = &signed_aggregate.message.aggregate;
+        let aggregator_index = signed_aggregate.message.aggregator_index;
+        let attestation_root = match Self::verify_early_checks(&signed_aggregate, chain) {
+            Ok(root) => root,
+            Err(e) => return Err(SignatureNotChecked(signed_aggregate.message.aggregate, e)),
+        };
+
         let indexed_attestation =
-            map_attestation_committee(chain, attestation, |(committee, _)| {
+            match map_attestation_committee(chain, attestation, |(committee, _)| {
                 // Note: this clones the signature which is known to be a relatively slow operation.
                 //
                 // Future optimizations should remove this clone.
@@ -361,40 +531,29 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
                     return Err(Error::AggregatorNotInCommittee { aggregator_index });
                 }
 
-                get_indexed_attestation(committee.committee, &attestation)
+                get_indexed_attestation(committee.committee, attestation)
                     .map_err(|e| BeaconChainError::from(e).into())
-            })?;
+            }) {
+                Ok(indexed_attestation) => indexed_attestation,
+                Err(e) => return Err(SignatureNotChecked(signed_aggregate.message.aggregate, e)),
+            };
 
         // Ensure that all signatures are valid.
-        if !verify_signed_aggregate_signatures(chain, &signed_aggregate, &indexed_attestation)? {
-            return Err(Error::InvalidSignature);
+        if let Err(e) =
+            verify_signed_aggregate_signatures(chain, &signed_aggregate, &indexed_attestation)
+                .and_then(|is_valid| {
+                    if !is_valid {
+                        Err(Error::InvalidSignature)
+                    } else {
+                        Ok(())
+                    }
+                })
+        {
+            return Err(SignatureInvalid(e));
         }
 
-        // Observe the valid attestation so we do not re-process it.
-        //
-        // It's important to double check that the attestation is not already known, otherwise two
-        // attestations processed at the same time could be published.
-        if let ObserveOutcome::AlreadyKnown = chain
-            .observed_attestations
-            .observe_attestation(attestation, Some(attestation_root))
-            .map_err(|e| Error::BeaconChainError(e.into()))?
-        {
-            return Err(Error::AttestationAlreadyKnown(attestation_root));
-        }
-
-        // Observe the aggregator so we don't process another aggregate from them.
-        //
-        // It's important to double check that the attestation is not already known, otherwise two
-        // attestations processed at the same time could be published.
-        if chain
-            .observed_aggregators
-            .observe_validator(&attestation, aggregator_index as usize)
-            .map_err(BeaconChainError::from)?
-        {
-            return Err(Error::PriorAttestationKnown {
-                validator_index: aggregator_index,
-                epoch: attestation.data.target.epoch,
-            });
+        if let Err(e) = Self::verify_late_checks(&signed_aggregate, attestation_root, chain) {
+            return Err(SignatureValid(indexed_attestation, e));
         }
 
         Ok(VerifiedAggregatedAttestation {
@@ -412,19 +571,29 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
     pub fn attestation(&self) -> &Attestation<T::EthSpec> {
         &self.signed_aggregate.message.aggregate
     }
+
+    /// Returns the underlying `signed_aggregate`.
+    pub fn aggregate(&self) -> &SignedAggregateAndProof<T::EthSpec> {
+        &self.signed_aggregate
+    }
 }
 
 impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
-    /// Returns `Ok(Self)` if the `attestation` is valid to be (re)published on the gossip
-    /// network.
-    ///
-    /// `subnet_id` is the subnet from which we received this attestation. This function will
-    /// verify that it was received on the correct subnet.
-    pub fn verify(
-        attestation: Attestation<T::EthSpec>,
-        subnet_id: SubnetId,
+    /// Run the checks that happen before an indexed attestation is constructed.
+    pub fn verify_early_checks(
+        attestation: &Attestation<T::EthSpec>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, Error> {
+    ) -> Result<(), Error> {
+        let attestation_epoch = attestation.data.slot.epoch(T::EthSpec::slots_per_epoch());
+
+        // Check the attestation's epoch matches its target.
+        if attestation_epoch != attestation.data.target.epoch {
+            return Err(Error::InvalidTargetEpoch {
+                slot: attestation.data.slot,
+                epoch: attestation.data.target.epoch,
+            });
+        }
+
         // Ensure attestation is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (within a
         // MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance).
         //
@@ -433,20 +602,32 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 
         // Check to ensure that the attestation is "unaggregated". I.e., it has exactly one
         // aggregation bit set.
-        let num_aggreagtion_bits = attestation.aggregation_bits.num_set_bits();
-        if num_aggreagtion_bits != 1 {
-            return Err(Error::NotExactlyOneAggregationBitSet(num_aggreagtion_bits));
+        let num_aggregation_bits = attestation.aggregation_bits.num_set_bits();
+        if num_aggregation_bits != 1 {
+            return Err(Error::NotExactlyOneAggregationBitSet(num_aggregation_bits));
         }
 
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
         //
         // Enforce a maximum skip distance for unaggregated attestations.
-        verify_head_block_is_known(chain, &attestation, chain.config.import_max_skip_slots)?;
+        let head_block =
+            verify_head_block_is_known(chain, &attestation, chain.config.import_max_skip_slots)?;
 
-        let (indexed_attestation, committees_per_slot) =
-            obtain_indexed_attestation_and_committees_per_slot(chain, &attestation)?;
+        // Check the attestation target root is consistent with the head root.
+        verify_attestation_target_root::<T::EthSpec>(&head_block, &attestation)?;
 
+        Ok(())
+    }
+
+    /// Run the checks that apply to the indexed attestation before the signature is checked.
+    pub fn verify_middle_checks(
+        attestation: &Attestation<T::EthSpec>,
+        indexed_attestation: &IndexedAttestation<T::EthSpec>,
+        committees_per_slot: u64,
+        subnet_id: Option<SubnetId>,
+        chain: &BeaconChain<T>,
+    ) -> Result<(u64, SubnetId), Error> {
         let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::<T::EthSpec>(
             &indexed_attestation.data,
             committees_per_slot,
@@ -454,13 +635,15 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         )
         .map_err(BeaconChainError::from)?;
 
-        // Ensure the attestation is from the correct subnet.
-        if subnet_id != expected_subnet_id {
-            return Err(Error::InvalidSubnetId {
-                received: subnet_id,
-                expected: expected_subnet_id,
-            });
-        }
+        // If a subnet was specified, ensure that subnet is correct.
+        if let Some(subnet_id) = subnet_id {
+            if subnet_id != expected_subnet_id {
+                return Err(Error::InvalidSubnetId {
+                    received: subnet_id,
+                    expected: expected_subnet_id,
+                });
+            }
+        };
 
         let validator_index = *indexed_attestation
             .attesting_indices
@@ -473,6 +656,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
          */
         if chain
             .observed_attesters
+            .read()
             .validator_has_been_observed(&attestation, validator_index as usize)
             .map_err(BeaconChainError::from)?
         {
@@ -482,9 +666,15 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
             });
         }
 
-        // The aggregate signature of the attestation is valid.
-        verify_attestation_signature(chain, &indexed_attestation)?;
+        Ok((validator_index, expected_subnet_id))
+    }
 
+    /// Run the checks that apply after the signature has been checked.
+    fn verify_late_checks(
+        attestation: &Attestation<T::EthSpec>,
+        validator_index: u64,
+        chain: &BeaconChain<T>,
+    ) -> Result<(), Error> {
         // Now that the attestation has been fully verified, store that we have received a valid
         // attestation from this validator.
         //
@@ -493,6 +683,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         // process them in different threads.
         if chain
             .observed_attesters
+            .write()
             .observe_validator(&attestation, validator_index as usize)
             .map_err(BeaconChainError::from)?
         {
@@ -501,10 +692,73 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
                 epoch: attestation.data.target.epoch,
             });
         }
+        Ok(())
+    }
+
+    /// Returns `Ok(Self)` if the `attestation` is valid to be (re)published on the gossip
+    /// network.
+    ///
+    /// `subnet_id` is the subnet from which we received this attestation. This function will
+    /// verify that it was received on the correct subnet.
+    pub fn verify(
+        attestation: Attestation<T::EthSpec>,
+        subnet_id: Option<SubnetId>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, Error> {
+        Self::verify_slashable(attestation, subnet_id, chain)
+            .map(|verified_unaggregated| {
+                if let Some(slasher) = chain.slasher.as_ref() {
+                    slasher.accept_attestation(verified_unaggregated.indexed_attestation.clone());
+                }
+                verified_unaggregated
+            })
+            .map_err(|slash_info| process_slash_info(slash_info, chain))
+    }
+
+    /// Verify the attestation, producing extra information about whether it might be slashable.
+    pub fn verify_slashable(
+        attestation: Attestation<T::EthSpec>,
+        subnet_id: Option<SubnetId>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, AttestationSlashInfo<T, Error>> {
+        use AttestationSlashInfo::*;
+
+        if let Err(e) = Self::verify_early_checks(&attestation, chain) {
+            return Err(SignatureNotChecked(attestation, e));
+        }
+
+        let (indexed_attestation, committees_per_slot) =
+            match obtain_indexed_attestation_and_committees_per_slot(chain, &attestation) {
+                Ok(x) => x,
+                Err(e) => {
+                    return Err(SignatureNotChecked(attestation, e));
+                }
+            };
+
+        let (validator_index, expected_subnet_id) = match Self::verify_middle_checks(
+            &attestation,
+            &indexed_attestation,
+            committees_per_slot,
+            subnet_id,
+            chain,
+        ) {
+            Ok(t) => t,
+            Err(e) => return Err(SignatureNotCheckedIndexed(indexed_attestation, e)),
+        };
+
+        // The aggregate signature of the attestation is valid.
+        if let Err(e) = verify_attestation_signature(chain, &indexed_attestation) {
+            return Err(SignatureInvalid(e));
+        }
+
+        if let Err(e) = Self::verify_late_checks(&attestation, validator_index, chain) {
+            return Err(SignatureValid(indexed_attestation, e));
+        }
 
         Ok(Self {
             attestation,
             indexed_attestation,
+            subnet_id: expected_subnet_id,
         })
     }
 
@@ -513,6 +767,11 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         chain.add_to_naive_aggregation_pool(self)
     }
 
+    /// Returns the correct subnet for the attestation.
+    pub fn subnet_id(&self) -> SubnetId {
+        self.subnet_id
+    }
+
     /// Returns the wrapped `attestation`.
     pub fn attestation(&self) -> &Attestation<T::EthSpec> {
         &self.attestation
@@ -528,6 +787,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 }
 
 /// Returns `Ok(())` if the `attestation.data.beacon_block_root` is known to this chain.
+/// You can use this `shuffling_id` to read from the shuffling cache.
 ///
 /// The block root may not be known for two reasons:
 ///
@@ -541,7 +801,7 @@ fn verify_head_block_is_known<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
     max_skip_slots: Option<u64>,
-) -> Result<(), Error> {
+) -> Result<ProtoBlock, Error> {
     if let Some(block) = chain
         .fork_choice
         .read()
@@ -556,7 +816,8 @@ fn verify_head_block_is_known<T: BeaconChainTypes>(
                 });
             }
         }
-        Ok(())
+
+        Ok(block)
     } else {
         Err(Error::UnknownHeadBlock {
             beacon_block_root: attestation.data.beacon_block_root,
@@ -642,6 +903,57 @@ pub fn verify_attestation_signature<T: BeaconChainTypes>(
     }
 }
 
+/// Verifies that the `attestation.data.target.root` is indeed the target root of the block at
+/// `attestation.data.beacon_block_root`.
+pub fn verify_attestation_target_root<T: EthSpec>(
+    head_block: &ProtoBlock,
+    attestation: &Attestation<T>,
+) -> Result<(), Error> {
+    // Check the attestation target root.
+    let head_block_epoch = head_block.slot.epoch(T::slots_per_epoch());
+    let attestation_epoch = attestation.data.slot.epoch(T::slots_per_epoch());
+    if head_block_epoch > attestation_epoch {
+        // The epoch references an invalid head block from a future epoch.
+        //
+        // This check is not in the specification, however we guard against it since it opens us up
+        // to weird edge cases during verification.
+        //
+        // Whilst this attestation *technically* could be used to add value to a block, it is
+        // invalid in the spirit of the protocol. Here we choose safety over profit.
+        //
+        // Reference:
+        // https://github.com/ethereum/eth2.0-specs/pull/2001#issuecomment-699246659
+        return Err(Error::InvalidTargetRoot {
+            attestation: attestation.data.target.root,
+            // It is not clear what root we should expect in this case, since the attestation is
+            // fundamentally invalid.
+            expected: None,
+        });
+    } else {
+        let target_root = if head_block_epoch == attestation_epoch {
+            // If the block is in the same epoch as the attestation, then use the target root
+            // from the block.
+            head_block.target_root
+        } else {
+            // If the head block is from a previous epoch then skip slots will cause the head block
+            // root to become the target block root.
+            //
+            // We know the head block is from a previous epoch due to a previous check.
+            head_block.root
+        };
+
+        // Reject any attestation with an invalid target root.
+        if target_root != attestation.data.target.root {
+            return Err(Error::InvalidTargetRoot {
+                attestation: attestation.data.target.root,
+                expected: Some(target_root),
+            });
+        }
+    }
+
+    Ok(())
+}
+
 /// Verifies all the signatures in a `SignedAggregateAndProof` using BLS batch verification. This
 /// includes three signatures:
 ///
@@ -711,14 +1023,14 @@ type CommitteesPerSlot = u64;
 
 /// Returns the `indexed_attestation` and committee count per slot for the `attestation` using the
 /// public keys cached in the `chain`.
-pub fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
+fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
 ) -> Result<(IndexedAttestation<T::EthSpec>, CommitteesPerSlot), Error> {
     map_attestation_committee(chain, attestation, |(committee, committees_per_slot)| {
         get_indexed_attestation(committee.committee, &attestation)
             .map(|attestation| (attestation, committees_per_slot))
-            .map_err(|e| BeaconChainError::from(e).into())
+            .map_err(Error::Invalid)
     })
 }
 
@@ -731,8 +1043,8 @@ pub fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
 ///
 /// If the committee for `attestation` isn't found in the `shuffling_cache`, we will read a state
 /// from disk and then update the `shuffling_cache`.
-pub fn map_attestation_committee<'a, T, F, R>(
-    chain: &'a BeaconChain<T>,
+fn map_attestation_committee<T, F, R>(
+    chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
     map_fn: F,
 ) -> Result<R, Error>
@@ -750,104 +1062,23 @@ where
     // processing an attestation that does not include our latest finalized block in its chain.
     //
     // We do not delay consideration for later, we simply drop the attestation.
-    let target_block = chain
-        .fork_choice
-        .read()
-        .get_block(&target.root)
-        .ok_or_else(|| Error::UnknownTargetRoot(target.root))?;
-
-    // Obtain the shuffling cache, timing how long we wait.
-    let cache_wait_timer =
-        metrics::start_timer(&metrics::ATTESTATION_PROCESSING_SHUFFLING_CACHE_WAIT_TIMES);
-
-    let mut shuffling_cache = chain
-        .shuffling_cache
-        .try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
-        .ok_or_else(|| BeaconChainError::AttestationCacheLockTimeout)?;
-
-    metrics::stop_timer(cache_wait_timer);
-
-    if let Some(committee_cache) = shuffling_cache.get(attestation_epoch, target.root) {
-        let committees_per_slot = committee_cache.committees_per_slot();
-        committee_cache
-            .get_beacon_committee(attestation.data.slot, attestation.data.index)
-            .map(|committee| map_fn((committee, committees_per_slot)))
-            .unwrap_or_else(|| {
-                Err(Error::NoCommitteeForSlotAndIndex {
-                    slot: attestation.data.slot,
-                    index: attestation.data.index,
-                })
-            })
-    } else {
-        // Drop the shuffling cache to avoid holding the lock for any longer than
-        // required.
-        drop(shuffling_cache);
-
-        debug!(
-            chain.log,
-            "Attestation processing cache miss";
-            "attn_epoch" => attestation_epoch.as_u64(),
-            "target_block_epoch" => target_block.slot.epoch(T::EthSpec::slots_per_epoch()).as_u64(),
-        );
-
-        let state_read_timer =
-            metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_READ_TIMES);
-
-        let mut state = chain
-            .store
-            .get_inconsistent_state_for_attestation_verification_only(
-                &target_block.state_root,
-                Some(target_block.slot),
-            )
-            .map_err(BeaconChainError::from)?
-            .ok_or_else(|| BeaconChainError::MissingBeaconState(target_block.state_root))?;
-
-        metrics::stop_timer(state_read_timer);
-        let state_skip_timer =
-            metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_SKIP_TIMES);
-
-        while state.current_epoch() + 1 < attestation_epoch {
-            // Here we tell `per_slot_processing` to skip hashing the state and just
-            // use the zero hash instead.
-            //
-            // The state roots are not useful for the shuffling, so there's no need to
-            // compute them.
-            per_slot_processing(&mut state, Some(Hash256::zero()), &chain.spec)
-                .map_err(BeaconChainError::from)?;
-        }
-
-        metrics::stop_timer(state_skip_timer);
-        let committee_building_timer =
-            metrics::start_timer(&metrics::ATTESTATION_PROCESSING_COMMITTEE_BUILDING_TIMES);
-
-        let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), attestation_epoch)
-            .map_err(BeaconChainError::IncorrectStateForAttestation)?;
-
-        state
-            .build_committee_cache(relative_epoch, &chain.spec)
-            .map_err(BeaconChainError::from)?;
-
-        let committee_cache = state
-            .committee_cache(relative_epoch)
-            .map_err(BeaconChainError::from)?;
-
-        chain
-            .shuffling_cache
-            .try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
-            .ok_or_else(|| BeaconChainError::AttestationCacheLockTimeout)?
-            .insert(attestation_epoch, target.root, committee_cache);
-
-        metrics::stop_timer(committee_building_timer);
-
-        let committees_per_slot = committee_cache.committees_per_slot();
-        committee_cache
-            .get_beacon_committee(attestation.data.slot, attestation.data.index)
-            .map(|committee| map_fn((committee, committees_per_slot)))
-            .unwrap_or_else(|| {
-                Err(Error::NoCommitteeForSlotAndIndex {
-                    slot: attestation.data.slot,
-                    index: attestation.data.index,
-                })
-            })
+    if !chain.fork_choice.read().contains_block(&target.root) {
+        return Err(Error::UnknownTargetRoot(target.root));
     }
+
+    chain
+        .with_committee_cache(target.root, attestation_epoch, |committee_cache| {
+            let committees_per_slot = committee_cache.committees_per_slot();
+
+            Ok(committee_cache
+                .get_beacon_committee(attestation.data.slot, attestation.data.index)
+                .map(|committee| map_fn((committee, committees_per_slot)))
+                .unwrap_or_else(|| {
+                    Err(Error::NoCommitteeForSlotAndIndex {
+                        slot: attestation.data.slot,
+                        index: attestation.data.index,
+                    })
+                }))
+        })
+        .map_err(BeaconChainError::from)?
 }

beacon_node/beacon_chain/src/block_verification.rs

@@ -63,11 +63,11 @@ use std::borrow::Cow;
 use std::convert::TryFrom;
 use std::fs;
 use std::io::Write;
-use store::{Error as DBError, HotColdDB, HotStateSummary, StoreOp};
+use store::{Error as DBError, HotColdDB, HotStateSummary, KeyValueStore, StoreOp};
 use tree_hash::TreeHash;
 use types::{
     BeaconBlock, BeaconState, BeaconStateError, ChainSpec, CloneConfig, EthSpec, Hash256,
-    PublicKey, RelativeEpoch, SignedBeaconBlock, Slot,
+    PublicKey, RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
 };
 
 /// Maximum block slot number. Block with slots bigger than this constant will NOT be processed.
@@ -207,6 +207,13 @@ pub enum BlockError<T: EthSpec> {
     /// We were unable to process this block due to an internal error. It's unclear if the block is
     /// valid.
     BeaconChainError(BeaconChainError),
+    /// There was an error whilst verifying weak subjectivity. This block conflicts with the
+    /// configured weak subjectivity checkpoint and was not imported.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The block is invalid and the peer is faulty.
+    WeakSubjectivityConflict,
 }
 
 impl<T: EthSpec> std::fmt::Display for BlockError<T> {
@@ -261,6 +268,27 @@ impl<T: EthSpec> From<DBError> for BlockError<T> {
     }
 }
 
+/// Information about invalid blocks which might still be slashable despite being invalid.
+pub enum BlockSlashInfo<TErr> {
+    /// The block is invalid, but its proposer signature wasn't checked.
+    SignatureNotChecked(SignedBeaconBlockHeader, TErr),
+    /// The block's proposer signature is invalid, so it will never be slashable.
+    SignatureInvalid(TErr),
+    /// The signature is valid but the attestation is invalid in some other way.
+    SignatureValid(SignedBeaconBlockHeader, TErr),
+}
+
+impl<E: EthSpec> BlockSlashInfo<BlockError<E>> {
+    pub fn from_early_error(header: SignedBeaconBlockHeader, e: BlockError<E>) -> Self {
+        match e {
+            BlockError::ProposalSignatureInvalid => BlockSlashInfo::SignatureInvalid(e),
+            // `InvalidSignature` could indicate any signature in the block, so we want
+            // to recheck the proposer signature alone.
+            _ => BlockSlashInfo::SignatureNotChecked(header, e),
+        }
+    }
+}
+
 /// Verify all signatures (except deposit signatures) on all blocks in the `chain_segment`. If all
 /// signatures are valid, the `chain_segment` is mapped to a `Vec<SignatureVerifiedBlock>` that can
 /// later be transformed into a `FullyVerifiedBlock` without re-checking the signatures. If any
@@ -356,17 +384,57 @@ pub struct FullyVerifiedBlock<'a, T: BeaconChainTypes> {
     pub block_root: Hash256,
     pub state: BeaconState<T::EthSpec>,
     pub parent_block: SignedBeaconBlock<T::EthSpec>,
-    pub intermediate_states: Vec<StoreOp<'a, T::EthSpec>>,
+    pub confirmation_db_batch: Vec<StoreOp<'a, T::EthSpec>>,
 }
 
 /// Implemented on types that can be converted into a `FullyVerifiedBlock`.
 ///
 /// Used to allow functions to accept blocks at various stages of verification.
-pub trait IntoFullyVerifiedBlock<T: BeaconChainTypes> {
+pub trait IntoFullyVerifiedBlock<T: BeaconChainTypes>: Sized {
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>>;
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
+        self.into_fully_verified_block_slashable(chain)
+            .map(|fully_verified| {
+                // Supply valid block to slasher.
+                if let Some(slasher) = chain.slasher.as_ref() {
+                    slasher.accept_block_header(fully_verified.block.signed_block_header());
+                }
+                fully_verified
+            })
+            .map_err(|slash_info| {
+                // Process invalid blocks to see if they are suitable for the slasher.
+                if let Some(slasher) = chain.slasher.as_ref() {
+                    let (verified_header, error) = match slash_info {
+                        BlockSlashInfo::SignatureNotChecked(header, e) => {
+                            if verify_header_signature(chain, &header).is_ok() {
+                                (header, e)
+                            } else {
+                                return e;
+                            }
+                        }
+                        BlockSlashInfo::SignatureInvalid(e) => return e,
+                        BlockSlashInfo::SignatureValid(header, e) => (header, e),
+                    };
+
+                    slasher.accept_block_header(verified_header);
+                    error
+                } else {
+                    match slash_info {
+                        BlockSlashInfo::SignatureNotChecked(_, e)
+                        | BlockSlashInfo::SignatureInvalid(e)
+                        | BlockSlashInfo::SignatureValid(_, e) => e,
+                    }
+                }
+            })
+    }
+
+    /// Convert the block to fully-verified form while producing data to aid checking slashability.
+    fn into_fully_verified_block_slashable(
+        self,
+        chain: &BeaconChain<T>,
+    ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>>;
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec>;
 }
@@ -411,6 +479,7 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
         // Check that we have not already received a block with a valid signature for this slot.
         if chain
             .observed_block_producers
+            .read()
             .proposer_has_been_observed(&block.message)
             .map_err(|e| BlockError::BeaconChainError(e.into()))?
         {
@@ -465,6 +534,7 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
         // have a race-condition when verifying two blocks simultaneously.
         if chain
             .observed_block_producers
+            .write()
             .observe_proposer(&block.message)
             .map_err(|e| BlockError::BeaconChainError(e.into()))?
         {
@@ -497,12 +567,13 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
 
 impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for GossipVerifiedBlock<T> {
     /// Completes verification of the wrapped `block`.
-    fn into_fully_verified_block(
+    fn into_fully_verified_block_slashable(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
-        let fully_verified = SignatureVerifiedBlock::from_gossip_verified_block(self, chain)?;
-        fully_verified.into_fully_verified_block(chain)
+    ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        let fully_verified =
+            SignatureVerifiedBlock::from_gossip_verified_block_check_slashable(self, chain)?;
+        fully_verified.into_fully_verified_block_slashable(chain)
     }
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec> {
@@ -549,6 +620,15 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
         }
     }
 
+    /// As for `new` above but producing `BlockSlashInfo`.
+    pub fn check_slashable(
+        block: SignedBeaconBlock<T::EthSpec>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        let header = block.signed_block_header();
+        Self::new(block, chain).map_err(|e| BlockSlashInfo::from_early_error(header, e))
+    }
+
     /// Finishes signature verification on the provided `GossipVerifedBlock`. Does not re-verify
     /// the proposer signature.
     pub fn from_gossip_verified_block(
@@ -580,18 +660,30 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
             Err(BlockError::InvalidSignature)
         }
     }
+
+    /// Same as `from_gossip_verified_block` but producing slashing-relevant data as well.
+    pub fn from_gossip_verified_block_check_slashable(
+        from: GossipVerifiedBlock<T>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        let header = from.block.signed_block_header();
+        Self::from_gossip_verified_block(from, chain)
+            .map_err(|e| BlockSlashInfo::from_early_error(header, e))
+    }
 }
 
 impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignatureVerifiedBlock<T> {
     /// Completes verification of the wrapped `block`.
-    fn into_fully_verified_block(
+    fn into_fully_verified_block_slashable(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
+    ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        let header = self.block.signed_block_header();
         let (parent, block) = if let Some(parent) = self.parent {
             (parent, self.block)
         } else {
-            load_parent(self.block, chain)?
+            load_parent(self.block, chain)
+                .map_err(|e| BlockSlashInfo::SignatureValid(header.clone(), e))?
         };
 
         FullyVerifiedBlock::from_signature_verified_components(
@@ -600,6 +692,7 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignatureVerifiedBlock<T
             parent,
             chain,
         )
+        .map_err(|e| BlockSlashInfo::SignatureValid(header, e))
     }
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec> {
@@ -610,11 +703,12 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignatureVerifiedBlock<T
 impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignedBeaconBlock<T::EthSpec> {
     /// Verifies the `SignedBeaconBlock` by first transforming it into a `SignatureVerifiedBlock`
     /// and then using that implementation of `IntoFullyVerifiedBlock` to complete verification.
-    fn into_fully_verified_block(
+    fn into_fully_verified_block_slashable(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
-        SignatureVerifiedBlock::new(self, chain)?.into_fully_verified_block(chain)
+    ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        SignatureVerifiedBlock::check_slashable(self, chain)?
+            .into_fully_verified_block_slashable(chain)
     }
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec> {
@@ -669,9 +763,9 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
 
         let catchup_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CATCHUP_STATE);
 
-        // Keep a batch of any states that were "skipped" (block-less) in between the parent state
-        // slot and the block slot. These will be stored in the database.
-        let mut intermediate_states: Vec<StoreOp<T::EthSpec>> = Vec::new();
+        // Stage a batch of operations to be completed atomically if this block is imported
+        // successfully.
+        let mut confirmation_db_batch = vec![];
 
         // The block must have a higher slot than its parent.
         if block.slot() <= parent.beacon_state.slot {
@@ -695,18 +789,36 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
                 // processing, but we get early access to it.
                 let state_root = state.update_tree_hash_cache()?;
 
-                let op = if state.slot % T::EthSpec::slots_per_epoch() == 0 {
-                    StoreOp::PutState(
-                        state_root.into(),
-                        Cow::Owned(state.clone_with(CloneConfig::committee_caches_only())),
-                    )
+                // Store the state immediately, marking it as temporary, and staging the deletion
+                // of its temporary status as part of the larger atomic operation.
+                let txn_lock = chain.store.hot_db.begin_rw_transaction();
+                let state_already_exists =
+                    chain.store.load_hot_state_summary(&state_root)?.is_some();
+
+                let state_batch = if state_already_exists {
+                    // If the state exists, it could be temporary or permanent, but in neither case
+                    // should we rewrite it or store a new temporary flag for it. We *will* stage
+                    // the temporary flag for deletion because it's OK to double-delete the flag,
+                    // and we don't mind if another thread gets there first.
+                    vec![]
                 } else {
-                    StoreOp::PutStateSummary(
-                        state_root.into(),
-                        HotStateSummary::new(&state_root, &state)?,
-                    )
+                    vec![
+                        if state.slot % T::EthSpec::slots_per_epoch() == 0 {
+                            StoreOp::PutState(state_root, &state)
+                        } else {
+                            StoreOp::PutStateSummary(
+                                state_root,
+                                HotStateSummary::new(&state_root, &state)?,
+                            )
+                        },
+                        StoreOp::PutStateTemporaryFlag(state_root),
+                    ]
                 };
-                intermediate_states.push(op);
+                chain.store.do_atomically(state_batch)?;
+                drop(txn_lock);
+
+                confirmation_db_batch.push(StoreOp::DeleteStateTemporaryFlag(state_root));
+
                 state_root
             };
 
@@ -794,7 +906,7 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
             block_root,
             state,
             parent_block: parent.beacon_block,
-            intermediate_states,
+            confirmation_db_batch,
         })
     }
 }
@@ -980,14 +1092,17 @@ fn load_parent<T: BeaconChainTypes>(
         // exist in fork choice but not in the database yet. In such a case we simply
         // indicate that we don't yet know the parent.
         let root = block.parent_root();
-        let parent_block = if let Some(block) = chain
+        let parent_block = chain
             .get_block(&block.parent_root())
             .map_err(BlockError::BeaconChainError)?
-        {
-            block
-        } else {
-            return Err(BlockError::ParentUnknown(Box::new(block)));
-        };
+            .ok_or_else(|| {
+                // Return a `MissingBeaconBlock` error instead of a `ParentUnknown` error since
+                // we've already checked fork choice for this block.
+                //
+                // It's an internal error if the block exists in fork choice but not in the
+                // database.
+                BlockError::from(BeaconChainError::MissingBeaconBlock(block.parent_root()))
+            })?;
 
         // Load the parent blocks state from the database, returning an error if it is not found.
         // It is an error because if we know the parent block we should also know the parent state.
@@ -1095,6 +1210,38 @@ fn get_signature_verifier<'a, E: EthSpec>(
     )
 }
 
+/// Verify that `header` was signed with a valid signature from its proposer.
+///
+/// Return `Ok(())` if the signature is valid, and an `Err` otherwise.
+fn verify_header_signature<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    header: &SignedBeaconBlockHeader,
+) -> Result<(), BlockError<T::EthSpec>> {
+    let proposer_pubkey = get_validator_pubkey_cache(chain)?
+        .get(header.message.proposer_index as usize)
+        .cloned()
+        .ok_or_else(|| BlockError::UnknownValidator(header.message.proposer_index))?;
+    let (fork, genesis_validators_root) = chain
+        .with_head(|head| {
+            Ok((
+                head.beacon_state.fork,
+                head.beacon_state.genesis_validators_root,
+            ))
+        })
+        .map_err(|e: BlockError<T::EthSpec>| e)?;
+
+    if header.verify_signature::<T::EthSpec>(
+        &proposer_pubkey,
+        &fork,
+        genesis_validators_root,
+        &chain.spec,
+    ) {
+        Ok(())
+    } else {
+        Err(BlockError::ProposalSignatureInvalid)
+    }
+}
+
 fn expose_participation_metrics(summaries: &[EpochProcessingSummary]) {
     if !cfg!(feature = "participation_metrics") {
         return;

beacon_node/beacon_chain/src/builder.rs

@@ -4,7 +4,7 @@ use crate::beacon_chain::{
 use crate::eth1_chain::{CachingEth1Backend, SszEth1};
 use crate::events::NullEventHandler;
 use crate::head_tracker::HeadTracker;
-use crate::migrate::Migrate;
+use crate::migrate::{BackgroundMigrator, MigratorConfig};
 use crate::persisted_beacon_chain::PersistedBeaconChain;
 use crate::persisted_fork_choice::PersistedForkChoice;
 use crate::shuffling_cache::ShufflingCache;
@@ -18,9 +18,11 @@ use crate::{
 };
 use eth1::Config as Eth1Config;
 use fork_choice::ForkChoice;
+use futures::channel::mpsc::Sender;
 use operation_pool::{OperationPool, PersistedOperationPool};
 use parking_lot::RwLock;
-use slog::{info, Logger};
+use slasher::Slasher;
+use slog::{crit, info, Logger};
 use slot_clock::{SlotClock, TestingSlotClock};
 use std::marker::PhantomData;
 use std::path::PathBuf;
@@ -36,17 +38,8 @@ pub const PUBKEY_CACHE_FILENAME: &str = "pubkey_cache.ssz";
 
 /// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing
 /// functionality and only exists to satisfy the type system.
-pub struct Witness<
-    TStoreMigrator,
-    TSlotClock,
-    TEth1Backend,
-    TEthSpec,
-    TEventHandler,
-    THotStore,
-    TColdStore,
->(
+pub struct Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>(
     PhantomData<(
-        TStoreMigrator,
         TSlotClock,
         TEth1Backend,
         TEthSpec,
@@ -56,21 +49,11 @@ pub struct Witness<
     )>,
 );
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    BeaconChainTypes
-    for Witness<
-        TStoreMigrator,
-        TSlotClock,
-        TEth1Backend,
-        TEthSpec,
-        TEventHandler,
-        THotStore,
-        TColdStore,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore> BeaconChainTypes
+    for Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -78,7 +61,6 @@ where
 {
     type HotStore = THotStore;
     type ColdStore = TColdStore;
-    type StoreMigrator = TStoreMigrator;
     type SlotClock = TSlotClock;
     type Eth1Chain = TEth1Backend;
     type EthSpec = TEthSpec;
@@ -96,16 +78,19 @@ where
 pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     #[allow(clippy::type_complexity)]
     store: Option<Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>>,
-    store_migrator: Option<T::StoreMigrator>,
-    canonical_head: Option<BeaconSnapshot<T::EthSpec>>,
-    /// The finalized checkpoint to anchor the chain. May be genesis or a higher
-    /// checkpoint.
-    pub finalized_snapshot: Option<BeaconSnapshot<T::EthSpec>>,
+    store_migrator_config: Option<MigratorConfig>,
+    pub genesis_time: Option<u64>,
     genesis_block_root: Option<Hash256>,
+    genesis_state_root: Option<Hash256>,
+    #[allow(clippy::type_complexity)]
+    fork_choice: Option<
+        ForkChoice<BeaconForkChoiceStore<T::EthSpec, T::HotStore, T::ColdStore>, T::EthSpec>,
+    >,
     op_pool: Option<OperationPool<T::EthSpec>>,
     eth1_chain: Option<Eth1Chain<T::Eth1Chain, T::EthSpec>>,
     event_handler: Option<T::EventHandler>,
     slot_clock: Option<T::SlotClock>,
+    shutdown_sender: Option<Sender<&'static str>>,
     head_tracker: Option<HeadTracker>,
     data_dir: Option<PathBuf>,
     pubkey_cache_path: Option<PathBuf>,
@@ -115,24 +100,16 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     disabled_forks: Vec<String>,
     log: Option<Logger>,
     graffiti: Graffiti,
+    slasher: Option<Arc<Slasher<T::EthSpec>>>,
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
     BeaconChainBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+        Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>,
     >
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -145,14 +122,16 @@ where
     pub fn new(_eth_spec_instance: TEthSpec) -> Self {
         Self {
             store: None,
-            store_migrator: None,
-            canonical_head: None,
-            finalized_snapshot: None,
+            store_migrator_config: None,
+            genesis_time: None,
             genesis_block_root: None,
+            genesis_state_root: None,
+            fork_choice: None,
             op_pool: None,
             eth1_chain: None,
             event_handler: None,
             slot_clock: None,
+            shutdown_sender: None,
             head_tracker: None,
             pubkey_cache_path: None,
             data_dir: None,
@@ -162,6 +141,7 @@ where
             chain_config: ChainConfig::default(),
             log: None,
             graffiti: Graffiti::default(),
+            slasher: None,
         }
     }
 
@@ -191,9 +171,15 @@ where
         self
     }
 
-    /// Sets the store migrator.
-    pub fn store_migrator(mut self, store_migrator: TStoreMigrator) -> Self {
-        self.store_migrator = Some(store_migrator);
+    /// Sets the store migrator config (optional).
+    pub fn store_migrator_config(mut self, config: MigratorConfig) -> Self {
+        self.store_migrator_config = Some(config);
+        self
+    }
+
+    /// Sets the slasher.
+    pub fn slasher(mut self, slasher: Arc<Slasher<TEthSpec>>) -> Self {
+        self.slasher = Some(slasher);
         self
     }
 
@@ -228,7 +214,7 @@ where
             .ok_or_else(|| "get_persisted_eth1_backend requires a store.".to_string())?;
 
         store
-            .get_item::<SszEth1>(&Hash256::from_slice(&ETH1_CACHE_DB_KEY))
+            .get_item::<SszEth1>(&ETH1_CACHE_DB_KEY)
             .map_err(|e| format!("DB error whilst reading eth1 cache: {:?}", e))
     }
 
@@ -240,7 +226,7 @@ where
             .ok_or_else(|| "store_contains_beacon_chain requires a store.".to_string())?;
 
         Ok(store
-            .get_item::<PersistedBeaconChain>(&Hash256::from_slice(&BEACON_CHAIN_DB_KEY))
+            .get_item::<PersistedBeaconChain>(&BEACON_CHAIN_DB_KEY)
             .map_err(|e| format!("DB error when reading persisted beacon chain: {:?}", e))?
             .is_some())
     }
@@ -271,67 +257,58 @@ where
             .ok_or_else(|| "resume_from_db requires a store.".to_string())?;
 
         let chain = store
-            .get_item::<PersistedBeaconChain>(&Hash256::from_slice(&BEACON_CHAIN_DB_KEY))
+            .get_item::<PersistedBeaconChain>(&BEACON_CHAIN_DB_KEY)
             .map_err(|e| format!("DB error when reading persisted beacon chain: {:?}", e))?
             .ok_or_else(|| {
                 "No persisted beacon chain found in store. Try purging the beacon chain database."
                     .to_string()
             })?;
 
-        self.genesis_block_root = Some(chain.genesis_block_root);
-        self.head_tracker = Some(
-            HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
-                .map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
-        );
+        let persisted_fork_choice = store
+            .get_item::<PersistedForkChoice>(&FORK_CHOICE_DB_KEY)
+            .map_err(|e| format!("DB error when reading persisted fork choice: {:?}", e))?
+            .ok_or_else(|| "No persisted fork choice present in database.".to_string())?;
 
-        let head_block_root = chain.canonical_head_block_root;
-        let head_block = store
-            .get_item::<SignedBeaconBlock<TEthSpec>>(&head_block_root)
-            .map_err(|e| format!("DB error when reading head block: {:?}", e))?
-            .ok_or_else(|| "Head block not found in store".to_string())?;
-        let head_state_root = head_block.state_root();
-        let head_state = store
-            .get_state(&head_state_root, Some(head_block.slot()))
-            .map_err(|e| format!("DB error when reading head state: {:?}", e))?
-            .ok_or_else(|| "Head state not found in store".to_string())?;
+        let fc_store = BeaconForkChoiceStore::from_persisted(
+            persisted_fork_choice.fork_choice_store,
+            store.clone(),
+        )
+        .map_err(|e| format!("Unable to load ForkChoiceStore: {:?}", e))?;
+
+        let fork_choice =
+            ForkChoice::from_persisted(persisted_fork_choice.fork_choice, fc_store)
+                .map_err(|e| format!("Unable to parse persisted fork choice from disk: {:?}", e))?;
+
+        let genesis_block = store
+            .get_item::<SignedBeaconBlock<TEthSpec>>(&chain.genesis_block_root)
+            .map_err(|e| format!("DB error when reading genesis block: {:?}", e))?
+            .ok_or_else(|| "Genesis block not found in store".to_string())?;
+        let genesis_state = store
+            .get_state(&genesis_block.state_root(), Some(genesis_block.slot()))
+            .map_err(|e| format!("DB error when reading genesis state: {:?}", e))?
+            .ok_or_else(|| "Genesis block not found in store".to_string())?;
+
+        self.genesis_time = Some(genesis_state.genesis_time);
 
         self.op_pool = Some(
             store
-                .get_item::<PersistedOperationPool<TEthSpec>>(&Hash256::from_slice(&OP_POOL_DB_KEY))
+                .get_item::<PersistedOperationPool<TEthSpec>>(&OP_POOL_DB_KEY)
                 .map_err(|e| format!("DB error whilst reading persisted op pool: {:?}", e))?
                 .map(PersistedOperationPool::into_operation_pool)
                 .unwrap_or_else(OperationPool::new),
         );
 
-        let finalized_block_root = head_state.finalized_checkpoint.root;
-        let finalized_block = store
-            .get_item::<SignedBeaconBlock<TEthSpec>>(&finalized_block_root)
-            .map_err(|e| format!("DB error when reading finalized block: {:?}", e))?
-            .ok_or_else(|| "Finalized block not found in store".to_string())?;
-        let finalized_state_root = finalized_block.state_root();
-        let finalized_state = store
-            .get_state(&finalized_state_root, Some(finalized_block.slot()))
-            .map_err(|e| format!("DB error when reading finalized state: {:?}", e))?
-            .ok_or_else(|| "Finalized state not found in store".to_string())?;
-
-        self.finalized_snapshot = Some(BeaconSnapshot {
-            beacon_block_root: finalized_block_root,
-            beacon_block: finalized_block,
-            beacon_state_root: finalized_state_root,
-            beacon_state: finalized_state,
-        });
-
-        self.canonical_head = Some(BeaconSnapshot {
-            beacon_block_root: head_block_root,
-            beacon_block: head_block,
-            beacon_state_root: head_state_root,
-            beacon_state: head_state,
-        });
-
         let pubkey_cache = ValidatorPubkeyCache::load_from_file(pubkey_cache_path)
             .map_err(|e| format!("Unable to open persisted pubkey cache: {:?}", e))?;
 
+        self.genesis_block_root = Some(chain.genesis_block_root);
+        self.genesis_state_root = Some(genesis_block.state_root());
+        self.head_tracker = Some(
+            HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
+                .map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
+        );
         self.validator_pubkey_cache = Some(pubkey_cache);
+        self.fork_choice = Some(fork_choice);
 
         Ok(self)
     }
@@ -355,6 +332,7 @@ where
         let beacon_state_root = beacon_block.message.state_root;
         let beacon_block_root = beacon_block.canonical_root();
 
+        self.genesis_state_root = Some(beacon_state_root);
         self.genesis_block_root = Some(beacon_block_root);
 
         store
@@ -374,12 +352,25 @@ where
                 )
             })?;
 
-        self.finalized_snapshot = Some(BeaconSnapshot {
+        let genesis = BeaconSnapshot {
             beacon_block_root,
             beacon_block,
             beacon_state_root,
             beacon_state,
-        });
+        };
+
+        let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis);
+
+        let fork_choice = ForkChoice::from_genesis(
+            fc_store,
+            genesis.beacon_block_root,
+            &genesis.beacon_block.message,
+            &genesis.beacon_state,
+        )
+        .map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?;
+
+        self.fork_choice = Some(fork_choice);
+        self.genesis_time = Some(genesis.beacon_state.genesis_time);
 
         Ok(self.empty_op_pool())
     }
@@ -406,6 +397,12 @@ where
         self
     }
 
+    /// Sets a `Sender` to allow the beacon chain to send shutdown signals.
+    pub fn shutdown_sender(mut self, sender: Sender<&'static str>) -> Self {
+        self.shutdown_sender = Some(sender);
+        self
+    }
+
     /// Creates a new, empty operation pool.
     fn empty_op_pool(mut self) -> Self {
         self.op_pool = Some(OperationPool::new());
@@ -435,15 +432,7 @@ where
         self,
     ) -> Result<
         BeaconChain<
-            Witness<
-                TStoreMigrator,
-                TSlotClock,
-                TEth1Backend,
-                TEthSpec,
-                TEventHandler,
-                THotStore,
-                TColdStore,
-            >,
+            Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>,
         >,
         String,
     > {
@@ -457,23 +446,76 @@ where
             .store
             .clone()
             .ok_or_else(|| "Cannot build without a store.".to_string())?;
+        let mut fork_choice = self
+            .fork_choice
+            .ok_or_else(|| "Cannot build without fork choice.".to_string())?;
+        let genesis_block_root = self
+            .genesis_block_root
+            .ok_or_else(|| "Cannot build without a genesis block root".to_string())?;
+        let genesis_state_root = self
+            .genesis_state_root
+            .ok_or_else(|| "Cannot build without a genesis state root".to_string())?;
 
-        // If this beacon chain is being loaded from disk, use the stored head. Otherwise, just use
-        // the finalized checkpoint (which is probably genesis).
-        let mut canonical_head = if let Some(head) = self.canonical_head {
-            head
+        let current_slot = if slot_clock
+            .is_prior_to_genesis()
+            .ok_or_else(|| "Unable to read slot clock".to_string())?
+        {
+            self.spec.genesis_slot
         } else {
-            self.finalized_snapshot
-                .ok_or_else(|| "Cannot build without a state".to_string())?
+            slot_clock
+                .now()
+                .ok_or_else(|| "Unable to read slot".to_string())?
         };
 
+        let head_block_root = fork_choice
+            .get_head(current_slot)
+            .map_err(|e| format!("Unable to get fork choice head: {:?}", e))?;
+
+        let head_block = store
+            .get_item::<SignedBeaconBlock<TEthSpec>>(&head_block_root)
+            .map_err(|e| format!("DB error when reading head block: {:?}", e))?
+            .ok_or_else(|| "Head block not found in store".to_string())?;
+        let head_state_root = head_block.state_root();
+        let head_state = store
+            .get_state(&head_state_root, Some(head_block.slot()))
+            .map_err(|e| format!("DB error when reading head state: {:?}", e))?
+            .ok_or_else(|| "Head state not found in store".to_string())?;
+
+        let mut canonical_head = BeaconSnapshot {
+            beacon_block_root: head_block_root,
+            beacon_block: head_block,
+            beacon_state_root: head_state_root,
+            beacon_state: head_state,
+        };
+
+        if canonical_head.beacon_block.state_root() != canonical_head.beacon_state_root {
+            return Err("beacon_block.state_root != beacon_state".to_string());
+        }
+
         canonical_head
             .beacon_state
             .build_all_caches(&self.spec)
             .map_err(|e| format!("Failed to build state caches: {:?}", e))?;
 
-        if canonical_head.beacon_block.state_root() != canonical_head.beacon_state_root {
-            return Err("beacon_block.state_root != beacon_state".to_string());
+        // Perform a check to ensure that the finalization points of the head and fork choice are
+        // consistent.
+        //
+        // This is a sanity check to detect database corruption.
+        let fc_finalized = fork_choice.finalized_checkpoint();
+        let head_finalized = canonical_head.beacon_state.finalized_checkpoint;
+        if fc_finalized != head_finalized {
+            if head_finalized.root == Hash256::zero()
+                && head_finalized.epoch == fc_finalized.epoch
+                && fc_finalized.root == genesis_block_root
+            {
+                // This is a legal edge-case encountered during genesis.
+            } else {
+                return Err(format!(
+                    "Database corrupt: fork choice is finalized at {:?} whilst head is finalized at \
+                    {:?}",
+                    fc_finalized, head_finalized
+                ));
+            }
         }
 
         let pubkey_cache_path = self
@@ -485,33 +527,19 @@ where
                 .map_err(|e| format!("Unable to init validator pubkey cache: {:?}", e))
         })?;
 
-        let persisted_fork_choice = store
-            .get_item::<PersistedForkChoice>(&Hash256::from_slice(&FORK_CHOICE_DB_KEY))
-            .map_err(|e| format!("DB error when reading persisted fork choice: {:?}", e))?;
-
-        let fork_choice = if let Some(persisted) = persisted_fork_choice {
-            let fc_store =
-                BeaconForkChoiceStore::from_persisted(persisted.fork_choice_store, store.clone())
-                    .map_err(|e| format!("Unable to load ForkChoiceStore: {:?}", e))?;
-
-            ForkChoice::from_persisted(persisted.fork_choice, fc_store)
-                .map_err(|e| format!("Unable to parse persisted fork choice from disk: {:?}", e))?
-        } else {
-            let genesis = &canonical_head;
-
-            let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), genesis);
-
-            ForkChoice::from_genesis(fc_store, &genesis.beacon_block.message)
-                .map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?
-        };
+        let migrator_config = self.store_migrator_config.unwrap_or_default();
+        let store_migrator = BackgroundMigrator::new(
+            store.clone(),
+            migrator_config,
+            genesis_block_root,
+            log.clone(),
+        );
 
         let beacon_chain = BeaconChain {
             spec: self.spec,
             config: self.chain_config,
             store,
-            store_migrator: self
-                .store_migrator
-                .ok_or_else(|| "Cannot build without store migrator".to_string())?,
+            store_migrator,
             slot_clock,
             op_pool: self
                 .op_pool
@@ -533,9 +561,8 @@ where
             eth1_chain: self.eth1_chain,
             genesis_validators_root: canonical_head.beacon_state.genesis_validators_root,
             canonical_head: TimeoutRwLock::new(canonical_head.clone()),
-            genesis_block_root: self
-                .genesis_block_root
-                .ok_or_else(|| "Cannot build without a genesis block root".to_string())?,
+            genesis_block_root,
+            genesis_state_root,
             fork_choice: RwLock::new(fork_choice),
             event_handler: self
                 .event_handler
@@ -548,14 +575,39 @@ where
             shuffling_cache: TimeoutRwLock::new(ShufflingCache::new()),
             validator_pubkey_cache: TimeoutRwLock::new(validator_pubkey_cache),
             disabled_forks: self.disabled_forks,
+            shutdown_sender: self
+                .shutdown_sender
+                .ok_or_else(|| "Cannot build without a shutdown sender.".to_string())?,
             log: log.clone(),
             graffiti: self.graffiti,
+            slasher: self.slasher.clone(),
         };
 
         let head = beacon_chain
             .head()
             .map_err(|e| format!("Failed to get head: {:?}", e))?;
 
+        // Only perform the check if it was configured.
+        if let Some(wss_checkpoint) = beacon_chain.config.weak_subjectivity_checkpoint {
+            if let Err(e) = beacon_chain.verify_weak_subjectivity_checkpoint(
+                wss_checkpoint,
+                head.beacon_block_root,
+                &head.beacon_state,
+            ) {
+                crit!(
+                    log,
+                    "Weak subjectivity checkpoint verification failed on startup!";
+                    "head_block_root" => format!("{}", head.beacon_block_root),
+                    "head_slot" => format!("{}", head.beacon_block.slot()),
+                    "finalized_epoch" => format!("{}", head.beacon_state.finalized_checkpoint.epoch),
+                    "wss_checkpoint_epoch" => format!("{}", wss_checkpoint.epoch),
+                    "error" => format!("{:?}", e),
+                );
+                crit!(log, "You must use the `--purge-db` flag to clear the database and restart sync. You may be on a hostile network.");
+                return Err(format!("Weak subjectivity verification failed: {:?}", e));
+            }
+        }
+
         info!(
             log,
             "Beacon chain initialized";
@@ -568,10 +620,9 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
     BeaconChainBuilder<
         Witness<
-            TStoreMigrator,
             TSlotClock,
             CachingEth1Backend<TEthSpec>,
             TEthSpec,
@@ -583,7 +634,6 @@ impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -609,22 +659,13 @@ where
     }
 }
 
-impl<TStoreMigrator, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
     BeaconChainBuilder<
-        Witness<
-            TStoreMigrator,
-            TestingSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+        Witness<TestingSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>,
     >
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -634,11 +675,8 @@ where
     /// Requires the state to be initialized.
     pub fn testing_slot_clock(self, slot_duration: Duration) -> Result<Self, String> {
         let genesis_time = self
-            .finalized_snapshot
-            .as_ref()
-            .ok_or_else(|| "testing_slot_clock requires an initialized state")?
-            .beacon_state
-            .genesis_time;
+            .genesis_time
+            .ok_or_else(|| "testing_slot_clock requires an initialized state")?;
 
         let slot_clock = TestingSlotClock::new(
             Slot::new(0),
@@ -650,10 +688,9 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
+impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
     BeaconChainBuilder<
         Witness<
-            TStoreMigrator,
             TSlotClock,
             TEth1Backend,
             TEthSpec,
@@ -665,7 +702,6 @@ impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -697,7 +733,6 @@ fn genesis_block<T: EthSpec>(
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::migrate::NullMigrator;
     use eth2_hashing::hash;
     use genesis::{generate_deterministic_keypairs, interop_genesis_state};
     use sloggers::{null::NullLoggerBuilder, Build};
@@ -737,10 +772,11 @@ mod test {
         )
         .expect("should create interop genesis state");
 
+        let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+
         let chain = BeaconChainBuilder::new(MinimalEthSpec)
             .logger(log.clone())
             .store(Arc::new(store))
-            .store_migrator(NullMigrator)
             .data_dir(data_dir.path().to_path_buf())
             .genesis_state(genesis_state)
             .expect("should build state using recent genesis")
@@ -749,6 +785,7 @@ mod test {
             .null_event_handler()
             .testing_slot_clock(Duration::from_secs(1))
             .expect("should configure testing slot clock")
+            .shutdown_sender(shutdown_tx)
             .build()
             .expect("should build");
 

beacon_node/beacon_chain/src/chain_config.rs

@@ -1,7 +1,5 @@
 use serde_derive::{Deserialize, Serialize};
-
-/// There is a 693 block skip in the current canonical Medalla chain, we use 700 to be safe.
-pub const DEFAULT_IMPORT_BLOCK_MAX_SKIP_SLOTS: u64 = 700;
+use types::Checkpoint;
 
 #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
 pub struct ChainConfig {
@@ -10,12 +8,17 @@ pub struct ChainConfig {
     ///
     /// If `None`, there is no limit.
     pub import_max_skip_slots: Option<u64>,
+    /// A user-input `Checkpoint` that must exist in the beacon chain's sync path.
+    ///
+    /// If `None`, there is no weak subjectivity verification.
+    pub weak_subjectivity_checkpoint: Option<Checkpoint>,
 }
 
 impl Default for ChainConfig {
     fn default() -> Self {
         Self {
-            import_max_skip_slots: Some(DEFAULT_IMPORT_BLOCK_MAX_SKIP_SLOTS),
+            import_max_skip_slots: None,
+            weak_subjectivity_checkpoint: None,
         }
     }
 }

beacon_node/beacon_chain/src/errors.rs

@@ -1,9 +1,11 @@
 use crate::beacon_chain::ForkChoiceError;
 use crate::eth1_chain::Error as Eth1ChainError;
+use crate::migrate::PruningError;
 use crate::naive_aggregation_pool::Error as NaiveAggregationError;
 use crate::observed_attestations::Error as ObservedAttestationsError;
 use crate::observed_attesters::Error as ObservedAttestersError;
 use crate::observed_block_producers::Error as ObservedBlockProducersError;
+use futures::channel::mpsc::TrySendError;
 use operation_pool::OpPoolError;
 use safe_arith::ArithError;
 use ssz_types::Error as SszTypesError;
@@ -61,6 +63,7 @@ pub enum BeaconChainError {
         requested_slot: Slot,
         max_task_runtime: Duration,
     },
+    MissingFinalizedStateRoot(Slot),
     /// Returned when an internal check fails, indicating corrupt data.
     InvariantViolated(String),
     SszTypesError(SszTypesError),
@@ -79,7 +82,14 @@ pub enum BeaconChainError {
     ObservedAttestationsError(ObservedAttestationsError),
     ObservedAttestersError(ObservedAttestersError),
     ObservedBlockProducersError(ObservedBlockProducersError),
+    PruningError(PruningError),
     ArithError(ArithError),
+    InvalidShufflingId {
+        shuffling_epoch: Epoch,
+        head_block_epoch: Epoch,
+    },
+    WeakSubjectivtyVerificationFailure,
+    WeakSubjectivtyShutdownError(TrySendError<&'static str>),
 }
 
 easy_from_to!(SlotProcessingError, BeaconChainError);
@@ -94,6 +104,7 @@ easy_from_to!(ObservedAttestationsError, BeaconChainError);
 easy_from_to!(ObservedAttestersError, BeaconChainError);
 easy_from_to!(ObservedBlockProducersError, BeaconChainError);
 easy_from_to!(BlockSignatureVerifierError, BeaconChainError);
+easy_from_to!(PruningError, BeaconChainError);
 easy_from_to!(ArithError, BeaconChainError);
 
 #[derive(Debug)]

beacon_node/beacon_chain/src/eth1_chain.rs

@@ -1,6 +1,6 @@
 use crate::metrics;
-use environment::TaskExecutor;
 use eth1::{Config as Eth1Config, Eth1Block, Service as HttpService};
+use eth2::lighthouse::Eth1SyncStatusData;
 use eth2_hashing::hash;
 use slog::{debug, error, trace, Logger};
 use ssz::{Decode, Encode};
@@ -10,7 +10,9 @@ use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::iter::DoubleEndedIterator;
 use std::marker::PhantomData;
+use std::time::{SystemTime, UNIX_EPOCH};
 use store::{DBColumn, Error as StoreError, StoreItem};
+use task_executor::TaskExecutor;
 use types::{
     BeaconState, BeaconStateError, ChainSpec, Deposit, Eth1Data, EthSpec, Hash256, Slot, Unsigned,
     DEPOSIT_TREE_DEPTH,
@@ -19,6 +21,11 @@ use types::{
 type BlockNumber = u64;
 type Eth1DataVoteCount = HashMap<(Eth1Data, BlockNumber), u64>;
 
+/// We will declare ourself synced with the Eth1 chain, even if we are this many blocks behind.
+///
+/// This number (8) was chosen somewhat arbitrarily.
+const ETH1_SYNC_TOLERANCE: u64 = 8;
+
 #[derive(Debug)]
 pub enum Error {
     /// Unable to return an Eth1Data for the given epoch.
@@ -53,6 +60,84 @@ impl From<safe_arith::ArithError> for Error {
     }
 }
 
+/// Returns an `Eth1SyncStatusData` given some parameters:
+///
+/// - `latest_cached_block`: The latest eth1 block in our cache, if any.
+/// - `head_block`: The block at the very head of our eth1 node (ignoring follow distance, etc).
+/// - `genesis_time`: beacon chain genesis time.
+/// - `current_slot`: current beacon chain slot.
+/// - `spec`: current beacon chain specification.
+fn get_sync_status<T: EthSpec>(
+    latest_cached_block: Option<&Eth1Block>,
+    head_block: Option<&Eth1Block>,
+    genesis_time: u64,
+    current_slot: Slot,
+    spec: &ChainSpec,
+) -> Option<Eth1SyncStatusData> {
+    let period = T::SlotsPerEth1VotingPeriod::to_u64();
+    // Since `period` is a "constant", we assume it is set sensibly.
+    let voting_period_start_slot = (current_slot / period) * period;
+    let voting_target_timestamp = {
+        let period_start = slot_start_seconds::<T>(
+            genesis_time,
+            spec.milliseconds_per_slot,
+            voting_period_start_slot,
+        );
+        let eth1_follow_distance_seconds = spec
+            .seconds_per_eth1_block
+            .saturating_mul(spec.eth1_follow_distance);
+
+        period_start.saturating_sub(eth1_follow_distance_seconds)
+    };
+
+    let latest_cached_block_number = latest_cached_block.map(|b| b.number);
+    let latest_cached_block_timestamp = latest_cached_block.map(|b| b.timestamp);
+    let head_block_number = head_block.map(|b| b.number);
+    let head_block_timestamp = head_block.map(|b| b.timestamp);
+
+    let eth1_node_sync_status_percentage = if let Some(head_block) = head_block {
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs();
+        let head_age = now.saturating_sub(head_block.timestamp);
+
+        if head_age < ETH1_SYNC_TOLERANCE * spec.seconds_per_eth1_block {
+            // Always indicate we are fully synced if it's within the sync threshold.
+            100.0
+        } else {
+            let blocks_behind = head_age
+                .checked_div(spec.seconds_per_eth1_block)
+                .unwrap_or(0);
+
+            let part = f64::from(head_block.number as u32);
+            let whole = f64::from(head_block.number.saturating_add(blocks_behind) as u32);
+
+            if whole > 0.0 {
+                (part / whole) * 100.0
+            } else {
+                // Avoids a divide-by-zero.
+                0.0
+            }
+        }
+    } else {
+        // Always return 0% synced if the head block of the eth1 chain is unknown.
+        0.0
+    };
+
+    // Lighthouse is "cached and ready" when it has cached enough blocks to cover the start of the
+    // current voting period.
+    let lighthouse_is_cached_and_ready =
+        latest_cached_block_timestamp.map_or(false, |t| t >= voting_target_timestamp);
+
+    Some(Eth1SyncStatusData {
+        head_block_number,
+        head_block_timestamp,
+        latest_cached_block_number,
+        latest_cached_block_timestamp,
+        voting_target_timestamp,
+        eth1_node_sync_status_percentage,
+        lighthouse_is_cached_and_ready,
+    })
+}
+
 #[derive(Encode, Decode, Clone)]
 pub struct SszEth1 {
     use_dummy_backend: bool,
@@ -143,6 +228,22 @@ where
         }
     }
 
+    /// Returns a status indicating how synced our caches are with the eth1 chain.
+    pub fn sync_status(
+        &self,
+        genesis_time: u64,
+        current_slot: Slot,
+        spec: &ChainSpec,
+    ) -> Option<Eth1SyncStatusData> {
+        get_sync_status::<E>(
+            self.backend.latest_cached_block().as_ref(),
+            self.backend.head_block().as_ref(),
+            genesis_time,
+            current_slot,
+            spec,
+        )
+    }
+
     /// Instantiate `Eth1Chain` from a persisted `SszEth1`.
     ///
     /// The `Eth1Chain` will have the same caches as the persisted `SszEth1`.
@@ -195,6 +296,14 @@ pub trait Eth1ChainBackend<T: EthSpec>: Sized + Send + Sync {
         spec: &ChainSpec,
     ) -> Result<Vec<Deposit>, Error>;
 
+    /// Returns the latest block stored in the cache. Used to obtain an idea of how up-to-date the
+    /// beacon node eth1 cache is.
+    fn latest_cached_block(&self) -> Option<Eth1Block>;
+
+    /// Returns the block at the head of the chain (ignoring follow distance, etc). Used to obtain
+    /// an idea of how up-to-date the remote eth1 node is.
+    fn head_block(&self) -> Option<Eth1Block>;
+
     /// Encode the `Eth1ChainBackend` instance to bytes.
     fn as_bytes(&self) -> Vec<u8>;
 
@@ -241,6 +350,14 @@ impl<T: EthSpec> Eth1ChainBackend<T> for DummyEth1ChainBackend<T> {
         Ok(vec![])
     }
 
+    fn latest_cached_block(&self) -> Option<Eth1Block> {
+        None
+    }
+
+    fn head_block(&self) -> Option<Eth1Block> {
+        None
+    }
+
     /// Return empty Vec<u8> for dummy backend.
     fn as_bytes(&self) -> Vec<u8> {
         Vec::new()
@@ -333,13 +450,14 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
             // If no votes exist, choose `state.eth1_data` as default vote.
             votes_to_consider
                 .iter()
-                .max_by(|(_, x), (_, y)| x.cmp(y))
+                .max_by_key(|(_, block_number)| *block_number)
                 .map(|vote| {
                     let vote = vote.0.clone();
                     debug!(
                         self.log,
                         "No valid eth1_data votes";
                         "outcome" => "Casting vote corresponding to last candidate eth1 block",
+                        "vote" => ?vote
                     );
                     vote
                 })
@@ -400,6 +518,14 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
         }
     }
 
+    fn latest_cached_block(&self) -> Option<Eth1Block> {
+        self.core.latest_cached_block()
+    }
+
+    fn head_block(&self) -> Option<Eth1Block> {
+        self.core.head_block()
+    }
+
     /// Return encoded byte representation of the block and deposit caches.
     fn as_bytes(&self) -> Vec<u8> {
         self.core.as_bytes()

beacon_node/beacon_chain/src/head_tracker.rs

@@ -15,7 +15,7 @@ pub enum Error {
 /// In order for this struct to be effective, every single block that is imported must be
 /// registered here.
 #[derive(Default, Debug)]
-pub struct HeadTracker(RwLock<HashMap<Hash256, Slot>>);
+pub struct HeadTracker(pub RwLock<HashMap<Hash256, Slot>>);
 
 impl HeadTracker {
     /// Register a block with `Self`, so it may or may not be included in a `Self::heads` call.
@@ -29,13 +29,6 @@ impl HeadTracker {
         map.insert(block_root, slot);
     }
 
-    /// Removes abandoned head.
-    pub fn remove_head(&self, block_root: Hash256) {
-        let mut map = self.0.write();
-        debug_assert!(map.contains_key(&block_root));
-        map.remove(&block_root);
-    }
-
     /// Returns true iff `block_root` is a recognized head.
     pub fn contains_head(&self, block_root: Hash256) -> bool {
         self.0.read().contains_key(&block_root)
@@ -53,14 +46,7 @@ impl HeadTracker {
     /// Returns a `SszHeadTracker`, which contains all necessary information to restore the state
     /// of `Self` at some later point.
     pub fn to_ssz_container(&self) -> SszHeadTracker {
-        let (roots, slots) = self
-            .0
-            .read()
-            .iter()
-            .map(|(hash, slot)| (*hash, *slot))
-            .unzip();
-
-        SszHeadTracker { roots, slots }
+        SszHeadTracker::from_map(&*self.0.read())
     }
 
     /// Creates a new `Self` from the given `SszHeadTracker`, restoring `Self` to the same state of
@@ -103,6 +89,13 @@ pub struct SszHeadTracker {
     slots: Vec<Slot>,
 }
 
+impl SszHeadTracker {
+    pub fn from_map(map: &HashMap<Hash256, Slot>) -> Self {
+        let (roots, slots) = map.iter().map(|(hash, slot)| (*hash, *slot)).unzip();
+        SszHeadTracker { roots, slots }
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

beacon_node/beacon_chain/src/lib.rs

@@ -1,7 +1,4 @@
 #![recursion_limit = "128"] // For lazy-static
-#[macro_use]
-extern crate lazy_static;
-
 pub mod attestation_verification;
 mod beacon_chain;
 mod beacon_fork_choice_store;

beacon_node/beacon_chain/src/metrics.rs

@@ -1,4 +1,5 @@
 use crate::{BeaconChain, BeaconChainTypes};
+use lazy_static::lazy_static;
 pub use lighthouse_metrics::*;
 use slot_clock::SlotClock;
 use types::{BeaconState, Epoch, EthSpec, Hash256, Slot};
@@ -418,6 +419,7 @@ fn scrape_attestation_observation<T: BeaconChainTypes>(slot_now: Slot, chain: &B
 
     if let Some(count) = chain
         .observed_attesters
+        .read()
         .observed_validator_count(prev_epoch)
     {
         set_gauge_by_usize(&ATTN_OBSERVATION_PREV_EPOCH_ATTESTERS, count);
@@ -425,6 +427,7 @@ fn scrape_attestation_observation<T: BeaconChainTypes>(slot_now: Slot, chain: &B
 
     if let Some(count) = chain
         .observed_aggregators
+        .read()
         .observed_validator_count(prev_epoch)
     {
         set_gauge_by_usize(&ATTN_OBSERVATION_PREV_EPOCH_AGGREGATORS, count);

beacon_node/beacon_chain/src/migrate.rs

@@ -1,76 +1,322 @@
+use crate::beacon_chain::BEACON_CHAIN_DB_KEY;
 use crate::errors::BeaconChainError;
-use crate::head_tracker::HeadTracker;
+use crate::head_tracker::{HeadTracker, SszHeadTracker};
+use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_BLOCK_ROOT};
 use parking_lot::Mutex;
-use slog::{debug, warn, Logger};
+use slog::{debug, error, info, warn, Logger};
 use std::collections::{HashMap, HashSet};
 use std::mem;
-use std::sync::mpsc;
-use std::sync::Arc;
+use std::sync::{mpsc, Arc};
 use std::thread;
-use store::hot_cold_store::{process_finalization, HotColdDBError};
-use store::iter::{ParentRootBlockIterator, RootsIterator};
-use store::{Error, ItemStore, StoreOp};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use store::hot_cold_store::{migrate_database, HotColdDBError};
+use store::iter::RootsIterator;
+use store::{Error, ItemStore, StoreItem, StoreOp};
 pub use store::{HotColdDB, MemoryStore};
-use types::*;
-use types::{BeaconState, EthSpec, Hash256, Slot};
+use types::{
+    BeaconState, BeaconStateError, BeaconStateHash, Checkpoint, Epoch, EthSpec, Hash256,
+    SignedBeaconBlockHash, Slot,
+};
 
-/// Trait for migration processes that update the database upon finalization.
-pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
-    Send + Sync + 'static
-{
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self;
+/// Compact at least this frequently, finalization permitting (7 days).
+const MAX_COMPACTION_PERIOD_SECONDS: u64 = 604800;
+/// Compact at *most* this frequently, to prevent over-compaction during sync (2 hours).
+const MIN_COMPACTION_PERIOD_SECONDS: u64 = 7200;
+/// Compact after a large finality gap, if we respect `MIN_COMPACTION_PERIOD_SECONDS`.
+const COMPACTION_FINALITY_DISTANCE: u64 = 1024;
 
-    fn process_finalization(
+/// The background migrator runs a thread to perform pruning and migrate state from the hot
+/// to the cold database.
+pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
+    db: Arc<HotColdDB<E, Hot, Cold>>,
+    #[allow(clippy::type_complexity)]
+    tx_thread: Option<Mutex<(mpsc::Sender<MigrationNotification>, thread::JoinHandle<()>)>>,
+    /// Genesis block root, for persisting the `PersistedBeaconChain`.
+    genesis_block_root: Hash256,
+    log: Logger,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct MigratorConfig {
+    pub blocking: bool,
+}
+
+impl MigratorConfig {
+    pub fn blocking(mut self) -> Self {
+        self.blocking = true;
+        self
+    }
+}
+
+/// Pruning can be successful, or in rare cases deferred to a later point.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PruningOutcome {
+    /// The pruning succeeded and updated the pruning checkpoint from `old_finalized_checkpoint`.
+    Successful {
+        old_finalized_checkpoint: Checkpoint,
+    },
+    DeferredConcurrentMutation,
+}
+
+/// Logic errors that can occur during pruning, none of these should ever happen.
+#[derive(Debug)]
+pub enum PruningError {
+    IncorrectFinalizedState {
+        state_slot: Slot,
+        new_finalized_slot: Slot,
+    },
+    MissingInfoForCanonicalChain {
+        slot: Slot,
+    },
+    UnexpectedEqualStateRoots,
+    UnexpectedUnequalStateRoots,
+}
+
+/// Message sent to the migration thread containing the information it needs to run.
+pub struct MigrationNotification {
+    finalized_state_root: BeaconStateHash,
+    finalized_checkpoint: Checkpoint,
+    head_tracker: Arc<HeadTracker>,
+    genesis_block_root: Hash256,
+}
+
+impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
+    /// Create a new `BackgroundMigrator` and spawn its thread if necessary.
+    pub fn new(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        config: MigratorConfig,
+        genesis_block_root: Hash256,
+        log: Logger,
+    ) -> Self {
+        let tx_thread = if config.blocking {
+            None
+        } else {
+            Some(Mutex::new(Self::spawn_thread(db.clone(), log.clone())))
+        };
+        Self {
+            db,
+            tx_thread,
+            genesis_block_root,
+            log,
+        }
+    }
+
+    /// Process a finalized checkpoint from the `BeaconChain`.
+    ///
+    /// If successful, all forks descending from before the `finalized_checkpoint` will be
+    /// pruned, and the split point of the database will be advanced to the slot of the finalized
+    /// checkpoint.
+    pub fn process_finalization(
         &self,
-        _state_root: Hash256,
-        _new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
-        _head_tracker: Arc<HeadTracker>,
-        _old_finalized_block_hash: SignedBeaconBlockHash,
-        _new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
+        finalized_state_root: BeaconStateHash,
+        finalized_checkpoint: Checkpoint,
+        head_tracker: Arc<HeadTracker>,
+    ) -> Result<(), BeaconChainError> {
+        let notif = MigrationNotification {
+            finalized_state_root,
+            finalized_checkpoint,
+            head_tracker,
+            genesis_block_root: self.genesis_block_root,
+        };
+
+        // Async path, on the background thread.
+        if let Some(tx_thread) = &self.tx_thread {
+            let (ref mut tx, ref mut thread) = *tx_thread.lock();
+
+            // Restart the background thread if it has crashed.
+            if let Err(tx_err) = tx.send(notif) {
+                let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
+
+                *tx = new_tx;
+                let old_thread = mem::replace(thread, new_thread);
+
+                // Join the old thread, which will probably have panicked, or may have
+                // halted normally just now as a result of us dropping the old `mpsc::Sender`.
+                if let Err(thread_err) = old_thread.join() {
+                    warn!(
+                        self.log,
+                        "Migration thread died, so it was restarted";
+                        "reason" => format!("{:?}", thread_err)
+                    );
+                }
+
+                // Retry at most once, we could recurse but that would risk overflowing the stack.
+                let _ = tx.send(tx_err.0);
+            }
+        }
+        // Synchronous path, on the current thread.
+        else {
+            Self::run_migration(self.db.clone(), notif, &self.log)
+        }
+
+        Ok(())
+    }
+
+    /// Perform the actual work of `process_finalization`.
+    fn run_migration(db: Arc<HotColdDB<E, Hot, Cold>>, notif: MigrationNotification, log: &Logger) {
+        let finalized_state_root = notif.finalized_state_root;
+
+        let finalized_state = match db.get_state(&finalized_state_root.into(), None) {
+            Ok(Some(state)) => state,
+            other => {
+                error!(
+                    log,
+                    "Migrator failed to load state";
+                    "state_root" => ?finalized_state_root,
+                    "error" => ?other
+                );
+                return;
+            }
+        };
+
+        let old_finalized_checkpoint = match Self::prune_abandoned_forks(
+            db.clone(),
+            notif.head_tracker,
+            finalized_state_root,
+            &finalized_state,
+            notif.finalized_checkpoint,
+            notif.genesis_block_root,
+            log,
+        ) {
+            Ok(PruningOutcome::Successful {
+                old_finalized_checkpoint,
+            }) => old_finalized_checkpoint,
+            Ok(PruningOutcome::DeferredConcurrentMutation) => {
+                warn!(
+                    log,
+                    "Pruning deferred because of a concurrent mutation";
+                    "message" => "this is expected only very rarely!"
+                );
+                return;
+            }
+            Err(e) => {
+                warn!(log, "Block pruning failed"; "error" => format!("{:?}", e));
+                return;
+            }
+        };
+
+        match migrate_database(db.clone(), finalized_state_root.into(), &finalized_state) {
+            Ok(()) => {}
+            Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
+                debug!(
+                    log,
+                    "Database migration postponed, unaligned finalized block";
+                    "slot" => slot.as_u64()
+                );
+            }
+            Err(e) => {
+                warn!(
+                    log,
+                    "Database migration failed";
+                    "error" => format!("{:?}", e)
+                );
+                return;
+            }
+        };
+
+        // Finally, compact the database so that new free space is properly reclaimed.
+        if let Err(e) = Self::run_compaction(
+            db,
+            old_finalized_checkpoint.epoch,
+            notif.finalized_checkpoint.epoch,
+            log,
+        ) {
+            warn!(log, "Database compaction failed"; "error" => format!("{:?}", e));
+        }
+    }
+
+    /// Spawn a new child thread to run the migration process.
+    ///
+    /// Return a channel handle for sending new finalized states to the thread.
+    fn spawn_thread(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        log: Logger,
+    ) -> (mpsc::Sender<MigrationNotification>, thread::JoinHandle<()>) {
+        let (tx, rx) = mpsc::channel();
+        let thread = thread::spawn(move || {
+            while let Ok(notif) = rx.recv() {
+                // Read the rest of the messages in the channel, ultimately choosing the `notif`
+                // with the highest finalized epoch.
+                let notif = rx
+                    .try_iter()
+                    .fold(notif, |best, other: MigrationNotification| {
+                        if other.finalized_checkpoint.epoch > best.finalized_checkpoint.epoch {
+                            other
+                        } else {
+                            best
+                        }
+                    });
+
+                Self::run_migration(db.clone(), notif, &log);
+            }
+        });
+        (tx, thread)
     }
 
     /// Traverses live heads and prunes blocks and states of chains that we know can't be built
-    /// upon because finalization would prohibit it.  This is an optimisation intended to save disk
+    /// upon because finalization would prohibit it. This is an optimisation intended to save disk
     /// space.
-    ///
-    /// Assumptions:
-    ///  * It is called after every finalization.
+    #[allow(clippy::too_many_arguments)]
     fn prune_abandoned_forks(
         store: Arc<HotColdDB<E, Hot, Cold>>,
         head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_slot: Slot,
-    ) -> Result<(), BeaconChainError> {
-        // There will never be any blocks to prune if there is only a single head in the chain.
-        if head_tracker.heads().len() == 1 {
-            return Ok(());
+        new_finalized_state_hash: BeaconStateHash,
+        new_finalized_state: &BeaconState<E>,
+        new_finalized_checkpoint: Checkpoint,
+        genesis_block_root: Hash256,
+        log: &Logger,
+    ) -> Result<PruningOutcome, BeaconChainError> {
+        let old_finalized_checkpoint =
+            store
+                .load_pruning_checkpoint()?
+                .unwrap_or_else(|| Checkpoint {
+                    epoch: Epoch::new(0),
+                    root: Hash256::zero(),
+                });
+
+        let old_finalized_slot = old_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_slot = new_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_block_hash = new_finalized_checkpoint.root.into();
+
+        // The finalized state must be for the epoch boundary slot, not the slot of the finalized
+        // block.
+        if new_finalized_state.slot != new_finalized_slot {
+            return Err(PruningError::IncorrectFinalizedState {
+                state_slot: new_finalized_state.slot,
+                new_finalized_slot,
+            }
+            .into());
         }
 
-        let old_finalized_slot = store
-            .get_block(&old_finalized_block_hash.into())?
-            .ok_or_else(|| BeaconChainError::MissingBeaconBlock(old_finalized_block_hash.into()))?
-            .slot();
-
-        // Collect hashes from new_finalized_block back to old_finalized_block (inclusive)
-        let mut found_block = false; // hack for `take_until`
-        let newly_finalized_blocks: HashMap<SignedBeaconBlockHash, Slot> =
-            ParentRootBlockIterator::new(&*store, new_finalized_block_hash.into())
-                .take_while(|result| match result {
-                    Ok((block_hash, _)) => {
-                        if found_block {
-                            false
-                        } else {
-                            found_block |= *block_hash == old_finalized_block_hash.into();
-                            true
-                        }
-                    }
-                    Err(_) => true,
-                })
-                .map(|result| result.map(|(block_hash, block)| (block_hash.into(), block.slot())))
-                .collect::<Result<_, _>>()?;
+        debug!(
+            log,
+            "Starting database pruning";
+            "old_finalized_epoch" => old_finalized_checkpoint.epoch,
+            "new_finalized_epoch" => new_finalized_checkpoint.epoch,
+        );
+        // For each slot between the new finalized checkpoint and the old finalized checkpoint,
+        // collect the beacon block root and state root of the canonical chain.
+        let newly_finalized_chain: HashMap<Slot, (SignedBeaconBlockHash, BeaconStateHash)> =
+            std::iter::once(Ok((
+                new_finalized_slot,
+                (new_finalized_block_hash, new_finalized_state_hash),
+            )))
+            .chain(
+                RootsIterator::new(store.clone(), new_finalized_state).map(|res| {
+                    res.map(|(block_root, state_root, slot)| {
+                        (slot, (block_root.into(), state_root.into()))
+                    })
+                }),
+            )
+            .take_while(|res| {
+                res.as_ref()
+                    .map_or(true, |(slot, _)| *slot >= old_finalized_slot)
+            })
+            .collect::<Result<_, _>>()?;
 
         // We don't know which blocks are shared among abandoned chains, so we buffer and delete
         // everything in one fell swoop.
@@ -78,284 +324,220 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
         let mut abandoned_states: HashSet<(Slot, BeaconStateHash)> = HashSet::new();
         let mut abandoned_heads: HashSet<Hash256> = HashSet::new();
 
-        for (head_hash, head_slot) in head_tracker.heads() {
-            let mut potentially_abandoned_head: Option<Hash256> = Some(head_hash);
-            let mut potentially_abandoned_blocks: Vec<(
-                Slot,
-                Option<SignedBeaconBlockHash>,
-                Option<BeaconStateHash>,
-            )> = Vec::new();
+        let heads = head_tracker.heads();
+        debug!(
+            log,
+            "Extra pruning information";
+            "old_finalized_root" => format!("{:?}", old_finalized_checkpoint.root),
+            "new_finalized_root" => format!("{:?}", new_finalized_checkpoint.root),
+            "head_count" => heads.len(),
+        );
+
+        for (head_hash, head_slot) in heads {
+            let mut potentially_abandoned_head = Some(head_hash);
+            let mut potentially_abandoned_blocks = vec![];
 
             let head_state_hash = store
                 .get_block(&head_hash)?
                 .ok_or_else(|| BeaconStateError::MissingBeaconBlock(head_hash.into()))?
                 .state_root();
 
+            // Iterate backwards from this head, staging blocks and states for deletion.
             let iter = std::iter::once(Ok((head_hash, head_state_hash, head_slot)))
-                .chain(RootsIterator::from_block(Arc::clone(&store), head_hash)?);
+                .chain(RootsIterator::from_block(store.clone(), head_hash)?);
+
             for maybe_tuple in iter {
-                let (block_hash, state_hash, slot) = maybe_tuple?;
-                if slot < old_finalized_slot {
-                    // We must assume here any candidate chains include old_finalized_block_hash,
-                    // i.e. there aren't any forks starting at a block that is a strict ancestor of
-                    // old_finalized_block_hash.
-                    break;
-                }
-                match newly_finalized_blocks.get(&block_hash.into()).copied() {
-                    // Block is not finalized, mark it and its state for deletion
+                let (block_root, state_root, slot) = maybe_tuple?;
+                let block_root = SignedBeaconBlockHash::from(block_root);
+                let state_root = BeaconStateHash::from(state_root);
+
+                match newly_finalized_chain.get(&slot) {
+                    // If there's no information about a slot on the finalized chain, then
+                    // it should be because it's ahead of the new finalized slot. Stage
+                    // the fork's block and state for possible deletion.
                     None => {
-                        potentially_abandoned_blocks.push((
-                            slot,
-                            Some(block_hash.into()),
-                            Some(state_hash.into()),
-                        ));
+                        if slot > new_finalized_slot {
+                            potentially_abandoned_blocks.push((
+                                slot,
+                                Some(block_root),
+                                Some(state_root),
+                            ));
+                        } else if slot >= old_finalized_slot {
+                            return Err(PruningError::MissingInfoForCanonicalChain { slot }.into());
+                        } else {
+                            // We must assume here any candidate chains include the old finalized
+                            // checkpoint, i.e. there aren't any forks starting at a block that is a
+                            // strict ancestor of old_finalized_checkpoint.
+                            warn!(
+                                log,
+                                "Found a chain that should already have been pruned";
+                                "head_block_root" => format!("{:?}", head_hash),
+                                "head_slot" => head_slot,
+                            );
+                            potentially_abandoned_head.take();
+                            break;
+                        }
                     }
-                    Some(finalized_slot) => {
-                        // Block root is finalized, and we have reached the slot it was finalized
-                        // at: we've hit a shared part of the chain.
-                        if finalized_slot == slot {
-                            // The first finalized block of a candidate chain lies after (in terms
-                            // of slots order) the newly finalized block.  It's not a candidate for
-                            // prunning.
-                            if finalized_slot == new_finalized_slot {
+                    Some((finalized_block_root, finalized_state_root)) => {
+                        // This fork descends from a newly finalized block, we can stop.
+                        if block_root == *finalized_block_root {
+                            // Sanity check: if the slot and block root match, then the
+                            // state roots should match too.
+                            if state_root != *finalized_state_root {
+                                return Err(PruningError::UnexpectedUnequalStateRoots.into());
+                            }
+
+                            // If the fork descends from the whole finalized chain,
+                            // do not prune it. Otherwise continue to delete all
+                            // of the blocks and states that have been staged for
+                            // deletion so far.
+                            if slot == new_finalized_slot {
                                 potentially_abandoned_blocks.clear();
                                 potentially_abandoned_head.take();
                             }
-
+                            // If there are skipped slots on the fork to be pruned, then
+                            // we will have just staged the common block for deletion.
+                            // Unstage it.
+                            else {
+                                for (_, block_root, _) in
+                                    potentially_abandoned_blocks.iter_mut().rev()
+                                {
+                                    if block_root.as_ref() == Some(finalized_block_root) {
+                                        *block_root = None;
+                                    } else {
+                                        break;
+                                    }
+                                }
+                            }
                             break;
-                        }
-                        // Block root is finalized, but we're at a skip slot: delete the state only.
-                        else {
+                        } else {
+                            if state_root == *finalized_state_root {
+                                return Err(PruningError::UnexpectedEqualStateRoots.into());
+                            }
                             potentially_abandoned_blocks.push((
                                 slot,
-                                None,
-                                Some(state_hash.into()),
+                                Some(block_root),
+                                Some(state_root),
                             ));
                         }
                     }
                 }
             }
 
-            abandoned_heads.extend(potentially_abandoned_head.into_iter());
-            if !potentially_abandoned_blocks.is_empty() {
+            if let Some(abandoned_head) = potentially_abandoned_head {
+                debug!(
+                    log,
+                    "Pruning head";
+                    "head_block_root" => format!("{:?}", abandoned_head),
+                    "head_slot" => head_slot,
+                );
+                abandoned_heads.insert(abandoned_head);
                 abandoned_blocks.extend(
                     potentially_abandoned_blocks
                         .iter()
                         .filter_map(|(_, maybe_block_hash, _)| *maybe_block_hash),
                 );
                 abandoned_states.extend(potentially_abandoned_blocks.iter().filter_map(
-                    |(slot, _, maybe_state_hash)| match maybe_state_hash {
-                        None => None,
-                        Some(state_hash) => Some((*slot, *state_hash)),
-                    },
+                    |(slot, _, maybe_state_hash)| maybe_state_hash.map(|sr| (*slot, sr)),
                 ));
             }
         }
 
+        // Update the head tracker before the database, so that we maintain the invariant
+        // that a block present in the head tracker is present in the database.
+        // See https://github.com/sigp/lighthouse/issues/1557
+        let mut head_tracker_lock = head_tracker.0.write();
+
+        // Check that all the heads to be deleted are still present. The absence of any
+        // head indicates a race, that will likely resolve itself, so we defer pruning until
+        // later.
+        for head_hash in &abandoned_heads {
+            if !head_tracker_lock.contains_key(head_hash) {
+                return Ok(PruningOutcome::DeferredConcurrentMutation);
+            }
+        }
+
+        // Then remove them for real.
+        for head_hash in abandoned_heads {
+            head_tracker_lock.remove(&head_hash);
+        }
+
         let batch: Vec<StoreOp<E>> = abandoned_blocks
             .into_iter()
+            .map(Into::into)
             .map(StoreOp::DeleteBlock)
             .chain(
                 abandoned_states
                     .into_iter()
-                    .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash, slot)),
+                    .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash.into(), Some(slot))),
             )
             .collect();
-        store.do_atomically(batch)?;
-        for head_hash in abandoned_heads.into_iter() {
-            head_tracker.remove_head(head_hash);
+
+        let mut kv_batch = store.convert_to_kv_batch(&batch)?;
+
+        // Persist the head in case the process is killed or crashes here. This prevents
+        // the head tracker reverting after our mutation above.
+        let persisted_head = PersistedBeaconChain {
+            _canonical_head_block_root: DUMMY_CANONICAL_HEAD_BLOCK_ROOT,
+            genesis_block_root,
+            ssz_head_tracker: SszHeadTracker::from_map(&*head_tracker_lock),
+        };
+        drop(head_tracker_lock);
+        kv_batch.push(persisted_head.as_kv_store_op(BEACON_CHAIN_DB_KEY));
+
+        // Persist the new finalized checkpoint as the pruning checkpoint.
+        kv_batch.push(store.pruning_checkpoint_store_op(new_finalized_checkpoint));
+
+        store.hot_db.do_atomically(kv_batch)?;
+        debug!(log, "Database pruning complete");
+
+        Ok(PruningOutcome::Successful {
+            old_finalized_checkpoint,
+        })
+    }
+
+    /// Compact the database if it has been more than `COMPACTION_PERIOD_SECONDS` since it
+    /// was last compacted.
+    pub fn run_compaction(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        old_finalized_epoch: Epoch,
+        new_finalized_epoch: Epoch,
+        log: &Logger,
+    ) -> Result<(), Error> {
+        if !db.compact_on_prune() {
+            return Ok(());
         }
 
+        let last_compaction_timestamp = db
+            .load_compaction_timestamp()?
+            .unwrap_or_else(|| Duration::from_secs(0));
+        let start_time = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or(last_compaction_timestamp);
+        let seconds_since_last_compaction = start_time
+            .checked_sub(last_compaction_timestamp)
+            .as_ref()
+            .map_or(0, Duration::as_secs);
+
+        if seconds_since_last_compaction > MAX_COMPACTION_PERIOD_SECONDS
+            || (new_finalized_epoch - old_finalized_epoch > COMPACTION_FINALITY_DISTANCE
+                && seconds_since_last_compaction > MIN_COMPACTION_PERIOD_SECONDS)
+        {
+            info!(
+                log,
+                "Starting database compaction";
+                "old_finalized_epoch" => old_finalized_epoch,
+                "new_finalized_epoch" => new_finalized_epoch,
+            );
+            db.compact()?;
+
+            let finish_time = SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap_or(start_time);
+            db.store_compaction_timestamp(finish_time)?;
+
+            info!(log, "Database compaction complete");
+        }
         Ok(())
     }
 }
-
-/// Migrator that does nothing, for stores that don't need migration.
-pub struct NullMigrator;
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold> for NullMigrator {
-    fn new(_: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
-        NullMigrator
-    }
-}
-
-/// Migrator that immediately calls the store's migration function, blocking the current execution.
-///
-/// Mostly useful for tests.
-pub struct BlockingMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
-    db: Arc<HotColdDB<E, Hot, Cold>>,
-}
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
-    for BlockingMigrator<E, Hot, Cold>
-{
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
-        BlockingMigrator { db }
-    }
-
-    fn process_finalization(
-        &self,
-        state_root: Hash256,
-        new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
-        head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
-        if let Err(e) = Self::prune_abandoned_forks(
-            self.db.clone(),
-            head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_state.slot,
-        ) {
-            eprintln!("Pruning error: {:?}", e);
-        }
-
-        if let Err(e) = process_finalization(self.db.clone(), state_root, &new_finalized_state) {
-            // This migrator is only used for testing, so we just log to stderr without a logger.
-            eprintln!("Migration error: {:?}", e);
-        }
-    }
-}
-
-type MpscSender<E> = mpsc::Sender<(
-    Hash256,
-    BeaconState<E>,
-    Arc<HeadTracker>,
-    SignedBeaconBlockHash,
-    SignedBeaconBlockHash,
-    Slot,
-)>;
-
-/// Migrator that runs a background thread to migrate state from the hot to the cold database.
-pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
-    db: Arc<HotColdDB<E, Hot, Cold>>,
-    tx_thread: Mutex<(MpscSender<E>, thread::JoinHandle<()>)>,
-    log: Logger,
-}
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
-    for BackgroundMigrator<E, Hot, Cold>
-{
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self {
-        let tx_thread = Mutex::new(Self::spawn_thread(db.clone(), log.clone()));
-        Self { db, tx_thread, log }
-    }
-
-    /// Perform the freezing operation on the database,
-    fn process_finalization(
-        &self,
-        finalized_state_root: Hash256,
-        new_finalized_state: BeaconState<E>,
-        max_finality_distance: u64,
-        head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
-        if !self.needs_migration(new_finalized_state.slot, max_finality_distance) {
-            return;
-        }
-
-        let (ref mut tx, ref mut thread) = *self.tx_thread.lock();
-
-        let new_finalized_slot = new_finalized_state.slot;
-        if let Err(tx_err) = tx.send((
-            finalized_state_root,
-            new_finalized_state,
-            head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_slot,
-        )) {
-            let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
-
-            drop(mem::replace(tx, new_tx));
-            let old_thread = mem::replace(thread, new_thread);
-
-            // Join the old thread, which will probably have panicked, or may have
-            // halted normally just now as a result of us dropping the old `mpsc::Sender`.
-            if let Err(thread_err) = old_thread.join() {
-                warn!(
-                    self.log,
-                    "Migration thread died, so it was restarted";
-                    "reason" => format!("{:?}", thread_err)
-                );
-            }
-
-            // Retry at most once, we could recurse but that would risk overflowing the stack.
-            let _ = tx.send(tx_err.0);
-        }
-    }
-}
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
-    /// Return true if a migration needs to be performed, given a new `finalized_slot`.
-    fn needs_migration(&self, finalized_slot: Slot, max_finality_distance: u64) -> bool {
-        let finality_distance = finalized_slot - self.db.get_split_slot();
-        finality_distance > max_finality_distance
-    }
-
-    #[allow(clippy::type_complexity)]
-    /// Spawn a new child thread to run the migration process.
-    ///
-    /// Return a channel handle for sending new finalized states to the thread.
-    fn spawn_thread(
-        db: Arc<HotColdDB<E, Hot, Cold>>,
-        log: Logger,
-    ) -> (
-        mpsc::Sender<(
-            Hash256,
-            BeaconState<E>,
-            Arc<HeadTracker>,
-            SignedBeaconBlockHash,
-            SignedBeaconBlockHash,
-            Slot,
-        )>,
-        thread::JoinHandle<()>,
-    ) {
-        let (tx, rx) = mpsc::channel();
-        let thread = thread::spawn(move || {
-            while let Ok((
-                state_root,
-                state,
-                head_tracker,
-                old_finalized_block_hash,
-                new_finalized_block_hash,
-                new_finalized_slot,
-            )) = rx.recv()
-            {
-                match Self::prune_abandoned_forks(
-                    db.clone(),
-                    head_tracker,
-                    old_finalized_block_hash,
-                    new_finalized_block_hash,
-                    new_finalized_slot,
-                ) {
-                    Ok(()) => {}
-                    Err(e) => warn!(log, "Block pruning failed: {:?}", e),
-                }
-
-                match process_finalization(db.clone(), state_root, &state) {
-                    Ok(()) => {}
-                    Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
-                        debug!(
-                            log,
-                            "Database migration postponed, unaligned finalized block";
-                            "slot" => slot.as_u64()
-                        );
-                    }
-                    Err(e) => {
-                        warn!(
-                            log,
-                            "Database migration failed";
-                            "error" => format!("{:?}", e)
-                        );
-                    }
-                };
-            }
-        });
-
-        (tx, thread)
-    }
-}

beacon_node/beacon_chain/src/naive_aggregation_pool.rs

@@ -1,7 +1,9 @@
 use crate::metrics;
 use std::collections::HashMap;
-use types::{Attestation, AttestationData, EthSpec, Slot};
+use tree_hash::TreeHash;
+use types::{Attestation, AttestationData, EthSpec, Hash256, Slot};
 
+type AttestationDataRoot = Hash256;
 /// The number of slots that will be stored in the pool.
 ///
 /// For example, if `SLOTS_RETAINED == 3` and the pool is pruned at slot `6`, then all attestations
@@ -53,7 +55,7 @@ pub enum Error {
 /// A collection of `Attestation` objects, keyed by their `attestation.data`. Enforces that all
 /// `attestation` are from the same slot.
 struct AggregatedAttestationMap<E: EthSpec> {
-    map: HashMap<AttestationData, Attestation<E>>,
+    map: HashMap<AttestationDataRoot, Attestation<E>>,
 }
 
 impl<E: EthSpec> AggregatedAttestationMap<E> {
@@ -87,7 +89,9 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
             return Err(Error::MoreThanOneAggregationBitSet(set_bits.len()));
         }
 
-        if let Some(existing_attestation) = self.map.get_mut(&a.data) {
+        let attestation_data_root = a.data.tree_hash_root();
+
+        if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) {
             if existing_attestation
                 .aggregation_bits
                 .get(committee_index)
@@ -107,7 +111,7 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
                 ));
             }
 
-            self.map.insert(a.data.clone(), a.clone());
+            self.map.insert(attestation_data_root, a.clone());
             Ok(InsertOutcome::NewAttestationData { committee_index })
         }
     }
@@ -115,8 +119,13 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
     /// Returns an aggregated `Attestation` with the given `data`, if any.
     ///
     /// The given `a.data.slot` must match the slot that `self` was initialized with.
-    pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
-        Ok(self.map.get(data).cloned())
+    pub fn get(&self, data: &AttestationData) -> Option<Attestation<E>> {
+        self.map.get(&data.tree_hash_root()).cloned()
+    }
+
+    /// Returns an aggregated `Attestation` with the given `root`, if any.
+    pub fn get_by_root(&self, root: &AttestationDataRoot) -> Option<&Attestation<E>> {
+        self.map.get(root)
     }
 
     /// Iterate all attestations in `self`.
@@ -220,12 +229,19 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
     }
 
     /// Returns an aggregated `Attestation` with the given `data`, if any.
-    pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
+    pub fn get(&self, data: &AttestationData) -> Option<Attestation<E>> {
+        self.maps.get(&data.slot).and_then(|map| map.get(data))
+    }
+
+    /// Returns an aggregated `Attestation` with the given `data`, if any.
+    pub fn get_by_slot_and_root(
+        &self,
+        slot: Slot,
+        root: &AttestationDataRoot,
+    ) -> Option<Attestation<E>> {
         self.maps
-            .iter()
-            .find(|(slot, _map)| **slot == data.slot)
-            .map(|(_slot, map)| map.get(data))
-            .unwrap_or_else(|| Ok(None))
+            .get(&slot)
+            .and_then(|map| map.get_by_root(root).cloned())
     }
 
     /// Iterate all attestations in all slots of `self`.
@@ -338,8 +354,7 @@ mod tests {
 
         let retrieved = pool
             .get(&a.data)
-            .expect("should not error while getting attestation")
-            .expect("should get an attestation");
+            .expect("should not error while getting attestation");
         assert_eq!(
             retrieved, a,
             "retrieved attestation should equal the one inserted"
@@ -378,8 +393,7 @@ mod tests {
 
         let retrieved = pool
             .get(&a_0.data)
-            .expect("should not error while getting attestation")
-            .expect("should get an attestation");
+            .expect("should not error while getting attestation");
 
         let mut a_01 = a_0.clone();
         a_01.aggregate(&a_1);
@@ -408,8 +422,7 @@ mod tests {
 
         assert_eq!(
             pool.get(&a_0.data)
-                .expect("should not error while getting attestation")
-                .expect("should get an attestation"),
+                .expect("should not error while getting attestation"),
             retrieved,
             "should not have aggregated different attestation data"
         );

beacon_node/beacon_chain/src/observed_attestations.rs

@@ -1,7 +1,6 @@
 //! Provides an `ObservedAttestations` struct which allows us to reject aggregated attestations if
 //! we've already seen the aggregated attestation.
 
-use parking_lot::RwLock;
 use std::collections::HashSet;
 use std::marker::PhantomData;
 use tree_hash::TreeHash;
@@ -116,16 +115,16 @@ impl SlotHashSet {
 /// Stores the roots of `Attestation` objects for some number of `Slots`, so we can determine if
 /// these have previously been seen on the network.
 pub struct ObservedAttestations<E: EthSpec> {
-    lowest_permissible_slot: RwLock<Slot>,
-    sets: RwLock<Vec<SlotHashSet>>,
+    lowest_permissible_slot: Slot,
+    sets: Vec<SlotHashSet>,
     _phantom: PhantomData<E>,
 }
 
 impl<E: EthSpec> Default for ObservedAttestations<E> {
     fn default() -> Self {
         Self {
-            lowest_permissible_slot: RwLock::new(Slot::new(0)),
-            sets: RwLock::new(vec![]),
+            lowest_permissible_slot: Slot::new(0),
+            sets: vec![],
             _phantom: PhantomData,
         }
     }
@@ -136,7 +135,7 @@ impl<E: EthSpec> ObservedAttestations<E> {
     ///
     /// `root` must equal `a.tree_hash_root()`.
     pub fn observe_attestation(
-        &self,
+        &mut self,
         a: &Attestation<E>,
         root_opt: Option<Hash256>,
     ) -> Result<ObserveOutcome, Error> {
@@ -144,7 +143,6 @@ impl<E: EthSpec> ObservedAttestations<E> {
         let root = root_opt.unwrap_or_else(|| a.tree_hash_root());
 
         self.sets
-            .write()
             .get_mut(index)
             .ok_or_else(|| Error::InvalidSetIndex(index))
             .and_then(|set| set.observe_attestation(a, root))
@@ -153,11 +151,10 @@ impl<E: EthSpec> ObservedAttestations<E> {
     /// Check to see if the `root` of `a` is in self.
     ///
     /// `root` must equal `a.tree_hash_root()`.
-    pub fn is_known(&self, a: &Attestation<E>, root: Hash256) -> Result<bool, Error> {
+    pub fn is_known(&mut self, a: &Attestation<E>, root: Hash256) -> Result<bool, Error> {
         let index = self.get_set_index(a.data.slot)?;
 
         self.sets
-            .read()
             .get(index)
             .ok_or_else(|| Error::InvalidSetIndex(index))
             .and_then(|set| set.is_known(a, root))
@@ -172,23 +169,21 @@ impl<E: EthSpec> ObservedAttestations<E> {
 
     /// Removes any attestations with a slot lower than `current_slot` and bars any future
     /// attestations with a slot lower than `current_slot - SLOTS_RETAINED`.
-    pub fn prune(&self, current_slot: Slot) {
+    pub fn prune(&mut self, current_slot: Slot) {
         // Taking advantage of saturating subtraction on `Slot`.
         let lowest_permissible_slot = current_slot - (self.max_capacity() - 1);
 
-        self.sets
-            .write()
-            .retain(|set| set.slot >= lowest_permissible_slot);
+        self.sets.retain(|set| set.slot >= lowest_permissible_slot);
 
-        *self.lowest_permissible_slot.write() = lowest_permissible_slot;
+        self.lowest_permissible_slot = lowest_permissible_slot;
     }
 
     /// Returns the index of `self.set` that matches `slot`.
     ///
     /// If there is no existing set for this slot one will be created. If `self.sets.len() >=
     /// Self::max_capacity()`, the set with the lowest slot will be replaced.
-    fn get_set_index(&self, slot: Slot) -> Result<usize, Error> {
-        let lowest_permissible_slot: Slot = *self.lowest_permissible_slot.read();
+    fn get_set_index(&mut self, slot: Slot) -> Result<usize, Error> {
+        let lowest_permissible_slot = self.lowest_permissible_slot;
 
         if slot < lowest_permissible_slot {
             return Err(Error::SlotTooLow {
@@ -202,15 +197,14 @@ impl<E: EthSpec> ObservedAttestations<E> {
             self.prune(slot)
         }
 
-        let mut sets = self.sets.write();
-
-        if let Some(index) = sets.iter().position(|set| set.slot == slot) {
+        if let Some(index) = self.sets.iter().position(|set| set.slot == slot) {
             return Ok(index);
         }
 
         // To avoid re-allocations, try and determine a rough initial capacity for the new set
         // by obtaining the mean size of all items in earlier epoch.
-        let (count, sum) = sets
+        let (count, sum) = self
+            .sets
             .iter()
             // Only include slots that are less than the given slot in the average. This should
             // generally avoid including recent slots that are still "filling up".
@@ -222,20 +216,21 @@ impl<E: EthSpec> ObservedAttestations<E> {
         // but considering it's approx. 128 * 32 bytes we're not wasting much.
         let initial_capacity = sum.checked_div(count).unwrap_or(128);
 
-        if sets.len() < self.max_capacity() as usize || sets.is_empty() {
-            let index = sets.len();
-            sets.push(SlotHashSet::new(slot, initial_capacity));
+        if self.sets.len() < self.max_capacity() as usize || self.sets.is_empty() {
+            let index = self.sets.len();
+            self.sets.push(SlotHashSet::new(slot, initial_capacity));
             return Ok(index);
         }
 
-        let index = sets
+        let index = self
+            .sets
             .iter()
             .enumerate()
             .min_by_key(|(_i, set)| set.slot)
             .map(|(i, _set)| i)
             .expect("sets cannot be empty due to previous .is_empty() check");
 
-        sets[index] = SlotHashSet::new(slot, initial_capacity);
+        self.sets[index] = SlotHashSet::new(slot, initial_capacity);
 
         Ok(index)
     }
@@ -259,7 +254,7 @@ mod tests {
         a
     }
 
-    fn single_slot_test(store: &ObservedAttestations<E>, slot: Slot) {
+    fn single_slot_test(store: &mut ObservedAttestations<E>, slot: Slot) {
         let attestations = (0..NUM_ELEMENTS as u64)
             .map(|i| get_attestation(slot, i))
             .collect::<Vec<_>>();
@@ -293,17 +288,13 @@ mod tests {
 
     #[test]
     fn single_slot() {
-        let store = ObservedAttestations::default();
+        let mut store = ObservedAttestations::default();
 
-        single_slot_test(&store, Slot::new(0));
+        single_slot_test(&mut store, Slot::new(0));
 
+        assert_eq!(store.sets.len(), 1, "should have a single set stored");
         assert_eq!(
-            store.sets.read().len(),
-            1,
-            "should have a single set stored"
-        );
-        assert_eq!(
-            store.sets.read()[0].len(),
+            store.sets[0].len(),
             NUM_ELEMENTS,
             "set should have NUM_ELEMENTS elements"
         );
@@ -311,13 +302,13 @@ mod tests {
 
     #[test]
     fn mulitple_contiguous_slots() {
-        let store = ObservedAttestations::default();
+        let mut store = ObservedAttestations::default();
         let max_cap = store.max_capacity();
 
         for i in 0..max_cap * 3 {
             let slot = Slot::new(i);
 
-            single_slot_test(&store, slot);
+            single_slot_test(&mut store, slot);
 
             /*
              * Ensure that the number of sets is correct.
@@ -325,14 +316,14 @@ mod tests {
 
             if i < max_cap {
                 assert_eq!(
-                    store.sets.read().len(),
+                    store.sets.len(),
                     i as usize + 1,
                     "should have a {} sets stored",
                     i + 1
                 );
             } else {
                 assert_eq!(
-                    store.sets.read().len(),
+                    store.sets.len(),
                     max_cap as usize,
                     "should have max_capacity sets stored"
                 );
@@ -342,7 +333,7 @@ mod tests {
              * Ensure that each set contains the correct number of elements.
              */
 
-            for set in &store.sets.read()[..] {
+            for set in &store.sets[..] {
                 assert_eq!(
                     set.len(),
                     NUM_ELEMENTS,
@@ -354,12 +345,7 @@ mod tests {
              *  Ensure that all the sets have the expected slots
              */
 
-            let mut store_slots = store
-                .sets
-                .read()
-                .iter()
-                .map(|set| set.slot)
-                .collect::<Vec<_>>();
+            let mut store_slots = store.sets.iter().map(|set| set.slot).collect::<Vec<_>>();
 
             assert!(
                 store_slots.len() <= store.max_capacity() as usize,
@@ -378,7 +364,7 @@ mod tests {
 
     #[test]
     fn mulitple_non_contiguous_slots() {
-        let store = ObservedAttestations::default();
+        let mut store = ObservedAttestations::default();
         let max_cap = store.max_capacity();
 
         let to_skip = vec![1_u64, 2, 3, 5, 6, 29, 30, 31, 32, 64];
@@ -394,13 +380,13 @@ mod tests {
 
             let slot = Slot::from(i);
 
-            single_slot_test(&store, slot);
+            single_slot_test(&mut store, slot);
 
             /*
              * Ensure that each set contains the correct number of elements.
              */
 
-            for set in &store.sets.read()[..] {
+            for set in &store.sets[..] {
                 assert_eq!(
                     set.len(),
                     NUM_ELEMENTS,
@@ -412,12 +398,7 @@ mod tests {
              *  Ensure that all the sets have the expected slots
              */
 
-            let mut store_slots = store
-                .sets
-                .read()
-                .iter()
-                .map(|set| set.slot)
-                .collect::<Vec<_>>();
+            let mut store_slots = store.sets.iter().map(|set| set.slot).collect::<Vec<_>>();
 
             store_slots.sort_unstable();
 
@@ -426,7 +407,7 @@ mod tests {
                 "store size should not exceed max"
             );
 
-            let lowest = store.lowest_permissible_slot.read().as_u64();
+            let lowest = store.lowest_permissible_slot.as_u64();
             let highest = slot.as_u64();
             let expected_slots = (lowest..=highest)
                 .filter(|i| !to_skip.contains(i))

beacon_node/beacon_chain/src/observed_attesters.rs

@@ -7,7 +7,6 @@
 //!   the same epoch
 
 use bitvec::vec::BitVec;
-use parking_lot::RwLock;
 use std::collections::{HashMap, HashSet};
 use std::marker::PhantomData;
 use types::{Attestation, Epoch, EthSpec, Unsigned};
@@ -148,16 +147,16 @@ impl Item for EpochHashSet {
 ///
 /// `T` should be set to a `EpochBitfield` or `EpochHashSet`.
 pub struct AutoPruningContainer<T, E: EthSpec> {
-    lowest_permissible_epoch: RwLock<Epoch>,
-    items: RwLock<HashMap<Epoch, T>>,
+    lowest_permissible_epoch: Epoch,
+    items: HashMap<Epoch, T>,
     _phantom: PhantomData<E>,
 }
 
 impl<T, E: EthSpec> Default for AutoPruningContainer<T, E> {
     fn default() -> Self {
         Self {
-            lowest_permissible_epoch: RwLock::new(Epoch::new(0)),
-            items: RwLock::new(HashMap::new()),
+            lowest_permissible_epoch: Epoch::new(0),
+            items: HashMap::new(),
             _phantom: PhantomData,
         }
     }
@@ -172,7 +171,7 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
     /// - `validator_index` is higher than `VALIDATOR_REGISTRY_LIMIT`.
     /// - `a.data.target.slot` is earlier than `self.earliest_permissible_slot`.
     pub fn observe_validator(
-        &self,
+        &mut self,
         a: &Attestation<E>,
         validator_index: usize,
     ) -> Result<bool, Error> {
@@ -182,14 +181,13 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
 
         self.prune(epoch);
 
-        let mut items = self.items.write();
-
-        if let Some(item) = items.get_mut(&epoch) {
+        if let Some(item) = self.items.get_mut(&epoch) {
             Ok(item.insert(validator_index))
         } else {
             // To avoid re-allocations, try and determine a rough initial capacity for the new item
             // by obtaining the mean size of all items in earlier epoch.
-            let (count, sum) = items
+            let (count, sum) = self
+                .items
                 .iter()
                 // Only include epochs that are less than the given slot in the average. This should
                 // generally avoid including recent epochs that are still "filling up".
@@ -201,7 +199,7 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
 
             let mut item = T::with_capacity(initial_capacity);
             item.insert(validator_index);
-            items.insert(epoch, item);
+            self.items.insert(epoch, item);
 
             Ok(false)
         }
@@ -223,7 +221,6 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
 
         let exists = self
             .items
-            .read()
             .get(&a.data.target.epoch)
             .map_or(false, |item| item.contains(validator_index));
 
@@ -233,10 +230,7 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
     /// Returns the number of validators that have been observed at the given `epoch`. Returns
     /// `None` if `self` does not have a cache for that epoch.
     pub fn observed_validator_count(&self, epoch: Epoch) -> Option<usize> {
-        self.items
-            .read()
-            .get(&epoch)
-            .map(|item| item.validator_count())
+        self.items.get(&epoch).map(|item| item.validator_count())
     }
 
     fn sanitize_request(&self, a: &Attestation<E>, validator_index: usize) -> Result<(), Error> {
@@ -245,7 +239,7 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
         }
 
         let epoch = a.data.target.epoch;
-        let lowest_permissible_epoch: Epoch = *self.lowest_permissible_epoch.read();
+        let lowest_permissible_epoch = self.lowest_permissible_epoch;
         if epoch < lowest_permissible_epoch {
             return Err(Error::EpochTooLow {
                 epoch,
@@ -270,14 +264,13 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
     ///
     /// Also sets `self.lowest_permissible_epoch` with relation to `current_epoch` and
     /// `Self::max_capacity`.
-    pub fn prune(&self, current_epoch: Epoch) {
+    pub fn prune(&mut self, current_epoch: Epoch) {
         // Taking advantage of saturating subtraction on `Slot`.
         let lowest_permissible_epoch = current_epoch - (self.max_capacity().saturating_sub(1));
 
-        *self.lowest_permissible_epoch.write() = lowest_permissible_epoch;
+        self.lowest_permissible_epoch = lowest_permissible_epoch;
 
         self.items
-            .write()
             .retain(|epoch, _item| *epoch >= lowest_permissible_epoch);
     }
 }
@@ -301,7 +294,7 @@ mod tests {
                     a
                 }
 
-                fn single_epoch_test(store: &$type<E>, epoch: Epoch) {
+                fn single_epoch_test(store: &mut $type<E>, epoch: Epoch) {
                     let attesters = [0, 1, 2, 3, 5, 6, 7, 18, 22];
                     let a = &get_attestation(epoch);
 
@@ -334,26 +327,22 @@ mod tests {
 
                 #[test]
                 fn single_epoch() {
-                    let store = $type::default();
+                    let mut store = $type::default();
 
-                    single_epoch_test(&store, Epoch::new(0));
+                    single_epoch_test(&mut store, Epoch::new(0));
 
-                    assert_eq!(
-                        store.items.read().len(),
-                        1,
-                        "should have a single bitfield stored"
-                    );
+                    assert_eq!(store.items.len(), 1, "should have a single bitfield stored");
                 }
 
                 #[test]
                 fn mulitple_contiguous_epochs() {
-                    let store = $type::default();
+                    let mut store = $type::default();
                     let max_cap = store.max_capacity();
 
                     for i in 0..max_cap * 3 {
                         let epoch = Epoch::new(i);
 
-                        single_epoch_test(&store, epoch);
+                        single_epoch_test(&mut store, epoch);
 
                         /*
                          * Ensure that the number of sets is correct.
@@ -361,14 +350,14 @@ mod tests {
 
                         if i < max_cap {
                             assert_eq!(
-                                store.items.read().len(),
+                                store.items.len(),
                                 i as usize + 1,
                                 "should have a {} items stored",
                                 i + 1
                             );
                         } else {
                             assert_eq!(
-                                store.items.read().len(),
+                                store.items.len(),
                                 max_cap as usize,
                                 "should have max_capacity items stored"
                             );
@@ -380,7 +369,6 @@ mod tests {
 
                         let mut store_epochs = store
                             .items
-                            .read()
                             .iter()
                             .map(|(epoch, _set)| *epoch)
                             .collect::<Vec<_>>();
@@ -402,7 +390,7 @@ mod tests {
 
                 #[test]
                 fn mulitple_non_contiguous_epochs() {
-                    let store = $type::default();
+                    let mut store = $type::default();
                     let max_cap = store.max_capacity();
 
                     let to_skip = vec![1_u64, 3, 4, 5];
@@ -418,7 +406,7 @@ mod tests {
 
                         let epoch = Epoch::from(i);
 
-                        single_epoch_test(&store, epoch);
+                        single_epoch_test(&mut store, epoch);
 
                         /*
                          *  Ensure that all the sets have the expected slots
@@ -426,7 +414,6 @@ mod tests {
 
                         let mut store_epochs = store
                             .items
-                            .read()
                             .iter()
                             .map(|(epoch, _)| *epoch)
                             .collect::<Vec<_>>();
@@ -438,7 +425,7 @@ mod tests {
                             "store size should not exceed max"
                         );
 
-                        let lowest = store.lowest_permissible_epoch.read().as_u64();
+                        let lowest = store.lowest_permissible_epoch.as_u64();
                         let highest = epoch.as_u64();
                         let expected_epochs = (lowest..=highest)
                             .filter(|i| !to_skip.contains(i))

beacon_node/beacon_chain/src/observed_block_producers.rs

@@ -1,7 +1,6 @@
 //! Provides the `ObservedBlockProducers` struct which allows for rejecting gossip blocks from
 //! validators that have already produced a block.
 
-use parking_lot::RwLock;
 use std::collections::{HashMap, HashSet};
 use std::marker::PhantomData;
 use types::{BeaconBlock, EthSpec, Slot, Unsigned};
@@ -27,8 +26,8 @@ pub enum Error {
 /// active_validator_count`, however in reality that is more like `slots_since_finality *
 /// known_distinct_shufflings` which is much smaller.
 pub struct ObservedBlockProducers<E: EthSpec> {
-    finalized_slot: RwLock<Slot>,
-    items: RwLock<HashMap<Slot, HashSet<u64>>>,
+    finalized_slot: Slot,
+    items: HashMap<Slot, HashSet<u64>>,
     _phantom: PhantomData<E>,
 }
 
@@ -36,8 +35,8 @@ impl<E: EthSpec> Default for ObservedBlockProducers<E> {
     /// Instantiates `Self` with `finalized_slot == 0`.
     fn default() -> Self {
         Self {
-            finalized_slot: RwLock::new(Slot::new(0)),
-            items: RwLock::new(HashMap::new()),
+            finalized_slot: Slot::new(0),
+            items: HashMap::new(),
             _phantom: PhantomData,
         }
     }
@@ -53,12 +52,11 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
     ///
     /// - `block.proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`.
     /// - `block.slot` is equal to or less than the latest pruned `finalized_slot`.
-    pub fn observe_proposer(&self, block: &BeaconBlock<E>) -> Result<bool, Error> {
+    pub fn observe_proposer(&mut self, block: &BeaconBlock<E>) -> Result<bool, Error> {
         self.sanitize_block(block)?;
 
         let did_not_exist = self
             .items
-            .write()
             .entry(block.slot)
             .or_insert_with(|| HashSet::with_capacity(E::SlotsPerEpoch::to_usize()))
             .insert(block.proposer_index);
@@ -79,7 +77,6 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
 
         let exists = self
             .items
-            .read()
             .get(&block.slot)
             .map_or(false, |set| set.contains(&block.proposer_index));
 
@@ -92,7 +89,7 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
             return Err(Error::ValidatorIndexTooHigh(block.proposer_index));
         }
 
-        let finalized_slot = *self.finalized_slot.read();
+        let finalized_slot = self.finalized_slot;
         if finalized_slot > 0 && block.slot <= finalized_slot {
             return Err(Error::FinalizedBlock {
                 slot: block.slot,
@@ -109,15 +106,13 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
     /// equal to or less than `finalized_slot`.
     ///
     /// No-op if `finalized_slot == 0`.
-    pub fn prune(&self, finalized_slot: Slot) {
+    pub fn prune(&mut self, finalized_slot: Slot) {
         if finalized_slot == 0 {
             return;
         }
 
-        *self.finalized_slot.write() = finalized_slot;
-        self.items
-            .write()
-            .retain(|slot, _set| *slot > finalized_slot);
+        self.finalized_slot = finalized_slot;
+        self.items.retain(|slot, _set| *slot > finalized_slot);
     }
 }
 
@@ -137,10 +132,10 @@ mod tests {
 
     #[test]
     fn pruning() {
-        let cache = ObservedBlockProducers::default();
+        let mut cache = ObservedBlockProducers::default();
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(cache.items.read().len(), 0, "no slots should be present");
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 0, "no slots should be present");
 
         // Slot 0, proposer 0
         let block_a = &get_block(0, 0);
@@ -155,16 +150,11 @@ mod tests {
          * Preconditions.
          */
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -178,16 +168,11 @@ mod tests {
 
         cache.prune(Slot::new(0));
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -201,11 +186,11 @@ mod tests {
 
         cache.prune(E::slots_per_epoch().into());
         assert_eq!(
-            *cache.finalized_slot.read(),
+            cache.finalized_slot,
             Slot::from(E::slots_per_epoch()),
             "finalized slot is updated"
         );
-        assert_eq!(cache.items.read().len(), 0, "no items left");
+        assert_eq!(cache.items.len(), 0, "no items left");
 
         /*
          * Check that we can't insert a finalized block
@@ -223,7 +208,7 @@ mod tests {
             "cant insert finalized block"
         );
 
-        assert_eq!(cache.items.read().len(), 0, "block was not added");
+        assert_eq!(cache.items.len(), 0, "block was not added");
 
         /*
          * Check that we _can_ insert a non-finalized block
@@ -240,15 +225,10 @@ mod tests {
             "can insert non-finalized block"
         );
 
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(three_epochs))
                 .expect("the three epochs slot should be present")
                 .len(),
@@ -264,20 +244,15 @@ mod tests {
         cache.prune(two_epochs.into());
 
         assert_eq!(
-            *cache.finalized_slot.read(),
+            cache.finalized_slot,
             Slot::from(two_epochs),
             "finalized slot is updated"
         );
 
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(three_epochs))
                 .expect("the three epochs slot should be present")
                 .len(),
@@ -288,7 +263,7 @@ mod tests {
 
     #[test]
     fn simple_observations() {
-        let cache = ObservedBlockProducers::default();
+        let mut cache = ObservedBlockProducers::default();
 
         // Slot 0, proposer 0
         let block_a = &get_block(0, 0);
@@ -314,16 +289,11 @@ mod tests {
             "observing again indicates true"
         );
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -355,12 +325,11 @@ mod tests {
             "observing slot 1 again indicates true"
         );
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(cache.items.read().len(), 2, "two slots should be present");
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -370,7 +339,6 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(1))
                 .expect("slot zero should be present")
                 .len(),
@@ -402,12 +370,11 @@ mod tests {
             "observing new proposer again indicates true"
         );
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(cache.items.read().len(), 2, "two slots should be present");
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -417,7 +384,6 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(1))
                 .expect("slot zero should be present")
                 .len(),

beacon_node/beacon_chain/src/observed_operations.rs

@@ -1,5 +1,4 @@
 use derivative::Derivative;
-use parking_lot::Mutex;
 use smallvec::SmallVec;
 use state_processing::{SigVerifiedOp, VerifyOperation};
 use std::collections::HashSet;
@@ -25,7 +24,7 @@ pub struct ObservedOperations<T: ObservableOperation<E>, E: EthSpec> {
     /// For attester slashings, this is the set of all validators who would be slashed by
     /// previously seen attester slashings, i.e. those validators in the intersection of
     /// `attestation_1.attester_indices` and `attestation_2.attester_indices`.
-    observed_validator_indices: Mutex<HashSet<u64>>,
+    observed_validator_indices: HashSet<u64>,
     _phantom: PhantomData<(T, E)>,
 }
 
@@ -71,12 +70,12 @@ impl<E: EthSpec> ObservableOperation<E> for AttesterSlashing<E> {
 
 impl<T: ObservableOperation<E>, E: EthSpec> ObservedOperations<T, E> {
     pub fn verify_and_observe(
-        &self,
+        &mut self,
         op: T,
         head_state: &BeaconState<E>,
         spec: &ChainSpec,
     ) -> Result<ObservationOutcome<T>, T::Error> {
-        let mut observed_validator_indices = self.observed_validator_indices.lock();
+        let observed_validator_indices = &mut self.observed_validator_indices;
         let new_validator_indices = op.observed_validators();
 
         // If all of the new validator indices have been previously observed, short-circuit

beacon_node/beacon_chain/src/persisted_beacon_chain.rs

@@ -4,9 +4,19 @@ use ssz_derive::{Decode, Encode};
 use store::{DBColumn, Error as StoreError, StoreItem};
 use types::Hash256;
 
+/// Dummy value to use for the canonical head block root, see below.
+pub const DUMMY_CANONICAL_HEAD_BLOCK_ROOT: Hash256 = Hash256::repeat_byte(0xff);
+
 #[derive(Clone, Encode, Decode)]
 pub struct PersistedBeaconChain {
-    pub canonical_head_block_root: Hash256,
+    /// This value is ignored to resolve the issue described here:
+    ///
+    /// https://github.com/sigp/lighthouse/pull/1639
+    ///
+    /// Its removal is tracked here:
+    ///
+    /// https://github.com/sigp/lighthouse/issues/1784
+    pub _canonical_head_block_root: Hash256,
     pub genesis_block_root: Hash256,
     pub ssz_head_tracker: SszHeadTracker,
 }

beacon_node/beacon_chain/src/shuffling_cache.rs

@@ -1,6 +1,6 @@
 use crate::metrics;
 use lru::LruCache;
-use types::{beacon_state::CommitteeCache, Epoch, Hash256};
+use types::{beacon_state::CommitteeCache, Epoch, Hash256, ShufflingId};
 
 /// The size of the LRU cache that stores committee caches for quicker verification.
 ///
@@ -14,7 +14,7 @@ const CACHE_SIZE: usize = 16;
 /// It has been named `ShufflingCache` because `CommitteeCacheCache` is a bit weird and looks like
 /// a find/replace error.
 pub struct ShufflingCache {
-    cache: LruCache<(Epoch, Hash256), CommitteeCache>,
+    cache: LruCache<ShufflingId, CommitteeCache>,
 }
 
 impl ShufflingCache {
@@ -24,8 +24,8 @@ impl ShufflingCache {
         }
     }
 
-    pub fn get(&mut self, epoch: Epoch, root: Hash256) -> Option<&CommitteeCache> {
-        let opt = self.cache.get(&(epoch, root));
+    pub fn get(&mut self, key: &ShufflingId) -> Option<&CommitteeCache> {
+        let opt = self.cache.get(key);
 
         if opt.is_some() {
             metrics::inc_counter(&metrics::SHUFFLING_CACHE_HITS);
@@ -36,11 +36,37 @@ impl ShufflingCache {
         opt
     }
 
-    pub fn insert(&mut self, epoch: Epoch, root: Hash256, committee_cache: &CommitteeCache) {
-        let key = (epoch, root);
+    pub fn contains(&self, key: &ShufflingId) -> bool {
+        self.cache.contains(key)
+    }
 
+    pub fn insert(&mut self, key: ShufflingId, committee_cache: &CommitteeCache) {
         if !self.cache.contains(&key) {
             self.cache.put(key, committee_cache.clone());
         }
     }
 }
+
+/// Contains the shuffling IDs for a beacon block.
+pub struct BlockShufflingIds {
+    pub current: ShufflingId,
+    pub next: ShufflingId,
+    pub block_root: Hash256,
+}
+
+impl BlockShufflingIds {
+    /// Returns the shuffling ID for the given epoch.
+    ///
+    /// Returns `None` if `epoch` is prior to `self.current.shuffling_epoch`.
+    pub fn id_for_epoch(&self, epoch: Epoch) -> Option<ShufflingId> {
+        if epoch == self.current.shuffling_epoch {
+            Some(self.current.clone())
+        } else if epoch == self.next.shuffling_epoch {
+            Some(self.next.clone())
+        } else if epoch > self.next.shuffling_epoch {
+            Some(ShufflingId::from_components(epoch, self.block_root))
+        } else {
+            None
+        }
+    }
+}

beacon_node/beacon_chain/src/validator_pubkey_cache.rs

@@ -39,13 +39,6 @@ impl ValidatorPubkeyCache {
         state: &BeaconState<T>,
         persistence_path: P,
     ) -> Result<Self, BeaconChainError> {
-        if persistence_path.as_ref().exists() {
-            return Err(BeaconChainError::ValidatorPubkeyCacheFileError(format!(
-                "Persistence file already exists: {:?}",
-                persistence_path.as_ref()
-            )));
-        }
-
         let mut cache = Self {
             persitence_file: ValidatorPubkeyCacheFile::create(persistence_path)?,
             pubkeys: vec![],
@@ -159,8 +152,9 @@ impl ValidatorPubkeyCacheFile {
     /// Creates a file for reading and writing.
     pub fn create<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
         OpenOptions::new()
-            .create_new(true)
+            .create(true)
             .write(true)
+            .truncate(true)
             .open(path)
             .map(Self)
             .map_err(Error::Io)

beacon_node/beacon_chain/tests/attestation_production.rs

@@ -26,7 +26,7 @@ lazy_static! {
 fn produces_attestations() {
     let num_blocks_produced = MainnetEthSpec::slots_per_epoch() * 4;
 
-    let harness = BeaconChainHarness::new(
+    let harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[..].to_vec(),
         StoreConfig::default(),
@@ -55,7 +55,7 @@ fn produces_attestations() {
     // Test all valid committee indices for all slots in the chain.
     for slot in 0..=current_slot.as_u64() + MainnetEthSpec::slots_per_epoch() * 3 {
         let slot = Slot::from(slot);
-        let state = chain
+        let mut state = chain
             .state_at_slot(slot, StateSkipConfig::WithStateRoots)
             .expect("should get state");
 
@@ -81,6 +81,9 @@ fn produces_attestations() {
                 .expect("should get target block root")
         };
 
+        state
+            .build_committee_cache(RelativeEpoch::Current, &harness.chain.spec)
+            .unwrap();
         let committee_cache = state
             .committee_cache(RelativeEpoch::Current)
             .expect("should get committee_cache");

beacon_node/beacon_chain/tests/attestation_verification.rs

@@ -5,17 +5,19 @@ extern crate lazy_static;
 
 use beacon_chain::{
     attestation_verification::Error as AttnError,
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
     BeaconChain, BeaconChainTypes,
 };
 use int_to_bytes::int_to_bytes32;
-use state_processing::per_slot_processing;
+use state_processing::{
+    per_block_processing::errors::AttestationValidationError, per_slot_processing,
+};
 use store::config::StoreConfig;
 use tree_hash::TreeHash;
 use types::{
-    test_utils::generate_deterministic_keypair, AggregateSignature, Attestation, EthSpec, Hash256,
-    Keypair, MainnetEthSpec, SecretKey, SelectionProof, SignedAggregateAndProof, SignedBeaconBlock,
-    SubnetId, Unsigned,
+    test_utils::generate_deterministic_keypair, AggregateSignature, Attestation, BeaconStateError,
+    BitList, EthSpec, Hash256, Keypair, MainnetEthSpec, SecretKey, SelectionProof,
+    SignedAggregateAndProof, SignedBeaconBlock, SubnetId, Unsigned,
 };
 
 pub type E = MainnetEthSpec;
@@ -30,7 +32,7 @@ lazy_static! {
 }
 
 /// Returns a beacon chain harness.
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<E>> {
     let harness = BeaconChainHarness::new_with_target_aggregators(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
@@ -185,7 +187,6 @@ fn get_non_aggregator<T: BeaconChainTypes>(
 #[test]
 fn aggregated_gossip_verification() {
     let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -197,7 +198,7 @@ fn aggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
+    let current_slot = harness.chain.slot().expect("should get slot");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -275,6 +276,21 @@ fn aggregated_gossip_verification() {
             && earliest_permissible_slot == current_slot - E::slots_per_epoch() - 1
     );
 
+    /*
+     * This is not in the specification for aggregate attestations (only unaggregates), but we
+     * check it anyway to avoid weird edge cases.
+     */
+    let unknown_root = Hash256::from_low_u64_le(424242);
+    assert_invalid!(
+        "attestation with invalid target root",
+        {
+            let mut a = valid_aggregate.clone();
+            a.message.aggregate.data.target.root = unknown_root;
+            a
+        },
+        AttnError::InvalidTargetRoot { .. }
+    );
+
     /*
      * The following test ensures:
      *
@@ -533,7 +549,6 @@ fn aggregated_gossip_verification() {
 #[test]
 fn unaggregated_gossip_verification() {
     let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -545,8 +560,8 @@ fn unaggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -568,7 +583,7 @@ fn unaggregated_gossip_verification() {
                 matches!(
                     harness
                         .chain
-                        .verify_unaggregated_attestation_for_gossip($attn_getter, $subnet_getter)
+                        .verify_unaggregated_attestation_for_gossip($attn_getter, Some($subnet_getter))
                         .err()
                         .expect(&format!(
                             "{} should error during verify_unaggregated_attestation_for_gossip",
@@ -582,6 +597,31 @@ fn unaggregated_gossip_verification() {
         };
     }
 
+    /*
+     * The following test ensures:
+     *
+     * Spec v0.12.3
+     *
+     * The committee index is within the expected range -- i.e. `data.index <
+     * get_committee_count_per_slot(state, data.target.epoch)`.
+     */
+    assert_invalid!(
+        "attestation with invalid committee index",
+        {
+            let mut a = valid_attestation.clone();
+            a.data.index = harness
+                .chain
+                .head()
+                .unwrap()
+                .beacon_state
+                .get_committee_count_at_slot(a.data.slot)
+                .unwrap();
+            a
+        },
+        subnet_id,
+        AttnError::NoCommitteeForSlotAndIndex { .. }
+    );
+
     /*
      * The following test ensures:
      *
@@ -642,6 +682,7 @@ fn unaggregated_gossip_verification() {
         {
             let mut a = valid_attestation.clone();
             a.data.slot = early_slot;
+            a.data.target.epoch = early_slot.epoch(E::slots_per_epoch());
             a
         },
         subnet_id,
@@ -654,6 +695,27 @@ fn unaggregated_gossip_verification() {
         if attestation_slot == early_slot && earliest_permissible_slot == current_slot - E::slots_per_epoch() - 1
     );
 
+    /*
+     * The following test ensures:
+     *
+     * Spec v0.12.3
+     *
+     * The attestation's epoch matches its target -- i.e. `attestation.data.target.epoch ==
+     *   compute_epoch_at_slot(attestation.data.slot)`
+     *
+     */
+
+    assert_invalid!(
+        "attestation with invalid target epoch",
+        {
+            let mut a = valid_attestation.clone();
+            a.data.target.epoch += 1;
+            a
+        },
+        subnet_id,
+        AttnError::InvalidTargetEpoch { .. }
+    );
+
     /*
      * The following two tests ensure:
      *
@@ -694,6 +756,32 @@ fn unaggregated_gossip_verification() {
         AttnError::NotExactlyOneAggregationBitSet(2)
     );
 
+    /*
+     * The following test ensures:
+     *
+     * Spec v0.12.3
+     *
+     * The number of aggregation bits matches the committee size -- i.e.
+     *   `len(attestation.aggregation_bits) == len(get_beacon_committee(state, data.slot,
+     *   data.index))`.
+     */
+    assert_invalid!(
+        "attestation with invalid bitfield",
+        {
+            let mut a = valid_attestation.clone();
+            let bits = a.aggregation_bits.iter().collect::<Vec<_>>();
+            a.aggregation_bits = BitList::with_capacity(bits.len() + 1).unwrap();
+            for (i, bit) in bits.into_iter().enumerate() {
+                a.aggregation_bits.set(i, bit).unwrap();
+            }
+            a
+        },
+        subnet_id,
+        AttnError::Invalid(AttestationValidationError::BeaconStateError(
+            BeaconStateError::InvalidBitfield
+        ))
+    );
+
     /*
      * The following test ensures that:
      *
@@ -717,6 +805,26 @@ fn unaggregated_gossip_verification() {
         if beacon_block_root == unknown_root
     );
 
+    /*
+     * The following test ensures that:
+     *
+     * Spec v0.12.3
+     *
+     * The attestation's target block is an ancestor of the block named in the LMD vote
+     */
+
+    let unknown_root = Hash256::from_low_u64_le(424242);
+    assert_invalid!(
+        "attestation with invalid target root",
+        {
+            let mut a = valid_attestation.clone();
+            a.data.target.root = unknown_root;
+            a
+        },
+        subnet_id,
+        AttnError::InvalidTargetRoot { .. }
+    );
+
     /*
      * The following test ensures that:
      *
@@ -742,7 +850,7 @@ fn unaggregated_gossip_verification() {
 
     harness
         .chain
-        .verify_unaggregated_attestation_for_gossip(valid_attestation.clone(), subnet_id)
+        .verify_unaggregated_attestation_for_gossip(valid_attestation.clone(), Some(subnet_id))
         .expect("valid attestation should be verified");
 
     /*
@@ -773,7 +881,6 @@ fn unaggregated_gossip_verification() {
 #[test]
 fn attestation_that_skips_epochs() {
     let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -782,16 +889,18 @@ fn attestation_that_skips_epochs() {
         AttestationStrategy::SomeValidators(vec![]),
     );
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     let earlier_slot = (current_epoch - 2).start_slot(MainnetEthSpec::slots_per_epoch());
-    let earlier_block = chain
+    let earlier_block = harness
+        .chain
         .block_at_slot(earlier_slot)
         .expect("should not error getting block at slot")
         .expect("should find block at slot");
 
-    let mut state = chain
+    let mut state = harness
+        .chain
         .get_state(&earlier_block.state_root(), Some(earlier_slot))
         .expect("should not error getting state")
         .expect("should find state");
@@ -830,6 +939,6 @@ fn attestation_that_skips_epochs() {
 
     harness
         .chain
-        .verify_unaggregated_attestation_for_gossip(attestation, subnet_id)
+        .verify_unaggregated_attestation_for_gossip(attestation, Some(subnet_id))
         .expect("should gossip verify attestation that skips slots");
 }

beacon_node/beacon_chain/tests/block_verification.rs

@@ -4,7 +4,7 @@
 extern crate lazy_static;
 
 use beacon_chain::{
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
     BeaconSnapshot, BlockError,
 };
 use store::config::StoreConfig;
@@ -48,8 +48,8 @@ fn get_chain_segment() -> Vec<BeaconSnapshot<E>> {
         .collect()
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<E>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -81,7 +81,7 @@ fn junk_aggregate_signature() -> AggregateSignature {
 
 fn update_proposal_signatures(
     snapshots: &mut [BeaconSnapshot<E>],
-    harness: &BeaconChainHarness<HarnessType<E>>,
+    harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
 ) {
     for snapshot in snapshots {
         let spec = &harness.chain.spec;
@@ -91,7 +91,7 @@ fn update_proposal_signatures(
             .get_beacon_proposer_index(slot, spec)
             .expect("should find proposer index");
         let keypair = harness
-            .keypairs
+            .validator_keypairs
             .get(proposer_index)
             .expect("proposer keypair should be available");
 
@@ -274,7 +274,7 @@ fn chain_segment_non_linear_slots() {
 }
 
 fn assert_invalid_signature(
-    harness: &BeaconChainHarness<HarnessType<E>>,
+    harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
     block_index: usize,
     snapshots: &[BeaconSnapshot<E>],
     item: &str,
@@ -325,7 +325,7 @@ fn assert_invalid_signature(
     // slot) tuple.
 }
 
-fn get_invalid_sigs_harness() -> BeaconChainHarness<HarnessType<E>> {
+fn get_invalid_sigs_harness() -> BeaconChainHarness<EphemeralHarnessType<E>> {
     let harness = get_harness(VALIDATOR_COUNT);
     harness
         .chain

beacon_node/beacon_chain/tests/op_verification.rs

@@ -58,7 +58,7 @@ fn voluntary_exit() {
     let db_path = tempdir().unwrap();
     let store = get_store(&db_path);
     let harness = get_harness(store.clone(), VALIDATOR_COUNT);
-    let spec = &harness.chain.spec;
+    let spec = &harness.chain.spec.clone();
 
     harness.extend_chain(
         (E::slots_per_epoch() * (spec.shard_committee_period + 1)) as usize,

beacon_node/beacon_chain/tests/persistence_tests.rs

@@ -153,8 +153,11 @@ fn assert_chains_pretty_much_the_same<T: BeaconChainTypes>(a: &BeaconChain<T>, b
         a.genesis_block_root, b.genesis_block_root,
         "genesis_block_root should be equal"
     );
+
+    let slot = a.slot().unwrap();
     assert!(
-        *a.fork_choice.read() == *b.fork_choice.read(),
-        "fork_choice should be equal"
+        a.fork_choice.write().get_head(slot).unwrap()
+            == b.fork_choice.write().get_head(slot).unwrap(),
+        "fork_choice heads should be equal"
     );
 }

beacon_node/beacon_chain/tests/tests.rs

@@ -6,7 +6,8 @@ extern crate lazy_static;
 use beacon_chain::{
     attestation_verification::Error as AttnError,
     test_utils::{
-        AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType, OP_POOL_DB_KEY,
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType,
+        OP_POOL_DB_KEY,
     },
 };
 use operation_pool::PersistedOperationPool;
@@ -24,8 +25,8 @@ lazy_static! {
     static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<MinimalEthSpec>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<MinimalEthSpec>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MinimalEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -354,11 +355,10 @@ fn roundtrip_operation_pool() {
         .persist_op_pool()
         .expect("should persist op pool");
 
-    let key = Hash256::from_slice(&OP_POOL_DB_KEY);
     let restored_op_pool = harness
         .chain
         .store
-        .get_item::<PersistedOperationPool<MinimalEthSpec>>(&key)
+        .get_item::<PersistedOperationPool<MinimalEthSpec>>(&OP_POOL_DB_KEY)
         .expect("should read db")
         .expect("should find op pool")
         .into_operation_pool();
@@ -436,8 +436,8 @@ fn attestations_with_increasing_slots() {
             AttestationStrategy::SomeValidators(vec![]),
         );
 
-        attestations.append(
-            &mut harness.get_unaggregated_attestations(
+        attestations.extend(
+            harness.get_unaggregated_attestations(
                 &AttestationStrategy::AllValidators,
                 &harness.chain.head().expect("should get head").beacon_state,
                 harness
@@ -460,7 +460,7 @@ fn attestations_with_increasing_slots() {
     for (attestation, subnet_id) in attestations.into_iter().flatten() {
         let res = harness
             .chain
-            .verify_unaggregated_attestation_for_gossip(attestation.clone(), subnet_id);
+            .verify_unaggregated_attestation_for_gossip(attestation.clone(), Some(subnet_id));
 
         let current_slot = harness.chain.slot().expect("should get slot");
         let expected_attestation_slot = attestation.data.slot;

beacon_node/client/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
 [dev-dependencies]
-sloggers = "1.0.0"
+sloggers = "1.0.1"
 toml = "0.5.6"
 
 [dependencies]
@@ -14,30 +14,34 @@ store = { path = "../store" }
 network = { path = "../network" }
 timer = { path = "../timer" }
 eth2_libp2p = { path = "../eth2_libp2p" }
-rest_api = { path = "../rest_api" }
 parking_lot = "0.11.0"
 websocket_server = { path = "../websocket_server" }
-prometheus = "0.9.0"
+prometheus = "0.10.0"
 types = { path = "../../consensus/types" }
-tree_hash = "0.1.0"
+tree_hash = "0.1.1"
 eth2_config = { path = "../../common/eth2_config" }
 slot_clock = { path = "../../common/slot_clock" }
-serde = "1.0.110"
-serde_derive = "1.0.110"
-error-chain = "0.12.2"
-serde_yaml = "0.8.11"
+serde = "1.0.116"
+serde_derive = "1.0.116"
+error-chain = "0.12.4"
+serde_yaml = "0.8.13"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 slog-async = "2.5.0"
-tokio = "0.2.21"
-dirs = "2.0.2"
+tokio = "0.2.22"
+dirs = "3.0.1"
 futures = "0.3.5"
-reqwest = { version = "0.10.4", features = ["native-tls-vendored"] }
+reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
 url = "2.1.1"
 eth1 = { path = "../eth1" }
 genesis = { path = "../genesis" }
+task_executor = { path = "../../common/task_executor" }
 environment = { path = "../../lighthouse/environment" }
 eth2_ssz = "0.1.2"
 lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-time = "0.2.16"
+time = "0.2.22"
 bus = "2.2.3"
+directory = {path = "../../common/directory"}
+http_api = { path = "../http_api" }
+http_metrics = { path = "../http_metrics" }
+slasher = { path = "../../slasher" }

beacon_node/client/src/builder.rs

@@ -5,7 +5,6 @@ use beacon_chain::events::TeeEventHandler;
 use beacon_chain::{
     builder::{BeaconChainBuilder, Witness},
     eth1_chain::{CachingEth1Backend, Eth1Chain},
-    migrate::{BackgroundMigrator, Migrate},
     slot_clock::{SlotClock, SystemTimeSlotClock},
     store::{HotColdDB, ItemStore, LevelDB, StoreConfig},
     BeaconChain, BeaconChainTypes, Eth1ChainBackend, EventHandler,
@@ -13,19 +12,20 @@ use beacon_chain::{
 use bus::Bus;
 use environment::RuntimeContext;
 use eth1::{Config as Eth1Config, Service as Eth1Service};
-use eth2_config::Eth2Config;
 use eth2_libp2p::NetworkGlobals;
 use genesis::{interop_genesis_state, Eth1GenesisService};
 use network::{NetworkConfig, NetworkMessage, NetworkService};
 use parking_lot::Mutex;
-use slog::info;
+use slasher::{Slasher, SlasherServer};
+use slog::{debug, info, warn};
 use ssz::Decode;
 use std::net::SocketAddr;
-use std::path::Path;
+use std::net::TcpListener;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 use timer::spawn_timer;
-use tokio::sync::mpsc::UnboundedSender;
+use tokio::sync::{mpsc::UnboundedSender, oneshot};
 use types::{
     test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec,
     SignedBeaconBlockHash,
@@ -52,7 +52,6 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
     slot_clock: Option<T::SlotClock>,
     #[allow(clippy::type_complexity)]
     store: Option<Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>>,
-    store_migrator: Option<T::StoreMigrator>,
     runtime_context: Option<RuntimeContext<T::EthSpec>>,
     chain_spec: Option<ChainSpec>,
     beacon_chain_builder: Option<BeaconChainBuilder<T>>,
@@ -61,25 +60,18 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
     event_handler: Option<T::EventHandler>,
     network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
     network_send: Option<UnboundedSender<NetworkMessage<T::EthSpec>>>,
-    http_listen_addr: Option<SocketAddr>,
+    db_path: Option<PathBuf>,
+    freezer_db_path: Option<PathBuf>,
+    http_api_config: http_api::Config,
+    http_metrics_config: http_metrics::Config,
     websocket_listen_addr: Option<SocketAddr>,
+    slasher: Option<Arc<Slasher<T::EthSpec>>>,
     eth_spec_instance: T::EthSpec,
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+    ClientBuilder<Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>>
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + Clone + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -94,7 +86,6 @@ where
         Self {
             slot_clock: None,
             store: None,
-            store_migrator: None,
             runtime_context: None,
             chain_spec: None,
             beacon_chain_builder: None,
@@ -103,8 +94,12 @@ where
             event_handler: None,
             network_globals: None,
             network_send: None,
-            http_listen_addr: None,
+            db_path: None,
+            freezer_db_path: None,
+            http_api_config: <_>::default(),
+            http_metrics_config: <_>::default(),
             websocket_listen_addr: None,
+            slasher: None,
             eth_spec_instance,
         }
     }
@@ -121,6 +116,11 @@ where
         self
     }
 
+    pub fn slasher(mut self, slasher: Arc<Slasher<TEthSpec>>) -> Self {
+        self.slasher = Some(slasher);
+        self
+    }
+
     /// Initializes the `BeaconChainBuilder`. The `build_beacon_chain` method will need to be
     /// called later in order to actually instantiate the `BeaconChain`.
     pub async fn beacon_chain_builder(
@@ -129,7 +129,6 @@ where
         config: ClientConfig,
     ) -> Result<Self, String> {
         let store = self.store.clone();
-        let store_migrator = self.store_migrator.take();
         let chain_spec = self.chain_spec.clone();
         let runtime_context = self.runtime_context.clone();
         let eth_spec_instance = self.eth_spec_instance.clone();
@@ -140,8 +139,6 @@ where
 
         let store =
             store.ok_or_else(|| "beacon_chain_start_method requires a store".to_string())?;
-        let store_migrator = store_migrator
-            .ok_or_else(|| "beacon_chain_start_method requires a store migrator".to_string())?;
         let context = runtime_context
             .ok_or_else(|| "beacon_chain_start_method requires a runtime context".to_string())?
             .service_context("beacon".into());
@@ -151,13 +148,18 @@ where
         let builder = BeaconChainBuilder::new(eth_spec_instance)
             .logger(context.log().clone())
             .store(store)
-            .store_migrator(store_migrator)
             .data_dir(data_dir)
             .custom_spec(spec.clone())
             .chain_config(chain_config)
             .disabled_forks(disabled_forks)
             .graffiti(graffiti);
 
+        let builder = if let Some(slasher) = self.slasher.clone() {
+            builder.slasher(slasher)
+        } else {
+            builder
+        };
+
         let chain_exists = builder
             .store_contains_beacon_chain()
             .unwrap_or_else(|_| false);
@@ -204,7 +206,7 @@ where
                 info!(
                     context.log(),
                     "Waiting for eth2 genesis from eth1";
-                    "eth1_endpoint" => &config.eth1.endpoint,
+                    "eth1_endpoints" => format!("{:?}", &config.eth1.endpoints),
                     "contract_deploy_block" => config.eth1.deposit_contract_deploy_block,
                     "deposit_contract" => &config.eth1.deposit_contract_address
                 );
@@ -215,6 +217,53 @@ where
                     context.eth2_config().spec.clone(),
                 );
 
+                // If the HTTP API server is enabled, start an instance of it where it only
+                // contains a reference to the eth1 service (all non-eth1 endpoints will fail
+                // gracefully).
+                //
+                // Later in this function we will shutdown this temporary "waiting for genesis"
+                // server so the real one can be started later.
+                let (exit_tx, exit_rx) = oneshot::channel::<()>();
+                let http_listen_opt = if self.http_api_config.enabled {
+                    #[allow(clippy::type_complexity)]
+                    let ctx: Arc<
+                        http_api::Context<
+                            Witness<
+                                TSlotClock,
+                                TEth1Backend,
+                                TEthSpec,
+                                TEventHandler,
+                                THotStore,
+                                TColdStore,
+                            >,
+                        >,
+                    > = Arc::new(http_api::Context {
+                        config: self.http_api_config.clone(),
+                        chain: None,
+                        network_tx: None,
+                        network_globals: None,
+                        eth1_service: Some(genesis_service.eth1_service.clone()),
+                        log: context.log().clone(),
+                    });
+
+                    // Discard the error from the oneshot.
+                    let exit_future = async {
+                        let _ = exit_rx.await;
+                    };
+
+                    let (listen_addr, server) = http_api::serve(ctx, exit_future)
+                        .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
+
+                    context
+                        .clone()
+                        .executor
+                        .spawn_without_exit(async move { server.await }, "http-api");
+
+                    Some(listen_addr)
+                } else {
+                    None
+                };
+
                 let genesis_state = genesis_service
                     .wait_for_genesis_state(
                         Duration::from_millis(ETH1_GENESIS_UPDATE_INTERVAL_MILLIS),
@@ -222,6 +271,22 @@ where
                     )
                     .await?;
 
+                let _ = exit_tx.send(());
+
+                if let Some(http_listen) = http_listen_opt {
+                    // This is a bit of a hack to ensure that the HTTP server has indeed shutdown.
+                    //
+                    // We will restart it again after we've finished setting up for genesis.
+                    while TcpListener::bind(http_listen).is_err() {
+                        warn!(
+                            context.log(),
+                            "Waiting for HTTP server port to open";
+                            "port" => http_listen
+                        );
+                        tokio::time::delay_for(Duration::from_secs(1)).await;
+                    }
+                }
+
                 builder
                     .genesis_state(genesis_state)
                     .map(|v| (v, Some(genesis_service.into_core_service())))?
@@ -280,55 +345,37 @@ where
         Ok(self)
     }
 
-    /// Immediately starts the beacon node REST API http server.
-    pub fn http_server(
-        mut self,
-        client_config: &ClientConfig,
-        eth2_config: &Eth2Config,
-        events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
-    ) -> Result<Self, String> {
-        let beacon_chain = self
-            .beacon_chain
-            .clone()
-            .ok_or_else(|| "http_server requires a beacon chain")?;
+    /// Provides configuration for the HTTP API.
+    pub fn http_api_config(mut self, config: http_api::Config) -> Self {
+        self.http_api_config = config;
+        self
+    }
+
+    /// Provides configuration for the HTTP server that serves Prometheus metrics.
+    pub fn http_metrics_config(mut self, config: http_metrics::Config) -> Self {
+        self.http_metrics_config = config;
+        self
+    }
+
+    /// Immediately start the slasher service.
+    ///
+    /// Error if no slasher is configured.
+    pub fn start_slasher_server(&self) -> Result<(), String> {
         let context = self
             .runtime_context
             .as_ref()
-            .ok_or_else(|| "http_server requires a runtime_context")?
-            .service_context("http".into());
-        let network_globals = self
-            .network_globals
+            .ok_or_else(|| "slasher requires a runtime_context")?
+            .service_context("slasher_server_ctxt".into());
+        let slasher = self
+            .slasher
             .clone()
-            .ok_or_else(|| "http_server requires a libp2p network")?;
-        let network_send = self
-            .network_send
+            .ok_or_else(|| "slasher server requires a slasher")?;
+        let slot_clock = self
+            .slot_clock
             .clone()
-            .ok_or_else(|| "http_server requires a libp2p network sender")?;
-
-        let network_info = rest_api::NetworkInfo {
-            network_globals,
-            network_chan: network_send,
-        };
-
-        let listening_addr = rest_api::start_server(
-            context.executor,
-            &client_config.rest_api,
-            beacon_chain,
-            network_info,
-            client_config
-                .create_db_path()
-                .map_err(|_| "unable to read data dir")?,
-            client_config
-                .create_freezer_db_path()
-                .map_err(|_| "unable to read freezer DB dir")?,
-            eth2_config.clone(),
-            events,
-        )
-        .map_err(|e| format!("Failed to start HTTP API: {:?}", e))?;
-
-        self.http_listen_addr = Some(listening_addr);
-
-        Ok(self)
+            .ok_or_else(|| "slasher server requires a slot clock")?;
+        SlasherServer::run(slasher, slot_clock, &context.executor);
+        Ok(())
     }
 
     /// Immediately starts the service that periodically logs information each slot.
@@ -367,42 +414,86 @@ where
     /// specified.
     ///
     /// If type inference errors are being raised, see the comment on the definition of `Self`.
+    #[allow(clippy::type_complexity)]
     pub fn build(
         self,
-    ) -> Client<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+    ) -> Result<
+        Client<Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>>,
+        String,
     > {
-        Client {
+        let runtime_context = self
+            .runtime_context
+            .as_ref()
+            .ok_or_else(|| "build requires a runtime context".to_string())?;
+        let log = runtime_context.log().clone();
+
+        let http_api_listen_addr = if self.http_api_config.enabled {
+            let ctx = Arc::new(http_api::Context {
+                config: self.http_api_config.clone(),
+                chain: self.beacon_chain.clone(),
+                network_tx: self.network_send.clone(),
+                network_globals: self.network_globals.clone(),
+                eth1_service: self.eth1_service.clone(),
+                log: log.clone(),
+            });
+
+            let exit = runtime_context.executor.exit();
+
+            let (listen_addr, server) = http_api::serve(ctx, exit)
+                .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
+
+            runtime_context
+                .clone()
+                .executor
+                .spawn_without_exit(async move { server.await }, "http-api");
+
+            Some(listen_addr)
+        } else {
+            info!(log, "HTTP server is disabled");
+            None
+        };
+
+        let http_metrics_listen_addr = if self.http_metrics_config.enabled {
+            let ctx = Arc::new(http_metrics::Context {
+                config: self.http_metrics_config.clone(),
+                chain: self.beacon_chain.clone(),
+                db_path: self.db_path.clone(),
+                freezer_db_path: self.freezer_db_path.clone(),
+                log: log.clone(),
+            });
+
+            let exit = runtime_context.executor.exit();
+
+            let (listen_addr, server) = http_metrics::serve(ctx, exit)
+                .map_err(|e| format!("Unable to start HTTP metrics server: {:?}", e))?;
+
+            runtime_context
+                .executor
+                .spawn_without_exit(async move { server.await }, "http-metrics");
+
+            Some(listen_addr)
+        } else {
+            debug!(log, "Metrics server is disabled");
+            None
+        };
+
+        if self.slasher.is_some() {
+            self.start_slasher_server()?;
+        }
+
+        Ok(Client {
             beacon_chain: self.beacon_chain,
             network_globals: self.network_globals,
-            http_listen_addr: self.http_listen_addr,
+            http_api_listen_addr,
+            http_metrics_listen_addr,
             websocket_listen_addr: self.websocket_listen_addr,
-        }
+        })
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+    ClientBuilder<Witness<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>>
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + Clone + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -412,6 +503,12 @@ where
 {
     /// Consumes the internal `BeaconChainBuilder`, attaching the resulting `BeaconChain` to self.
     pub fn build_beacon_chain(mut self) -> Result<Self, String> {
+        let context = self
+            .runtime_context
+            .as_ref()
+            .ok_or_else(|| "beacon_chain requires a runtime context")?
+            .clone();
+
         let chain = self
             .beacon_chain_builder
             .ok_or_else(|| "beacon_chain requires a beacon_chain_builder")?
@@ -424,6 +521,7 @@ where
                     .clone()
                     .ok_or_else(|| "beacon_chain requires a slot clock")?,
             )
+            .shutdown_sender(context.executor.shutdown_sender())
             .build()
             .map_err(|e| format!("Failed to build beacon chain: {}", e))?;
 
@@ -436,10 +534,9 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
+impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
     ClientBuilder<
         Witness<
-            TStoreMigrator,
             TSlotClock,
             TEth1Backend,
             TEthSpec,
@@ -449,7 +546,6 @@ impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
         >,
     >
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
@@ -484,10 +580,9 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
     ClientBuilder<
         Witness<
-            TStoreMigrator,
             TSlotClock,
             TEth1Backend,
             TEthSpec,
@@ -498,7 +593,6 @@ impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
     >
 where
     TSlotClock: SlotClock + 'static,
-    TStoreMigrator: Migrate<TEthSpec, LevelDB<TEthSpec>, LevelDB<TEthSpec>> + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -520,6 +614,9 @@ where
             .clone()
             .ok_or_else(|| "disk_store requires a chain spec".to_string())?;
 
+        self.db_path = Some(hot_path.into());
+        self.freezer_db_path = Some(cold_path.into());
+
         let store = HotColdDB::open(hot_path, cold_path, config, spec, context.log().clone())
             .map_err(|e| format!("Unable to open database: {:?}", e))?;
         self.store = Some(Arc::new(store));
@@ -527,44 +624,9 @@ where
     }
 }
 
-impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
     ClientBuilder<
         Witness<
-            BackgroundMigrator<TEthSpec, THotStore, TColdStore>,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
-where
-    TSlotClock: SlotClock + 'static,
-    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
-    TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
-    THotStore: ItemStore<TEthSpec> + 'static,
-    TColdStore: ItemStore<TEthSpec> + 'static,
-{
-    pub fn background_migrator(mut self) -> Result<Self, String> {
-        let context = self
-            .runtime_context
-            .as_ref()
-            .ok_or_else(|| "disk_store requires a log".to_string())?
-            .service_context("freezer_db".into());
-        let store = self.store.clone().ok_or_else(|| {
-            "background_migrator requires the store to be initialized".to_string()
-        })?;
-        self.store_migrator = Some(BackgroundMigrator::new(store, context.log().clone()));
-        Ok(self)
-    }
-}
-
-impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
             TSlotClock,
             CachingEth1Backend<TEthSpec>,
             TEthSpec,
@@ -574,7 +636,6 @@ impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
         >,
     >
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -584,7 +645,7 @@ where
     /// Specifies that the `BeaconChain` should cache eth1 blocks/logs from a remote eth1 node
     /// (e.g., Parity/Geth) and refer to that cache when collecting deposits or eth1 votes during
     /// block production.
-    pub fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
+    pub async fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
         let context = self
             .runtime_context
             .as_ref()
@@ -633,7 +694,7 @@ where
                 })?
         };
 
-        self.eth1_service = None;
+        self.eth1_service = Some(backend.core.clone());
 
         // Starts the service that connects to an eth1 node and periodically updates caches.
         backend.start(context.executor);
@@ -674,20 +735,11 @@ where
     }
 }
 
-impl<TStoreMigrator, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
     ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            SystemTimeSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+        Witness<SystemTimeSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>,
     >
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
     TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -702,11 +754,8 @@ where
             .ok_or_else(|| "system_time_slot_clock requires a beacon_chain_builder")?;
 
         let genesis_time = beacon_chain_builder
-            .finalized_snapshot
-            .as_ref()
-            .ok_or_else(|| "system_time_slot_clock requires an initialized beacon state")?
-            .beacon_state
-            .genesis_time;
+            .genesis_time
+            .ok_or_else(|| "system_time_slot_clock requires an initialized beacon state")?;
 
         let spec = self
             .chain_spec

beacon_node/client/src/config.rs

@@ -1,14 +1,10 @@
+use directory::DEFAULT_ROOT_DIR;
 use network::NetworkConfig;
 use serde_derive::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
 use types::Graffiti;
 
-pub const DEFAULT_DATADIR: &str = ".lighthouse";
-
-/// The number initial validators when starting the `Minimal`.
-const TESTNET_SPEC_CONSTANTS: &str = "minimal";
-
 /// Default directory name for the freezer database under the top-level data dir.
 const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db";
 
@@ -47,7 +43,6 @@ pub struct Config {
     /// Path where the freezer database will be located.
     pub freezer_db_path: Option<PathBuf>,
     pub log_file: PathBuf,
-    pub spec_constants: String,
     /// If true, the node will use co-ordinated junk for eth1 values.
     ///
     /// This is the method used for the 2019 client interop in Canada.
@@ -63,16 +58,18 @@ pub struct Config {
     pub genesis: ClientGenesis,
     pub store: store::StoreConfig,
     pub network: network::NetworkConfig,
-    pub rest_api: rest_api::Config,
     pub chain: beacon_chain::ChainConfig,
     pub websocket_server: websocket_server::Config,
     pub eth1: eth1::Config,
+    pub http_api: http_api::Config,
+    pub http_metrics: http_metrics::Config,
+    pub slasher: Option<slasher::Config>,
 }
 
 impl Default for Config {
     fn default() -> Self {
         Self {
-            data_dir: PathBuf::from(DEFAULT_DATADIR),
+            data_dir: PathBuf::from(DEFAULT_ROOT_DIR),
             db_name: "chain_db".to_string(),
             freezer_db_path: None,
             log_file: PathBuf::from(""),
@@ -80,14 +77,15 @@ impl Default for Config {
             store: <_>::default(),
             network: NetworkConfig::default(),
             chain: <_>::default(),
-            rest_api: <_>::default(),
             websocket_server: <_>::default(),
-            spec_constants: TESTNET_SPEC_CONSTANTS.into(),
             dummy_eth1_backend: false,
             sync_eth1_chain: false,
             eth1: <_>::default(),
             disabled_forks: Vec::new(),
             graffiti: Graffiti::default(),
+            http_api: <_>::default(),
+            http_metrics: <_>::default(),
+            slasher: None,
         }
     }
 }

beacon_node/client/src/lib.rs

@@ -23,7 +23,10 @@ pub use eth2_config::Eth2Config;
 pub struct Client<T: BeaconChainTypes> {
     beacon_chain: Option<Arc<BeaconChain<T>>>,
     network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
-    http_listen_addr: Option<SocketAddr>,
+    /// Listen address for the standard eth2.0 API, if the service was started.
+    http_api_listen_addr: Option<SocketAddr>,
+    /// Listen address for the HTTP server which serves Prometheus metrics.
+    http_metrics_listen_addr: Option<SocketAddr>,
     websocket_listen_addr: Option<SocketAddr>,
 }
 
@@ -33,9 +36,14 @@ impl<T: BeaconChainTypes> Client<T> {
         self.beacon_chain.clone()
     }
 
-    /// Returns the address of the client's HTTP API server, if it was started.
-    pub fn http_listen_addr(&self) -> Option<SocketAddr> {
-        self.http_listen_addr
+    /// Returns the address of the client's standard eth2.0 API server, if it was started.
+    pub fn http_api_listen_addr(&self) -> Option<SocketAddr> {
+        self.http_api_listen_addr
+    }
+
+    /// Returns the address of the client's HTTP Prometheus metrics server, if it was started.
+    pub fn http_metrics_listen_addr(&self) -> Option<SocketAddr> {
+        self.http_metrics_listen_addr
     }
 
     /// Returns the address of the client's WebSocket API server, if it was started.

beacon_node/client/src/notifier.rs

@@ -22,7 +22,7 @@ const SPEEDO_OBSERVATIONS: usize = 4;
 
 /// Spawns a notifier service which periodically logs information about the node.
 pub fn spawn_notifier<T: BeaconChainTypes>(
-    executor: environment::TaskExecutor,
+    executor: task_executor::TaskExecutor,
     beacon_chain: Arc<BeaconChain<T>>,
     network: Arc<NetworkGlobals<T::EthSpec>>,
     milliseconds_per_slot: u64,
@@ -171,6 +171,38 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                     "current_slot" => current_slot,
                 );
             }
+
+            // Perform some logging about the eth1 chain
+            if let Some(eth1_chain) = beacon_chain.eth1_chain.as_ref() {
+                if let Some(status) =
+                    eth1_chain.sync_status(head_info.genesis_time, current_slot, &beacon_chain.spec)
+                {
+                    debug!(
+                        log,
+                        "Eth1 cache sync status";
+                        "eth1_head_block" => status.head_block_number,
+                        "latest_cached_block_number" => status.latest_cached_block_number,
+                        "latest_cached_timestamp" => status.latest_cached_block_timestamp,
+                        "voting_target_timestamp" => status.voting_target_timestamp,
+                        "ready" => status.lighthouse_is_cached_and_ready
+                    );
+
+                    if !status.lighthouse_is_cached_and_ready {
+                        warn!(
+                            log,
+                            "Syncing eth1 block cache";
+                            "target_timestamp" => status.voting_target_timestamp,
+                            "latest_timestamp" => status.latest_cached_block_timestamp,
+                            "msg" => "block production temporarily impaired"
+                        );
+                    }
+                } else {
+                    error!(
+                        log,
+                        "Unable to determine eth1 sync status";
+                    );
+                }
+            }
         }
         Ok::<(), ()>(())
     };

beacon_node/eth1/Cargo.toml

@@ -8,25 +8,28 @@ edition = "2018"
 eth1_test_rig = { path = "../../testing/eth1_test_rig" }
 toml = "0.5.6"
 web3 = "0.11.0"
-sloggers = "1.0.0"
+sloggers = "1.0.1"
+environment = { path = "../../lighthouse/environment" }
 
 [dependencies]
-reqwest = { version = "0.10.4", features = ["native-tls-vendored"] }
+reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
 futures = { version = "0.3.5", features = ["compat"] }
-serde_json = "1.0.52"
-serde = { version = "1.0.110", features = ["derive"] }
+serde_json = "1.0.58"
+serde = { version = "1.0.116", features = ["derive"] }
 hex = "0.4.2"
 types = { path = "../../consensus/types"}
 merkle_proof = { path = "../../consensus/merkle_proof"}
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
-tree_hash = "0.1.0"
+tree_hash = "0.1.1"
 eth2_hashing = "0.1.0"
 parking_lot = "0.11.0"
 slog = "2.5.2"
-tokio = { version = "0.2.21", features = ["full"] }
+tokio = { version = "0.2.22", features = ["full"] }
 state_processing = { path = "../../consensus/state_processing" }
-libflate = "1.0.0"
+libflate = "1.0.2"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics"}
 lazy_static = "1.4.0"
-environment = { path = "../../lighthouse/environment" }
+task_executor = { path = "../../common/task_executor" }
+eth2 = { path = "../../common/eth2" }
+fallback = { path = "../../common/fallback" }

beacon_node/eth1/src/block_cache.rs

@@ -1,6 +1,7 @@
 use ssz_derive::{Decode, Encode};
 use std::ops::RangeInclusive;
-use types::{Eth1Data, Hash256};
+
+pub use eth2::lighthouse::Eth1Block;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Error {
@@ -15,28 +16,6 @@ pub enum Error {
     Internal(String),
 }
 
-/// A block of the eth1 chain.
-///
-/// Contains all information required to add a `BlockCache` entry.
-#[derive(Debug, PartialEq, Clone, Eq, Hash, Encode, Decode)]
-pub struct Eth1Block {
-    pub hash: Hash256,
-    pub timestamp: u64,
-    pub number: u64,
-    pub deposit_root: Option<Hash256>,
-    pub deposit_count: Option<u64>,
-}
-
-impl Eth1Block {
-    pub fn eth1_data(self) -> Option<Eth1Data> {
-        Some(Eth1Data {
-            deposit_root: self.deposit_root?,
-            deposit_count: self.deposit_count?,
-            block_hash: self.hash,
-        })
-    }
-}
-
 /// Stores block and deposit contract information and provides queries based upon the block
 /// timestamp.
 #[derive(Debug, PartialEq, Clone, Default, Encode, Decode)]
@@ -55,6 +34,16 @@ impl BlockCache {
         self.blocks.is_empty()
     }
 
+    /// Returns the earliest (lowest timestamp) block, if any.
+    pub fn earliest_block(&self) -> Option<&Eth1Block> {
+        self.blocks.first()
+    }
+
+    /// Returns the latest (highest timestamp) block, if any.
+    pub fn latest_block(&self) -> Option<&Eth1Block> {
+        self.blocks.last()
+    }
+
     /// Returns the timestamp of the earliest block in the cache (if any).
     pub fn earliest_block_timestamp(&self) -> Option<u64> {
         self.blocks.first().map(|block| block.timestamp)
@@ -181,6 +170,7 @@ impl BlockCache {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use types::Hash256;
 
     fn get_block(i: u64, interval_secs: u64) -> Eth1Block {
         Eth1Block {

beacon_node/eth1/src/deposit_cache.rs

@@ -304,7 +304,7 @@ pub mod tests {
             block_number: 42,
             data: EXAMPLE_LOG.to_vec(),
         };
-        DepositLog::from_log(&log, &spec).expect("should decode log")
+        log.to_deposit_log(&spec).expect("should decode log")
     }
 
     #[test]

beacon_node/eth1/src/deposit_log.rs

@@ -1,11 +1,10 @@
 use super::http::Log;
 use ssz::Decode;
-use ssz_derive::{Decode, Encode};
-use state_processing::per_block_processing::signature_sets::{
-    deposit_pubkey_signature_message, deposit_signature_set,
-};
+use state_processing::per_block_processing::signature_sets::deposit_pubkey_signature_message;
 use types::{ChainSpec, DepositData, Hash256, PublicKeyBytes, SignatureBytes};
 
+pub use eth2::lighthouse::DepositLog;
+
 /// The following constants define the layout of bytes in the deposit contract `DepositEvent`. The
 /// event bytes are formatted according to the  Ethereum ABI.
 const PUBKEY_START: usize = 192;
@@ -19,22 +18,10 @@ const SIG_LEN: usize = 96;
 const INDEX_START: usize = SIG_START + 96 + 32;
 const INDEX_LEN: usize = 8;
 
-/// A fully parsed eth1 deposit contract log.
-#[derive(Debug, PartialEq, Clone, Encode, Decode)]
-pub struct DepositLog {
-    pub deposit_data: DepositData,
-    /// The block number of the log that included this `DepositData`.
-    pub block_number: u64,
-    /// The index included with the deposit log.
-    pub index: u64,
-    /// True if the signature is valid.
-    pub signature_is_valid: bool,
-}
-
-impl DepositLog {
+impl Log {
     /// Attempts to parse a raw `Log` from the deposit contract into a `DepositLog`.
-    pub fn from_log(log: &Log, spec: &ChainSpec) -> Result<Self, String> {
-        let bytes = &log.data;
+    pub fn to_deposit_log(&self, spec: &ChainSpec) -> Result<DepositLog, String> {
+        let bytes = &self.data;
 
         let pubkey = bytes
             .get(PUBKEY_START..PUBKEY_START + PUBKEY_LEN)
@@ -64,11 +51,13 @@ impl DepositLog {
         };
 
         let signature_is_valid = deposit_pubkey_signature_message(&deposit_data, spec)
-            .map_or(false, |msg| deposit_signature_set(&msg).verify());
+            .map_or(false, |(public_key, signature, msg)| {
+                signature.verify(&public_key, msg)
+            });
 
         Ok(DepositLog {
             deposit_data,
-            block_number: log.block_number,
+            block_number: self.block_number,
             index: u64::from_ssz_bytes(index).map_err(|e| format!("Invalid index ssz: {:?}", e))?,
             signature_is_valid,
         })
@@ -77,7 +66,6 @@ impl DepositLog {
 
 #[cfg(test)]
 pub mod tests {
-    use super::*;
     use crate::http::Log;
     use types::{EthSpec, MainnetEthSpec};
 
@@ -113,6 +101,7 @@ pub mod tests {
             block_number: 42,
             data: EXAMPLE_LOG.to_vec(),
         };
-        DepositLog::from_log(&log, &MainnetEthSpec::default_spec()).expect("should decode log");
+        log.to_deposit_log(&MainnetEthSpec::default_spec())
+            .expect("should decode log");
     }
 }

beacon_node/eth1/src/http.rs

@@ -12,8 +12,10 @@
 
 use futures::future::TryFutureExt;
 use reqwest::{header::CONTENT_TYPE, ClientBuilder, StatusCode};
+use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
 use std::ops::Range;
+use std::str::FromStr;
 use std::time::Duration;
 use types::Hash256;
 
@@ -30,6 +32,75 @@ pub const DEPOSIT_COUNT_RESPONSE_BYTES: usize = 96;
 /// Number of bytes in deposit contract deposit root (value only).
 pub const DEPOSIT_ROOT_BYTES: usize = 32;
 
+/// Represents an eth1 chain/network id.
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub enum Eth1Id {
+    Goerli,
+    Mainnet,
+    Custom(u64),
+}
+
+/// Used to identify a block when querying the Eth1 node.
+#[derive(Clone, Copy)]
+pub enum BlockQuery {
+    Number(u64),
+    Latest,
+}
+
+impl Into<u64> for Eth1Id {
+    fn into(self) -> u64 {
+        match self {
+            Eth1Id::Mainnet => 1,
+            Eth1Id::Goerli => 5,
+            Eth1Id::Custom(id) => id,
+        }
+    }
+}
+
+impl From<u64> for Eth1Id {
+    fn from(id: u64) -> Self {
+        let into = |x: Eth1Id| -> u64 { x.into() };
+        match id {
+            id if id == into(Eth1Id::Mainnet) => Eth1Id::Mainnet,
+            id if id == into(Eth1Id::Goerli) => Eth1Id::Goerli,
+            id => Eth1Id::Custom(id),
+        }
+    }
+}
+
+impl FromStr for Eth1Id {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        u64::from_str_radix(s, 10)
+            .map(Into::into)
+            .map_err(|e| format!("Failed to parse eth1 network id {}", e))
+    }
+}
+
+/// Get the eth1 network id of the given endpoint.
+pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id, String> {
+    let response_body = send_rpc_request(endpoint, "net_version", json!([]), timeout).await?;
+    Eth1Id::from_str(
+        response_result(&response_body)?
+            .ok_or_else(|| "No result was returned for network id".to_string())?
+            .as_str()
+            .ok_or_else(|| "Data was not string")?,
+    )
+}
+
+/// Get the eth1 chain id of the given endpoint.
+pub async fn get_chain_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id, String> {
+    let response_body = send_rpc_request(endpoint, "eth_chainId", json!([]), timeout).await?;
+    hex_to_u64_be(
+        response_result(&response_body)?
+            .ok_or_else(|| "No result was returned for chain id".to_string())?
+            .as_str()
+            .ok_or_else(|| "Data was not string")?,
+    )
+    .map(Into::into)
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct Block {
     pub hash: Hash256,
@@ -56,11 +127,15 @@ pub async fn get_block_number(endpoint: &str, timeout: Duration) -> Result<u64,
 /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
 pub async fn get_block(
     endpoint: &str,
-    block_number: u64,
+    query: BlockQuery,
     timeout: Duration,
 ) -> Result<Block, String> {
+    let query_param = match query {
+        BlockQuery::Number(block_number) => format!("0x{:x}", block_number),
+        BlockQuery::Latest => "latest".to_string(),
+    };
     let params = json!([
-        format!("0x{:x}", block_number),
+        query_param,
         false // do not return full tx objects.
     ]);
 

beacon_node/eth1/src/inner.rs

@@ -1,6 +1,6 @@
 use crate::Config;
 use crate::{
-    block_cache::BlockCache,
+    block_cache::{BlockCache, Eth1Block},
     deposit_cache::{DepositCache, SszDepositCache},
 };
 use parking_lot::RwLock;
@@ -29,6 +29,7 @@ pub struct Inner {
     pub block_cache: RwLock<BlockCache>,
     pub deposit_cache: RwLock<DepositUpdater>,
     pub config: RwLock<Config>,
+    pub remote_head_block: RwLock<Option<Eth1Block>>,
     pub spec: ChainSpec,
 }
 
@@ -86,6 +87,9 @@ impl SszEth1Cache {
                 cache: self.deposit_cache.to_deposit_cache()?,
                 last_processed_block: self.last_processed_block,
             }),
+            // Set the remote head_block zero when creating a new instance. We only care about
+            // present and future eth1 nodes.
+            remote_head_block: RwLock::new(None),
             config: RwLock::new(config),
             spec,
         })

beacon_node/eth1/src/lib.rs

@@ -13,4 +13,7 @@ pub use block_cache::{BlockCache, Eth1Block};
 pub use deposit_cache::DepositCache;
 pub use deposit_log::DepositLog;
 pub use inner::SszEth1Cache;
-pub use service::{BlockCacheUpdateOutcome, Config, DepositCacheUpdateOutcome, Error, Service};
+pub use service::{
+    BlockCacheUpdateOutcome, Config, DepositCacheUpdateOutcome, Error, Service, DEFAULT_CHAIN_ID,
+    DEFAULT_NETWORK_ID,
+};

beacon_node/eth1/src/metrics.rs

@@ -16,4 +16,14 @@ lazy_static! {
         try_create_int_gauge("eth1_deposit_cache_len", "Number of deposits in the eth1 cache");
     pub static ref HIGHEST_PROCESSED_DEPOSIT_BLOCK: Result<IntGauge> =
         try_create_int_gauge("eth1_highest_processed_deposit_block", "Number of the last block checked for deposits");
+
+    /*
+     * Eth1 endpoint errors
+     */
+    pub static ref ENDPOINT_ERRORS: Result<IntCounterVec> = try_create_int_counter_vec(
+        "eth1_endpoint_errors", "The number of eth1 request errors for each endpoint", &["endpoint"]
+    );
+    pub static ref ENDPOINT_REQUESTS: Result<IntCounterVec> = try_create_int_counter_vec(
+        "eth1_endpoint_requests", "The number of eth1 requests for each endpoint", &["endpoint"]
+    );
 }

beacon_node/eth1/src/service.rs

@@ -2,19 +2,30 @@ use crate::metrics;
 use crate::{
     block_cache::{BlockCache, Error as BlockCacheError, Eth1Block},
     deposit_cache::Error as DepositCacheError,
-    http::{get_block, get_block_number, get_deposit_logs_in_range, Log},
+    http::{
+        get_block, get_block_number, get_chain_id, get_deposit_logs_in_range, get_network_id,
+        BlockQuery, Eth1Id, Log,
+    },
     inner::{DepositUpdater, Inner},
-    DepositLog,
 };
+use fallback::{Fallback, FallbackError};
 use futures::{future::TryFutureExt, stream, stream::TryStreamExt, StreamExt};
 use parking_lot::{RwLock, RwLockReadGuard};
 use serde::{Deserialize, Serialize};
-use slog::{debug, error, info, trace, Logger};
+use slog::{crit, debug, error, info, trace, warn, Logger};
+use std::fmt::Debug;
+use std::future::Future;
 use std::ops::{Range, RangeInclusive};
 use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
+use tokio::sync::RwLock as TRwLock;
 use tokio::time::{interval_at, Duration, Instant};
-use types::ChainSpec;
+use types::{ChainSpec, EthSpec, Unsigned};
+
+/// Indicates the default eth1 network id we use for the deposit contract.
+pub const DEFAULT_NETWORK_ID: Eth1Id = Eth1Id::Goerli;
+/// Indicates the default eth1 chain id we use for the deposit contract.
+pub const DEFAULT_CHAIN_ID: Eth1Id = Eth1Id::Goerli;
 
 const STANDARD_TIMEOUT_MILLIS: u64 = 15_000;
 
@@ -25,14 +36,235 @@ const GET_BLOCK_TIMEOUT_MILLIS: u64 = STANDARD_TIMEOUT_MILLIS;
 /// Timeout when doing an eth_getLogs to read the deposit contract logs.
 const GET_DEPOSIT_LOG_TIMEOUT_MILLIS: u64 = STANDARD_TIMEOUT_MILLIS;
 
+const WARNING_MSG: &str = "BLOCK PROPOSALS WILL FAIL WITHOUT VALID, SYNCED ETH1 CONNECTION";
+
+/// A factor used to reduce the eth1 follow distance to account for discrepancies in the block time.
+const ETH1_BLOCK_TIME_TOLERANCE_FACTOR: u64 = 4;
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum EndpointError {
+    NotReachable,
+    WrongNetworkId,
+    WrongChainId,
+    FarBehind,
+}
+
+type EndpointState = Result<(), EndpointError>;
+
+type EndpointWithState = (String, TRwLock<Option<EndpointState>>);
+
+/// A cache structure to lazily check usability of endpoints. An endpoint is usable if it is
+/// reachable and has the correct network id and chain id. Emits a `WARN` log if a checked endpoint
+/// is not usable.
+pub struct EndpointsCache {
+    pub fallback: Fallback<EndpointWithState>,
+    pub config_network_id: Eth1Id,
+    pub config_chain_id: Eth1Id,
+    pub log: Logger,
+}
+
+impl EndpointsCache {
+    /// Checks the usability of an endpoint. Results get cached and therefore only the first call
+    /// for each endpoint does the real check.
+    async fn state(&self, endpoint: &EndpointWithState) -> EndpointState {
+        if let Some(result) = *endpoint.1.read().await {
+            return result;
+        }
+        let mut value = endpoint.1.write().await;
+        if let Some(result) = *value {
+            return result;
+        }
+        crate::metrics::inc_counter_vec(&crate::metrics::ENDPOINT_REQUESTS, &[&endpoint.0]);
+        let state = endpoint_state(
+            &endpoint.0,
+            &self.config_network_id,
+            &self.config_chain_id,
+            &self.log,
+        )
+        .await;
+        *value = Some(state);
+        if state.is_err() {
+            crate::metrics::inc_counter_vec(&crate::metrics::ENDPOINT_ERRORS, &[&endpoint.0]);
+        }
+        state
+    }
+
+    pub async fn first_success<'a, F, O, R>(
+        &'a self,
+        func: F,
+    ) -> Result<O, FallbackError<SingleEndpointError>>
+    where
+        F: Fn(&'a str) -> R,
+        R: Future<Output = Result<O, SingleEndpointError>>,
+    {
+        let func = &func;
+        self.fallback
+            .first_success(|endpoint| async move {
+                match self.state(endpoint).await {
+                    Ok(()) => {
+                        let endpoint_str = &endpoint.0;
+                        crate::metrics::inc_counter_vec(
+                            &crate::metrics::ENDPOINT_REQUESTS,
+                            &[endpoint_str],
+                        );
+                        match func(&endpoint.0).await {
+                            Ok(t) => Ok(t),
+                            Err(t) => {
+                                crate::metrics::inc_counter_vec(
+                                    &crate::metrics::ENDPOINT_ERRORS,
+                                    &[endpoint_str],
+                                );
+                                if let SingleEndpointError::EndpointError(e) = &t {
+                                    *endpoint.1.write().await = Some(Err(*e));
+                                }
+                                Err(t)
+                            }
+                        }
+                    }
+                    Err(e) => Err(SingleEndpointError::EndpointError(e)),
+                }
+            })
+            .await
+    }
+}
+
+/// Returns `Ok` if the endpoint is usable, i.e. is reachable and has a correct network id and
+/// chain id. Otherwise it returns `Err`.
+async fn endpoint_state(
+    endpoint: &str,
+    config_network_id: &Eth1Id,
+    config_chain_id: &Eth1Id,
+    log: &Logger,
+) -> EndpointState {
+    let error_connecting = |_| {
+        warn!(
+            log,
+            "Error connecting to eth1 node. Trying fallback ...";
+            "endpoint" => endpoint,
+        );
+        EndpointError::NotReachable
+    };
+    let network_id = get_network_id(endpoint, Duration::from_millis(STANDARD_TIMEOUT_MILLIS))
+        .await
+        .map_err(error_connecting)?;
+    if &network_id != config_network_id {
+        warn!(
+            log,
+            "Invalid eth1 network id. Please switch to correct network id. Trying \
+             fallback ...";
+            "endpoint" => endpoint,
+            "expected" => format!("{:?}",config_network_id),
+            "received" => format!("{:?}",network_id),
+        );
+        return Err(EndpointError::WrongNetworkId);
+    }
+    let chain_id = get_chain_id(endpoint, Duration::from_millis(STANDARD_TIMEOUT_MILLIS))
+        .await
+        .map_err(error_connecting)?;
+    // Eth1 nodes return chain_id = 0 if the node is not synced
+    // Handle the special case
+    if chain_id == Eth1Id::Custom(0) {
+        warn!(
+            log,
+            "Remote eth1 node is not synced";
+            "endpoint" => endpoint,
+        );
+        return Err(EndpointError::FarBehind);
+    }
+    if &chain_id != config_chain_id {
+        warn!(
+            log,
+            "Invalid eth1 chain id. Please switch to correct chain id. Trying \
+             fallback ...";
+            "endpoint" => endpoint,
+            "expected" => format!("{:?}",config_chain_id),
+            "received" => format!("{:?}", chain_id),
+        );
+        Err(EndpointError::WrongChainId)
+    } else {
+        Ok(())
+    }
+}
+
+/// Enum for the two internal (maybe different) cached heads for cached deposits and for the block
+/// cache.
+pub enum HeadType {
+    Deposit,
+    BlockCache,
+}
+
+/// Returns the head block and the new block ranges relevant for deposits and the block cache
+/// from the given endpoint.
+async fn get_remote_head_and_new_block_ranges(
+    endpoint: &str,
+    service: &Service,
+    node_far_behind_seconds: u64,
+) -> Result<
+    (
+        Eth1Block,
+        Option<RangeInclusive<u64>>,
+        Option<RangeInclusive<u64>>,
+    ),
+    SingleEndpointError,
+> {
+    let remote_head_block = download_eth1_block(endpoint, service.inner.clone(), None).await?;
+    let now = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(u64::MAX);
+    if remote_head_block.timestamp + node_far_behind_seconds < now {
+        warn!(
+            service.log,
+            "Eth1 endpoint is far behind. Trying fallback ...";
+            "endpoint" => endpoint,
+            "last_seen_block_unix_timestamp" => remote_head_block.timestamp
+        );
+        return Err(SingleEndpointError::EndpointError(EndpointError::FarBehind));
+    }
+
+    let handle_remote_not_synced = |e| {
+        if let SingleEndpointError::RemoteNotSynced { .. } = e {
+            warn!(service.log, "Eth1 node not synced. Trying fallback..."; "endpoint" => endpoint);
+        }
+        e
+    };
+    let new_deposit_block_numbers = service
+        .relevant_new_block_numbers(remote_head_block.number, HeadType::Deposit)
+        .map_err(handle_remote_not_synced)?;
+    let new_block_cache_numbers = service
+        .relevant_new_block_numbers(remote_head_block.number, HeadType::BlockCache)
+        .map_err(handle_remote_not_synced)?;
+    Ok((
+        remote_head_block,
+        new_deposit_block_numbers,
+        new_block_cache_numbers,
+    ))
+}
+
+/// Returns the range of new block numbers to be considered for the given head type from the given
+/// endpoint.
+async fn relevant_new_block_numbers_from_endpoint(
+    endpoint: &str,
+    service: &Service,
+    head_type: HeadType,
+) -> Result<Option<RangeInclusive<u64>>, SingleEndpointError> {
+    let remote_highest_block =
+        get_block_number(endpoint, Duration::from_millis(BLOCK_NUMBER_TIMEOUT_MILLIS))
+            .map_err(SingleEndpointError::GetBlockNumberFailed)
+            .await?;
+    service.relevant_new_block_numbers(remote_highest_block, head_type)
+}
+
 #[derive(Debug, PartialEq)]
-pub enum Error {
+pub enum SingleEndpointError {
+    /// Endpoint is currently not functional.
+    EndpointError(EndpointError),
     /// The remote node is less synced that we expect, it is not useful until has done more
     /// syncing.
     RemoteNotSynced {
         next_required_block: u64,
         remote_highest_block: u64,
-        follow_distance: u64,
+        reduced_follow_distance: u64,
     },
     /// Failed to download a block from the eth1 node.
     BlockDownloadFailed(String),
@@ -44,6 +276,10 @@ pub enum Error {
     GetDepositCountFailed(String),
     /// Failed to read the deposit contract root from the eth1 node.
     GetDepositLogsFailed(String),
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
     /// There was an inconsistency when adding a block to the cache.
     FailedToInsertEth1Block(BlockCacheError),
     /// There was an inconsistency when adding a deposit to the cache.
@@ -53,6 +289,8 @@ pub enum Error {
         block_range: Range<u64>,
         error: String,
     },
+    /// All possible endpoints returned a `SingleEndpointError`.
+    FallbackError(FallbackError<SingleEndpointError>),
     /// There was an unexpected internal error.
     Internal(String),
 }
@@ -73,9 +311,13 @@ pub struct DepositCacheUpdateOutcome {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Config {
     /// An Eth1 node (e.g., Geth) running a HTTP JSON-RPC endpoint.
-    pub endpoint: String,
+    pub endpoints: Vec<String>,
     /// The address the `BlockCache` and `DepositCache` should assume is the canonical deposit contract.
     pub deposit_contract_address: String,
+    /// The eth1 network id where the deposit contract is deployed (Goerli/Mainnet).
+    pub network_id: Eth1Id,
+    /// The eth1 chain id where the deposit contract is deployed (Goerli/Mainnet).
+    pub chain_id: Eth1Id,
     /// Defines the first block that the `DepositCache` will start searching for deposit logs.
     ///
     /// Setting too high can result in missed logs. Setting too low will result in unnecessary
@@ -87,6 +329,9 @@ pub struct Config {
     ///
     /// Note: this should be less than or equal to the specification's `ETH1_FOLLOW_DISTANCE`.
     pub follow_distance: u64,
+    /// Specifies the seconds when we consider the head of a node far behind.
+    /// This should be less than `ETH1_FOLLOW_DISTANCE * SECONDS_PER_ETH1_BLOCK`.
+    pub node_far_behind_seconds: u64,
     /// Defines the number of blocks that should be retained each time the `BlockCache` calls truncate on
     /// itself.
     pub block_cache_truncation: Option<usize>,
@@ -100,14 +345,42 @@ pub struct Config {
     pub max_blocks_per_update: Option<usize>,
 }
 
+impl Config {
+    /// Sets the block cache to a length that is suitable for the given `EthSpec` and `ChainSpec`.
+    pub fn set_block_cache_truncation<E: EthSpec>(&mut self, spec: &ChainSpec) {
+        // Compute the number of eth1 blocks in an eth1 voting period.
+        let seconds_per_voting_period =
+            E::SlotsPerEth1VotingPeriod::to_u64() * spec.milliseconds_per_slot / 1000;
+        let eth1_blocks_per_voting_period = seconds_per_voting_period / spec.seconds_per_eth1_block;
+
+        // Compute the number of extra blocks we store prior to the voting period start blocks.
+        let follow_distance_tolerance_blocks =
+            spec.eth1_follow_distance / ETH1_BLOCK_TIME_TOLERANCE_FACTOR;
+
+        // Ensure we can store two full windows of voting blocks.
+        let voting_windows = eth1_blocks_per_voting_period * 2;
+
+        // Extend the cache to account for varying eth1 block times and the follow distance
+        // tolerance blocks.
+        let length = voting_windows
+            + (voting_windows / ETH1_BLOCK_TIME_TOLERANCE_FACTOR)
+            + follow_distance_tolerance_blocks;
+
+        self.block_cache_truncation = Some(length as usize);
+    }
+}
+
 impl Default for Config {
     fn default() -> Self {
         Self {
-            endpoint: "http://localhost:8545".into(),
+            endpoints: vec!["http://localhost:8545".into()],
             deposit_contract_address: "0x0000000000000000000000000000000000000000".into(),
+            network_id: DEFAULT_NETWORK_ID,
+            chain_id: DEFAULT_CHAIN_ID,
             deposit_contract_deploy_block: 1,
             lowest_cached_block_number: 1,
             follow_distance: 128,
+            node_far_behind_seconds: 128 * 14,
             block_cache_truncation: Some(4_096),
             auto_update_interval_millis: 7_000,
             blocks_per_log_query: 1_000,
@@ -138,6 +411,7 @@ impl Service {
                 deposit_cache: RwLock::new(DepositUpdater::new(
                     config.deposit_contract_deploy_block,
                 )),
+                remote_head_block: RwLock::new(None),
                 config: RwLock::new(config),
                 spec,
             }),
@@ -145,6 +419,18 @@ impl Service {
         }
     }
 
+    /// Returns the follow distance that has been shortened to accommodate for differences in the
+    /// spacing between blocks.
+    ///
+    /// ## Notes
+    ///
+    /// This is useful since the spec declares `SECONDS_PER_ETH1_BLOCK` to be `14`, whilst it is
+    /// actually `15` on Goerli.
+    pub fn reduced_follow_distance(&self) -> u64 {
+        let full = self.config().follow_distance;
+        full.saturating_sub(full / ETH1_BLOCK_TIME_TOLERANCE_FACTOR)
+    }
+
     /// Return byte representation of deposit and block caches.
     pub fn as_bytes(&self) -> Vec<u8> {
         self.inner.as_bytes()
@@ -196,6 +482,21 @@ impl Service {
         self.inner.block_cache.read().latest_block_timestamp()
     }
 
+    /// Returns the latest head block returned from an Eth1 node.
+    ///
+    /// ## Note
+    ///
+    /// This is the simply the head of the Eth1 chain, with no regard to follow distance or the
+    /// voting period start.
+    pub fn head_block(&self) -> Option<Eth1Block> {
+        self.inner.remote_head_block.read().as_ref().cloned()
+    }
+
+    /// Returns the latest cached block.
+    pub fn latest_cached_block(&self) -> Option<Eth1Block> {
+        self.inner.block_cache.read().latest_block().cloned()
+    }
+
     /// Returns the lowest block number stored.
     pub fn lowest_block_number(&self) -> Option<u64> {
         self.inner.block_cache.read().lowest_block_number()
@@ -280,6 +581,23 @@ impl Service {
         self.inner.config.write().lowest_cached_block_number = block_number;
     }
 
+    pub fn init_endpoints(&self) -> EndpointsCache {
+        let endpoints = self.config().endpoints.clone();
+        let config_network_id = self.config().network_id.clone();
+        let config_chain_id = self.config().chain_id.clone();
+        EndpointsCache {
+            fallback: Fallback::new(
+                endpoints
+                    .into_iter()
+                    .map(|s| (s, TRwLock::new(None)))
+                    .collect(),
+            ),
+            config_network_id,
+            config_chain_id,
+            log: self.log.clone(),
+        }
+    }
+
     /// Update the deposit and block cache, returning an error if either fail.
     ///
     /// ## Returns
@@ -291,11 +609,57 @@ impl Service {
     pub async fn update(
         &self,
     ) -> Result<(DepositCacheUpdateOutcome, BlockCacheUpdateOutcome), String> {
+        let endpoints = self.init_endpoints();
+        let node_far_behind_seconds = self.inner.config.read().node_far_behind_seconds;
+
+        let process_single_err = |e: &FallbackError<SingleEndpointError>| {
+            match e {
+                FallbackError::AllErrored(errors) => {
+                    if errors
+                        .iter()
+                        .all(|error| matches!(error, SingleEndpointError::EndpointError(_)))
+                    {
+                        crit!(
+                            self.log,
+                            "Couldn't connect to any eth1 node. Please ensure that you have an \
+                             eth1 http server running locally on http://localhost:8545 or specify \
+                             one or more (remote) endpoints using \
+                             `--eth1-endpoints <COMMA-SEPARATED-SERVER-ADDRESSES>`. \
+                             Also ensure that `eth` and `net` apis are enabled on the eth1 http \
+                             server";
+                             "warning" => WARNING_MSG
+                        );
+                    }
+                }
+            }
+            endpoints.fallback.map_format_error(|s| &s.0, &e)
+        };
+
+        let process_err = |e: Error| match &e {
+            Error::FallbackError(f) => process_single_err(f),
+            e => format!("{:?}", e),
+        };
+
+        let (remote_head_block, new_block_numbers_deposit, new_block_numbers_block_cache) =
+            endpoints
+                .first_success(|e| async move {
+                    get_remote_head_and_new_block_ranges(e, &self, node_far_behind_seconds).await
+                })
+                .await
+                .map_err(|e| {
+                    format!(
+                        "Failed to update Eth1 service: {:?}",
+                        process_single_err(&e)
+                    )
+                })?;
+
+        *self.inner.remote_head_block.write() = Some(remote_head_block);
+
         let update_deposit_cache = async {
             let outcome = self
-                .update_deposit_cache()
+                .update_deposit_cache(Some(new_block_numbers_deposit), &endpoints)
                 .await
-                .map_err(|e| format!("Failed to update eth1 cache: {:?}", e))?;
+                .map_err(|e| format!("Failed to update eth1 cache: {:?}", process_err(e)))?;
 
             trace!(
                 self.log,
@@ -304,14 +668,14 @@ impl Service {
                 "logs_imported" => outcome.logs_imported,
                 "last_processed_eth1_block" => self.inner.deposit_cache.read().last_processed_block,
             );
-            Ok(outcome)
+            Ok::<_, String>(outcome)
         };
 
         let update_block_cache = async {
             let outcome = self
-                .update_block_cache()
+                .update_block_cache(Some(new_block_numbers_block_cache), &endpoints)
                 .await
-                .map_err(|e| format!("Failed to update eth1 cache: {:?}", e))?;
+                .map_err(|e| format!("Failed to update eth1 cache: {:?}", process_err(e)))?;
 
             trace!(
                 self.log,
@@ -320,10 +684,13 @@ impl Service {
                 "blocks_imported" => outcome.blocks_imported,
                 "head_block" => outcome.head_block_number,
             );
-            Ok(outcome)
+            Ok::<_, String>(outcome)
         };
 
-        futures::try_join!(update_deposit_cache, update_block_cache)
+        let (deposit_outcome, block_outcome) =
+            futures::try_join!(update_deposit_cache, update_block_cache)?;
+
+        Ok((deposit_outcome, block_outcome))
     }
 
     /// A looping future that updates the cache, then waits `config.auto_update_interval` before
@@ -335,7 +702,7 @@ impl Service {
     /// - Err(_) if there is an error.
     ///
     /// Emits logs for debugging and errors.
-    pub fn auto_update(self, handle: environment::TaskExecutor) {
+    pub fn auto_update(self, handle: task_executor::TaskExecutor) {
         let update_interval = Duration::from_millis(self.config().auto_update_interval_millis);
 
         let mut interval = interval_at(Instant::now(), update_interval);
@@ -369,21 +736,52 @@ impl Service {
         Ok(())
     }
 
+    /// Returns the range of new block numbers to be considered for the given head type.
+    fn relevant_new_block_numbers(
+        &self,
+        remote_highest_block: u64,
+        head_type: HeadType,
+    ) -> Result<Option<RangeInclusive<u64>>, SingleEndpointError> {
+        let follow_distance = self.reduced_follow_distance();
+        let next_required_block = match head_type {
+            HeadType::Deposit => self
+                .deposits()
+                .read()
+                .last_processed_block
+                .map(|n| n + 1)
+                .unwrap_or_else(|| self.config().deposit_contract_deploy_block),
+            HeadType::BlockCache => self
+                .inner
+                .block_cache
+                .read()
+                .highest_block_number()
+                .map(|n| n + 1)
+                .unwrap_or_else(|| self.config().lowest_cached_block_number),
+        };
+
+        relevant_block_range(remote_highest_block, next_required_block, follow_distance)
+    }
+
     /// Contacts the remote eth1 node and attempts to import deposit logs up to the configured
     /// follow-distance block.
     ///
     /// Will process no more than `BLOCKS_PER_LOG_QUERY * MAX_LOG_REQUESTS_PER_UPDATE` blocks in a
     /// single update.
     ///
+    /// If `remote_highest_block_opt` is `Some`, use that value instead of querying `self.endpoint`
+    /// for the head of the eth1 chain.
+    ///
     /// ## Resolves with
     ///
     /// - Ok(_) if the update was successful (the cache may or may not have been modified).
     /// - Err(_) if there is an error.
     ///
     /// Emits logs for debugging and errors.
-    pub async fn update_deposit_cache(&self) -> Result<DepositCacheUpdateOutcome, Error> {
-        let endpoint = self.config().endpoint.clone();
-        let follow_distance = self.config().follow_distance;
+    pub async fn update_deposit_cache(
+        &self,
+        new_block_numbers: Option<Option<RangeInclusive<u64>>>,
+        endpoints: &EndpointsCache,
+    ) -> Result<DepositCacheUpdateOutcome, Error> {
         let deposit_contract_address = self.config().deposit_contract_address.clone();
 
         let blocks_per_log_query = self.config().blocks_per_log_query;
@@ -392,14 +790,17 @@ impl Service {
             .max_log_requests_per_update
             .unwrap_or_else(usize::max_value);
 
-        let next_required_block = self
-            .deposits()
-            .read()
-            .last_processed_block
-            .map(|n| n + 1)
-            .unwrap_or_else(|| self.config().deposit_contract_deploy_block);
-
-        let range = get_new_block_numbers(&endpoint, next_required_block, follow_distance).await?;
+        let range = {
+            match new_block_numbers {
+                Some(range) => range,
+                None => endpoints
+                    .first_success(|e| async move {
+                        relevant_new_block_numbers_from_endpoint(e, &self, HeadType::Deposit).await
+                    })
+                    .await
+                    .map_err(Error::FallbackError)?,
+            }
+        };
 
         let block_number_chunks = if let Some(range) = range {
             range
@@ -416,28 +817,32 @@ impl Service {
             Vec::new()
         };
 
+        let deposit_contract_address_ref: &str = &deposit_contract_address;
         let logs: Vec<(Range<u64>, Vec<Log>)> =
             stream::try_unfold(block_number_chunks.into_iter(), |mut chunks| async {
                 match chunks.next() {
                     Some(chunk) => {
-                        let chunk_1 = chunk.clone();
-                        match get_deposit_logs_in_range(
-                            &endpoint,
-                            &deposit_contract_address,
-                            chunk,
-                            Duration::from_millis(GET_DEPOSIT_LOG_TIMEOUT_MILLIS),
-                        )
-                        .await
-                        {
-                            Ok(logs) => Ok(Some(((chunk_1, logs), chunks))),
-                            Err(e) => Err(Error::GetDepositLogsFailed(e)),
-                        }
+                        let chunk_ref = &chunk;
+                        endpoints
+                            .first_success(|e| async move {
+                                get_deposit_logs_in_range(
+                                    e,
+                                    deposit_contract_address_ref,
+                                    chunk_ref.clone(),
+                                    Duration::from_millis(GET_DEPOSIT_LOG_TIMEOUT_MILLIS),
+                                )
+                                .await
+                                .map_err(SingleEndpointError::GetDepositLogsFailed)
+                            })
+                            .await
+                            .map(|logs| Some(((chunk, logs), chunks)))
                     }
                     None => Ok(None),
                 }
             })
             .try_collect()
-            .await?;
+            .await
+            .map_err(Error::FallbackError)?;
 
         let mut logs_imported = 0;
         for (block_range, log_chunk) in logs.iter() {
@@ -445,7 +850,7 @@ impl Service {
             log_chunk
                 .iter()
                 .map(|raw_log| {
-                    DepositLog::from_log(&raw_log, self.inner.spec()).map_err(|error| {
+                    raw_log.to_deposit_log(self.inner.spec()).map_err(|error| {
                         Error::FailedToParseDepositLog {
                             block_range: block_range.clone(),
                             error,
@@ -510,31 +915,39 @@ impl Service {
     ///
     /// If configured, prunes the block cache after importing new blocks.
     ///
+    /// If `remote_highest_block_opt` is `Some`, use that value instead of querying `self.endpoint`
+    /// for the head of the eth1 chain.
+    ///
     /// ## Resolves with
     ///
     /// - Ok(_) if the update was successful (the cache may or may not have been modified).
     /// - Err(_) if there is an error.
     ///
     /// Emits logs for debugging and errors.
-    pub async fn update_block_cache(&self) -> Result<BlockCacheUpdateOutcome, Error> {
+    pub async fn update_block_cache(
+        &self,
+        new_block_numbers: Option<Option<RangeInclusive<u64>>>,
+        endpoints: &EndpointsCache,
+    ) -> Result<BlockCacheUpdateOutcome, Error> {
         let block_cache_truncation = self.config().block_cache_truncation;
         let max_blocks_per_update = self
             .config()
             .max_blocks_per_update
             .unwrap_or_else(usize::max_value);
 
-        let next_required_block = self
-            .inner
-            .block_cache
-            .read()
-            .highest_block_number()
-            .map(|n| n + 1)
-            .unwrap_or_else(|| self.config().lowest_cached_block_number);
+        let range = {
+            match new_block_numbers {
+                Some(range) => range,
+                None => endpoints
+                    .first_success(|e| async move {
+                        relevant_new_block_numbers_from_endpoint(e, &self, HeadType::BlockCache)
+                            .await
+                    })
+                    .await
+                    .map_err(Error::FallbackError)?,
+            }
+        };
 
-        let endpoint = self.config().endpoint.clone();
-        let follow_distance = self.config().follow_distance;
-
-        let range = get_new_block_numbers(&endpoint, next_required_block, follow_distance).await?;
         // Map the range of required blocks into a Vec.
         //
         // If the required range is larger than the size of the cache, drop the exiting cache
@@ -585,7 +998,12 @@ impl Service {
             |mut block_numbers| async {
                 match block_numbers.next() {
                     Some(block_number) => {
-                        match download_eth1_block(self.inner.clone(), block_number).await {
+                        match endpoints
+                            .first_success(|e| async move {
+                                download_eth1_block(e, self.inner.clone(), Some(block_number)).await
+                            })
+                            .await
+                        {
                             Ok(eth1_block) => Ok(Some((eth1_block, block_numbers))),
                             Err(e) => Err(e),
                         }
@@ -595,7 +1013,8 @@ impl Service {
             },
         )
         .try_collect()
-        .await?;
+        .await
+        .map_err(Error::FallbackError)?;
 
         let mut blocks_imported = 0;
         for eth1_block in eth1_blocks {
@@ -666,18 +1085,16 @@ impl Service {
     }
 }
 
-/// Determine the range of blocks that need to be downloaded, given the remotes best block and
-/// the locally stored best block.
-async fn get_new_block_numbers<'a>(
-    endpoint: &str,
+/// Returns the range of blocks starting from `next_required_block` that are at least
+/// `follow_distance` many blocks before `remote_highest_block`.
+/// Returns an error if `next_required_block > remote_highest_block + 1` which means the remote went
+/// backwards.
+fn relevant_block_range(
+    remote_highest_block: u64,
     next_required_block: u64,
-    follow_distance: u64,
-) -> Result<Option<RangeInclusive<u64>>, Error> {
-    let remote_highest_block =
-        get_block_number(endpoint, Duration::from_millis(BLOCK_NUMBER_TIMEOUT_MILLIS))
-            .map_err(Error::GetBlockNumberFailed)
-            .await?;
-    let remote_follow_block = remote_highest_block.saturating_sub(follow_distance);
+    reduced_follow_distance: u64,
+) -> Result<Option<RangeInclusive<u64>>, SingleEndpointError> {
+    let remote_follow_block = remote_highest_block.saturating_sub(reduced_follow_distance);
 
     if next_required_block <= remote_follow_block {
         Ok(Some(next_required_block..=remote_follow_block))
@@ -685,12 +1102,12 @@ async fn get_new_block_numbers<'a>(
         // If this is the case, the node must have gone "backwards" in terms of it's sync
         // (i.e., it's head block is lower than it was before).
         //
-        // We assume that the `follow_distance` should be sufficient to ensure this never
+        // We assume that the `reduced_follow_distance` should be sufficient to ensure this never
         // happens, otherwise it is an error.
-        Err(Error::RemoteNotSynced {
+        Err(SingleEndpointError::RemoteNotSynced {
             next_required_block,
             remote_highest_block,
-            follow_distance,
+            reduced_follow_distance,
         })
     } else {
         // Return an empty range.
@@ -701,29 +1118,39 @@ async fn get_new_block_numbers<'a>(
 /// Downloads the `(block, deposit_root, deposit_count)` tuple from an eth1 node for the given
 /// `block_number`.
 ///
+/// Set `block_number_opt = None` to get the "latest" eth1 block (i.e., the head).
+///
 /// Performs three async calls to an Eth1 HTTP JSON RPC endpoint.
-async fn download_eth1_block(cache: Arc<Inner>, block_number: u64) -> Result<Eth1Block, Error> {
-    let endpoint = cache.config.read().endpoint.clone();
+async fn download_eth1_block(
+    endpoint: &str,
+    cache: Arc<Inner>,
+    block_number_opt: Option<u64>,
+) -> Result<Eth1Block, SingleEndpointError> {
+    let deposit_root = block_number_opt.and_then(|block_number| {
+        cache
+            .deposit_cache
+            .read()
+            .cache
+            .get_deposit_root_from_cache(block_number)
+    });
 
-    let deposit_root = cache
-        .deposit_cache
-        .read()
-        .cache
-        .get_deposit_root_from_cache(block_number);
-
-    let deposit_count = cache
-        .deposit_cache
-        .read()
-        .cache
-        .get_deposit_count_from_cache(block_number);
+    let deposit_count = block_number_opt.and_then(|block_number| {
+        cache
+            .deposit_cache
+            .read()
+            .cache
+            .get_deposit_count_from_cache(block_number)
+    });
 
     // Performs a `get_blockByNumber` call to an eth1 node.
     let http_block = get_block(
-        &endpoint,
-        block_number,
+        endpoint,
+        block_number_opt
+            .map(BlockQuery::Number)
+            .unwrap_or_else(|| BlockQuery::Latest),
         Duration::from_millis(GET_BLOCK_TIMEOUT_MILLIS),
     )
-    .map_err(Error::BlockDownloadFailed)
+    .map_err(SingleEndpointError::BlockDownloadFailed)
     .await?;
 
     Ok(Eth1Block {
@@ -739,6 +1166,7 @@ async fn download_eth1_block(cache: Arc<Inner>, block_number: u64) -> Result<Eth
 mod tests {
     use super::*;
     use toml;
+    use types::MainnetEthSpec;
 
     #[test]
     fn serde_serialize() {
@@ -746,4 +1174,26 @@ mod tests {
             toml::to_string(&Config::default()).expect("Should serde encode default config");
         toml::from_str::<Config>(&serialized).expect("Should serde decode default config");
     }
+
+    #[test]
+    fn block_cache_size() {
+        let mut config = Config::default();
+
+        let spec = MainnetEthSpec::default_spec();
+
+        config.set_block_cache_truncation::<MainnetEthSpec>(&spec);
+
+        let len = config.block_cache_truncation.unwrap();
+
+        let seconds_per_voting_period =
+            <MainnetEthSpec as EthSpec>::SlotsPerEth1VotingPeriod::to_u64()
+                * (spec.milliseconds_per_slot / 1000);
+        let eth1_blocks_per_voting_period = seconds_per_voting_period / spec.seconds_per_eth1_block;
+        let reduce_follow_distance_blocks =
+            config.follow_distance / ETH1_BLOCK_TIME_TOLERANCE_FACTOR;
+
+        let minimum_len = eth1_blocks_per_voting_period * 2 + reduce_follow_distance_blocks;
+
+        assert!(len > minimum_len as usize);
+    }
 }

beacon_node/eth1/tests/test.rs

@@ -2,7 +2,7 @@
 use environment::{Environment, EnvironmentBuilder};
 use eth1::http::{get_deposit_count, get_deposit_logs_in_range, get_deposit_root, Block, Log};
 use eth1::{Config, Service};
-use eth1::{DepositCache, DepositLog};
+use eth1::{DepositCache, DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID};
 use eth1_test_rig::GanacheEth1Instance;
 use futures::compat::Future01CompatExt;
 use merkle_proof::verify_merkle_proof;
@@ -97,6 +97,10 @@ async fn get_block_number(web3: &Web3<Http>) -> u64 {
         .expect("should get block number")
 }
 
+async fn new_ganache_instance() -> Result<GanacheEth1Instance, String> {
+    GanacheEth1Instance::new(DEFAULT_NETWORK_ID.into(), DEFAULT_CHAIN_ID.into()).await
+}
+
 mod eth1_cache {
     use super::*;
     use types::{EthSpec, MainnetEthSpec};
@@ -106,7 +110,7 @@ mod eth1_cache {
         let log = null_logger();
 
         for follow_distance in 0..2 {
-            let eth1 = GanacheEth1Instance::new()
+            let eth1 = new_ganache_instance()
                 .await
                 .expect("should start eth1 environment");
             let deposit_contract = &eth1.deposit_contract;
@@ -116,7 +120,7 @@ mod eth1_cache {
 
             let service = Service::new(
                 Config {
-                    endpoint: eth1.endpoint(),
+                    endpoints: vec![eth1.endpoint()],
                     deposit_contract_address: deposit_contract.address(),
                     lowest_cached_block_number: initial_block_number,
                     follow_distance,
@@ -145,17 +149,19 @@ mod eth1_cache {
                     eth1.ganache.evm_mine().await.expect("should mine block");
                 }
 
+                let endpoints = service.init_endpoints();
+
                 service
-                    .update_deposit_cache()
+                    .update_deposit_cache(None, &endpoints)
                     .await
                     .expect("should update deposit cache");
                 service
-                    .update_block_cache()
+                    .update_block_cache(None, &endpoints)
                     .await
                     .expect("should update block cache");
 
                 service
-                    .update_block_cache()
+                    .update_block_cache(None, &endpoints)
                     .await
                     .expect("should update cache when nothing has changed");
 
@@ -181,7 +187,7 @@ mod eth1_cache {
     async fn big_skip() {
         let log = null_logger();
 
-        let eth1 = GanacheEth1Instance::new()
+        let eth1 = new_ganache_instance()
             .await
             .expect("should start eth1 environment");
         let deposit_contract = &eth1.deposit_contract;
@@ -191,7 +197,7 @@ mod eth1_cache {
 
         let service = Service::new(
             Config {
-                endpoint: eth1.endpoint(),
+                endpoints: vec![eth1.endpoint()],
                 deposit_contract_address: deposit_contract.address(),
                 lowest_cached_block_number: get_block_number(&web3).await,
                 follow_distance: 0,
@@ -208,12 +214,14 @@ mod eth1_cache {
             eth1.ganache.evm_mine().await.expect("should mine block")
         }
 
+        let endpoints = service.init_endpoints();
+
         service
-            .update_deposit_cache()
+            .update_deposit_cache(None, &endpoints)
             .await
             .expect("should update deposit cache");
         service
-            .update_block_cache()
+            .update_block_cache(None, &endpoints)
             .await
             .expect("should update block cache");
 
@@ -230,7 +238,7 @@ mod eth1_cache {
     async fn pruning() {
         let log = null_logger();
 
-        let eth1 = GanacheEth1Instance::new()
+        let eth1 = new_ganache_instance()
             .await
             .expect("should start eth1 environment");
         let deposit_contract = &eth1.deposit_contract;
@@ -240,7 +248,7 @@ mod eth1_cache {
 
         let service = Service::new(
             Config {
-                endpoint: eth1.endpoint(),
+                endpoints: vec![eth1.endpoint()],
                 deposit_contract_address: deposit_contract.address(),
                 lowest_cached_block_number: get_block_number(&web3).await,
                 follow_distance: 0,
@@ -255,12 +263,13 @@ mod eth1_cache {
             for _ in 0..cache_len / 2 {
                 eth1.ganache.evm_mine().await.expect("should mine block")
             }
+            let endpoints = service.init_endpoints();
             service
-                .update_deposit_cache()
+                .update_deposit_cache(None, &endpoints)
                 .await
                 .expect("should update deposit cache");
             service
-                .update_block_cache()
+                .update_block_cache(None, &endpoints)
                 .await
                 .expect("should update block cache");
         }
@@ -278,7 +287,7 @@ mod eth1_cache {
 
         let n = 16;
 
-        let eth1 = GanacheEth1Instance::new()
+        let eth1 = new_ganache_instance()
             .await
             .expect("should start eth1 environment");
         let deposit_contract = &eth1.deposit_contract;
@@ -286,7 +295,7 @@ mod eth1_cache {
 
         let service = Service::new(
             Config {
-                endpoint: eth1.endpoint(),
+                endpoints: vec![eth1.endpoint()],
                 deposit_contract_address: deposit_contract.address(),
                 lowest_cached_block_number: get_block_number(&web3).await,
                 follow_distance: 0,
@@ -299,13 +308,18 @@ mod eth1_cache {
         for _ in 0..n {
             eth1.ganache.evm_mine().await.expect("should mine block")
         }
+
+        let endpoints = service.init_endpoints();
         futures::try_join!(
-            service.update_deposit_cache(),
-            service.update_deposit_cache()
+            service.update_deposit_cache(None, &endpoints),
+            service.update_deposit_cache(None, &endpoints)
         )
         .expect("should perform two simultaneous updates of deposit cache");
-        futures::try_join!(service.update_block_cache(), service.update_block_cache())
-            .expect("should perform two simultaneous updates of block cache");
+        futures::try_join!(
+            service.update_block_cache(None, &endpoints),
+            service.update_block_cache(None, &endpoints)
+        )
+        .expect("should perform two simultaneous updates of block cache");
 
         assert!(service.block_cache_len() >= n, "should grow the cache");
     }
@@ -320,7 +334,7 @@ mod deposit_tree {
 
         let n = 4;
 
-        let eth1 = GanacheEth1Instance::new()
+        let eth1 = new_ganache_instance()
             .await
             .expect("should start eth1 environment");
         let deposit_contract = &eth1.deposit_contract;
@@ -330,7 +344,7 @@ mod deposit_tree {
 
         let service = Service::new(
             Config {
-                endpoint: eth1.endpoint(),
+                endpoints: vec![eth1.endpoint()],
                 deposit_contract_address: deposit_contract.address(),
                 deposit_contract_deploy_block: start_block,
                 follow_distance: 0,
@@ -350,13 +364,15 @@ mod deposit_tree {
                     .expect("should perform a deposit");
             }
 
+            let endpoints = service.init_endpoints();
+
             service
-                .update_deposit_cache()
+                .update_deposit_cache(None, &endpoints)
                 .await
                 .expect("should perform update");
 
             service
-                .update_deposit_cache()
+                .update_deposit_cache(None, &endpoints)
                 .await
                 .expect("should perform update when nothing has changed");
 
@@ -395,7 +411,7 @@ mod deposit_tree {
 
         let n = 8;
 
-        let eth1 = GanacheEth1Instance::new()
+        let eth1 = new_ganache_instance()
             .await
             .expect("should start eth1 environment");
         let deposit_contract = &eth1.deposit_contract;
@@ -405,7 +421,7 @@ mod deposit_tree {
 
         let service = Service::new(
             Config {
-                endpoint: eth1.endpoint(),
+                endpoints: vec![eth1.endpoint()],
                 deposit_contract_address: deposit_contract.address(),
                 deposit_contract_deploy_block: start_block,
                 lowest_cached_block_number: start_block,
@@ -425,9 +441,10 @@ mod deposit_tree {
                 .expect("should perform a deposit");
         }
 
+        let endpoints = service.init_endpoints();
         futures::try_join!(
-            service.update_deposit_cache(),
-            service.update_deposit_cache()
+            service.update_deposit_cache(None, &endpoints),
+            service.update_deposit_cache(None, &endpoints)
         )
         .expect("should perform two updates concurrently");
 
@@ -442,7 +459,7 @@ mod deposit_tree {
 
         let deposits: Vec<_> = (0..n).map(|_| random_deposit_data()).collect();
 
-        let eth1 = GanacheEth1Instance::new()
+        let eth1 = new_ganache_instance()
             .await
             .expect("should start eth1 environment");
         let deposit_contract = &eth1.deposit_contract;
@@ -477,7 +494,7 @@ mod deposit_tree {
         let logs: Vec<_> = blocking_deposit_logs(&eth1, 0..block_number)
             .await
             .iter()
-            .map(|raw| DepositLog::from_log(raw, spec).expect("should parse deposit log"))
+            .map(|raw| raw.to_deposit_log(spec).expect("should parse deposit log"))
             .inspect(|log| {
                 tree.insert_log(log.clone())
                     .expect("should add consecutive logs")
@@ -535,16 +552,21 @@ mod deposit_tree {
 /// Tests for the base HTTP requests and response handlers.
 mod http {
     use super::*;
+    use eth1::http::BlockQuery;
 
     async fn get_block(eth1: &GanacheEth1Instance, block_number: u64) -> Block {
-        eth1::http::get_block(&eth1.endpoint(), block_number, timeout())
-            .await
-            .expect("should get block number")
+        eth1::http::get_block(
+            &eth1.endpoint(),
+            BlockQuery::Number(block_number),
+            timeout(),
+        )
+        .await
+        .expect("should get block number")
     }
 
     #[tokio::test]
     async fn incrementing_deposits() {
-        let eth1 = GanacheEth1Instance::new()
+        let eth1 = new_ganache_instance()
             .await
             .expect("should start eth1 environment");
         let deposit_contract = &eth1.deposit_contract;
@@ -636,7 +658,7 @@ mod fast {
     async fn deposit_cache_query() {
         let log = null_logger();
 
-        let eth1 = GanacheEth1Instance::new()
+        let eth1 = new_ganache_instance()
             .await
             .expect("should start eth1 environment");
         let deposit_contract = &eth1.deposit_contract;
@@ -645,7 +667,7 @@ mod fast {
         let now = get_block_number(&web3).await;
         let service = Service::new(
             Config {
-                endpoint: eth1.endpoint(),
+                endpoints: vec![eth1.endpoint()],
                 deposit_contract_address: deposit_contract.address(),
                 deposit_contract_deploy_block: now,
                 lowest_cached_block_number: now,
@@ -667,8 +689,9 @@ mod fast {
             eth1.ganache.evm_mine().await.expect("should mine block");
         }
 
+        let endpoints = service.init_endpoints();
         service
-            .update_deposit_cache()
+            .update_deposit_cache(None, &endpoints)
             .await
             .expect("should perform update");
 
@@ -709,7 +732,7 @@ mod persist {
     async fn test_persist_caches() {
         let log = null_logger();
 
-        let eth1 = GanacheEth1Instance::new()
+        let eth1 = new_ganache_instance()
             .await
             .expect("should start eth1 environment");
         let deposit_contract = &eth1.deposit_contract;
@@ -717,7 +740,7 @@ mod persist {
 
         let now = get_block_number(&web3).await;
         let config = Config {
-            endpoint: eth1.endpoint(),
+            endpoints: vec![eth1.endpoint()],
             deposit_contract_address: deposit_contract.address(),
             deposit_contract_deploy_block: now,
             lowest_cached_block_number: now,
@@ -735,8 +758,9 @@ mod persist {
                 .expect("should perform a deposit");
         }
 
+        let endpoints = service.init_endpoints();
         service
-            .update_deposit_cache()
+            .update_deposit_cache(None, &endpoints)
             .await
             .expect("should perform update");
 
@@ -748,7 +772,7 @@ mod persist {
         let deposit_count = service.deposit_cache_len();
 
         service
-            .update_block_cache()
+            .update_block_cache(None, &endpoints)
             .await
             .expect("should perform update");
 
@@ -778,3 +802,273 @@ mod persist {
         );
     }
 }
+
+/// Tests for eth1 fallback
+mod fallbacks {
+    use super::*;
+    use tokio::time::delay_for;
+
+    #[tokio::test]
+    async fn test_fallback_when_offline() {
+        let log = null_logger();
+        let endpoint2 = new_ganache_instance()
+            .await
+            .expect("should start eth1 environment");
+        let deposit_contract = &endpoint2.deposit_contract;
+
+        let initial_block_number = get_block_number(&endpoint2.web3()).await;
+
+        // Create some blocks and then consume them, performing the test `rounds` times.
+        let new_blocks = 4;
+
+        for _ in 0..new_blocks {
+            endpoint2
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+        }
+
+        let endpoint1 = endpoint2
+            .ganache
+            .fork()
+            .expect("should start eth1 environment");
+
+        //mine additional blocks on top of the original endpoint
+        for _ in 0..new_blocks {
+            endpoint2
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+        }
+
+        let service = Service::new(
+            Config {
+                endpoints: vec![endpoint1.endpoint(), endpoint2.endpoint()],
+                deposit_contract_address: deposit_contract.address(),
+                lowest_cached_block_number: initial_block_number,
+                follow_distance: 0,
+                ..Config::default()
+            },
+            log.clone(),
+            MainnetEthSpec::default_spec(),
+        );
+
+        let endpoint1_block_number = get_block_number(&endpoint1.web3).await;
+        //the first call will only query endpoint1
+        service.update().await.expect("should update deposit cache");
+        assert_eq!(
+            service.deposits().read().last_processed_block.unwrap(),
+            endpoint1_block_number
+        );
+
+        drop(endpoint1);
+
+        let endpoint2_block_number = get_block_number(&endpoint2.web3()).await;
+        assert!(endpoint1_block_number < endpoint2_block_number);
+        //endpoint1 is offline => query will import blocks from endpoint2
+        service.update().await.expect("should update deposit cache");
+        assert_eq!(
+            service.deposits().read().last_processed_block.unwrap(),
+            endpoint2_block_number
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fallback_when_wrong_network_id() {
+        let log = null_logger();
+        let correct_network_id: u64 = DEFAULT_NETWORK_ID.into();
+        let wrong_network_id = correct_network_id + 1;
+        let endpoint1 = GanacheEth1Instance::new(wrong_network_id, DEFAULT_CHAIN_ID.into())
+            .await
+            .expect("should start eth1 environment");
+        let endpoint2 = new_ganache_instance()
+            .await
+            .expect("should start eth1 environment");
+        let deposit_contract = &endpoint2.deposit_contract;
+
+        let initial_block_number = get_block_number(&endpoint2.web3()).await;
+
+        // Create some blocks and then consume them, performing the test `rounds` times.
+        let new_blocks = 4;
+
+        for _ in 0..new_blocks {
+            endpoint1
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+            endpoint2
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+        }
+
+        //additional blocks for endpoint1 to be able to distinguish
+        for _ in 0..new_blocks {
+            endpoint1
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+        }
+
+        let service = Service::new(
+            Config {
+                endpoints: vec![endpoint2.endpoint(), endpoint1.endpoint()],
+                deposit_contract_address: deposit_contract.address(),
+                lowest_cached_block_number: initial_block_number,
+                follow_distance: 0,
+                ..Config::default()
+            },
+            log.clone(),
+            MainnetEthSpec::default_spec(),
+        );
+
+        let endpoint1_block_number = get_block_number(&endpoint1.web3()).await;
+        let endpoint2_block_number = get_block_number(&endpoint2.web3()).await;
+        assert!(endpoint2_block_number < endpoint1_block_number);
+        //the call will fallback to endpoint2
+        service.update().await.expect("should update deposit cache");
+        assert_eq!(
+            service.deposits().read().last_processed_block.unwrap(),
+            endpoint2_block_number
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fallback_when_wrong_chain_id() {
+        let log = null_logger();
+        let correct_chain_id: u64 = DEFAULT_CHAIN_ID.into();
+        let wrong_chain_id = correct_chain_id + 1;
+        let endpoint1 = GanacheEth1Instance::new(DEFAULT_NETWORK_ID.into(), wrong_chain_id)
+            .await
+            .expect("should start eth1 environment");
+        let endpoint2 = new_ganache_instance()
+            .await
+            .expect("should start eth1 environment");
+        let deposit_contract = &endpoint2.deposit_contract;
+
+        let initial_block_number = get_block_number(&endpoint2.web3()).await;
+
+        // Create some blocks and then consume them, performing the test `rounds` times.
+        let new_blocks = 4;
+
+        for _ in 0..new_blocks {
+            endpoint1
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+            endpoint2
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+        }
+
+        //additional blocks for endpoint1 to be able to distinguish
+        for _ in 0..new_blocks {
+            endpoint1
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+        }
+
+        let service = Service::new(
+            Config {
+                endpoints: vec![endpoint2.endpoint(), endpoint1.endpoint()],
+                deposit_contract_address: deposit_contract.address(),
+                lowest_cached_block_number: initial_block_number,
+                follow_distance: 0,
+                ..Config::default()
+            },
+            log.clone(),
+            MainnetEthSpec::default_spec(),
+        );
+
+        let endpoint1_block_number = get_block_number(&endpoint1.web3()).await;
+        let endpoint2_block_number = get_block_number(&endpoint2.web3()).await;
+        assert!(endpoint2_block_number < endpoint1_block_number);
+        //the call will fallback to endpoint2
+        service.update().await.expect("should update deposit cache");
+        assert_eq!(
+            service.deposits().read().last_processed_block.unwrap(),
+            endpoint2_block_number
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fallback_when_node_far_behind() {
+        let log = null_logger();
+        let endpoint2 = new_ganache_instance()
+            .await
+            .expect("should start eth1 environment");
+        let deposit_contract = &endpoint2.deposit_contract;
+
+        let initial_block_number = get_block_number(&endpoint2.web3()).await;
+
+        // Create some blocks and then consume them, performing the test `rounds` times.
+        let new_blocks = 4;
+
+        for _ in 0..new_blocks {
+            endpoint2
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+        }
+
+        let endpoint1 = endpoint2
+            .ganache
+            .fork()
+            .expect("should start eth1 environment");
+
+        let service = Service::new(
+            Config {
+                endpoints: vec![endpoint1.endpoint(), endpoint2.endpoint()],
+                deposit_contract_address: deposit_contract.address(),
+                lowest_cached_block_number: initial_block_number,
+                follow_distance: 0,
+                node_far_behind_seconds: 5,
+                ..Config::default()
+            },
+            log.clone(),
+            MainnetEthSpec::default_spec(),
+        );
+
+        let endpoint1_block_number = get_block_number(&endpoint1.web3).await;
+        //the first call will only query endpoint1
+        service.update().await.expect("should update deposit cache");
+        assert_eq!(
+            service.deposits().read().last_processed_block.unwrap(),
+            endpoint1_block_number
+        );
+
+        delay_for(Duration::from_secs(7)).await;
+
+        //both endpoints don't have recent blocks => should return error
+        assert!(service.update().await.is_err());
+
+        //produce some new blocks on endpoint2
+        for _ in 0..new_blocks {
+            endpoint2
+                .ganache
+                .evm_mine()
+                .await
+                .expect("should mine block");
+        }
+
+        let endpoint2_block_number = get_block_number(&endpoint2.web3()).await;
+
+        //endpoint1 is far behind + endpoint2 not => update will import blocks from endpoint2
+        service.update().await.expect("should update deposit cache");
+        assert_eq!(
+            service.deposits().read().last_processed_block.unwrap(),
+            endpoint2_block_number
+        );
+    }
+}

beacon_node/eth2_libp2p/Cargo.toml

@@ -5,50 +5,50 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
 [dependencies]
-hex = "0.4.2"
+discv5 = { git = "https://github.com/sigp/discv5", rev = "fba7ceb5cfebd219ebbad6ffdb5d8c31dc8e4bc0", features = ["libp2p"] }
 types = { path =  "../../consensus/types" }
 hashset_delay = { path = "../../common/hashset_delay" }
 eth2_ssz_types = { path =  "../../consensus/ssz_types" }
-serde = { version = "1.0.110", features = ["derive"] }
-serde_derive = "1.0.110"
+serde = { version = "1.0.116", features = ["derive"] }
+serde_derive = "1.0.116"
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 lighthouse_version = { path = "../../common/lighthouse_version" }
-tokio = { version = "0.2.21", features = ["time", "macros"] }
+tokio = { version = "0.2.22", features = ["time", "macros"] }
 futures = "0.3.5"
-error-chain = "0.12.2"
-dirs = "2.0.2"
+error-chain = "0.12.4"
+dirs = "3.0.1"
 fnv = "1.0.7"
 unsigned-varint = { git = "https://github.com/sigp/unsigned-varint", branch = "latest-codecs", features = ["codec"] }
 lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-smallvec = "1.4.1"
-lru = "0.5.1"
+smallvec = "1.4.2"
+lru = "0.6.0"
 parking_lot = "0.11.0"
 sha2 = "0.9.1"
-base64 = "0.12.1"
-snap = "1.0.0"
+base64 = "0.13.0"
+snap = "1.0.1"
 void = "1.0.2"
+hex = "0.4.2"
 tokio-io-timeout = "0.4.0"
 tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
-discv5 = { version = "0.1.0-alpha.8", features = ["libp2p", "openssl-vendored"] }
 tiny-keccak = "2.0.2"
-environment = { path = "../../lighthouse/environment" }
-# TODO: Remove rand crate for mainnet
+task_executor = { path = "../../common/task_executor" }
 rand = "0.7.3"
+directory = { path = "../../common/directory" }
+regex = "1.3.9"
 
 [dependencies.libp2p]
 #version = "0.23.0"
 git = "https://github.com/sigp/rust-libp2p"
-rev = "bbf0cfbaff2f733b3ae7bfed3caba8b7ee542803"
+rev = "f53d02bc873fef2bf52cd31e3d5ce366a41d8a8c"
 default-features = false
-features = ["websocket", "identify", "mplex", "noise", "gossipsub", "dns", "tcp-tokio"]
+features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "tcp-tokio"]
 
 [dev-dependencies]
-tokio = { version = "0.2.21", features = ["full"] }
-slog-stdlog = "4.0.0"
-slog-term = "2.5.0"
+tokio = { version = "0.2.22", features = ["full"] }
+slog-term = "2.6.0"
 slog-async = "2.5.0"
 tempdir = "0.3.7"
 exit-future = "0.2.0"

beacon_node/eth2_libp2p/src/behaviour/gossipsub_scoring_parameters.rs

@@ -0,0 +1,346 @@
+use crate::types::{GossipEncoding, GossipKind, GossipTopic};
+use crate::{error, TopicHash};
+use libp2p::gossipsub::{
+    GenericGossipsubConfig, IdentTopic as Topic, PeerScoreParams, PeerScoreThresholds,
+    TopicScoreParams,
+};
+use std::cmp::max;
+use std::collections::HashMap;
+use std::marker::PhantomData;
+use std::time::Duration;
+use types::{ChainSpec, EnrForkId, EthSpec, Slot, SubnetId};
+
+const MAX_IN_MESH_SCORE: f64 = 10.0;
+const MAX_FIRST_MESSAGE_DELIVERIES_SCORE: f64 = 40.0;
+const BEACON_BLOCK_WEIGHT: f64 = 0.5;
+const BEACON_AGGREGATE_PROOF_WEIGHT: f64 = 0.5;
+const VOLUNTARY_EXIT_WEIGHT: f64 = 0.05;
+const PROPOSER_SLASHING_WEIGHT: f64 = 0.05;
+const ATTESTER_SLASHING_WEIGHT: f64 = 0.05;
+
+pub struct PeerScoreSettings<TSpec: EthSpec> {
+    slot: Duration,
+    epoch: Duration,
+
+    beacon_attestation_subnet_weight: f64,
+    max_positive_score: f64,
+
+    decay_interval: Duration,
+    decay_to_zero: f64,
+
+    mesh_n: usize,
+    max_committees_per_slot: usize,
+    target_committee_size: usize,
+    target_aggregators_per_committee: usize,
+    attestation_subnet_count: u64,
+    phantom: PhantomData<TSpec>,
+}
+
+impl<TSpec: EthSpec> PeerScoreSettings<TSpec> {
+    pub fn new<T>(
+        chain_spec: &ChainSpec,
+        gs_config: &GenericGossipsubConfig<T>,
+    ) -> PeerScoreSettings<TSpec> {
+        let slot = Duration::from_millis(chain_spec.milliseconds_per_slot);
+        let beacon_attestation_subnet_weight = 1.0 / chain_spec.attestation_subnet_count as f64;
+        let max_positive_score = (MAX_IN_MESH_SCORE + MAX_FIRST_MESSAGE_DELIVERIES_SCORE)
+            * (BEACON_BLOCK_WEIGHT
+                + BEACON_AGGREGATE_PROOF_WEIGHT
+                + beacon_attestation_subnet_weight * chain_spec.attestation_subnet_count as f64
+                + VOLUNTARY_EXIT_WEIGHT
+                + PROPOSER_SLASHING_WEIGHT
+                + ATTESTER_SLASHING_WEIGHT);
+
+        PeerScoreSettings {
+            slot,
+            epoch: slot * TSpec::slots_per_epoch() as u32,
+            beacon_attestation_subnet_weight,
+            max_positive_score,
+            decay_interval: slot,
+            decay_to_zero: 0.01,
+            mesh_n: gs_config.mesh_n(),
+            max_committees_per_slot: chain_spec.max_committees_per_slot,
+            target_committee_size: chain_spec.target_committee_size,
+            target_aggregators_per_committee: chain_spec.target_aggregators_per_committee as usize,
+            attestation_subnet_count: chain_spec.attestation_subnet_count,
+            phantom: PhantomData,
+        }
+    }
+
+    pub fn get_peer_score_params(
+        &self,
+        active_validators: usize,
+        thresholds: &PeerScoreThresholds,
+        enr_fork_id: &EnrForkId,
+        current_slot: Slot,
+    ) -> error::Result<PeerScoreParams> {
+        let mut params = PeerScoreParams::default();
+
+        params.decay_interval = self.decay_interval;
+        params.decay_to_zero = self.decay_to_zero;
+        params.retain_score = self.epoch * 100;
+        params.app_specific_weight = 1.0;
+        params.ip_colocation_factor_threshold = 3.0;
+        params.behaviour_penalty_threshold = 6.0;
+
+        params.behaviour_penalty_decay = self.score_parameter_decay(self.epoch * 10);
+
+        let target_value = Self::decay_convergence(
+            params.behaviour_penalty_decay,
+            10.0 / TSpec::slots_per_epoch() as f64,
+        ) - params.behaviour_penalty_threshold;
+        params.behaviour_penalty_weight = thresholds.gossip_threshold / target_value.powi(2);
+
+        params.topic_score_cap = self.max_positive_score * 0.5;
+        params.ip_colocation_factor_weight = -params.topic_score_cap;
+
+        params.topics = HashMap::new();
+
+        let get_hash = |kind: GossipKind| -> TopicHash {
+            let topic: Topic =
+                GossipTopic::new(kind, GossipEncoding::default(), enr_fork_id.fork_digest).into();
+            topic.hash()
+        };
+
+        //first all fixed topics
+        params.topics.insert(
+            get_hash(GossipKind::VoluntaryExit),
+            Self::get_topic_params(
+                self,
+                VOLUNTARY_EXIT_WEIGHT,
+                4.0 / TSpec::slots_per_epoch() as f64,
+                self.epoch * 100,
+                None,
+            ),
+        );
+        params.topics.insert(
+            get_hash(GossipKind::AttesterSlashing),
+            Self::get_topic_params(
+                self,
+                ATTESTER_SLASHING_WEIGHT,
+                1.0 / 5.0 / TSpec::slots_per_epoch() as f64,
+                self.epoch * 100,
+                None,
+            ),
+        );
+        params.topics.insert(
+            get_hash(GossipKind::ProposerSlashing),
+            Self::get_topic_params(
+                self,
+                PROPOSER_SLASHING_WEIGHT,
+                1.0 / 5.0 / TSpec::slots_per_epoch() as f64,
+                self.epoch * 100,
+                None,
+            ),
+        );
+
+        //dynamic topics
+        let (beacon_block_params, beacon_aggregate_proof_params, beacon_attestation_subnet_params) =
+            self.get_dynamic_topic_params(active_validators, current_slot)?;
+
+        params
+            .topics
+            .insert(get_hash(GossipKind::BeaconBlock), beacon_block_params);
+
+        params.topics.insert(
+            get_hash(GossipKind::BeaconAggregateAndProof),
+            beacon_aggregate_proof_params,
+        );
+
+        for i in 0..self.attestation_subnet_count {
+            params.topics.insert(
+                get_hash(GossipKind::Attestation(SubnetId::new(i))),
+                beacon_attestation_subnet_params.clone(),
+            );
+        }
+
+        Ok(params)
+    }
+
+    pub fn get_dynamic_topic_params(
+        &self,
+        active_validators: usize,
+        current_slot: Slot,
+    ) -> error::Result<(TopicScoreParams, TopicScoreParams, TopicScoreParams)> {
+        let (aggregators_per_slot, committees_per_slot) =
+            self.expected_aggregator_count_per_slot(active_validators)?;
+        let multiple_bursts_per_subnet_per_epoch = committees_per_slot as u64
+            >= 2 * self.attestation_subnet_count / TSpec::slots_per_epoch();
+
+        let beacon_block_params = Self::get_topic_params(
+            self,
+            BEACON_BLOCK_WEIGHT,
+            1.0,
+            self.epoch * 20,
+            Some((TSpec::slots_per_epoch() * 5, 3.0, self.epoch, current_slot)),
+        );
+
+        let beacon_aggregate_proof_params = Self::get_topic_params(
+            self,
+            BEACON_AGGREGATE_PROOF_WEIGHT,
+            aggregators_per_slot,
+            self.epoch,
+            Some((TSpec::slots_per_epoch() * 2, 4.0, self.epoch, current_slot)),
+        );
+        let beacon_attestation_subnet_params = Self::get_topic_params(
+            self,
+            self.beacon_attestation_subnet_weight,
+            active_validators as f64
+                / self.attestation_subnet_count as f64
+                / TSpec::slots_per_epoch() as f64,
+            self.epoch
+                * (if multiple_bursts_per_subnet_per_epoch {
+                    1
+                } else {
+                    4
+                }),
+            Some((
+                TSpec::slots_per_epoch()
+                    * (if multiple_bursts_per_subnet_per_epoch {
+                        4
+                    } else {
+                        16
+                    }),
+                16.0,
+                if multiple_bursts_per_subnet_per_epoch {
+                    self.slot * (TSpec::slots_per_epoch() as u32 / 2 + 1)
+                } else {
+                    self.epoch * 3
+                },
+                current_slot,
+            )),
+        );
+
+        Ok((
+            beacon_block_params,
+            beacon_aggregate_proof_params,
+            beacon_attestation_subnet_params,
+        ))
+    }
+
+    pub fn attestation_subnet_count(&self) -> u64 {
+        self.attestation_subnet_count
+    }
+
+    fn score_parameter_decay_with_base(
+        decay_time: Duration,
+        decay_interval: Duration,
+        decay_to_zero: f64,
+    ) -> f64 {
+        let ticks = decay_time.as_secs_f64() / decay_interval.as_secs_f64();
+        decay_to_zero.powf(1.0 / ticks)
+    }
+
+    fn decay_convergence(decay: f64, rate: f64) -> f64 {
+        rate / (1.0 - decay)
+    }
+
+    fn threshold(decay: f64, rate: f64) -> f64 {
+        Self::decay_convergence(decay, rate) * decay
+    }
+
+    fn expected_aggregator_count_per_slot(
+        &self,
+        active_validators: usize,
+    ) -> error::Result<(f64, usize)> {
+        let committees_per_slot = TSpec::get_committee_count_per_slot_with(
+            active_validators,
+            self.max_committees_per_slot,
+            self.target_committee_size,
+        )
+        .map_err(|e| format!("Could not get committee count from spec: {:?}", e))?;
+
+        let committees = committees_per_slot * TSpec::slots_per_epoch() as usize;
+
+        let smaller_committee_size = active_validators / committees;
+        let num_larger_committees = active_validators - smaller_committee_size * committees;
+
+        let modulo_smaller = max(
+            1,
+            smaller_committee_size / self.target_aggregators_per_committee as usize,
+        );
+        let modulo_larger = max(
+            1,
+            (smaller_committee_size + 1) / self.target_aggregators_per_committee as usize,
+        );
+
+        Ok((
+            (((committees - num_larger_committees) * smaller_committee_size) as f64
+                / modulo_smaller as f64
+                + (num_larger_committees * (smaller_committee_size + 1)) as f64
+                    / modulo_larger as f64)
+                / TSpec::slots_per_epoch() as f64,
+            committees_per_slot,
+        ))
+    }
+
+    fn score_parameter_decay(&self, decay_time: Duration) -> f64 {
+        Self::score_parameter_decay_with_base(decay_time, self.decay_interval, self.decay_to_zero)
+    }
+
+    fn get_topic_params(
+        &self,
+        topic_weight: f64,
+        expected_message_rate: f64,
+        first_message_decay_time: Duration,
+        // decay slots (decay time in slots), cap factor, activation window, current slot
+        mesh_message_info: Option<(u64, f64, Duration, Slot)>,
+    ) -> TopicScoreParams {
+        let mut t_params = TopicScoreParams::default();
+
+        t_params.topic_weight = topic_weight;
+
+        t_params.time_in_mesh_quantum = self.slot;
+        t_params.time_in_mesh_cap = 3600.0 / t_params.time_in_mesh_quantum.as_secs_f64();
+        t_params.time_in_mesh_weight = 10.0 / t_params.time_in_mesh_cap;
+
+        t_params.first_message_deliveries_decay =
+            self.score_parameter_decay(first_message_decay_time);
+        t_params.first_message_deliveries_cap = Self::decay_convergence(
+            t_params.first_message_deliveries_decay,
+            2.0 * expected_message_rate / self.mesh_n as f64,
+        );
+        t_params.first_message_deliveries_weight = 40.0 / t_params.first_message_deliveries_cap;
+
+        if let Some((decay_slots, cap_factor, activation_window, current_slot)) = mesh_message_info
+        {
+            let decay_time = self.slot * decay_slots as u32;
+            t_params.mesh_message_deliveries_decay = self.score_parameter_decay(decay_time);
+            t_params.mesh_message_deliveries_threshold = Self::threshold(
+                t_params.mesh_message_deliveries_decay,
+                expected_message_rate / 50.0,
+            );
+            t_params.mesh_message_deliveries_cap =
+                if cap_factor * t_params.mesh_message_deliveries_threshold < 2.0 {
+                    2.0
+                } else {
+                    cap_factor * t_params.mesh_message_deliveries_threshold
+                };
+            t_params.mesh_message_deliveries_activation = activation_window;
+            t_params.mesh_message_deliveries_window = Duration::from_secs(2);
+            t_params.mesh_failure_penalty_decay = t_params.mesh_message_deliveries_decay;
+            t_params.mesh_message_deliveries_weight = -self.max_positive_score
+                / (t_params.topic_weight * t_params.mesh_message_deliveries_threshold.powi(2));
+            t_params.mesh_failure_penalty_weight = t_params.mesh_message_deliveries_weight;
+            if decay_slots >= current_slot.as_u64() {
+                t_params.mesh_message_deliveries_threshold = 0.0;
+                t_params.mesh_message_deliveries_weight = 0.0;
+            }
+        } else {
+            t_params.mesh_message_deliveries_weight = 0.0;
+            t_params.mesh_message_deliveries_threshold = 0.0;
+            t_params.mesh_message_deliveries_decay = 0.0;
+            t_params.mesh_message_deliveries_cap = 0.0;
+            t_params.mesh_message_deliveries_window = Duration::from_secs(0);
+            t_params.mesh_message_deliveries_activation = Duration::from_secs(0);
+            t_params.mesh_failure_penalty_decay = 0.0;
+            t_params.mesh_failure_penalty_weight = 0.0;
+        }
+
+        t_params.invalid_message_deliveries_weight =
+            -self.max_positive_score / t_params.topic_weight;
+        t_params.invalid_message_deliveries_decay = self.score_parameter_decay(self.epoch * 50);
+
+        t_params
+    }
+}

beacon_node/eth2_libp2p/src/behaviour/handler/delegate.rs

@@ -1,8 +1,8 @@
+use crate::behaviour::Gossipsub;
 use crate::rpc::*;
 use libp2p::{
     core::either::{EitherError, EitherOutput},
     core::upgrade::{EitherUpgrade, InboundUpgrade, OutboundUpgrade, SelectUpgrade, UpgradeError},
-    gossipsub::Gossipsub,
     identify::Identify,
     swarm::{
         protocols_handler::{
@@ -54,8 +54,6 @@ impl<TSpec: EthSpec> DelegatingHandler<TSpec> {
     }
 }
 
-// TODO: this can all be created with macros
-
 /// Wrapper around the `ProtocolsHandler::InEvent` types of the handlers.
 /// Simply delegated to the corresponding behaviour's handler.
 #[derive(Debug, Clone)]
@@ -115,7 +113,6 @@ pub type DelegateOutProto<TSpec> = EitherUpgrade<
     >,
 >;
 
-// TODO: prob make this an enum
 pub type DelegateOutInfo<TSpec> = EitherOutput<
     <GossipHandler as ProtocolsHandler>::OutboundOpenInfo,
     EitherOutput<
@@ -131,8 +128,9 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         let gossip_proto = self.gossip_handler.listen_protocol();
         let rpc_proto = self.rpc_handler.listen_protocol();
         let identify_proto = self.identify_handler.listen_protocol();
@@ -147,24 +145,27 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             SelectUpgrade::new(rpc_proto.into_upgrade().1, identify_proto.into_upgrade().1),
         );
 
-        SubstreamProtocol::new(select).with_timeout(timeout)
+        SubstreamProtocol::new(select, ()).with_timeout(timeout)
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         match out {
             // Gossipsub
-            EitherOutput::First(out) => self.gossip_handler.inject_fully_negotiated_inbound(out),
+            EitherOutput::First(out) => {
+                self.gossip_handler.inject_fully_negotiated_inbound(out, ())
+            }
             // RPC
             EitherOutput::Second(EitherOutput::First(out)) => {
-                self.rpc_handler.inject_fully_negotiated_inbound(out)
+                self.rpc_handler.inject_fully_negotiated_inbound(out, ())
             }
             // Identify
-            EitherOutput::Second(EitherOutput::Second(out)) => {
-                self.identify_handler.inject_fully_negotiated_inbound(out)
-            }
+            EitherOutput::Second(EitherOutput::Second(out)) => self
+                .identify_handler
+                .inject_fully_negotiated_inbound(out, ()),
         }
     }
 
@@ -212,7 +213,6 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             <Self::OutboundProtocol as OutboundUpgrade<NegotiatedSubstream>>::Error,
         >,
     ) {
-        // TODO: find how to clean up
         match info {
             // Gossipsub
             EitherOutput::First(info) => match error {
@@ -317,10 +317,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
                     event,
                 )));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(EitherUpgrade::A),
-                    info: EitherOutput::First(info),
+                    protocol: protocol
+                        .map_upgrade(EitherUpgrade::A)
+                        .map_info(EitherOutput::First),
                 });
             }
             Poll::Pending => (),
@@ -333,10 +334,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::RPC(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u))),
-                    info: EitherOutput::Second(EitherOutput::First(info)),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u)))
+                        .map_info(|info| EitherOutput::Second(EitherOutput::First(info))),
                 });
             }
             Poll::Pending => (),
@@ -351,10 +353,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::Identify(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u))),
-                    info: EitherOutput::Second(EitherOutput::Second(())),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u)))
+                        .map_info(|_| EitherOutput::Second(EitherOutput::Second(()))),
                 });
             }
             Poll::Pending => (),

beacon_node/eth2_libp2p/src/behaviour/handler/mod.rs

@@ -1,3 +1,4 @@
+use crate::behaviour::Gossipsub;
 use crate::rpc::*;
 use delegate::DelegatingHandler;
 pub(super) use delegate::{
@@ -5,7 +6,6 @@ pub(super) use delegate::{
 };
 use libp2p::{
     core::upgrade::{InboundUpgrade, OutboundUpgrade},
-    gossipsub::Gossipsub,
     identify::Identify,
     swarm::protocols_handler::{
         KeepAlive, ProtocolsHandlerEvent, ProtocolsHandlerUpgrErr, SubstreamProtocol,
@@ -41,29 +41,25 @@ pub enum BehaviourHandlerIn<TSpec: EthSpec> {
     Shutdown(Option<(RequestId, RPCRequest<TSpec>)>),
 }
 
-pub enum BehaviourHandlerOut<TSpec: EthSpec> {
-    Delegate(Box<DelegateOut<TSpec>>),
-    // TODO: replace custom with events to send
-    Custom,
-}
-
 impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
     type InEvent = BehaviourHandlerIn<TSpec>;
-    type OutEvent = BehaviourHandlerOut<TSpec>;
+    type OutEvent = DelegateOut<TSpec>;
     type Error = DelegateError<TSpec>;
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.delegate.listen_protocol()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
-        self.delegate.inject_fully_negotiated_inbound(out)
+        self.delegate.inject_fully_negotiated_inbound(out, ())
     }
 
     fn inject_fully_negotiated_outbound(
@@ -120,18 +116,13 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
 
         match self.delegate.poll(cx) {
             Poll::Ready(ProtocolsHandlerEvent::Custom(event)) => {
-                return Poll::Ready(ProtocolsHandlerEvent::Custom(
-                    BehaviourHandlerOut::Delegate(Box::new(event)),
-                ))
+                return Poll::Ready(ProtocolsHandlerEvent::Custom(event))
             }
             Poll::Ready(ProtocolsHandlerEvent::Close(err)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(err))
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
-                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol,
-                    info,
-                });
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
+                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol });
             }
             Poll::Pending => (),
         }

beacon_node/eth2_libp2p/src/behaviour/mod.rs

@@ -1,17 +1,26 @@
-use crate::peer_manager::{score::PeerAction, PeerManager, PeerManagerEvent};
+use crate::behaviour::gossipsub_scoring_parameters::PeerScoreSettings;
+use crate::peer_manager::{score::PeerAction, ConnectionDirection, PeerManager, PeerManagerEvent};
 use crate::rpc::*;
-use crate::types::{EnrBitfield, GossipEncoding, GossipKind, GossipTopic, SubnetDiscovery};
+use crate::service::METADATA_FILENAME;
+use crate::types::{GossipEncoding, GossipKind, GossipTopic, MessageData, SubnetDiscovery};
 use crate::Eth2Enr;
 use crate::{error, metrics, Enr, NetworkConfig, NetworkGlobals, PubsubMessage, TopicHash};
 use futures::prelude::*;
-use handler::{BehaviourHandler, BehaviourHandlerIn, BehaviourHandlerOut, DelegateIn, DelegateOut};
+use handler::{BehaviourHandler, BehaviourHandlerIn, DelegateIn, DelegateOut};
+use libp2p::gossipsub::subscription_filter::{
+    MaxCountSubscriptionFilter, WhitelistSubscriptionFilter,
+};
+use libp2p::gossipsub::PeerScoreThresholds;
 use libp2p::{
     core::{
         connection::{ConnectedPoint, ConnectionId, ListenerId},
         identity::Keypair,
         Multiaddr,
     },
-    gossipsub::{Gossipsub, GossipsubEvent, MessageAuthenticity, MessageId},
+    gossipsub::{
+        GenericGossipsub, GenericGossipsubEvent, IdentTopic as Topic, MessageAcceptance,
+        MessageAuthenticity, MessageId,
+    },
     identify::{Identify, IdentifyEvent},
     swarm::{
         NetworkBehaviour, NetworkBehaviourAction as NBAction, NotifyHandler, PollParameters,
@@ -20,9 +29,10 @@ use libp2p::{
     PeerId,
 };
 use slog::{crit, debug, o, trace, warn};
-use ssz::{Decode, Encode};
+use ssz::Encode;
+use std::collections::HashSet;
 use std::fs::File;
-use std::io::{Read, Write};
+use std::io::Write;
 use std::path::PathBuf;
 use std::{
     collections::VecDeque,
@@ -30,12 +40,68 @@ use std::{
     sync::Arc,
     task::{Context, Poll},
 };
-use types::{EnrForkId, EthSpec, SignedBeaconBlock, SubnetId};
+use types::{ChainSpec, EnrForkId, EthSpec, SignedBeaconBlock, Slot, SubnetId};
 
+mod gossipsub_scoring_parameters;
 mod handler;
 
 const MAX_IDENTIFY_ADDRESSES: usize = 10;
-const METADATA_FILENAME: &str = "metadata";
+pub const GOSSIPSUB_GREYLIST_THRESHOLD: f64 = -16000.0;
+
+/// Identifier of requests sent by a peer.
+pub type PeerRequestId = (ConnectionId, SubstreamId);
+
+pub type SubscriptionFilter = MaxCountSubscriptionFilter<WhitelistSubscriptionFilter>;
+pub type Gossipsub = GenericGossipsub<MessageData, SubscriptionFilter>;
+pub type GossipsubEvent = GenericGossipsubEvent<MessageData>;
+
+/// The types of events than can be obtained from polling the behaviour.
+#[derive(Debug)]
+pub enum BehaviourEvent<TSpec: EthSpec> {
+    /// We have successfully dialed and connected to a peer.
+    PeerDialed(PeerId),
+    /// A peer has successfully dialed and connected to us.
+    PeerConnected(PeerId),
+    /// A peer has disconnected.
+    PeerDisconnected(PeerId),
+    /// An RPC Request that was sent failed.
+    RPCFailed {
+        /// The id of the failed request.
+        id: RequestId,
+        /// The peer to which this request was sent.
+        peer_id: PeerId,
+    },
+    RequestReceived {
+        /// The peer that sent the request.
+        peer_id: PeerId,
+        /// Identifier of the request. All responses to this request must use this id.
+        id: PeerRequestId,
+        /// Request the peer sent.
+        request: Request,
+    },
+    ResponseReceived {
+        /// Peer that sent the response.
+        peer_id: PeerId,
+        /// Id of the request to which the peer is responding.
+        id: RequestId,
+        /// Response the peer sent.
+        response: Response<TSpec>,
+    },
+    PubsubMessage {
+        /// The gossipsub message id. Used when propagating blocks after validation.
+        id: MessageId,
+        /// The peer from which we received this message, not the peer that published it.
+        source: PeerId,
+        /// The topic that this message was sent on.
+        topic: TopicHash,
+        /// The message itself.
+        message: PubsubMessage<TSpec>,
+    },
+    /// Subscribed to peer for given topic
+    PeerSubscribed(PeerId, TopicHash),
+    /// Inform the network to send a Status to this peer.
+    StatusPeer(PeerId),
+}
 
 /// Builds the network behaviour that manages the core protocols of eth2.
 /// This core behaviour is managed by `Behaviour` which adds peer management to all core
@@ -46,7 +112,7 @@ pub struct Behaviour<TSpec: EthSpec> {
     /// The Eth2 RPC specified in the wire-0 protocol.
     eth2_rpc: RPC<TSpec>,
     /// Keep regular connection to peers and disconnect if absent.
-    // TODO: Using id for initial interop. This will be removed by mainnet.
+    // NOTE: The id protocol is used for initial interop. This will be removed by mainnet.
     /// Provides IP addresses and peer information.
     identify: Identify,
     /// The peer manager that keeps track of peer's reputation and status.
@@ -55,8 +121,6 @@ pub struct Behaviour<TSpec: EthSpec> {
     events: VecDeque<BehaviourEvent<TSpec>>,
     /// Queue of peers to disconnect and an optional reason for the disconnection.
     peers_to_dc: VecDeque<(PeerId, Option<GoodbyeReason>)>,
-    /// The current meta data of the node, so respond to pings and get metadata
-    meta_data: MetaData<TSpec>,
     /// A collections of variables accessible outside the network service.
     network_globals: Arc<NetworkGlobals<TSpec>>,
     /// Keeps track of the current EnrForkId for upgrading gossipsub topics.
@@ -69,6 +133,11 @@ pub struct Behaviour<TSpec: EthSpec> {
     network_dir: PathBuf,
     /// Logger for behaviour actions.
     log: slog::Logger,
+
+    score_settings: PeerScoreSettings<TSpec>,
+
+    /// The interval for updating gossipsub scores
+    update_gossipsub_scores: tokio::time::Interval,
 }
 
 /// Implements the combined behaviour for the libp2p service.
@@ -78,6 +147,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
         net_conf: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
         log: &slog::Logger,
+        chain_spec: &ChainSpec,
     ) -> error::Result<Self> {
         let behaviour_log = log.new(o!());
 
@@ -92,31 +162,108 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .eth2()
             .expect("Local ENR must have a fork id");
 
-        let meta_data = load_or_build_metadata(&net_conf.network_dir, &log);
+        let possible_fork_digests = vec![enr_fork_id.fork_digest];
+        let filter = MaxCountSubscriptionFilter {
+            filter: Self::create_whitelist_filter(possible_fork_digests, 64), //TODO change this to a constant
+            max_subscribed_topics: 200, //TODO change this to a constant
+            max_subscriptions_per_request: 100, //this is according to the current go implementation
+        };
 
-        // TODO: Until other clients support no author, we will use a 0 peer_id as our author.
-        let message_author = PeerId::from_bytes(vec![0, 1, 0]).expect("Valid peer id");
+        let mut gossipsub = Gossipsub::new_with_subscription_filter(
+            MessageAuthenticity::Anonymous,
+            net_conf.gs_config.clone(),
+            filter,
+        )
+        .map_err(|e| format!("Could not construct gossipsub: {:?}", e))?;
+
+        //we don't know the number of active validators and the current slot yet
+        let active_validators = TSpec::minimum_validator_count();
+        let current_slot = Slot::new(0);
+
+        let thresholds = PeerScoreThresholds {
+            gossip_threshold: -4000.0,
+            publish_threshold: -8000.0,
+            graylist_threshold: GOSSIPSUB_GREYLIST_THRESHOLD,
+            accept_px_threshold: 100.0,
+            opportunistic_graft_threshold: 5.0,
+        };
+
+        let score_settings = PeerScoreSettings::new(chain_spec, &net_conf.gs_config);
+
+        //Prepare scoring parameters
+        let params = score_settings.get_peer_score_params(
+            active_validators,
+            &thresholds,
+            &enr_fork_id,
+            current_slot,
+        )?;
+
+        trace!(behaviour_log, "Using peer score params"; "params" => format!("{:?}", params));
+
+        let update_gossipsub_scores = tokio::time::interval(params.decay_interval);
+
+        gossipsub
+            .with_peer_score(params.clone(), thresholds)
+            .expect("Valid score params and thresholds");
 
         Ok(Behaviour {
             eth2_rpc: RPC::new(log.clone()),
-            gossipsub: Gossipsub::new(
-                MessageAuthenticity::Author(message_author),
-                net_conf.gs_config.clone(),
-            ),
+            gossipsub,
             identify,
             peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)
                 .await?,
             events: VecDeque::new(),
             peers_to_dc: VecDeque::new(),
-            meta_data,
             network_globals,
             enr_fork_id,
             waker: None,
             network_dir: net_conf.network_dir.clone(),
             log: behaviour_log,
+            score_settings,
+            update_gossipsub_scores,
         })
     }
 
+    pub fn update_gossipsub_parameters(
+        &mut self,
+        active_validators: usize,
+        current_slot: Slot,
+    ) -> error::Result<()> {
+        let (beacon_block_params, beacon_aggregate_proof_params, beacon_attestation_subnet_params) =
+            self.score_settings
+                .get_dynamic_topic_params(active_validators, current_slot)?;
+
+        let fork_digest = self.enr_fork_id.fork_digest;
+        let get_topic = |kind: GossipKind| -> Topic {
+            GossipTopic::new(kind, GossipEncoding::default(), fork_digest).into()
+        };
+
+        debug!(self.log, "Updating gossipsub score parameters";
+            "active_validators" => active_validators);
+        trace!(self.log, "Updated gossipsub score parameters";
+            "beacon_block_params" => format!("{:?}", beacon_block_params),
+            "beacon_aggregate_proof_params" => format!("{:?}", beacon_aggregate_proof_params),
+            "beacon_attestation_subnet_params" => format!("{:?}", beacon_attestation_subnet_params),
+        );
+
+        self.gossipsub
+            .set_topic_params(get_topic(GossipKind::BeaconBlock), beacon_block_params)?;
+
+        self.gossipsub.set_topic_params(
+            get_topic(GossipKind::BeaconAggregateAndProof),
+            beacon_aggregate_proof_params,
+        )?;
+
+        for i in 0..self.score_settings.attestation_subnet_count() {
+            self.gossipsub.set_topic_params(
+                get_topic(GossipKind::Attestation(SubnetId::new(i))),
+                beacon_attestation_subnet_params.clone(),
+            )?;
+        }
+
+        Ok(())
+    }
+
     /// Attempts to connect to a libp2p peer.
     ///
     /// This MUST be used over Swarm::dial() as this keeps track of the peer in the peer manager.
@@ -147,6 +294,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             GossipEncoding::default(),
             self.enr_fork_id.fork_digest,
         );
+
         self.subscribe(gossip_topic)
     }
 
@@ -189,9 +337,18 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .write()
             .insert(topic.clone());
 
-        let topic_str: String = topic.clone().into();
-        debug!(self.log, "Subscribed to topic"; "topic" => topic_str);
-        self.gossipsub.subscribe(topic.into())
+        let topic: Topic = topic.into();
+
+        match self.gossipsub.subscribe(&topic) {
+            Err(_) => {
+                warn!(self.log, "Failed to subscribe to topic"; "topic" => topic.to_string());
+                false
+            }
+            Ok(v) => {
+                debug!(self.log, "Subscribed to topic"; "topic" => topic.to_string());
+                v
+            }
+        }
     }
 
     /// Unsubscribe from a gossipsub topic.
@@ -201,8 +358,20 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .gossipsub_subscriptions
             .write()
             .remove(&topic);
+
         // unsubscribe from the topic
-        self.gossipsub.unsubscribe(topic.into())
+        let topic: Topic = topic.into();
+
+        match self.gossipsub.unsubscribe(&topic) {
+            Err(_) => {
+                warn!(self.log, "Failed to unsubscribe from topic"; "topic" => topic.to_string());
+                false
+            }
+            Ok(v) => {
+                debug!(self.log, "Unsubscribed to topic"; "topic" => topic.to_string());
+                v
+            }
+        }
     }
 
     /// Publishes a list of messages on the pubsub (gossipsub) behaviour, choosing the encoding.
@@ -211,8 +380,29 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) {
                 match message.encode(GossipEncoding::default()) {
                     Ok(message_data) => {
-                        if let Err(e) = self.gossipsub.publish(&topic.into(), message_data) {
-                            slog::warn!(self.log, "Could not publish message"; "error" => format!("{:?}", e));
+                        if let Err(e) = self.gossipsub.publish(topic.clone().into(), message_data) {
+                            slog::warn!(self.log, "Could not publish message";
+                                        "error" => format!("{:?}", e));
+
+                            // add to metrics
+                            match topic.kind() {
+                                GossipKind::Attestation(subnet_id) => {
+                                    if let Some(v) = metrics::get_int_gauge(
+                                        &metrics::FAILED_ATTESTATION_PUBLISHES_PER_SUBNET,
+                                        &[&subnet_id.to_string()],
+                                    ) {
+                                        v.inc()
+                                    };
+                                }
+                                kind => {
+                                    if let Some(v) = metrics::get_int_gauge(
+                                        &metrics::FAILED_PUBLISHES_PER_MAIN_TOPIC,
+                                        &[&format!("{:?}", kind)],
+                                    ) {
+                                        v.inc()
+                                    };
+                                }
+                            }
                         }
                     }
                     Err(e) => crit!(self.log, "Could not publish message"; "error" => e),
@@ -221,11 +411,21 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
         }
     }
 
-    /// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated
-    /// once validated by the beacon chain.
-    pub fn validate_message(&mut self, propagation_source: &PeerId, message_id: MessageId) {
-        self.gossipsub
-            .validate_message(&message_id, propagation_source);
+    /// Informs the gossipsub about the result of a message validation.
+    /// If the message is valid it will get propagated by gossipsub.
+    pub fn report_message_validation_result(
+        &mut self,
+        propagation_source: &PeerId,
+        message_id: MessageId,
+        validation_result: MessageAcceptance,
+    ) {
+        if let Err(e) = self.gossipsub.report_message_validation_result(
+            &message_id,
+            propagation_source,
+            validation_result,
+        ) {
+            warn!(self.log, "Failed to report message validation"; "message_id" => message_id.to_string(), "peer_id" => propagation_source.to_string(), "error" => format!("{:?}", e));
+        }
     }
 
     /* Eth2 RPC behaviour functions */
@@ -339,21 +539,31 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Updates the current meta data of the node to match the local ENR.
     fn update_metadata(&mut self) {
-        self.meta_data.seq_number += 1;
-        self.meta_data.attnets = self
+        let local_attnets = self
             .peer_manager
             .discovery()
             .local_enr()
             .bitfield::<TSpec>()
             .expect("Local discovery must have bitfield");
+
+        {
+            // write lock scope
+            let mut meta_data = self.network_globals.local_metadata.write();
+            meta_data.seq_number += 1;
+            meta_data.attnets = local_attnets;
+        }
         // Save the updated metadata to disk
-        save_metadata_to_disk(&self.network_dir, self.meta_data.clone(), &self.log);
+        save_metadata_to_disk(
+            &self.network_dir,
+            self.network_globals.local_metadata.read().clone(),
+            &self.log,
+        );
     }
 
     /// Sends a Ping request to the peer.
     fn ping(&mut self, id: RequestId, peer_id: PeerId) {
         let ping = crate::rpc::Ping {
-            data: self.meta_data.seq_number,
+            data: self.network_globals.local_metadata.read().seq_number,
         };
         trace!(self.log, "Sending Ping"; "request_id" => id, "peer_id" => peer_id.to_string());
 
@@ -364,7 +574,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
     /// Sends a Pong response to the peer.
     fn pong(&mut self, id: PeerRequestId, peer_id: PeerId) {
         let ping = crate::rpc::Ping {
-            data: self.meta_data.seq_number,
+            data: self.network_globals.local_metadata.read().seq_number,
         };
         trace!(self.log, "Sending Pong"; "request_id" => id.1, "peer_id" => peer_id.to_string());
         let event = RPCCodedResponse::Success(RPCResponse::Pong(ping));
@@ -380,7 +590,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Sends a METADATA response to a peer.
     fn send_meta_data_response(&mut self, id: PeerRequestId, peer_id: PeerId) {
-        let event = RPCCodedResponse::Success(RPCResponse::MetaData(self.meta_data.clone()));
+        let event = RPCCodedResponse::Success(RPCResponse::MetaData(
+            self.network_globals.local_metadata.read().clone(),
+        ));
         self.eth2_rpc.send_response(peer_id, id, event);
     }
 
@@ -392,40 +604,40 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     fn on_gossip_event(&mut self, event: GossipsubEvent) {
         match event {
-            GossipsubEvent::Message(propagation_source, id, gs_msg) => {
+            GossipsubEvent::Message {
+                propagation_source,
+                message_id: id,
+                message: gs_msg,
+            } => {
                 // Note: We are keeping track here of the peer that sent us the message, not the
                 // peer that originally published the message.
-                match PubsubMessage::decode(&gs_msg.topics, &gs_msg.data) {
-                    Err(e) => debug!(self.log, "Could not decode gossipsub message"; "error" => e),
+                match PubsubMessage::decode(&gs_msg.topic, gs_msg.data()) {
+                    Err(e) => {
+                        debug!(self.log, "Could not decode gossipsub message"; "error" => e);
+                        //reject the message
+                        if let Err(e) = self.gossipsub.report_message_validation_result(
+                            &id,
+                            &propagation_source,
+                            MessageAcceptance::Reject,
+                        ) {
+                            warn!(self.log, "Failed to report message validation"; "message_id" => id.to_string(), "peer_id" => propagation_source.to_string(), "error" => format!("{:?}", e));
+                        }
+                    }
                     Ok(msg) => {
                         // Notify the network
                         self.add_event(BehaviourEvent::PubsubMessage {
                             id,
                             source: propagation_source,
-                            topics: gs_msg.topics,
+                            topic: gs_msg.topic,
                             message: msg,
                         });
                     }
                 }
             }
             GossipsubEvent::Subscribed { peer_id, topic } => {
-                if let Some(topic_metric) = metrics::get_int_gauge(
-                    &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_COUNT,
-                    &[topic.as_str()],
-                ) {
-                    topic_metric.inc()
-                }
-
                 self.add_event(BehaviourEvent::PeerSubscribed(peer_id, topic));
             }
-            GossipsubEvent::Unsubscribed { peer_id: _, topic } => {
-                if let Some(topic_metric) = metrics::get_int_gauge(
-                    &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_COUNT,
-                    &[topic.as_str()],
-                ) {
-                    topic_metric.dec()
-                }
-            }
+            GossipsubEvent::Unsubscribed { .. } => {}
         }
     }
 
@@ -451,6 +663,17 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     fn on_rpc_event(&mut self, message: RPCMessage<TSpec>) {
         let peer_id = message.peer_id;
+
+        if !self.peer_manager.is_connected(&peer_id) {
+            //ignore this event
+            debug!(
+                self.log,
+                "Ignoring rpc message of disconnected peer";
+                "peer" => peer_id.to_string()
+            );
+            return;
+        }
+
         let handler_id = message.conn_id;
         // The METADATA and PING RPC responses are handled within the behaviour and not propagated
         match message.event {
@@ -463,19 +686,28 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                     } => {
                         if matches!(error, RPCError::HandlerRejected) {
                             // this peer's request got canceled
-                            // TODO: cancel processing for this request
                         }
                         // Inform the peer manager of the error.
                         // An inbound error here means we sent an error to the peer, or the stream
                         // timed out.
-                        self.peer_manager.handle_rpc_error(&peer_id, proto, &error);
+                        self.peer_manager.handle_rpc_error(
+                            &peer_id,
+                            proto,
+                            &error,
+                            ConnectionDirection::Incoming,
+                        );
                     }
                     HandlerErr::Outbound { id, proto, error } => {
                         // Inform the peer manager that a request we sent to the peer failed
-                        self.peer_manager.handle_rpc_error(&peer_id, proto, &error);
+                        self.peer_manager.handle_rpc_error(
+                            &peer_id,
+                            proto,
+                            &error,
+                            ConnectionDirection::Outgoing,
+                        );
                         // inform failures of requests comming outside the behaviour
                         if !matches!(id, RequestId::Behaviour) {
-                            self.add_event(BehaviourEvent::RPCFailed { peer_id, id, error });
+                            self.add_event(BehaviourEvent::RPCFailed { peer_id, id });
                         }
                     }
                 }
@@ -493,11 +725,8 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                     RPCRequest::MetaData(_) => {
                         // send the requested meta-data
                         self.send_meta_data_response((handler_id, id), peer_id);
-                        // TODO: inform the peer manager?
                     }
                     RPCRequest::Goodbye(reason) => {
-                        // let the peer manager know this peer is in the process of disconnecting
-                        self.peer_manager._disconnecting_peer(&peer_id);
                         // queue for disconnection without a goodbye message
                         debug!(
                             self.log, "Peer sent Goodbye";
@@ -627,6 +856,11 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             return Poll::Ready(NBAction::GenerateEvent(event));
         }
 
+        // perform gossipsub score updates when necessary
+        while let Poll::Ready(Some(_)) = self.update_gossipsub_scores.poll_next_unpin(cx) {
+            self.peer_manager.update_gossipsub_scores(&self.gossipsub);
+        }
+
         Poll::Pending
     }
 
@@ -667,6 +901,33 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             waker.wake_by_ref();
         }
     }
+
+    /// Creates a whitelist topic filter that covers all possible topics using the given set of
+    /// possible fork digests.
+    fn create_whitelist_filter(
+        possible_fork_digests: Vec<[u8; 4]>,
+        attestation_subnet_count: u64,
+    ) -> WhitelistSubscriptionFilter {
+        let mut possible_hashes = HashSet::new();
+        for fork_digest in possible_fork_digests {
+            let mut add = |kind| {
+                let topic: Topic =
+                    GossipTopic::new(kind, GossipEncoding::SSZSnappy, fork_digest).into();
+                possible_hashes.insert(topic.hash());
+            };
+
+            use GossipKind::*;
+            add(BeaconBlock);
+            add(BeaconAggregateAndProof);
+            add(VoluntaryExit);
+            add(ProposerSlashing);
+            add(AttesterSlashing);
+            for id in 0..attestation_subnet_count {
+                add(Attestation(SubnetId::new(id)));
+            }
+        }
+        WhitelistSubscriptionFilter(possible_hashes)
+    }
 }
 
 /// Calls the given function with the given args on all sub behaviours.
@@ -690,45 +951,10 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
         self.peer_manager.addresses_of_peer(peer_id)
     }
 
-    // This gets called every time a connection is closed.
-    fn inject_connection_closed(
-        &mut self,
-        peer_id: &PeerId,
-        conn_id: &ConnectionId,
-        endpoint: &ConnectedPoint,
-    ) {
-        // If the peer manager (and therefore the behaviour's) believe this peer connected, inform
-        // about the disconnection.
-        if self.network_globals.peers.read().is_connected(&peer_id) {
-            return;
-        }
-        delegate_to_behaviours!(self, inject_connection_closed, peer_id, conn_id, endpoint);
-    }
-
-    // This gets called once there are no more active connections.
-    fn inject_disconnected(&mut self, peer_id: &PeerId) {
-        // If the application/behaviour layers thinks this peer has connected inform it of the disconnect.
-        if self.network_globals.peers.read().is_connected(&peer_id) {
-            // Inform the application.
-            self.add_event(BehaviourEvent::PeerDisconnected(peer_id.clone()));
-            // Inform the behaviour.
-            delegate_to_behaviours!(self, inject_disconnected, peer_id);
-        }
-        // Inform the peer manager.
-        // NOTE: It may be the case that a rejected node, due to too many peers is disconnected
-        // here and the peer manager has no knowledge of its connection. We insert it here for
-        // reference so that peer manager can track this peer.
-        self.peer_manager.notify_disconnect(&peer_id);
-
-        // Update the prometheus metrics
-        metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT);
-        metrics::set_gauge(
-            &metrics::PEERS_CONNECTED,
-            self.network_globals.connected_peers() as i64,
-        );
-    }
-
     // This gets called every time a connection is established.
+    // NOTE: The current logic implies that we would reject extra connections for already connected
+    // peers if we have reached our peer limit. This is fine for the time being as we currently
+    // only allow a single connection per peer.
     fn inject_connection_established(
         &mut self,
         peer_id: &PeerId,
@@ -737,6 +963,9 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
     ) {
         let goodbye_reason: Option<GoodbyeReason> = if self.peer_manager.is_banned(peer_id) {
             // If the peer is banned, send goodbye with reason banned.
+            // A peer that has recently transitioned to the banned state should be in the
+            // disconnecting state, but the `is_banned()` function is dependent on score so should
+            // be true here in this case.
             Some(GoodbyeReason::Banned)
         } else if self.peer_manager.peer_limit_reached()
             && self
@@ -753,22 +982,27 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
             None
         };
 
-        if goodbye_reason.is_some() {
-            debug!(self.log, "Disconnecting newly connected peer"; "peer_id" => peer_id.to_string(), "reason" => goodbye_reason.as_ref().expect("Is some").to_string());
+        if let Some(goodbye_reason) = goodbye_reason {
+            debug!(self.log, "Disconnecting newly connected peer"; "peer_id" => peer_id.to_string(), "reason" => goodbye_reason.to_string());
             self.peers_to_dc
-                .push_back((peer_id.clone(), goodbye_reason));
+                .push_back((peer_id.clone(), Some(goodbye_reason)));
+            // NOTE: We don't inform the peer manager that this peer is disconnecting. It is simply
+            // rejected with a goodbye.
             return;
         }
 
-        // notify the peer manager of a successful connection
+        // All peers at this point will be registered as being connected.
+        // Notify the peer manager of a successful connection
         match endpoint {
-            ConnectedPoint::Listener { .. } => {
-                self.peer_manager.connect_ingoing(&peer_id);
+            ConnectedPoint::Listener { send_back_addr, .. } => {
+                self.peer_manager
+                    .connect_ingoing(&peer_id, send_back_addr.clone());
                 self.add_event(BehaviourEvent::PeerConnected(peer_id.clone()));
                 debug!(self.log, "Connection established"; "peer_id" => peer_id.to_string(), "connection" => "Incoming");
             }
-            ConnectedPoint::Dialer { .. } => {
-                self.peer_manager.connect_outgoing(&peer_id);
+            ConnectedPoint::Dialer { address } => {
+                self.peer_manager
+                    .connect_outgoing(&peer_id, address.clone());
                 self.add_event(BehaviourEvent::PeerDialed(peer_id.clone()));
                 debug!(self.log, "Connection established"; "peer_id" => peer_id.to_string(), "connection" => "Dialed");
             }
@@ -784,6 +1018,8 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
     }
 
     // This gets called on the initial connection establishment.
+    // NOTE: This gets called after inject_connection_established. Therefore the logic in that
+    // function dictates the logic here.
     fn inject_connected(&mut self, peer_id: &PeerId) {
         // If the PeerManager has connected this peer, inform the behaviours
         if !self.network_globals.peers.read().is_connected(&peer_id) {
@@ -800,6 +1036,79 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
         delegate_to_behaviours!(self, inject_connected, peer_id);
     }
 
+    // This gets called every time a connection is closed.
+    // NOTE: The peer manager state can be modified in the lifetime of the peer. Due to the scoring
+    // mechanism. Peers can become banned. In this case, we still want to inform the behaviours.
+    fn inject_connection_closed(
+        &mut self,
+        peer_id: &PeerId,
+        conn_id: &ConnectionId,
+        endpoint: &ConnectedPoint,
+    ) {
+        // If the peer manager (and therefore the behaviour's) believe this peer connected, inform
+        // about the disconnection.
+        // It could be the peer was in the process of being disconnected. In this case the
+        // sub-behaviours are expecting this peer to be connected and we inform them.
+        if self
+            .network_globals
+            .peers
+            .read()
+            .is_connected_or_disconnecting(peer_id)
+        {
+            // We are disconnecting the peer or the peer has already been connected.
+            // Both these cases, the peer has been previously registered in the sub protocols.
+            delegate_to_behaviours!(self, inject_connection_closed, peer_id, conn_id, endpoint);
+        }
+    }
+
+    // This gets called once there are no more active connections.
+    fn inject_disconnected(&mut self, peer_id: &PeerId) {
+        // If the application/behaviour layers thinks this peer has connected inform it of the disconnect.
+
+        if self
+            .network_globals
+            .peers
+            .read()
+            .is_connected_or_disconnecting(peer_id)
+        {
+            // We are disconnecting the peer or the peer has already been connected.
+            // Both these cases, the peer has been previously registered in the sub protocols and
+            // potentially the application layer.
+            // Inform the application.
+            self.add_event(BehaviourEvent::PeerDisconnected(peer_id.clone()));
+            // Inform the behaviour.
+            delegate_to_behaviours!(self, inject_disconnected, peer_id);
+
+            // Decrement the PEERS_PER_CLIENT metric
+            if let Some(kind) = self
+                .network_globals
+                .peers
+                .read()
+                .peer_info(peer_id)
+                .map(|info| info.client.kind.clone())
+            {
+                if let Some(v) =
+                    metrics::get_int_gauge(&metrics::PEERS_PER_CLIENT, &[&kind.to_string()])
+                {
+                    v.dec()
+                };
+            }
+        }
+
+        // Inform the peer manager.
+        // NOTE: It may be the case that a rejected node, due to too many peers is disconnected
+        // here and the peer manager has no knowledge of its connection. We insert it here for
+        // reference so that peer manager can track this peer.
+        self.peer_manager.notify_disconnect(&peer_id);
+
+        // Update the prometheus metrics
+        metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT);
+        metrics::set_gauge(
+            &metrics::PEERS_CONNECTED,
+            self.network_globals.connected_peers() as i64,
+        );
+    }
+
     fn inject_addr_reach_failure(
         &mut self,
         peer_id: Option<&PeerId>,
@@ -846,17 +1155,11 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
             return;
         }
 
+        // Events comming from the handler, redirected to each behaviour
         match event {
-            // Events comming from the handler, redirected to each behaviour
-            BehaviourHandlerOut::Delegate(delegate) => match *delegate {
-                DelegateOut::Gossipsub(ev) => self.gossipsub.inject_event(peer_id, conn_id, ev),
-                DelegateOut::RPC(ev) => self.eth2_rpc.inject_event(peer_id, conn_id, ev),
-                DelegateOut::Identify(ev) => self.identify.inject_event(peer_id, conn_id, *ev),
-            },
-            /* Custom events sent BY the handler */
-            BehaviourHandlerOut::Custom => {
-                // TODO: implement
-            }
+            DelegateOut::Gossipsub(ev) => self.gossipsub.inject_event(peer_id, conn_id, ev),
+            DelegateOut::RPC(ev) => self.eth2_rpc.inject_event(peer_id, conn_id, ev),
+            DelegateOut::Identify(ev) => self.identify.inject_event(peer_id, conn_id, *ev),
         }
     }
 
@@ -874,7 +1177,6 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
             self.waker = Some(cx.waker().clone());
         }
 
-        // TODO: move where it's less distracting
         macro_rules! poll_behaviour {
             /* $behaviour:  The sub-behaviour being polled.
              * $on_event_fn:  Function to call if we get an event from the sub-behaviour.
@@ -986,98 +1288,8 @@ impl<TSpec: EthSpec> std::convert::From<Response<TSpec>> for RPCCodedResponse<TS
     }
 }
 
-/// Identifier of requests sent by a peer.
-pub type PeerRequestId = (ConnectionId, SubstreamId);
-
-/// The types of events than can be obtained from polling the behaviour.
-#[derive(Debug)]
-pub enum BehaviourEvent<TSpec: EthSpec> {
-    /// We have successfully dialed and connected to a peer.
-    PeerDialed(PeerId),
-    /// A peer has successfully dialed and connected to us.
-    PeerConnected(PeerId),
-    /// A peer has disconnected.
-    PeerDisconnected(PeerId),
-    /// An RPC Request that was sent failed.
-    RPCFailed {
-        /// The id of the failed request.
-        id: RequestId,
-        /// The peer to which this request was sent.
-        peer_id: PeerId,
-        /// The error that occurred.
-        error: RPCError,
-    },
-    RequestReceived {
-        /// The peer that sent the request.
-        peer_id: PeerId,
-        /// Identifier of the request. All responses to this request must use this id.
-        id: PeerRequestId,
-        /// Request the peer sent.
-        request: Request,
-    },
-    ResponseReceived {
-        /// Peer that sent the response.
-        peer_id: PeerId,
-        /// Id of the request to which the peer is responding.
-        id: RequestId,
-        /// Response the peer sent.
-        response: Response<TSpec>,
-    },
-    PubsubMessage {
-        /// The gossipsub message id. Used when propagating blocks after validation.
-        id: MessageId,
-        /// The peer from which we received this message, not the peer that published it.
-        source: PeerId,
-        /// The topics that this message was sent on.
-        topics: Vec<TopicHash>,
-        /// The message itself.
-        message: PubsubMessage<TSpec>,
-    },
-    /// Subscribed to peer for given topic
-    PeerSubscribed(PeerId, TopicHash),
-    /// Inform the network to send a Status to this peer.
-    StatusPeer(PeerId),
-}
-
-/// Load metadata from persisted file. Return default metadata if loading fails.
-fn load_or_build_metadata<E: EthSpec>(network_dir: &PathBuf, log: &slog::Logger) -> MetaData<E> {
-    // Default metadata
-    let mut meta_data = MetaData {
-        seq_number: 0,
-        attnets: EnrBitfield::<E>::default(),
-    };
-    // Read metadata from persisted file if available
-    let metadata_path = network_dir.join(METADATA_FILENAME);
-    if let Ok(mut metadata_file) = File::open(metadata_path) {
-        let mut metadata_ssz = Vec::new();
-        if metadata_file.read_to_end(&mut metadata_ssz).is_ok() {
-            match MetaData::<E>::from_ssz_bytes(&metadata_ssz) {
-                Ok(persisted_metadata) => {
-                    meta_data.seq_number = persisted_metadata.seq_number;
-                    // Increment seq number if persisted attnet is not default
-                    if persisted_metadata.attnets != meta_data.attnets {
-                        meta_data.seq_number += 1;
-                    }
-                    debug!(log, "Loaded metadata from disk");
-                }
-                Err(e) => {
-                    debug!(
-                        log,
-                        "Metadata from file could not be decoded";
-                        "error" => format!("{:?}", e),
-                    );
-                }
-            }
-        }
-    };
-
-    debug!(log, "Metadata sequence number"; "seq_num" => meta_data.seq_number);
-    save_metadata_to_disk(network_dir, meta_data.clone(), &log);
-    meta_data
-}
-
 /// Persist metadata to disk
-fn save_metadata_to_disk<E: EthSpec>(dir: &PathBuf, metadata: MetaData<E>, log: &slog::Logger) {
+pub fn save_metadata_to_disk<E: EthSpec>(dir: &PathBuf, metadata: MetaData<E>, log: &slog::Logger) {
     let _ = std::fs::create_dir_all(&dir);
     match File::create(dir.join(METADATA_FILENAME))
         .and_then(|mut f| f.write_all(&metadata.as_ssz_bytes()))

beacon_node/eth2_libp2p/src/config.rs

@@ -1,8 +1,12 @@
-use crate::types::GossipKind;
-use crate::Enr;
+use crate::types::{GossipKind, MessageData};
+use crate::{Enr, PeerIdSerialized};
+use directory::{
+    DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_TESTNET, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR,
+};
 use discv5::{Discv5Config, Discv5ConfigBuilder};
 use libp2p::gossipsub::{
-    GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode,
+    FastMessageId, GenericGossipsubConfig, GenericGossipsubConfigBuilder, GenericGossipsubMessage,
+    MessageId, RawGossipsubMessage, ValidationMode,
 };
 use libp2p::Multiaddr;
 use serde_derive::{Deserialize, Serialize};
@@ -11,6 +15,13 @@ use std::path::PathBuf;
 use std::time::Duration;
 
 pub const GOSSIP_MAX_SIZE: usize = 1_048_576;
+const MESSAGE_DOMAIN_INVALID_SNAPPY: [u8; 4] = [0, 0, 0, 0];
+const MESSAGE_DOMAIN_VALID_SNAPPY: [u8; 4] = [1, 0, 0, 0];
+pub const MESH_N_LOW: usize = 6;
+
+pub type GossipsubConfig = GenericGossipsubConfig<MessageData>;
+pub type GossipsubConfigBuilder = GenericGossipsubConfigBuilder<MessageData>;
+pub type GossipsubMessage = GenericGossipsubMessage<MessageData>;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
@@ -58,12 +69,25 @@ pub struct Config {
     /// List of libp2p nodes to initially connect to.
     pub libp2p_nodes: Vec<Multiaddr>,
 
+    /// List of trusted libp2p nodes which are not scored.
+    pub trusted_peers: Vec<PeerIdSerialized>,
+
     /// Client version
     pub client_version: String,
 
     /// Disables the discovery protocol from starting.
     pub disable_discovery: bool,
 
+    /// Attempt to construct external port mappings with UPnP.
+    pub upnp_enabled: bool,
+
+    /// Subscribe to all subnets for the duration of the runtime.
+    pub subscribe_all_subnets: bool,
+
+    /// Import/aggregate all attestations recieved on subscribed subnets for the duration of the
+    /// runtime.
+    pub import_all_attestations: bool,
+
     /// List of extra topics to initially subscribe to as strings.
     pub topics: Vec<GossipKind>,
 }
@@ -71,26 +95,40 @@ pub struct Config {
 impl Default for Config {
     /// Generate a default network configuration.
     fn default() -> Self {
-        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
-        network_dir.push(".lighthouse");
-        network_dir.push("network");
-
-        // The default topics that we will initially subscribe to
-        let topics = vec![
-            GossipKind::BeaconBlock,
-            GossipKind::BeaconAggregateAndProof,
-            GossipKind::VoluntaryExit,
-            GossipKind::ProposerSlashing,
-            GossipKind::AttesterSlashing,
-        ];
+        // WARNING: this directory default should be always overwritten with parameters
+        // from cli for specific networks.
+        let network_dir = dirs::home_dir()
+            .unwrap_or_else(|| PathBuf::from("."))
+            .join(DEFAULT_ROOT_DIR)
+            .join(DEFAULT_HARDCODED_TESTNET)
+            .join(DEFAULT_BEACON_NODE_DIR)
+            .join(DEFAULT_NETWORK_DIR);
 
         // The function used to generate a gossipsub message id
-        // We use base64(SHA256(data)) for content addressing
+        // We use the first 8 bytes of SHA256(data) for content addressing
+        let fast_gossip_message_id =
+            |message: &RawGossipsubMessage| FastMessageId::from(&Sha256::digest(&message.data)[..]);
+
+        fn prefix(prefix: [u8; 4], data: &[u8]) -> Vec<u8> {
+            prefix
+                .to_vec()
+                .into_iter()
+                .chain(data.iter().cloned())
+                .collect()
+        }
+
         let gossip_message_id = |message: &GossipsubMessage| {
-            MessageId::from(base64::encode_config(
-                &Sha256::digest(&message.data),
-                base64::URL_SAFE_NO_PAD,
-            ))
+            MessageId::from(
+                &Sha256::digest(
+                    {
+                        match &message.data.decompressed {
+                            Ok(decompressed) => prefix(MESSAGE_DOMAIN_VALID_SNAPPY, decompressed),
+                            _ => prefix(MESSAGE_DOMAIN_INVALID_SNAPPY, &message.data.raw),
+                        }
+                    }
+                    .as_slice(),
+                )[..20],
+            )
         };
 
         // gossipsub configuration
@@ -99,30 +137,34 @@ impl Default for Config {
         let gs_config = GossipsubConfigBuilder::new()
             .max_transmit_size(GOSSIP_MAX_SIZE)
             .heartbeat_interval(Duration::from_millis(700))
-            .mesh_n(6)
-            .mesh_n_low(5)
+            .mesh_n(8)
+            .mesh_n_low(MESH_N_LOW)
             .mesh_n_high(12)
             .gossip_lazy(6)
             .fanout_ttl(Duration::from_secs(60))
             .history_length(6)
             .history_gossip(3)
             .validate_messages() // require validation before propagation
-            .validation_mode(ValidationMode::Permissive)
+            .validation_mode(ValidationMode::Anonymous)
             // prevent duplicates for 550 heartbeats(700millis * 550) = 385 secs
             .duplicate_cache_time(Duration::from_secs(385))
             .message_id_fn(gossip_message_id)
-            .build();
+            .fast_message_id_fn(fast_gossip_message_id)
+            .allow_self_origin(true)
+            .build()
+            .expect("valid gossipsub configuration");
 
         // discv5 configuration
         let discv5_config = Discv5ConfigBuilder::new()
             .enable_packet_filter()
             .session_cache_capacity(1000)
-            .request_timeout(Duration::from_secs(4))
+            .request_timeout(Duration::from_secs(1))
+            .query_peer_timeout(Duration::from_secs(2))
+            .query_timeout(Duration::from_secs(30))
             .request_retries(1)
             .enr_peer_update_min(10)
             .query_parallelism(5)
-            .query_timeout(Duration::from_secs(30))
-            .query_peer_timeout(Duration::from_secs(2))
+            .disable_report_discovered_peers()
             .ip_limit() // limits /24 IP's in buckets.
             .ping_interval(Duration::from_secs(300))
             .build();
@@ -142,9 +184,13 @@ impl Default for Config {
             boot_nodes_enr: vec![],
             boot_nodes_multiaddr: vec![],
             libp2p_nodes: vec![],
+            trusted_peers: vec![],
             client_version: lighthouse_version::version_with_platform(),
             disable_discovery: false,
-            topics,
+            upnp_enabled: true,
+            subscribe_all_subnets: false,
+            import_all_attestations: false,
+            topics: Vec::new(),
         }
     }
 }

beacon_node/eth2_libp2p/src/discovery/enr.rs

@@ -6,6 +6,7 @@ use super::enr_ext::CombinedKeyExt;
 use super::ENR_FILENAME;
 use crate::types::{Enr, EnrBitfield};
 use crate::NetworkConfig;
+use discv5::enr::EnrKey;
 use libp2p::core::identity::Keypair;
 use slog::{debug, warn};
 use ssz::{Decode, Encode};
@@ -48,6 +49,56 @@ impl Eth2Enr for Enr {
     }
 }
 
+/// Either use the given ENR or load an ENR from file if it exists and matches the current NodeId
+/// and sequence number.
+/// If an ENR exists, with the same NodeId, this function checks to see if the loaded ENR from
+/// disk is suitable to use, otherwise we increment the given ENR's sequence number.
+pub fn use_or_load_enr(
+    enr_key: &CombinedKey,
+    local_enr: &mut Enr,
+    config: &NetworkConfig,
+    log: &slog::Logger,
+) -> Result<(), String> {
+    let enr_f = config.network_dir.join(ENR_FILENAME);
+    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
+        let mut enr_string = String::new();
+        match enr_file.read_to_string(&mut enr_string) {
+            Err(_) => debug!(log, "Could not read ENR from file"),
+            Ok(_) => {
+                match Enr::from_str(&enr_string) {
+                    Ok(disk_enr) => {
+                        // if the same node id, then we may need to update our sequence number
+                        if local_enr.node_id() == disk_enr.node_id() {
+                            if compare_enr(&local_enr, &disk_enr) {
+                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
+                                // the stored ENR has the same configuration, use it
+                                *local_enr = disk_enr;
+                                return Ok(());
+                            }
+
+                            // same node id, different configuration - update the sequence number
+                            // Note: local_enr is generated with default(0) attnets value,
+                            // so a non default value in persisted enr will also update sequence number.
+                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
+                            local_enr.set_seq(new_seq_no, enr_key).map_err(|e| {
+                                format!("Could not update ENR sequence number: {:?}", e)
+                            })?;
+                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
+                        }
+                    }
+                    Err(e) => {
+                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
+                    }
+                }
+            }
+        }
+    }
+
+    save_enr_to_disk(&config.network_dir, &local_enr, log);
+
+    Ok(())
+}
+
 /// Loads an ENR from file if it exists and matches the current NodeId and sequence number. If none
 /// exists, generates a new one.
 ///
@@ -65,51 +116,14 @@ pub fn build_or_load_enr<T: EthSpec>(
     let enr_key = CombinedKey::from_libp2p(&local_key)?;
     let mut local_enr = build_enr::<T>(&enr_key, config, enr_fork_id)?;
 
-    let enr_f = config.network_dir.join(ENR_FILENAME);
-    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
-        let mut enr_string = String::new();
-        match enr_file.read_to_string(&mut enr_string) {
-            Err(_) => debug!(log, "Could not read ENR from file"),
-            Ok(_) => {
-                match Enr::from_str(&enr_string) {
-                    Ok(disk_enr) => {
-                        // if the same node id, then we may need to update our sequence number
-                        if local_enr.node_id() == disk_enr.node_id() {
-                            if compare_enr(&local_enr, &disk_enr) {
-                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
-                                // the stored ENR has the same configuration, use it
-                                return Ok(disk_enr);
-                            }
-
-                            // same node id, different configuration - update the sequence number
-                            // Note: local_enr is generated with default(0) attnets value,
-                            // so a non default value in persisted enr will also update sequence number.
-                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
-                            local_enr.set_seq(new_seq_no, &enr_key).map_err(|e| {
-                                format!("Could not update ENR sequence number: {:?}", e)
-                            })?;
-                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
-                        }
-                    }
-                    Err(e) => {
-                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
-                    }
-                }
-            }
-        }
-    }
-
-    save_enr_to_disk(&config.network_dir, &local_enr, log);
-
+    use_or_load_enr(&enr_key, &mut local_enr, config, log)?;
     Ok(local_enr)
 }
 
-/// Builds a lighthouse ENR given a `NetworkConfig`.
-pub fn build_enr<T: EthSpec>(
-    enr_key: &CombinedKey,
+pub fn create_enr_builder_from_config<T: EnrKey>(
     config: &NetworkConfig,
-    enr_fork_id: EnrForkId,
-) -> Result<Enr, String> {
+    enable_tcp: bool,
+) -> EnrBuilder<T> {
     let mut builder = EnrBuilder::new("v4");
     if let Some(enr_address) = config.enr_address {
         builder.ip(enr_address);
@@ -118,20 +132,30 @@ pub fn build_enr<T: EthSpec>(
         builder.udp(udp_port);
     }
     // we always give it our listening tcp port
-    // TODO: Add uPnP support to map udp and tcp ports
-    let tcp_port = config.enr_tcp_port.unwrap_or_else(|| config.libp2p_port);
-    builder.tcp(tcp_port);
+    if enable_tcp {
+        let tcp_port = config.enr_tcp_port.unwrap_or_else(|| config.libp2p_port);
+        builder.tcp(tcp_port);
+    }
+    builder
+}
+
+/// Builds a lighthouse ENR given a `NetworkConfig`.
+pub fn build_enr<T: EthSpec>(
+    enr_key: &CombinedKey,
+    config: &NetworkConfig,
+    enr_fork_id: EnrForkId,
+) -> Result<Enr, String> {
+    let mut builder = create_enr_builder_from_config(config, true);
 
     // set the `eth2` field on our ENR
-    builder.add_value(ETH2_ENR_KEY.into(), enr_fork_id.as_ssz_bytes());
+    builder.add_value(ETH2_ENR_KEY, &enr_fork_id.as_ssz_bytes());
 
     // set the "attnets" field on our ENR
     let bitfield = BitVector::<T::SubnetBitfieldLength>::new();
 
-    builder.add_value(BITFIELD_ENR_KEY.into(), bitfield.as_ssz_bytes());
+    builder.add_value(BITFIELD_ENR_KEY, &bitfield.as_ssz_bytes());
 
     builder
-        .tcp(config.libp2p_port)
         .build(enr_key)
         .map_err(|e| format!("Could not build Local ENR: {:?}", e))
 }

beacon_node/eth2_libp2p/src/discovery/enr_ext.rs

@@ -13,9 +13,15 @@ pub trait EnrExt {
     /// The vector remains empty if these fields are not defined.
     fn multiaddr(&self) -> Vec<Multiaddr>;
 
-    /// Returns the multiaddr with the `PeerId` prepended.
+    /// Returns a list of multiaddrs with the `PeerId` prepended.
     fn multiaddr_p2p(&self) -> Vec<Multiaddr>;
 
+    /// Returns any multiaddrs that contain the TCP protocol with the `PeerId` prepended.
+    fn multiaddr_p2p_tcp(&self) -> Vec<Multiaddr>;
+
+    /// Returns any multiaddrs that contain the UDP protocol with the `PeerId` prepended.
+    fn multiaddr_p2p_udp(&self) -> Vec<Multiaddr>;
+
     /// Returns any multiaddrs that contain the TCP protocol.
     fn multiaddr_tcp(&self) -> Vec<Multiaddr>;
 }
@@ -111,6 +117,58 @@ impl EnrExt for Enr {
         multiaddrs
     }
 
+    /// Returns a list of multiaddrs if the ENR has an `ip` and a `tcp` key **or** an `ip6` and a `tcp6`.
+    /// The vector remains empty if these fields are not defined.
+    ///
+    /// This also prepends the `PeerId` into each multiaddr with the `P2p` protocol.
+    fn multiaddr_p2p_tcp(&self) -> Vec<Multiaddr> {
+        let peer_id = self.peer_id();
+        let mut multiaddrs: Vec<Multiaddr> = Vec::new();
+        if let Some(ip) = self.ip() {
+            if let Some(tcp) = self.tcp() {
+                let mut multiaddr: Multiaddr = ip.into();
+                multiaddr.push(Protocol::Tcp(tcp));
+                multiaddr.push(Protocol::P2p(peer_id.clone().into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        if let Some(ip6) = self.ip6() {
+            if let Some(tcp6) = self.tcp6() {
+                let mut multiaddr: Multiaddr = ip6.into();
+                multiaddr.push(Protocol::Tcp(tcp6));
+                multiaddr.push(Protocol::P2p(peer_id.into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        multiaddrs
+    }
+
+    /// Returns a list of multiaddrs if the ENR has an `ip` and a `udp` key **or** an `ip6` and a `udp6`.
+    /// The vector remains empty if these fields are not defined.
+    ///
+    /// This also prepends the `PeerId` into each multiaddr with the `P2p` protocol.
+    fn multiaddr_p2p_udp(&self) -> Vec<Multiaddr> {
+        let peer_id = self.peer_id();
+        let mut multiaddrs: Vec<Multiaddr> = Vec::new();
+        if let Some(ip) = self.ip() {
+            if let Some(udp) = self.udp() {
+                let mut multiaddr: Multiaddr = ip.into();
+                multiaddr.push(Protocol::Udp(udp));
+                multiaddr.push(Protocol::P2p(peer_id.clone().into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        if let Some(ip6) = self.ip6() {
+            if let Some(udp6) = self.udp6() {
+                let mut multiaddr: Multiaddr = ip6.into();
+                multiaddr.push(Protocol::Udp(udp6));
+                multiaddr.push(Protocol::P2p(peer_id.into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        multiaddrs
+    }
+
     /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`.
     /// The vector remains empty if these fields are not defined.
     fn multiaddr_tcp(&self) -> Vec<Multiaddr> {
@@ -140,7 +198,7 @@ impl CombinedKeyPublicExt for CombinedPublicKey {
     fn into_peer_id(&self) -> PeerId {
         match self {
             Self::Secp256k1(pk) => {
-                let pk_bytes = pk.serialize_compressed();
+                let pk_bytes = pk.to_bytes();
                 let libp2p_pk = libp2p::core::PublicKey::Secp256k1(
                     libp2p::core::identity::secp256k1::PublicKey::decode(&pk_bytes)
                         .expect("valid public key"),
@@ -163,7 +221,7 @@ impl CombinedKeyExt for CombinedKey {
     fn from_libp2p(key: &libp2p::core::identity::Keypair) -> Result<CombinedKey, &'static str> {
         match key {
             Keypair::Secp256k1(key) => {
-                let secret = discv5::enr::secp256k1::SecretKey::parse(&key.secret().to_bytes())
+                let secret = discv5::enr::k256::ecdsa::SigningKey::new(&key.secret().to_bytes())
                     .expect("libp2p key must be valid");
                 Ok(CombinedKey::Secp256k1(secret))
             }
@@ -219,7 +277,7 @@ mod tests {
     fn test_secp256k1_peer_id_conversion() {
         let sk_hex = "df94a73d528434ce2309abb19c16aedb535322797dbd59c157b1e04095900f48";
         let sk_bytes = hex::decode(sk_hex).unwrap();
-        let secret_key = discv5::enr::secp256k1::SecretKey::parse_slice(&sk_bytes).unwrap();
+        let secret_key = discv5::enr::k256::ecdsa::SigningKey::new(&sk_bytes).unwrap();
 
         let libp2p_sk = libp2p::identity::secp256k1::SecretKey::from_bytes(sk_bytes).unwrap();
         let secp256k1_kp: libp2p::identity::secp256k1::Keypair = libp2p_sk.into();

beacon_node/eth2_libp2p/src/discovery/mod.rs

@@ -3,11 +3,11 @@ pub(crate) mod enr;
 pub mod enr_ext;
 
 // Allow external use of the lighthouse ENR builder
-pub use enr::{build_enr, CombinedKey, Eth2Enr};
-pub use enr_ext::{CombinedKeyExt, EnrExt};
-pub use libp2p::core::identity::Keypair;
+pub use enr::{build_enr, create_enr_builder_from_config, use_or_load_enr, CombinedKey, Eth2Enr};
+pub use enr_ext::{peer_id_to_node_id, CombinedKeyExt, EnrExt};
+pub use libp2p::core::identity::{Keypair, PublicKey};
 
-use crate::metrics;
+use crate::{config, metrics};
 use crate::{error, Enr, NetworkConfig, NetworkGlobals, SubnetDiscovery};
 use discv5::{enr::NodeId, Discv5, Discv5Event};
 use enr::{BITFIELD_ENR_KEY, ETH2_ENR_KEY};
@@ -20,7 +20,7 @@ use ssz::{Decode, Encode};
 use ssz_types::BitVector;
 use std::{
     collections::{HashMap, VecDeque},
-    net::SocketAddr,
+    net::{IpAddr, SocketAddr},
     path::Path,
     pin::Pin,
     sync::Arc,
@@ -31,12 +31,12 @@ use tokio::sync::mpsc;
 use types::{EnrForkId, EthSpec, SubnetId};
 
 mod subnet_predicate;
-use subnet_predicate::subnet_predicate;
+pub use subnet_predicate::subnet_predicate;
 
 /// Local ENR storage filename.
 pub const ENR_FILENAME: &str = "enr.dat";
 /// Target number of peers we'd like to have connected to a given long-lived subnet.
-const TARGET_SUBNET_PEERS: usize = 3;
+pub const TARGET_SUBNET_PEERS: usize = config::MESH_N_LOW;
 /// Target number of peers to search for given a grouped subnet query.
 const TARGET_PEERS_FOR_GROUPED_QUERY: usize = 6;
 /// Number of times to attempt a discovery request.
@@ -155,7 +155,7 @@ pub struct Discovery<TSpec: EthSpec> {
 
     /// Indicates if the discovery service has been started. When the service is disabled, this is
     /// always false.
-    started: bool,
+    pub started: bool,
 
     /// Logger for the discovery behaviour.
     log: slog::Logger,
@@ -240,7 +240,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         while let Some((result, original_addr)) = fut_coll.next().await {
             match result {
-                Ok(Some(enr)) => {
+                Ok(enr) => {
                     debug!(
                         log,
                         "Adding node to routing table";
@@ -259,9 +259,6 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                         )
                     });
                 }
-                Ok(None) => {
-                    error!(log, "No ENR found for MultiAddr"; "addr" => original_addr.to_string())
-                }
                 Err(e) => {
                     error!(log, "Error getting mapping to ENR"; "multiaddr" => original_addr.to_string(), "error" => e.to_string())
                 }
@@ -287,6 +284,11 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         self.discv5.local_enr()
     }
 
+    /// Return the cached enrs.
+    pub fn cached_enrs(&self) -> impl Iterator<Item = (&PeerId, &Enr)> {
+        self.cached_enrs.iter()
+    }
+
     /// This adds a new `FindPeers` query to the queue if one doesn't already exist.
     pub fn discover_peers(&mut self) {
         // If the discv5 service isn't running or we are in the process of a query, don't bother queuing a new one.
@@ -353,6 +355,54 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         }
     }
 
+    /// Updates the local ENR TCP port.
+    /// There currently isn't a case to update the address here. We opt for discovery to
+    /// automatically update the external address.
+    ///
+    /// If the external address needs to be modified, use `update_enr_udp_socket.
+    pub fn update_enr_tcp_port(&mut self, port: u16) -> Result<(), String> {
+        self.discv5
+            .enr_insert("tcp", &port.to_be_bytes())
+            .map_err(|e| format!("{:?}", e))?;
+
+        // replace the global version
+        *self.network_globals.local_enr.write() = self.discv5.local_enr();
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
+        Ok(())
+    }
+
+    /// Updates the local ENR UDP socket.
+    ///
+    /// This is with caution. Discovery should automatically maintain this. This should only be
+    /// used when automatic discovery is disabled.
+    pub fn update_enr_udp_socket(&mut self, socket_addr: SocketAddr) -> Result<(), String> {
+        match socket_addr {
+            SocketAddr::V4(socket) => {
+                self.discv5
+                    .enr_insert("ip", &socket.ip().octets())
+                    .map_err(|e| format!("{:?}", e))?;
+                self.discv5
+                    .enr_insert("udp", &socket.port().to_be_bytes())
+                    .map_err(|e| format!("{:?}", e))?;
+            }
+            SocketAddr::V6(socket) => {
+                self.discv5
+                    .enr_insert("ip6", &socket.ip().octets())
+                    .map_err(|e| format!("{:?}", e))?;
+                self.discv5
+                    .enr_insert("udp6", &socket.port().to_be_bytes())
+                    .map_err(|e| format!("{:?}", e))?;
+            }
+        }
+
+        // replace the global version
+        *self.network_globals.local_enr.write() = self.discv5.local_enr();
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
+        Ok(())
+    }
+
     /// Adds/Removes a subnet from the ENR Bitfield
     pub fn update_enr_bitfield(&mut self, subnet_id: SubnetId, value: bool) -> Result<(), String> {
         let id = *subnet_id as usize;
@@ -385,9 +435,9 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             .map_err(|_| String::from("Subnet ID out of bounds, could not set subnet ID"))?;
 
         // insert the bitfield into the ENR record
-        let _ = self
-            .discv5
-            .enr_insert(BITFIELD_ENR_KEY, current_bitfield.as_ssz_bytes());
+        self.discv5
+            .enr_insert(BITFIELD_ENR_KEY, &current_bitfield.as_ssz_bytes())
+            .map_err(|e| format!("{:?}", e))?;
 
         // replace the global version
         *self.network_globals.local_enr.write() = self.discv5.local_enr();
@@ -415,7 +465,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         let _ = self
             .discv5
-            .enr_insert(ETH2_ENR_KEY, enr_fork_id.as_ssz_bytes())
+            .enr_insert(ETH2_ENR_KEY, &enr_fork_id.as_ssz_bytes())
             .map_err(|e| {
                 warn!(
                     self.log,
@@ -431,6 +481,33 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
     }
 
+    // Bans a peer and it's associated seen IP addresses.
+    pub fn ban_peer(&mut self, peer_id: &PeerId, ip_addresses: Vec<IpAddr>) {
+        // first try and convert the peer_id to a node_id.
+        if let Ok(node_id) = peer_id_to_node_id(peer_id) {
+            // If we could convert this peer id, remove it from the DHT and ban it from discovery.
+            self.discv5.ban_node(&node_id);
+            // Remove the node from the routing table.
+            self.discv5.remove_node(&node_id);
+        }
+
+        for ip_address in ip_addresses {
+            self.discv5.ban_ip(ip_address);
+        }
+    }
+
+    pub fn unban_peer(&mut self, peer_id: &PeerId, ip_addresses: Vec<IpAddr>) {
+        // first try and convert the peer_id to a node_id.
+        if let Ok(node_id) = peer_id_to_node_id(peer_id) {
+            // If we could convert this peer id, remove it from the DHT and ban it from discovery.
+            self.discv5.permit_node(&node_id);
+        }
+
+        for ip_address in ip_addresses {
+            self.discv5.permit_ip(ip_address);
+        }
+    }
+
     /* Internal Functions */
 
     /// Adds a subnet query if one doesn't exist. If a subnet query already exists, this
@@ -479,13 +556,15 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
     /// Consume the discovery queue and initiate queries when applicable.
     ///
     /// This also sanitizes the queue removing out-dated queries.
-    fn process_queue(&mut self) {
+    /// Returns `true` if any of the queued queries is processed and a discovery
+    /// query (Subnet or FindPeers) is started.
+    fn process_queue(&mut self) -> bool {
         // Sanitize the queue, removing any out-dated subnet queries
         self.queued_queries.retain(|query| !query.expired());
 
         // use this to group subnet queries together for a single discovery request
         let mut subnet_queries: Vec<SubnetQuery> = Vec::new();
-
+        let mut processed = false;
         // Check that we are within our query concurrency limit
         while !self.at_capacity() && !self.queued_queries.is_empty() {
             // consume and process the query queue
@@ -502,6 +581,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                             FIND_NODE_QUERY_CLOSEST_PEERS,
                             |_| true,
                         );
+                        processed = true;
                     } else {
                         self.queued_queries.push_back(QueryType::FindPeers);
                     }
@@ -527,6 +607,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                             "subnets" => format!("{:?}", grouped_queries.iter().map(|q| q.subnet_id).collect::<Vec<_>>()),
                         );
                         self.start_subnet_query(grouped_queries);
+                        processed = true;
                     }
                 }
                 None => {} // Queue is empty
@@ -534,6 +615,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         }
         // Update the queue metric
         metrics::set_gauge(&metrics::DISCOVERY_QUEUE, self.queued_queries.len() as i64);
+        processed
     }
 
     // Returns a boolean indicating if we are currently processing the maximum number of
@@ -555,10 +637,10 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                     .network_globals
                     .peers
                     .read()
-                    .peers_on_subnet(subnet_query.subnet_id)
+                    .good_peers_on_subnet(subnet_query.subnet_id)
                     .count();
 
-                if peers_on_subnet > TARGET_SUBNET_PEERS {
+                if peers_on_subnet >= TARGET_SUBNET_PEERS {
                     debug!(self.log, "Discovery ignored";
                         "reason" => "Already connected to desired peers",
                         "connected_peers_on_subnet" => peers_on_subnet,
@@ -647,111 +729,122 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         self.active_queries.push(Box::pin(query_future));
     }
 
+    /// Process the completed QueryResult returned from discv5.
+    fn process_completed_queries(
+        &mut self,
+        query_result: QueryResult,
+    ) -> Option<HashMap<PeerId, Option<Instant>>> {
+        match query_result.0 {
+            GroupedQueryType::FindPeers => {
+                self.find_peer_active = false;
+                match query_result.1 {
+                    Ok(r) if r.is_empty() => {
+                        debug!(self.log, "Discovery query yielded no results.");
+                    }
+                    Ok(r) => {
+                        debug!(self.log, "Discovery query completed"; "peers_found" => r.len());
+                        let mut results: HashMap<PeerId, Option<Instant>> = HashMap::new();
+                        r.iter().for_each(|enr| {
+                            // cache the found ENR's
+                            self.cached_enrs.put(enr.peer_id(), enr.clone());
+                            results.insert(enr.peer_id(), None);
+                        });
+                        return Some(results);
+                    }
+                    Err(e) => {
+                        warn!(self.log, "Discovery query failed"; "error" => e.to_string());
+                    }
+                }
+            }
+            GroupedQueryType::Subnet(queries) => {
+                let subnets_searched_for: Vec<SubnetId> =
+                    queries.iter().map(|query| query.subnet_id).collect();
+                match query_result.1 {
+                    Ok(r) if r.is_empty() => {
+                        debug!(self.log, "Grouped subnet discovery query yielded no results."; "subnets_searched_for" => format!("{:?}",subnets_searched_for));
+                    }
+                    Ok(r) => {
+                        debug!(self.log, "Peer grouped subnet discovery request completed"; "peers_found" => r.len(), "subnets_searched_for" => format!("{:?}",subnets_searched_for));
+
+                        let mut mapped_results: HashMap<PeerId, Option<Instant>> = HashMap::new();
+
+                        // cache the found ENR's
+                        for enr in r.iter().cloned() {
+                            self.cached_enrs.put(enr.peer_id(), enr);
+                        }
+
+                        // Map each subnet query's min_ttl to the set of ENR's returned for that subnet.
+                        queries.iter().for_each(|query| {
+                            // A subnet query has completed. Add back to the queue, incrementing retries.
+                            self.add_subnet_query(
+                                query.subnet_id,
+                                query.min_ttl,
+                                query.retries + 1,
+                            );
+
+                            // Check the specific subnet against the enr
+                            let subnet_predicate =
+                                subnet_predicate::<TSpec>(vec![query.subnet_id], &self.log);
+
+                            r.iter()
+                                .filter(|enr| subnet_predicate(enr))
+                                .map(|enr| enr.peer_id())
+                                .for_each(|peer_id| {
+                                    let other_min_ttl = mapped_results.get_mut(&peer_id);
+
+                                    // map peer IDs to the min_ttl furthest in the future
+                                    match (query.min_ttl, other_min_ttl) {
+                                        // update the mapping if the min_ttl is greater
+                                        (
+                                            Some(min_ttl_instant),
+                                            Some(Some(other_min_ttl_instant)),
+                                        ) => {
+                                            if min_ttl_instant
+                                                .saturating_duration_since(*other_min_ttl_instant)
+                                                > DURATION_DIFFERENCE
+                                            {
+                                                *other_min_ttl_instant = min_ttl_instant;
+                                            }
+                                        }
+                                        // update the mapping if we have a specified min_ttl
+                                        (Some(min_ttl), Some(None)) => {
+                                            mapped_results.insert(peer_id, Some(min_ttl));
+                                        }
+                                        // first seen min_ttl for this enr
+                                        (Some(min_ttl), None) => {
+                                            mapped_results.insert(peer_id, Some(min_ttl));
+                                        }
+                                        // first seen min_ttl for this enr
+                                        (None, None) => {
+                                            mapped_results.insert(peer_id, None);
+                                        }
+                                        (None, Some(Some(_))) => {} // Don't replace the existing specific min_ttl
+                                        (None, Some(None)) => {} // No-op because this is a duplicate
+                                    }
+                                });
+                        });
+
+                        if mapped_results.is_empty() {
+                            return None;
+                        } else {
+                            return Some(mapped_results);
+                        }
+                    }
+                    Err(e) => {
+                        warn!(self.log,"Grouped subnet discovery query failed"; "subnets_searched_for" => format!("{:?}",subnets_searched_for), "error" => e.to_string());
+                    }
+                }
+            }
+        }
+        None
+    }
+
     /// Drives the queries returning any results from completed queries.
     fn poll_queries(&mut self, cx: &mut Context) -> Option<HashMap<PeerId, Option<Instant>>> {
-        while let Poll::Ready(Some(query_future)) = self.active_queries.poll_next_unpin(cx) {
-            match query_future.0 {
-                GroupedQueryType::FindPeers => {
-                    self.find_peer_active = false;
-                    match query_future.1 {
-                        Ok(r) if r.is_empty() => {
-                            debug!(self.log, "Discovery query yielded no results.");
-                        }
-                        Ok(r) => {
-                            debug!(self.log, "Discovery query completed"; "peers_found" => r.len());
-                            let mut results: HashMap<PeerId, Option<Instant>> = HashMap::new();
-                            r.iter().for_each(|enr| {
-                                // cache the found ENR's
-                                self.cached_enrs.put(enr.peer_id(), enr.clone());
-                                results.insert(enr.peer_id(), None);
-                            });
-                            return Some(results);
-                        }
-                        Err(e) => {
-                            warn!(self.log, "Discovery query failed"; "error" => e.to_string());
-                        }
-                    }
-                }
-                GroupedQueryType::Subnet(queries) => {
-                    let subnets_searched_for: Vec<SubnetId> =
-                        queries.iter().map(|query| query.subnet_id).collect();
-                    match query_future.1 {
-                        Ok(r) if r.is_empty() => {
-                            debug!(self.log, "Grouped subnet discovery query yielded no results."; "subnets_searched_for" => format!("{:?}",subnets_searched_for));
-                        }
-                        Ok(r) => {
-                            debug!(self.log, "Peer grouped subnet discovery request completed"; "peers_found" => r.len(), "subnets_searched_for" => format!("{:?}",subnets_searched_for));
-
-                            let mut mapped_results: HashMap<PeerId, Option<Instant>> =
-                                HashMap::new();
-
-                            // cache the found ENR's
-                            for enr in r.iter().cloned() {
-                                self.cached_enrs.put(enr.peer_id(), enr);
-                            }
-
-                            // Map each subnet query's min_ttl to the set of ENR's returned for that subnet.
-                            queries.iter().for_each(|query| {
-                                // A subnet query has completed. Add back to the queue, incrementing retries.
-                                self.add_subnet_query(
-                                    query.subnet_id,
-                                    query.min_ttl,
-                                    query.retries + 1,
-                                );
-
-                                // Check the specific subnet against the enr
-                                let subnet_predicate =
-                                    subnet_predicate::<TSpec>(vec![query.subnet_id], &self.log);
-
-                                r.iter()
-                                    .filter(|enr| subnet_predicate(enr))
-                                    .map(|enr| enr.peer_id())
-                                    .for_each(|peer_id| {
-                                        let other_min_ttl = mapped_results.get_mut(&peer_id);
-
-                                        // map peer IDs to the min_ttl furthest in the future
-                                        match (query.min_ttl, other_min_ttl) {
-                                            // update the mapping if the min_ttl is greater
-                                            (
-                                                Some(min_ttl_instant),
-                                                Some(Some(other_min_ttl_instant)),
-                                            ) => {
-                                                if min_ttl_instant.saturating_duration_since(
-                                                    *other_min_ttl_instant,
-                                                ) > DURATION_DIFFERENCE
-                                                {
-                                                    *other_min_ttl_instant = min_ttl_instant;
-                                                }
-                                            }
-                                            // update the mapping if we have a specified min_ttl
-                                            (Some(min_ttl), Some(None)) => {
-                                                mapped_results.insert(peer_id, Some(min_ttl));
-                                            }
-                                            // first seen min_ttl for this enr
-                                            (Some(min_ttl), None) => {
-                                                mapped_results.insert(peer_id, Some(min_ttl));
-                                            }
-                                            // first seen min_ttl for this enr
-                                            (None, None) => {
-                                                mapped_results.insert(peer_id, None);
-                                            }
-                                            (None, Some(Some(_))) => {} // Don't replace the existing specific min_ttl
-                                            (None, Some(None)) => {} // No-op because this is a duplicate
-                                        }
-                                    });
-                            });
-
-                            if mapped_results.is_empty() {
-                                return None;
-                            } else {
-                                return Some(mapped_results);
-                            }
-                        }
-                        Err(e) => {
-                            warn!(self.log,"Grouped subnet discovery query failed"; "subnets_searched_for" => format!("{:?}",subnets_searched_for), "error" => e.to_string());
-                        }
-                    }
-                }
+        while let Poll::Ready(Some(query_result)) = self.active_queries.poll_next_unpin(cx) {
+            let result = self.process_completed_queries(query_result);
+            if result.is_some() {
+                return result;
             }
         }
         None
@@ -778,7 +871,10 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                 // Still awaiting the event stream, poll it
                 if let Poll::Ready(event_stream) = fut.poll_unpin(cx) {
                     match event_stream {
-                        Ok(stream) => self.event_stream = EventStream::Present(stream),
+                        Ok(stream) => {
+                            debug!(self.log, "Discv5 event stream ready");
+                            self.event_stream = EventStream::Present(stream);
+                        }
                         Err(e) => {
                             slog::crit!(self.log, "Discv5 event stream failed"; "error" => e.to_string());
                             self.event_stream = EventStream::InActive;
@@ -824,3 +920,184 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         Poll::Pending
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::rpc::methods::MetaData;
+    use enr::EnrBuilder;
+    use slog::{o, Drain};
+    use std::net::UdpSocket;
+    use types::MinimalEthSpec;
+
+    type E = MinimalEthSpec;
+
+    pub fn unused_port() -> u16 {
+        let socket = UdpSocket::bind("127.0.0.1:0").expect("should create udp socket");
+        let local_addr = socket.local_addr().expect("should read udp socket");
+        local_addr.port()
+    }
+
+    pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger {
+        let decorator = slog_term::TermDecorator::new().build();
+        let drain = slog_term::FullFormat::new(decorator).build().fuse();
+        let drain = slog_async::Async::new(drain).build().fuse();
+
+        if enabled {
+            slog::Logger::root(drain.filter_level(level).fuse(), o!())
+        } else {
+            slog::Logger::root(drain.filter(|_| false).fuse(), o!())
+        }
+    }
+
+    async fn build_discovery() -> Discovery<E> {
+        let keypair = libp2p::identity::Keypair::generate_secp256k1();
+        let mut config = NetworkConfig::default();
+        config.discovery_port = unused_port();
+        let enr_key: CombinedKey = CombinedKey::from_libp2p(&keypair).unwrap();
+        let enr: Enr = build_enr::<E>(&enr_key, &config, EnrForkId::default()).unwrap();
+        let log = build_log(slog::Level::Debug, false);
+        let globals = NetworkGlobals::new(
+            enr,
+            9000,
+            9000,
+            MetaData {
+                seq_number: 0,
+                attnets: Default::default(),
+            },
+            vec![],
+            &log,
+        );
+        Discovery::new(&keypair, &config, Arc::new(globals), &log)
+            .await
+            .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_add_subnet_query() {
+        let mut discovery = build_discovery().await;
+        let now = Instant::now();
+        let mut subnet_query = SubnetQuery {
+            subnet_id: SubnetId::new(1),
+            min_ttl: Some(now),
+            retries: 0,
+        };
+        discovery.add_subnet_query(
+            subnet_query.subnet_id,
+            subnet_query.min_ttl,
+            subnet_query.retries,
+        );
+        assert_eq!(
+            discovery.queued_queries.back(),
+            Some(&QueryType::Subnet(subnet_query.clone()))
+        );
+
+        // New query should replace old query
+        subnet_query.min_ttl = Some(now + Duration::from_secs(1));
+        discovery.add_subnet_query(subnet_query.subnet_id, subnet_query.min_ttl, 1);
+
+        subnet_query.retries += 1;
+
+        assert_eq!(discovery.queued_queries.len(), 1);
+        assert_eq!(
+            discovery.queued_queries.pop_back(),
+            Some(QueryType::Subnet(subnet_query.clone()))
+        );
+
+        // Retries > MAX_DISCOVERY_RETRY must return immediately without adding
+        // anything.
+        discovery.add_subnet_query(
+            subnet_query.subnet_id,
+            subnet_query.min_ttl,
+            MAX_DISCOVERY_RETRY + 1,
+        );
+
+        assert_eq!(discovery.queued_queries.len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_process_queue() {
+        let mut discovery = build_discovery().await;
+
+        // FindPeers query is processed if there is no subnet query
+        discovery.queued_queries.push_back(QueryType::FindPeers);
+        assert!(discovery.process_queue());
+
+        let now = Instant::now();
+        let subnet_query = SubnetQuery {
+            subnet_id: SubnetId::new(1),
+            min_ttl: Some(now + Duration::from_secs(10)),
+            retries: 0,
+        };
+
+        // Refresh active queries
+        discovery.active_queries = Default::default();
+
+        // SubnetQuery is processed if it's the only queued query
+        discovery
+            .queued_queries
+            .push_back(QueryType::Subnet(subnet_query.clone()));
+        assert!(discovery.process_queue());
+
+        // SubnetQuery is processed if it's there is also 1 queued discovery query
+        discovery.queued_queries.push_back(QueryType::FindPeers);
+        discovery
+            .queued_queries
+            .push_back(QueryType::Subnet(subnet_query.clone()));
+        // Process Subnet query and FindPeers afterwards.
+        assert!(discovery.process_queue());
+    }
+
+    fn make_enr(subnet_ids: Vec<usize>) -> Enr {
+        let mut builder = EnrBuilder::new("v4");
+        let keypair = libp2p::identity::Keypair::generate_secp256k1();
+        let enr_key: CombinedKey = CombinedKey::from_libp2p(&keypair).unwrap();
+
+        // set the "attnets" field on our ENR
+        let mut bitfield = BitVector::<ssz_types::typenum::U64>::new();
+        for id in subnet_ids {
+            bitfield.set(id, true).unwrap();
+        }
+
+        builder.add_value(BITFIELD_ENR_KEY, &bitfield.as_ssz_bytes());
+        builder.build(&enr_key).unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_completed_subnet_queries() {
+        let mut discovery = build_discovery().await;
+        let now = Instant::now();
+        let instant1 = Some(now + Duration::from_secs(10));
+        let instant2 = Some(now + Duration::from_secs(5));
+
+        let query = GroupedQueryType::Subnet(vec![
+            SubnetQuery {
+                subnet_id: SubnetId::new(1),
+                min_ttl: instant1,
+                retries: 0,
+            },
+            SubnetQuery {
+                subnet_id: SubnetId::new(2),
+                min_ttl: instant2,
+                retries: 0,
+            },
+        ]);
+
+        // Create enr which is subscribed to subnets 1 and 2
+        let enr1 = make_enr(vec![1, 2]);
+        let enr2 = make_enr(vec![2]);
+        // Unwanted enr for the given grouped query
+        let enr3 = make_enr(vec![3]);
+
+        let enrs: Vec<Enr> = vec![enr1.clone(), enr2.clone(), enr3.clone()];
+        let results = discovery
+            .process_completed_queries(QueryResult(query, Ok(enrs)))
+            .unwrap();
+
+        // enr1 and enr2 are required peers based on the requested subnet ids
+        assert_eq!(results.len(), 2);
+
+        // when a peer belongs to multiple subnet ids, we use the highest ttl.
+        assert_eq!(results.get(&enr1.peer_id()).unwrap(), &instant1);
+    }
+}

beacon_node/eth2_libp2p/src/discovery/subnet_predicate.rs

@@ -1,6 +1,6 @@
 ///! The subnet predicate used for searching for a particular subnet.
 use super::*;
-use slog::{debug, trace};
+use slog::trace;
 use std::ops::Deref;
 
 /// Returns the predicate for a given subnet.
@@ -38,7 +38,7 @@ where
                 );
                 return false;
             } else {
-                debug!(
+                trace!(
                    log_clone,
                    "Peer found on desired subnet(s)";
                    "peer_id" => format!("{}", enr.peer_id()),

beacon_node/eth2_libp2p/src/lib.rs

@@ -14,16 +14,62 @@ pub mod rpc;
 mod service;
 pub mod types;
 
+use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
+use std::str::FromStr;
+
+/// Wrapper over a libp2p `PeerId` which implements `Serialize` and `Deserialize`
+#[derive(Clone, Debug)]
+pub struct PeerIdSerialized(libp2p::PeerId);
+
+impl From<PeerIdSerialized> for PeerId {
+    fn from(peer_id: PeerIdSerialized) -> Self {
+        peer_id.0
+    }
+}
+
+impl FromStr for PeerIdSerialized {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Self(
+            PeerId::from_str(s).map_err(|e| format!("Invalid peer id: {}", e))?,
+        ))
+    }
+}
+
+impl Serialize for PeerIdSerialized {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.0.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for PeerIdSerialized {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s: String = Deserialize::deserialize(deserializer)?;
+        Ok(Self(PeerId::from_str(&s).map_err(|e| {
+            de::Error::custom(format!("Failed to deserialise peer id: {:?}", e))
+        })?))
+    }
+}
+
 pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage, SubnetDiscovery};
-pub use behaviour::{BehaviourEvent, PeerRequestId, Request, Response};
+pub use behaviour::{BehaviourEvent, Gossipsub, PeerRequestId, Request, Response};
 pub use config::Config as NetworkConfig;
+pub use config::{GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage};
 pub use discovery::{CombinedKeyExt, EnrExt, Eth2Enr};
 pub use discv5;
-pub use libp2p::gossipsub::{MessageId, Topic, TopicHash};
+pub use libp2p::gossipsub::{MessageAcceptance, MessageId, Topic, TopicHash};
 pub use libp2p::{core::ConnectedPoint, PeerId, Swarm};
 pub use libp2p::{multiaddr, Multiaddr};
 pub use metrics::scrape_discovery_metrics;
 pub use peer_manager::{
-    client::Client, score::PeerAction, PeerDB, PeerInfo, PeerSyncStatus, SyncInfo,
+    client::Client, score::PeerAction, ConnectionDirection, PeerConnectionStatus, PeerDB, PeerInfo,
+    PeerSyncStatus, SyncInfo,
 };
-pub use service::{Libp2pEvent, Service, NETWORK_KEY_FILENAME};
+pub use service::{load_private_key, Libp2pEvent, Service, NETWORK_KEY_FILENAME};

beacon_node/eth2_libp2p/src/metrics.rs

@@ -34,9 +34,20 @@ lazy_static! {
         "Unsolicited discovery requests per ip per second",
         &["Addresses"]
     );
-    pub static ref GOSSIPSUB_SUBSCRIBED_PEERS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
-        "gossipsub_peers_per_topic_count",
-        "Peers subscribed per topic",
+    pub static ref PEERS_PER_CLIENT: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "libp2p_peers_per_client",
+        "The connected peers via client implementation",
+        &["Client"]
+    );
+    pub static ref FAILED_ATTESTATION_PUBLISHES_PER_SUBNET: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "gossipsub_failed_attestation_publishes_per_subnet",
+            "Failed attestation publishes per subnet",
+            &["subnet"]
+        );
+    pub static ref FAILED_PUBLISHES_PER_MAIN_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_failed_publishes_per_main_topic",
+        "Failed gossip publishes",
         &["topic_hash"]
     );
 }

beacon_node/eth2_libp2p/src/peer_manager/client.rs

@@ -20,7 +20,7 @@ pub struct Client {
     pub agent_string: Option<String>,
 }
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, PartialEq)]
 pub enum ClientKind {
     /// A lighthouse node (the best kind).
     Lighthouse,
@@ -98,6 +98,32 @@ impl std::fmt::Display for Client {
     }
 }
 
+impl ClientKind {
+    pub fn as_static_ref(&self) -> &'static str {
+        use ClientKind::*;
+        match self {
+            Lighthouse => "Lighthouse",
+            Nimbus => "Nimbus",
+            Teku => "Teku",
+            Prysm => "Prysm",
+            Lodestar => "Lodestar",
+            Unknown => "Unknown",
+        }
+    }
+}
+
+impl AsRef<str> for ClientKind {
+    fn as_ref(&self) -> &str {
+        self.as_static_ref()
+    }
+}
+
+impl std::fmt::Display for ClientKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_ref())
+    }
+}
+
 // helper function to identify clients from their agent_version. Returns the client
 // kind and it's associated version and the OS kind.
 fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String) {

beacon_node/eth2_libp2p/src/peer_manager/mod.rs

@@ -1,16 +1,17 @@
 //! Implementation of a Lighthouse's peer management system.
 
 pub use self::peerdb::*;
-use crate::discovery::{Discovery, DiscoveryEvent};
+use crate::discovery::{subnet_predicate, Discovery, DiscoveryEvent, TARGET_SUBNET_PEERS};
 use crate::rpc::{GoodbyeReason, MetaData, Protocol, RPCError, RPCResponseErrorCode};
-use crate::{error, metrics};
+use crate::types::SyncState;
+use crate::{error, metrics, Gossipsub};
 use crate::{EnrExt, NetworkConfig, NetworkGlobals, PeerId, SubnetDiscovery};
 use futures::prelude::*;
 use futures::Stream;
 use hashset_delay::HashSetDelay;
 use libp2p::core::multiaddr::Protocol as MProtocol;
 use libp2p::identify::IdentifyInfo;
-use slog::{crit, debug, error};
+use slog::{crit, debug, error, warn};
 use smallvec::SmallVec;
 use std::{
     net::SocketAddr,
@@ -19,7 +20,7 @@ use std::{
     task::{Context, Poll},
     time::{Duration, Instant},
 };
-use types::EthSpec;
+use types::{EthSpec, SubnetId};
 
 pub use libp2p::core::{identity::Keypair, Multiaddr};
 
@@ -29,15 +30,21 @@ mod peer_sync_status;
 mod peerdb;
 pub(crate) mod score;
 
-pub use peer_info::{PeerConnectionStatus::*, PeerInfo};
+pub use peer_info::{ConnectionDirection, PeerConnectionStatus, PeerConnectionStatus::*, PeerInfo};
 pub use peer_sync_status::{PeerSyncStatus, SyncInfo};
 use score::{PeerAction, ScoreState};
+use std::cmp::Ordering;
 use std::collections::HashMap;
+
 /// The time in seconds between re-status's peers.
 const STATUS_INTERVAL: u64 = 300;
-/// The time in seconds between PING events. We do not send a ping if the other peer as PING'd us within
-/// this time frame (Seconds)
-const PING_INTERVAL: u64 = 30;
+/// The time in seconds between PING events. We do not send a ping if the other peer has PING'd us
+/// within this time frame (Seconds)
+/// This is asymmetric to avoid simultaneous pings.
+/// The interval for outbound connections.
+const PING_INTERVAL_OUTBOUND: u64 = 30;
+/// The interval for inbound connections.
+const PING_INTERVAL_INBOUND: u64 = 35;
 
 /// The heartbeat performs regular updates such as updating reputations and performing discovery
 /// requests. This defines the interval in seconds.
@@ -48,14 +55,20 @@ const HEARTBEAT_INTERVAL: u64 = 30;
 /// PEER_EXCESS_FACTOR = 0.1 we allow 10% more nodes, i.e 55.
 const PEER_EXCESS_FACTOR: f32 = 0.1;
 
+/// Relative factor of peers that are allowed to have a negative gossipsub score without penalizing
+/// them in lighthouse.
+const ALLOWED_NEGATIVE_GOSSIPSUB_FACTOR: f32 = 0.1;
+
 /// The main struct that handles peer's reputation and connection status.
 pub struct PeerManager<TSpec: EthSpec> {
     /// Storage of network globals to access the `PeerDB`.
     network_globals: Arc<NetworkGlobals<TSpec>>,
     /// A queue of events that the `PeerManager` is waiting to produce.
     events: SmallVec<[PeerManagerEvent; 16]>,
-    /// A collection of peers awaiting to be Ping'd.
-    ping_peers: HashSetDelay<PeerId>,
+    /// A collection of inbound-connected peers awaiting to be Ping'd.
+    inbound_ping_peers: HashSetDelay<PeerId>,
+    /// A collection of outbound-connected peers awaiting to be Ping'd.
+    outbound_ping_peers: HashSetDelay<PeerId>,
     /// A collection of peers awaiting to be Status'd.
     status_peers: HashSetDelay<PeerId>,
     /// The target number of peers we would like to connect to.
@@ -105,7 +118,8 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         Ok(PeerManager {
             network_globals,
             events: SmallVec::new(),
-            ping_peers: HashSetDelay::new(Duration::from_secs(PING_INTERVAL)),
+            inbound_ping_peers: HashSetDelay::new(Duration::from_secs(PING_INTERVAL_INBOUND)),
+            outbound_ping_peers: HashSetDelay::new(Duration::from_secs(PING_INTERVAL_OUTBOUND)),
             status_peers: HashSetDelay::new(Duration::from_secs(STATUS_INTERVAL)),
             target_peers: config.target_peers,
             max_peers: (config.target_peers as f32 * (1.0 + PEER_EXCESS_FACTOR)).ceil() as usize,
@@ -133,13 +147,25 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     pub fn goodbye_peer(&mut self, peer_id: &PeerId, reason: GoodbyeReason) {
         // get the peer info
         if let Some(info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
-            debug!(self.log, "Sending goodbye to peer"; "peer_id" => peer_id.to_string(), "reason" => reason.to_string(), "score" => info.score.to_string());
-            // Goodbye's are fatal
-            info.score.apply_peer_action(PeerAction::Fatal);
-            if info.connection_status.is_connected_or_dialing() {
-                self.events
-                    .push(PeerManagerEvent::DisconnectPeer(peer_id.clone(), reason));
+            debug!(self.log, "Sending goodbye to peer"; "peer_id" => %peer_id, "reason" => %reason, "score" => %info.score());
+            if matches!(reason, GoodbyeReason::IrrelevantNetwork) {
+                info.sync_status.update(PeerSyncStatus::IrrelevantPeer);
             }
+
+            // Goodbye's are fatal
+            info.apply_peer_action_to_score(PeerAction::Fatal);
+        }
+
+        // Update the peerdb and peer state accordingly
+        if self
+            .network_globals
+            .peers
+            .write()
+            .disconnect_and_ban(peer_id)
+        {
+            // update the state of the peer.
+            self.events
+                .push(PeerManagerEvent::DisconnectPeer(peer_id.clone(), reason));
         }
     }
 
@@ -147,57 +173,31 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// If the peer doesn't exist, log a warning and insert defaults.
     pub fn report_peer(&mut self, peer_id: &PeerId, action: PeerAction) {
-        // TODO: Remove duplicate code  - This is duplicated in the update_peer_scores()
-        // function.
+        // Helper function to avoid any potential deadlocks.
+        let mut to_ban_peers = Vec::with_capacity(1);
+        let mut to_unban_peers = Vec::with_capacity(1);
 
-        // Variables to update the PeerDb if required.
-        let mut ban_peer = None;
-        let mut unban_peer = None;
-
-        if let Some(info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
-            let previous_state = info.score.state();
-            info.score.apply_peer_action(action);
-            if previous_state != info.score.state() {
-                match info.score.state() {
-                    ScoreState::Banned => {
-                        debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string());
-                        ban_peer = Some(peer_id.clone());
-                        if info.connection_status.is_connected_or_dialing() {
-                            self.events.push(PeerManagerEvent::DisconnectPeer(
-                                peer_id.clone(),
-                                GoodbyeReason::BadScore,
-                            ));
-                        }
-                    }
-                    ScoreState::Disconnected => {
-                        debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
-                        // disconnect the peer if it's currently connected or dialing
-                        unban_peer = Some(peer_id.clone());
-                        if info.connection_status.is_connected_or_dialing() {
-                            self.events.push(PeerManagerEvent::DisconnectPeer(
-                                peer_id.clone(),
-                                GoodbyeReason::BadScore,
-                            ));
-                        }
-                        // TODO: Update the peer manager to inform that the peer is disconnecting.
-                    }
-                    ScoreState::Healthy => {
-                        debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
-                        // unban the peer if it was previously banned.
-                        unban_peer = Some(peer_id.clone());
-                    }
+        {
+            let mut peer_db = self.network_globals.peers.write();
+            if let Some(info) = peer_db.peer_info_mut(peer_id) {
+                let previous_state = info.score_state();
+                info.apply_peer_action_to_score(action);
+                Self::handle_score_transitions(
+                    previous_state,
+                    peer_id,
+                    info,
+                    &mut to_ban_peers,
+                    &mut to_unban_peers,
+                    &mut self.events,
+                    &self.log,
+                );
+                if previous_state == info.score_state() {
+                    debug!(self.log, "Peer score adjusted"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string());
                 }
-            } else {
-                debug!(self.log, "Peer score adjusted"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string());
             }
-        }
+        } // end write lock
 
-        // Update the PeerDB state.
-        if let Some(peer_id) = ban_peer.take() {
-            self.network_globals.peers.write().ban(&peer_id);
-        } else if let Some(peer_id) = unban_peer.take() {
-            self.network_globals.peers.write().unban(&peer_id);
-        }
+        self.ban_and_unban_peers(to_ban_peers, to_unban_peers);
     }
 
     /* Discovery Requests */
@@ -214,18 +214,52 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
 
     /// A request to find peers on a given subnet.
     pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
-        // Extend the time to maintain peers if required.
-        for s in subnets_to_discover.iter() {
-            if let Some(min_ttl) = s.min_ttl {
-                self.network_globals
-                    .peers
-                    .write()
-                    .extend_peers_on_subnet(s.subnet_id, min_ttl);
-            }
+        // If discovery is not started or disabled, ignore the request
+        if !self.discovery.started {
+            return;
         }
 
+        let filtered: Vec<SubnetDiscovery> = subnets_to_discover
+            .into_iter()
+            .filter(|s| {
+                // Extend min_ttl of connected peers on required subnets
+                if let Some(min_ttl) = s.min_ttl {
+                    self.network_globals
+                        .peers
+                        .write()
+                        .extend_peers_on_subnet(s.subnet_id, min_ttl);
+                }
+                // Already have target number of peers, no need for subnet discovery
+                let peers_on_subnet = self
+                    .network_globals
+                    .peers
+                    .read()
+                    .good_peers_on_subnet(s.subnet_id)
+                    .count();
+                if peers_on_subnet >= TARGET_SUBNET_PEERS {
+                    debug!(
+                        self.log,
+                        "Discovery query ignored";
+                        "subnet_id" => format!("{:?}",s.subnet_id),
+                        "reason" => "Already connected to desired peers",
+                        "connected_peers_on_subnet" => peers_on_subnet,
+                        "target_subnet_peers" => TARGET_SUBNET_PEERS,
+                    );
+                    false
+                // Queue an outgoing connection request to the cached peers that are on `s.subnet_id`.
+                // If we connect to the cached peers before the discovery query starts, then we potentially
+                // save a costly discovery query.
+                } else {
+                    self.dial_cached_enrs_in_subnet(s.subnet_id);
+                    true
+                }
+            })
+            .collect();
+
         // request the subnet query from discovery
-        self.discovery.discover_subnet_peers(subnets_to_discover);
+        if !filtered.is_empty() {
+            self.discovery.discover_subnet_peers(filtered);
+        }
     }
 
     /// A STATUS message has been received from a peer. This resets the status timer.
@@ -239,16 +273,15 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// This is also called when dialing a peer fails.
     pub fn notify_disconnect(&mut self, peer_id: &PeerId) {
-        self.network_globals.peers.write().disconnect(peer_id);
+        self.network_globals
+            .peers
+            .write()
+            .notify_disconnect(peer_id);
 
         // remove the ping and status timer for the peer
-        self.ping_peers.remove(peer_id);
+        self.inbound_ping_peers.remove(peer_id);
+        self.outbound_ping_peers.remove(peer_id);
         self.status_peers.remove(peer_id);
-        metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT);
-        metrics::set_gauge(
-            &metrics::PEERS_CONNECTED,
-            self.network_globals.connected_peers() as i64,
-        );
     }
 
     /// A dial attempt has failed.
@@ -264,20 +297,14 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
 
     /// Sets a peer as connected as long as their reputation allows it
     /// Informs if the peer was accepted
-    pub fn connect_ingoing(&mut self, peer_id: &PeerId) -> bool {
-        self.connect_peer(peer_id, ConnectingType::IngoingConnected)
+    pub fn connect_ingoing(&mut self, peer_id: &PeerId, multiaddr: Multiaddr) -> bool {
+        self.connect_peer(peer_id, ConnectingType::IngoingConnected { multiaddr })
     }
 
     /// Sets a peer as connected as long as their reputation allows it
     /// Informs if the peer was accepted
-    pub fn connect_outgoing(&mut self, peer_id: &PeerId) -> bool {
-        self.connect_peer(peer_id, ConnectingType::OutgoingConnected)
-    }
-
-    /// Updates the database informing that a peer is being disconnected.
-    pub fn _disconnecting_peer(&mut self, _peer_id: &PeerId) -> bool {
-        // TODO: implement
-        true
+    pub fn connect_outgoing(&mut self, peer_id: &PeerId, multiaddr: Multiaddr) -> bool {
+        self.connect_peer(peer_id, ConnectingType::OutgoingConnected { multiaddr })
     }
 
     /// Reports if a peer is banned or not.
@@ -287,6 +314,10 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         self.network_globals.peers.read().is_banned(peer_id)
     }
 
+    pub fn is_connected(&self, peer_id: &PeerId) -> bool {
+        self.network_globals.peers.read().is_connected(peer_id)
+    }
+
     /// Reports whether the peer limit is reached in which case we stop allowing new incoming
     /// connections.
     pub fn peer_limit_reached(&self) -> bool {
@@ -296,8 +327,25 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// Updates `PeerInfo` with `identify` information.
     pub fn identify(&mut self, peer_id: &PeerId, info: &IdentifyInfo) {
         if let Some(peer_info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
+            let previous_kind = peer_info.client.kind.clone();
             peer_info.client = client::Client::from_identify_info(info);
             peer_info.listening_addresses = info.listen_addrs.clone();
+
+            if previous_kind != peer_info.client.kind {
+                // update the peer client kind metric
+                if let Some(v) = metrics::get_int_gauge(
+                    &metrics::PEERS_PER_CLIENT,
+                    &[&peer_info.client.kind.to_string()],
+                ) {
+                    v.inc()
+                };
+                if let Some(v) = metrics::get_int_gauge(
+                    &metrics::PEERS_PER_CLIENT,
+                    &[&previous_kind.to_string()],
+                ) {
+                    v.dec()
+                };
+            }
         } else {
             crit!(self.log, "Received an Identify response from an unknown peer"; "peer_id" => peer_id.to_string());
         }
@@ -306,10 +354,17 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// An error has occured in the RPC.
     ///
     /// This adjusts a peer's score based on the error.
-    pub fn handle_rpc_error(&mut self, peer_id: &PeerId, protocol: Protocol, err: &RPCError) {
+    pub fn handle_rpc_error(
+        &mut self,
+        peer_id: &PeerId,
+        protocol: Protocol,
+        err: &RPCError,
+        direction: ConnectionDirection,
+    ) {
         let client = self.network_globals.client(peer_id);
         let score = self.network_globals.peers.read().score(peer_id);
-        debug!(self.log, "RPC Error"; "protocol" => protocol.to_string(), "err" => err.to_string(), "client" => client.to_string(), "peer_id" => peer_id.to_string(), "score" => score.to_string());
+        debug!(self.log, "RPC Error"; "protocol" => %protocol, "err" => %err, "client" => %client,
+            "peer_id" => %peer_id, "score" => %score, "direction" => ?direction);
 
         // Map this error to a `PeerAction` (if any)
         let peer_action = match err {
@@ -340,10 +395,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                 // Not supporting a protocol shouldn't be considered a malicious action, but
                 // it is an action that in some cases will make the peer unfit to continue
                 // communicating.
-                // TODO: To avoid punishing a peer repeatedly for not supporting a protocol, this
-                // information could be stored and used to prevent sending requests for the given
-                // protocol to this peer. Similarly, to avoid blacklisting a peer for a protocol
-                // forever, if stored this information should expire.
+
                 match protocol {
                     Protocol::Ping => PeerAction::Fatal,
                     Protocol::BlocksByRange => return,
@@ -353,13 +405,20 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                     Protocol::Status => PeerAction::LowToleranceError,
                 }
             }
-            RPCError::StreamTimeout => match protocol {
-                Protocol::Ping => PeerAction::LowToleranceError,
-                Protocol::BlocksByRange => PeerAction::MidToleranceError,
-                Protocol::BlocksByRoot => PeerAction::MidToleranceError,
-                Protocol::Goodbye => return,
-                Protocol::MetaData => return,
-                Protocol::Status => return,
+            RPCError::StreamTimeout => match direction {
+                ConnectionDirection::Incoming => {
+                    // we timed out
+                    warn!(self.log, "Timed out to a peer's request. Likely too many resources, reduce peer count");
+                    return;
+                }
+                ConnectionDirection::Outgoing => match protocol {
+                    Protocol::Ping => PeerAction::LowToleranceError,
+                    Protocol::BlocksByRange => PeerAction::MidToleranceError,
+                    Protocol::BlocksByRoot => PeerAction::MidToleranceError,
+                    Protocol::Goodbye => return,
+                    Protocol::MetaData => return,
+                    Protocol::Status => return,
+                },
             },
             RPCError::NegotiationTimeout => PeerAction::HighToleranceError,
             RPCError::RateLimited => match protocol {
@@ -377,13 +436,22 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
 
     /// A ping request has been received.
     // NOTE: The behaviour responds with a PONG automatically
-    // TODO: Update last seen
     pub fn ping_request(&mut self, peer_id: &PeerId, seq: u64) {
         if let Some(peer_info) = self.network_globals.peers.read().peer_info(peer_id) {
             // received a ping
             // reset the to-ping timer for this peer
             debug!(self.log, "Received a ping request"; "peer_id" => peer_id.to_string(), "seq_no" => seq);
-            self.ping_peers.insert(peer_id.clone());
+            match peer_info.connection_direction {
+                Some(ConnectionDirection::Incoming) => {
+                    self.inbound_ping_peers.insert(peer_id.clone());
+                }
+                Some(ConnectionDirection::Outgoing) => {
+                    self.outbound_ping_peers.insert(peer_id.clone());
+                }
+                None => {
+                    warn!(self.log, "Received a ping from a peer with an unknown connection direction"; "peer_id" => %peer_id);
+                }
+            }
 
             // if the sequence number is unknown send an update the meta data of the peer.
             if let Some(meta_data) = &peer_info.meta_data {
@@ -407,7 +475,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     }
 
     /// A PONG has been returned from a peer.
-    // TODO: Update last seen
     pub fn pong_response(&mut self, peer_id: &PeerId, seq: u64) {
         if let Some(peer_info) = self.network_globals.peers.read().peer_info(peer_id) {
             // received a pong
@@ -433,7 +500,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     }
 
     /// Received a metadata response from a peer.
-    // TODO: Update last seen
     pub fn meta_data_response(&mut self, peer_id: &PeerId, meta_data: MetaData<TSpec>) {
         if let Some(peer_info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
             if let Some(known_meta_data) = &peer_info.meta_data {
@@ -466,10 +532,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             let mut out_list = enr.multiaddr();
             out_list.retain(|addr| {
                 addr.iter()
-                    .find(|v| match v {
-                        MProtocol::Udp(_) => true,
-                        _ => false,
-                    })
+                    .find(|v| matches!(v, MProtocol::Udp(_)))
                     .is_none()
             });
 
@@ -480,6 +543,59 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         }
     }
 
+    pub(crate) fn update_gossipsub_scores(&mut self, gossipsub: &Gossipsub) {
+        let mut to_ban_peers = Vec::new();
+        let mut to_unban_peers = Vec::new();
+
+        {
+            //collect peers with scores
+            let mut guard = self.network_globals.peers.write();
+            let mut peers: Vec<_> = guard
+                .peers_mut()
+                .filter_map(|(peer_id, info)| {
+                    gossipsub
+                        .peer_score(peer_id)
+                        .map(|score| (peer_id, info, score))
+                })
+                .collect();
+
+            // sort descending by score
+            peers.sort_unstable_by(|(.., s1), (.., s2)| {
+                s2.partial_cmp(s1).unwrap_or(Ordering::Equal)
+            });
+
+            let mut to_ignore_negative_peers =
+                (self.target_peers as f32 * ALLOWED_NEGATIVE_GOSSIPSUB_FACTOR).ceil() as usize;
+
+            for (peer_id, info, score) in peers {
+                let previous_state = info.score_state();
+                info.update_gossipsub_score(
+                    score,
+                    if score < 0.0 && to_ignore_negative_peers > 0 {
+                        to_ignore_negative_peers -= 1;
+                        // We ignore the negative score for the best negative peers so that their
+                        // gossipsub score can recover without getting disconnected.
+                        true
+                    } else {
+                        false
+                    },
+                );
+
+                Self::handle_score_transitions(
+                    previous_state,
+                    peer_id,
+                    info,
+                    &mut to_ban_peers,
+                    &mut to_unban_peers,
+                    &mut self.events,
+                    &self.log,
+                );
+            }
+        } // end write lock
+
+        self.ban_and_unban_peers(to_ban_peers, to_unban_peers);
+    }
+
     /* Internal functions */
 
     // The underlying discovery server has updated our external IP address. We send this up to
@@ -493,6 +609,30 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         self.events.push(PeerManagerEvent::SocketUpdated(multiaddr));
     }
 
+    /// Dial cached enrs in discovery service that are in the given `subnet_id` and aren't
+    /// in Connected, Dialing or Banned state.
+    fn dial_cached_enrs_in_subnet(&mut self, subnet_id: SubnetId) {
+        let predicate = subnet_predicate::<TSpec>(vec![subnet_id], &self.log);
+        let peers_to_dial: Vec<PeerId> = self
+            .discovery()
+            .cached_enrs()
+            .filter_map(|(peer_id, enr)| {
+                let peers = self.network_globals.peers.read();
+                if predicate(enr)
+                    && !peers.is_connected_or_dialing(peer_id)
+                    && !peers.is_banned(peer_id)
+                {
+                    Some(peer_id.clone())
+                } else {
+                    None
+                }
+            })
+            .collect();
+        for peer in &peers_to_dial {
+            self.dial_peer(peer);
+        }
+    }
+
     /// Peers that have been returned by discovery requests are dialed here if they are suitable.
     ///
     /// NOTE: By dialing `PeerId`s and not multiaddrs, libp2p requests the multiaddr associated
@@ -505,7 +645,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         let connected_or_dialing = self.network_globals.connected_or_dialing_peers();
         for (peer_id, min_ttl) in results {
             // we attempt a connection if this peer is a subnet peer or if the max peer count
-            // is not yet filled (including dialling peers)
+            // is not yet filled (including dialing peers)
             if (min_ttl.is_some() || connected_or_dialing + to_dial_peers.len() < self.max_peers)
                 && !self
                     .network_globals
@@ -518,7 +658,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                     .read()
                     .is_banned_or_disconnected(&peer_id)
             {
-                // TODO: Update output
                 // This should be updated with the peer dialing. In fact created once the peer is
                 // dialed
                 if let Some(min_ttl) = min_ttl {
@@ -545,20 +684,32 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     fn connect_peer(&mut self, peer_id: &PeerId, connection: ConnectingType) -> bool {
         {
             let mut peerdb = self.network_globals.peers.write();
-            if peerdb.connection_status(peer_id).map(|c| c.is_banned()) == Some(true) {
+            if peerdb.is_banned(&peer_id) {
                 // don't connect if the peer is banned
                 slog::crit!(self.log, "Connection has been allowed to a banned peer"; "peer_id" => peer_id.to_string());
             }
 
+            let enr = self.discovery.enr_of_peer(peer_id);
+
             match connection {
-                ConnectingType::Dialing => peerdb.dialing_peer(peer_id),
-                ConnectingType::IngoingConnected => peerdb.connect_outgoing(peer_id),
-                ConnectingType::OutgoingConnected => peerdb.connect_ingoing(peer_id),
+                ConnectingType::Dialing => {
+                    peerdb.dialing_peer(peer_id, enr);
+                    return true;
+                }
+                ConnectingType::IngoingConnected { multiaddr } => {
+                    peerdb.connect_ingoing(peer_id, multiaddr, enr);
+                    // start a timer to ping inbound peers.
+                    self.inbound_ping_peers.insert(peer_id.clone());
+                }
+                ConnectingType::OutgoingConnected { multiaddr } => {
+                    peerdb.connect_outgoing(peer_id, multiaddr, enr);
+                    // start a timer for to ping outbound peers.
+                    self.outbound_ping_peers.insert(peer_id.clone());
+                }
             }
         }
 
         // start a ping and status timer for the peer
-        self.ping_peers.insert(peer_id.clone());
         self.status_peers.insert(peer_id.clone());
 
         // increment prometheus metrics
@@ -568,117 +719,152 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             self.network_globals.connected_peers() as i64,
         );
 
+        // Increment the PEERS_PER_CLIENT metric
+        if let Some(kind) = self
+            .network_globals
+            .peers
+            .read()
+            .peer_info(peer_id)
+            .map(|peer_info| peer_info.client.kind.clone())
+        {
+            if let Some(v) =
+                metrics::get_int_gauge(&metrics::PEERS_PER_CLIENT, &[&kind.to_string()])
+            {
+                v.inc()
+            };
+        }
+
         true
     }
 
+    fn handle_score_transitions(
+        previous_state: ScoreState,
+        peer_id: &PeerId,
+        info: &mut PeerInfo<TSpec>,
+        to_ban_peers: &mut Vec<PeerId>,
+        to_unban_peers: &mut Vec<PeerId>,
+        events: &mut SmallVec<[PeerManagerEvent; 16]>,
+        log: &slog::Logger,
+    ) {
+        if previous_state != info.score_state() {
+            match info.score_state() {
+                ScoreState::Banned => {
+                    debug!(log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string());
+                    to_ban_peers.push(peer_id.clone());
+                }
+                ScoreState::Disconnected => {
+                    debug!(log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
+                    // disconnect the peer if it's currently connected or dialing
+                    if info.is_connected_or_dialing() {
+                        // Change the state to inform that we are disconnecting the peer.
+                        info.disconnecting(false);
+                        events.push(PeerManagerEvent::DisconnectPeer(
+                            peer_id.clone(),
+                            GoodbyeReason::BadScore,
+                        ));
+                    } else if info.is_banned() {
+                        to_unban_peers.push(peer_id.clone());
+                    }
+                }
+                ScoreState::Healthy => {
+                    debug!(log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
+                    // unban the peer if it was previously banned.
+                    if info.is_banned() {
+                        to_unban_peers.push(peer_id.clone());
+                    }
+                }
+            }
+        }
+    }
+
+    fn ban_and_unban_peers(&mut self, to_ban_peers: Vec<PeerId>, to_unban_peers: Vec<PeerId>) {
+        // process banning peers
+        for peer_id in to_ban_peers {
+            self.ban_peer(&peer_id);
+        }
+        // process unbanning peers
+        for peer_id in to_unban_peers {
+            if let Err(e) = self.unban_peer(&peer_id) {
+                error!(self.log, "{}", e; "peer_id" => %peer_id);
+            }
+        }
+    }
+
     /// Updates the scores of known peers according to their connection
     /// status and the time that has passed.
     /// NOTE: This is experimental and will likely be adjusted
     fn update_peer_scores(&mut self) {
         /* Check how long have peers been in this state and update their reputations if needed */
-        let mut pdb = self.network_globals.peers.write();
 
         let mut to_ban_peers = Vec::new();
         let mut to_unban_peers = Vec::new();
 
-        for (peer_id, info) in pdb.peers_mut() {
-            let previous_state = info.score.state();
+        for (peer_id, info) in self.network_globals.peers.write().peers_mut() {
+            let previous_state = info.score_state();
             // Update scores
-            info.score.update();
+            info.score_update();
 
-            /* TODO: Implement logic about connection lifetimes
-            match info.connection_status {
-                Connected { .. } => {
-                    // Connected peers gain reputation by sending useful messages
-                }
-                Disconnected { since } | Banned { since } => {
-                    // For disconnected peers, lower their reputation by 1 for every hour they
-                    // stay disconnected. This helps us slowly forget disconnected peers.
-                    // In the same way, slowly allow banned peers back again.
-                    let dc_hours = now
-                        .checked_duration_since(since)
-                        .unwrap_or_else(|| Duration::from_secs(0))
-                        .as_secs()
-                        / 3600;
-                    let last_dc_hours = self
-                        ._last_updated
-                        .checked_duration_since(since)
-                        .unwrap_or_else(|| Duration::from_secs(0))
-                        .as_secs()
-                        / 3600;
-                    if dc_hours > last_dc_hours {
-                        // this should be 1 most of the time
-                        let rep_dif = (dc_hours - last_dc_hours)
-                            .try_into()
-                            .unwrap_or(Rep::max_value());
+            Self::handle_score_transitions(
+                previous_state,
+                peer_id,
+                info,
+                &mut to_ban_peers,
+                &mut to_unban_peers,
+                &mut self.events,
+                &self.log,
+            );
+        }
+        self.ban_and_unban_peers(to_ban_peers, to_unban_peers);
+    }
 
-                        info.reputation = if info.connection_status.is_banned() {
-                            info.reputation.saturating_add(rep_dif)
-                        } else {
-                            info.reputation.saturating_sub(rep_dif)
-                        };
-                    }
-                }
-                Dialing { since } => {
-                    // A peer shouldn't be dialing for more than 2 minutes
-                    if since.elapsed().as_secs() > 120 {
-                        warn!(self.log,"Peer has been dialing for too long"; "peer_id" => id.to_string());
-                        // TODO: decide how to handle this
-                    }
-                }
-                Unknown => {} //TODO: Handle this case
-            }
-            // Check if the peer gets banned or unbanned and if it should be disconnected
-            if info.reputation < _MIN_REP_BEFORE_BAN && !info.connection_status.is_banned() {
-                // This peer gets banned. Check if we should request disconnection
-                ban_queue.push(id.clone());
-            } else if info.reputation >= _MIN_REP_BEFORE_BAN && info.connection_status.is_banned() {
-                // This peer gets unbanned
-                unban_queue.push(id.clone());
-            }
-            */
+    /// Bans a peer.
+    ///
+    /// Records updates the peers connection status and updates the peer db as well as blocks the
+    /// peer from participating in discovery and removes them from the routing table.
+    fn ban_peer(&mut self, peer_id: &PeerId) {
+        {
+            // write lock scope
+            let mut peer_db = self.network_globals.peers.write();
 
-            // handle score transitions
-            if previous_state != info.score.state() {
-                match info.score.state() {
-                    ScoreState::Banned => {
-                        debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string());
-                        to_ban_peers.push(peer_id.clone());
-                        if info.connection_status.is_connected_or_dialing() {
-                            self.events.push(PeerManagerEvent::DisconnectPeer(
-                                peer_id.clone(),
-                                GoodbyeReason::BadScore,
-                            ));
-                        }
-                    }
-                    ScoreState::Disconnected => {
-                        debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
-                        // disconnect the peer if it's currently connected or dialing
-                        to_unban_peers.push(peer_id.clone());
-                        if info.connection_status.is_connected_or_dialing() {
-                            self.events.push(PeerManagerEvent::DisconnectPeer(
-                                peer_id.clone(),
-                                GoodbyeReason::BadScore,
-                            ));
-                        }
-                        // TODO: Update peer manager to report that it's disconnecting.
-                    }
-                    ScoreState::Healthy => {
-                        debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
-                        // unban the peer if it was previously banned.
-                        to_unban_peers.push(peer_id.clone());
-                    }
-                }
+            if peer_db.disconnect_and_ban(peer_id) {
+                // The peer was currently connected, so we start a disconnection.
+                self.events.push(PeerManagerEvent::DisconnectPeer(
+                    peer_id.clone(),
+                    GoodbyeReason::BadScore,
+                ));
             }
-        }
-        // process banning peers
-        for peer_id in to_ban_peers {
-            pdb.ban(&peer_id);
-        }
-        // process unbanning peers
-        for peer_id in to_unban_peers {
-            pdb.unban(&peer_id);
-        }
+        } // end write lock
+
+        // take a read lock
+        let peer_db = self.network_globals.peers.read();
+
+        let banned_ip_addresses = peer_db
+            .peer_info(peer_id)
+            .map(|info| {
+                info.seen_addresses()
+                    .filter(|ip| peer_db.is_ip_banned(ip))
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default();
+
+        self.discovery.ban_peer(&peer_id, banned_ip_addresses);
+    }
+
+    /// Unbans a peer.
+    ///
+    /// Records updates the peers connection status and updates the peer db as well as removes
+    /// previous bans from discovery.
+    fn unban_peer(&mut self, peer_id: &PeerId) -> Result<(), &'static str> {
+        let mut peer_db = self.network_globals.peers.write();
+        peer_db.unban(&peer_id)?;
+
+        let seen_ip_addresses = peer_db
+            .peer_info(peer_id)
+            .map(|info| info.seen_addresses().collect::<Vec<_>>())
+            .unwrap_or_default();
+
+        self.discovery.unban_peer(&peer_id, seen_ip_addresses);
+        Ok(())
     }
 
     /// The Peer manager's heartbeat maintains the peer count and maintains peer reputations.
@@ -688,19 +874,21 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// NOTE: Discovery will only add a new query if one isn't already queued.
     fn heartbeat(&mut self) {
-        // TODO: Provide a back-off time for discovery queries. I.e Queue many initially, then only
-        // perform discoveries over a larger fixed interval. Perhaps one every 6 heartbeats. This
-        // is achievable with a leaky bucket
         let peer_count = self.network_globals.connected_or_dialing_peers();
         if peer_count < self.target_peers {
             // If we need more peers, queue a discovery lookup.
-            debug!(self.log, "Starting a new peer discovery query"; "connected_peers" => peer_count, "target_peers" => self.target_peers);
-            self.discovery.discover_peers();
+            if self.discovery.started {
+                debug!(self.log, "Starting a new peer discovery query"; "connected_peers" => peer_count, "target_peers" => self.target_peers);
+                self.discovery.discover_peers();
+            }
         }
 
         // Updates peer's scores.
         self.update_peer_scores();
 
+        // Keep a list of peers we are disconnecting
+        let mut disconnecting_peers = Vec::new();
+
         let connected_peer_count = self.network_globals.connected_peers();
         if connected_peer_count > self.target_peers {
             //remove excess peers with the worst scores, but keep subnet peers
@@ -714,14 +902,20 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                 .take(connected_peer_count - self.target_peers)
                 //we only need to disconnect peers with healthy scores, since the others got already
                 //disconnected in update_peer_scores
-                .filter(|(_, info)| info.score.state() == ScoreState::Healthy)
+                .filter(|(_, info)| info.score_state() == ScoreState::Healthy)
             {
-                self.events.push(PeerManagerEvent::DisconnectPeer(
-                    (*peer_id).clone(),
-                    GoodbyeReason::TooManyPeers,
-                ));
+                disconnecting_peers.push((*peer_id).clone());
             }
         }
+
+        let mut peer_db = self.network_globals.peers.write();
+        for peer_id in disconnecting_peers {
+            peer_db.notify_disconnecting(&peer_id);
+            self.events.push(PeerManagerEvent::DisconnectPeer(
+                peer_id,
+                GoodbyeReason::TooManyPeers,
+            ));
+        }
     }
 }
 
@@ -744,21 +938,33 @@ impl<TSpec: EthSpec> Stream for PeerManager<TSpec> {
 
         // poll the timeouts for pings and status'
         loop {
-            match self.ping_peers.poll_next_unpin(cx) {
+            match self.inbound_ping_peers.poll_next_unpin(cx) {
                 Poll::Ready(Some(Ok(peer_id))) => {
-                    self.ping_peers.insert(peer_id.clone());
+                    self.inbound_ping_peers.insert(peer_id.clone());
                     self.events.push(PeerManagerEvent::Ping(peer_id));
                 }
                 Poll::Ready(Some(Err(e))) => {
-                    error!(self.log, "Failed to check for peers to ping"; "error" => e.to_string())
+                    error!(self.log, "Failed to check for inbound peers to ping"; "error" => e.to_string())
                 }
                 Poll::Ready(None) | Poll::Pending => break,
             }
         }
 
-        // We don't want to update peers during syncing, since this may result in a new chain being
-        // synced which leads to inefficient re-downloads of blocks.
-        if !self.network_globals.is_syncing() {
+        loop {
+            match self.outbound_ping_peers.poll_next_unpin(cx) {
+                Poll::Ready(Some(Ok(peer_id))) => {
+                    self.outbound_ping_peers.insert(peer_id.clone());
+                    self.events.push(PeerManagerEvent::Ping(peer_id));
+                }
+                Poll::Ready(Some(Err(e))) => {
+                    error!(self.log, "Failed to check for outbound peers to ping"; "error" => e.to_string())
+                }
+                Poll::Ready(None) | Poll::Pending => break,
+            }
+        }
+
+        if !matches!(self.network_globals.sync_state(), SyncState::SyncingFinalized{..}|SyncState::SyncingHead{..})
+        {
             loop {
                 match self.status_peers.poll_next_unpin(cx) {
                     Poll::Ready(Some(Ok(peer_id))) => {
@@ -787,7 +993,13 @@ enum ConnectingType {
     /// We are in the process of dialing this peer.
     Dialing,
     /// A peer has dialed us.
-    IngoingConnected,
+    IngoingConnected {
+        // The multiaddr the peer connected to us on.
+        multiaddr: Multiaddr,
+    },
     /// We have successfully dialed a peer.
-    OutgoingConnected,
+    OutgoingConnected {
+        /// The multiaddr we dialed to reach the peer.
+        multiaddr: Multiaddr,
+    },
 }

beacon_node/eth2_libp2p/src/peer_manager/peer_info.rs

@@ -1,12 +1,15 @@
 use super::client::Client;
-use super::score::Score;
+use super::score::{PeerAction, Score, ScoreState};
 use super::PeerSyncStatus;
 use crate::rpc::MetaData;
 use crate::Multiaddr;
+use discv5::Enr;
 use serde::{
-    ser::{SerializeStructVariant, Serializer},
+    ser::{SerializeStruct, Serializer},
     Serialize,
 };
+use std::collections::HashSet;
+use std::net::{IpAddr, SocketAddr};
 use std::time::Instant;
 use types::{EthSpec, SubnetId};
 use PeerConnectionStatus::*;
@@ -18,13 +21,17 @@ pub struct PeerInfo<T: EthSpec> {
     /// The connection status of the peer
     _status: PeerStatus,
     /// The peers reputation
-    pub score: Score,
+    score: Score,
     /// Client managing this peer
     pub client: Client,
     /// Connection status of this peer
-    pub connection_status: PeerConnectionStatus,
-    /// The known listening addresses of this peer.
+    connection_status: PeerConnectionStatus,
+    /// The known listening addresses of this peer. This is given by identify and can be arbitrary
+    /// (including local IPs).
     pub listening_addresses: Vec<Multiaddr>,
+    /// This is addresses we have physically seen and this is what we use for banning/un-banning
+    /// peers.
+    pub seen_addresses: HashSet<SocketAddr>,
     /// The current syncing state of the peer. The state may be determined after it's initial
     /// connection.
     pub sync_status: PeerSyncStatus,
@@ -35,6 +42,13 @@ pub struct PeerInfo<T: EthSpec> {
     /// necessary.
     #[serde(skip)]
     pub min_ttl: Option<Instant>,
+    /// Is the peer a trusted peer.
+    pub is_trusted: bool,
+    /// Direction of the first connection of the last (or current) connected session with this peer.
+    /// None if this peer was never connected.
+    pub connection_direction: Option<ConnectionDirection>,
+    /// The enr of the peer, if known.
+    pub enr: Option<Enr>,
 }
 
 impl<TSpec: EthSpec> Default for PeerInfo<TSpec> {
@@ -44,15 +58,28 @@ impl<TSpec: EthSpec> Default for PeerInfo<TSpec> {
             score: Score::default(),
             client: Client::default(),
             connection_status: Default::default(),
-            listening_addresses: vec![],
+            listening_addresses: Vec::new(),
+            seen_addresses: HashSet::new(),
             sync_status: PeerSyncStatus::Unknown,
             meta_data: None,
             min_ttl: None,
+            is_trusted: false,
+            connection_direction: None,
+            enr: None,
         }
     }
 }
 
 impl<T: EthSpec> PeerInfo<T> {
+    /// Return a PeerInfo struct for a trusted peer.
+    pub fn trusted_peer_info() -> Self {
+        PeerInfo {
+            score: Score::max_score(),
+            is_trusted: true,
+            ..Default::default()
+        }
+    }
+
     /// Returns if the peer is subscribed to a given `SubnetId`
     pub fn on_subnet(&self, subnet_id: SubnetId) -> bool {
         if let Some(meta_data) = &self.meta_data {
@@ -64,10 +91,212 @@ impl<T: EthSpec> PeerInfo<T> {
         false
     }
 
+    /// Returns the seen IP addresses of the peer.
+    pub fn seen_addresses<'a>(&'a self) -> impl Iterator<Item = IpAddr> + 'a {
+        self.seen_addresses
+            .iter()
+            .map(|socket_addr| socket_addr.ip())
+    }
+
+    /// Returns the connection status of the peer.
+    pub fn connection_status(&self) -> &PeerConnectionStatus {
+        &self.connection_status
+    }
+
     /// Reports if this peer has some future validator duty in which case it is valuable to keep it.
     pub fn has_future_duty(&self) -> bool {
         self.min_ttl.map_or(false, |i| i >= Instant::now())
     }
+
+    /// Returns score of the peer.
+    pub fn score(&self) -> &Score {
+        &self.score
+    }
+
+    /// Returns the state of the peer based on the score.
+    pub(crate) fn score_state(&self) -> ScoreState {
+        self.score.state()
+    }
+
+    /// Applies decay rates to a non-trusted peer's score.
+    pub fn score_update(&mut self) {
+        if !self.is_trusted {
+            self.score.update()
+        }
+    }
+
+    /// Apply peer action to a non-trusted peer's score.
+    pub fn apply_peer_action_to_score(&mut self, peer_action: PeerAction) {
+        if !self.is_trusted {
+            self.score.apply_peer_action(peer_action)
+        }
+    }
+
+    pub(crate) fn update_gossipsub_score(&mut self, new_score: f64, ignore: bool) {
+        self.score.update_gossipsub_score(new_score, ignore);
+    }
+
+    pub fn is_good_gossipsub_peer(&self) -> bool {
+        self.score.is_good_gossipsub_peer()
+    }
+
+    #[cfg(test)]
+    /// Resets the peers score.
+    pub fn reset_score(&mut self) {
+        self.score.test_reset();
+    }
+
+    /* Peer connection status API */
+
+    /// Checks if the status is connected.
+    pub fn is_connected(&self) -> bool {
+        matches!(self.connection_status, PeerConnectionStatus::Connected { .. })
+    }
+
+    /// Checks if the status is connected.
+    pub fn is_dialing(&self) -> bool {
+        matches!(self.connection_status, PeerConnectionStatus::Dialing { .. })
+    }
+
+    /// The peer is either connected or in the process of being dialed.
+    pub fn is_connected_or_dialing(&self) -> bool {
+        self.is_connected() || self.is_dialing()
+    }
+
+    /// Checks if the status is banned.
+    pub fn is_banned(&self) -> bool {
+        matches!(self.connection_status, PeerConnectionStatus::Banned { .. })
+    }
+
+    /// Checks if the status is disconnected.
+    pub fn is_disconnected(&self) -> bool {
+        matches!(self.connection_status, Disconnected { .. })
+    }
+
+    /// Returns the number of connections with this peer.
+    pub fn connections(&self) -> (u8, u8) {
+        match self.connection_status {
+            Connected { n_in, n_out } => (n_in, n_out),
+            _ => (0, 0),
+        }
+    }
+
+    // Setters
+
+    /// Modifies the status to Disconnected and sets the last seen instant to now. Returns None if
+    /// no changes were made. Returns Some(bool) where the bool represents if peer became banned or
+    /// simply just disconnected.
+    pub fn notify_disconnect(&mut self) -> Option<bool> {
+        match self.connection_status {
+            Banned { .. } | Disconnected { .. } => None,
+            Disconnecting { to_ban } => {
+                // If we are disconnecting this peer in the process of banning, we now ban the
+                // peer.
+                if to_ban {
+                    self.connection_status = Banned {
+                        since: Instant::now(),
+                    };
+                    Some(true)
+                } else {
+                    self.connection_status = Disconnected {
+                        since: Instant::now(),
+                    };
+                    Some(false)
+                }
+            }
+            Connected { .. } | Dialing { .. } | Unknown => {
+                self.connection_status = Disconnected {
+                    since: Instant::now(),
+                };
+                Some(false)
+            }
+        }
+    }
+
+    /// Notify the we are currently disconnecting this peer, after which the peer will be
+    /// considered banned.
+    // This intermediate state is required to inform the network behaviours that the sub-protocols
+    // are aware this peer exists and it is in the process of being banned. Compared to nodes that
+    // try to connect to us and are already banned (sub protocols do not know of these peers).
+    pub fn disconnecting(&mut self, to_ban: bool) {
+        self.connection_status = Disconnecting { to_ban }
+    }
+
+    /// Modifies the status to Banned
+    pub fn ban(&mut self) {
+        self.connection_status = Banned {
+            since: Instant::now(),
+        };
+    }
+
+    /// The score system has unbanned the peer. Update the connection status
+    pub fn unban(&mut self) {
+        if let PeerConnectionStatus::Banned { since, .. } = self.connection_status {
+            self.connection_status = PeerConnectionStatus::Disconnected { since };
+        }
+    }
+
+    /// Modifies the status to Dialing
+    /// Returns an error if the current state is unexpected.
+    pub(crate) fn dialing_peer(&mut self) -> Result<(), &'static str> {
+        match &mut self.connection_status {
+            Connected { .. } => return Err("Dialing connected peer"),
+            Dialing { .. } => return Err("Dialing an already dialing peer"),
+            Disconnecting { .. } => return Err("Dialing a disconnecting peer"),
+            Disconnected { .. } | Banned { .. } | Unknown => {}
+        }
+        self.connection_status = Dialing {
+            since: Instant::now(),
+        };
+        Ok(())
+    }
+
+    /// Modifies the status to Connected and increases the number of ingoing
+    /// connections by one
+    pub(crate) fn connect_ingoing(&mut self, seen_address: Option<SocketAddr>) {
+        match &mut self.connection_status {
+            Connected { n_in, .. } => *n_in += 1,
+            Disconnected { .. }
+            | Banned { .. }
+            | Dialing { .. }
+            | Disconnecting { .. }
+            | Unknown => {
+                self.connection_status = Connected { n_in: 1, n_out: 0 };
+                self.connection_direction = Some(ConnectionDirection::Incoming);
+            }
+        }
+
+        if let Some(socket_addr) = seen_address {
+            self.seen_addresses.insert(socket_addr);
+        }
+    }
+
+    /// Modifies the status to Connected and increases the number of outgoing
+    /// connections by one
+    pub(crate) fn connect_outgoing(&mut self, seen_address: Option<SocketAddr>) {
+        match &mut self.connection_status {
+            Connected { n_out, .. } => *n_out += 1,
+            Disconnected { .. }
+            | Banned { .. }
+            | Dialing { .. }
+            | Disconnecting { .. }
+            | Unknown => {
+                self.connection_status = Connected { n_in: 0, n_out: 1 };
+                self.connection_direction = Some(ConnectionDirection::Outgoing);
+            }
+        }
+        if let Some(ip_addr) = seen_address {
+            self.seen_addresses.insert(ip_addr);
+        }
+    }
+
+    #[cfg(test)]
+    /// Add an f64 to a non-trusted peer's score abiding by the limits.
+    pub fn add_to_score(&mut self, score: f64) {
+        if !self.is_trusted {
+            self.score.test_add(score)
+        }
+    }
 }
 
 #[derive(Clone, Debug, Serialize)]
@@ -85,6 +314,13 @@ impl Default for PeerStatus {
     }
 }
 
+/// Connection Direction of connection.
+#[derive(Debug, Clone, Serialize)]
+pub enum ConnectionDirection {
+    Incoming,
+    Outgoing,
+}
+
 /// Connection Status of the peer.
 #[derive(Debug, Clone)]
 pub enum PeerConnectionStatus {
@@ -95,11 +331,17 @@ pub enum PeerConnectionStatus {
         /// number of outgoing connections.
         n_out: u8,
     },
+    /// The peer is being disconnected.
+    Disconnecting {
+        // After the disconnection the peer will be considered banned.
+        to_ban: bool,
+    },
     /// The peer has disconnected.
     Disconnected {
         /// last time the peer was connected or discovered.
         since: Instant,
     },
+
     /// The peer has been banned and is disconnected.
     Banned {
         /// moment when the peer was banned.
@@ -117,29 +359,51 @@ pub enum PeerConnectionStatus {
 /// Serialization for http requests.
 impl Serialize for PeerConnectionStatus {
     fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let mut s = serializer.serialize_struct("connection_status", 6)?;
         match self {
             Connected { n_in, n_out } => {
-                let mut s = serializer.serialize_struct_variant("", 0, "Connected", 2)?;
-                s.serialize_field("in", n_in)?;
-                s.serialize_field("out", n_out)?;
+                s.serialize_field("status", "connected")?;
+                s.serialize_field("connections_in", n_in)?;
+                s.serialize_field("connections_out", n_out)?;
+                s.serialize_field("last_seen", &0)?;
+                s.end()
+            }
+            Disconnecting { .. } => {
+                s.serialize_field("status", "disconnecting")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &0)?;
                 s.end()
             }
             Disconnected { since } => {
-                let mut s = serializer.serialize_struct_variant("", 1, "Disconnected", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "disconnected")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
                 s.end()
             }
             Banned { since } => {
-                let mut s = serializer.serialize_struct_variant("", 2, "Banned", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "banned")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
                 s.end()
             }
             Dialing { since } => {
-                let mut s = serializer.serialize_struct_variant("", 3, "Dialing", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "dialing")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.end()
+            }
+            Unknown => {
+                s.serialize_field("status", "unknown")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &0)?;
                 s.end()
             }
-            Unknown => serializer.serialize_unit_variant("", 4, "Unknown"),
         }
     }
 }
@@ -149,92 +413,3 @@ impl Default for PeerConnectionStatus {
         PeerConnectionStatus::Unknown
     }
 }
-
-impl PeerConnectionStatus {
-    /// Checks if the status is connected.
-    pub fn is_connected(&self) -> bool {
-        match self {
-            PeerConnectionStatus::Connected { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Checks if the status is connected.
-    pub fn is_dialing(&self) -> bool {
-        match self {
-            PeerConnectionStatus::Dialing { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// The peer is either connected or in the process of being dialed.
-    pub fn is_connected_or_dialing(&self) -> bool {
-        self.is_connected() || self.is_dialing()
-    }
-
-    /// Checks if the status is banned.
-    pub fn is_banned(&self) -> bool {
-        match self {
-            PeerConnectionStatus::Banned { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Checks if the status is disconnected.
-    pub fn is_disconnected(&self) -> bool {
-        match self {
-            Disconnected { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Modifies the status to Connected and increases the number of ingoing
-    /// connections by one
-    pub fn connect_ingoing(&mut self) {
-        match self {
-            Connected { n_in, .. } => *n_in += 1,
-            Disconnected { .. } | Banned { .. } | Dialing { .. } | Unknown => {
-                *self = Connected { n_in: 1, n_out: 0 }
-            }
-        }
-    }
-
-    /// Modifies the status to Connected and increases the number of outgoing
-    /// connections by one
-    pub fn connect_outgoing(&mut self) {
-        match self {
-            Connected { n_out, .. } => *n_out += 1,
-            Disconnected { .. } | Banned { .. } | Dialing { .. } | Unknown => {
-                *self = Connected { n_in: 0, n_out: 1 }
-            }
-        }
-    }
-
-    /// Modifies the status to Disconnected and sets the last seen instant to now
-    pub fn disconnect(&mut self) {
-        *self = Disconnected {
-            since: Instant::now(),
-        };
-    }
-
-    /// Modifies the status to Banned
-    pub fn ban(&mut self) {
-        *self = Banned {
-            since: Instant::now(),
-        };
-    }
-
-    /// The score system has unbanned the peer. Update the connection status
-    pub fn unban(&mut self) {
-        if let PeerConnectionStatus::Banned { since } = self {
-            *self = PeerConnectionStatus::Disconnected { since: *since }
-        }
-    }
-
-    pub fn connections(&self) -> (u8, u8) {
-        match self {
-            Connected { n_in, n_out } => (*n_in, *n_out),
-            _ => (0, 0),
-        }
-    }
-}

beacon_node/eth2_libp2p/src/peer_manager/peer_sync_status.rs

@@ -12,93 +12,73 @@ pub enum PeerSyncStatus {
     Advanced { info: SyncInfo },
     /// Is behind our current head and not useful for block downloads.
     Behind { info: SyncInfo },
+    /// This peer is in an incompatible network.
+    IrrelevantPeer,
     /// Not currently known as a STATUS handshake has not occurred.
     Unknown,
 }
 
-/// This is stored inside the PeerSyncStatus and is very similar to `PeerSyncInfo` in the
-/// `Network` crate.
+/// A relevant peer's sync information.
 #[derive(Clone, Debug, Serialize)]
 pub struct SyncInfo {
-    pub status_head_slot: Slot,
-    pub status_head_root: Hash256,
-    pub status_finalized_epoch: Epoch,
-    pub status_finalized_root: Hash256,
+    pub head_slot: Slot,
+    pub head_root: Hash256,
+    pub finalized_epoch: Epoch,
+    pub finalized_root: Hash256,
+}
+
+impl std::cmp::PartialEq for PeerSyncStatus {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (PeerSyncStatus::Synced { .. }, PeerSyncStatus::Synced { .. }) => true,
+            (PeerSyncStatus::Advanced { .. }, PeerSyncStatus::Advanced { .. }) => true,
+            (PeerSyncStatus::Behind { .. }, PeerSyncStatus::Behind { .. }) => true,
+            (PeerSyncStatus::IrrelevantPeer, PeerSyncStatus::IrrelevantPeer) => true,
+            (PeerSyncStatus::Unknown, PeerSyncStatus::Unknown) => true,
+            _ => false,
+        }
+    }
 }
 
 impl PeerSyncStatus {
     /// Returns true if the peer has advanced knowledge of the chain.
     pub fn is_advanced(&self) -> bool {
-        match self {
-            PeerSyncStatus::Advanced { .. } => true,
-            _ => false,
-        }
+        matches!(self, PeerSyncStatus::Advanced { .. })
     }
 
     /// Returns true if the peer is up to date with the current chain.
     pub fn is_synced(&self) -> bool {
-        match self {
-            PeerSyncStatus::Synced { .. } => true,
-            _ => false,
-        }
+        matches!(self, PeerSyncStatus::Synced { .. })
     }
 
     /// Returns true if the peer is behind the current chain.
     pub fn is_behind(&self) -> bool {
-        match self {
-            PeerSyncStatus::Behind { .. } => true,
-            _ => false,
+        matches!(self, PeerSyncStatus::Behind { .. })
+    }
+
+    pub fn update(&mut self, new_state: PeerSyncStatus) -> bool {
+        if *self == new_state {
+            *self = new_state;
+            false // state was not updated
+        } else {
+            *self = new_state;
+            true
         }
     }
 
-    /// Updates the sync state given a fully synced peer.
-    /// Returns true if the state has changed.
-    pub fn update_synced(&mut self, info: SyncInfo) -> bool {
-        let new_state = PeerSyncStatus::Synced { info };
-
+    pub fn as_str(&self) -> &'static str {
         match self {
-            PeerSyncStatus::Synced { .. } | PeerSyncStatus::Unknown => {
-                *self = new_state;
-                false // state was not updated
-            }
-            _ => {
-                *self = new_state;
-                true
-            }
-        }
-    }
-
-    /// Updates the sync state given a peer that is further ahead in the chain than us.
-    /// Returns true if the state has changed.
-    pub fn update_advanced(&mut self, info: SyncInfo) -> bool {
-        let new_state = PeerSyncStatus::Advanced { info };
-
-        match self {
-            PeerSyncStatus::Advanced { .. } | PeerSyncStatus::Unknown => {
-                *self = new_state;
-                false // state was not updated
-            }
-            _ => {
-                *self = new_state;
-                true
-            }
-        }
-    }
-
-    /// Updates the sync state given a peer that is behind us in the chain.
-    /// Returns true if the state has changed.
-    pub fn update_behind(&mut self, info: SyncInfo) -> bool {
-        let new_state = PeerSyncStatus::Behind { info };
-
-        match self {
-            PeerSyncStatus::Behind { .. } | PeerSyncStatus::Unknown => {
-                *self = new_state;
-                false // state was not updated
-            }
-            _ => {
-                *self = new_state;
-                true
-            }
+            PeerSyncStatus::Advanced { .. } => "Advanced",
+            PeerSyncStatus::Behind { .. } => "Behind",
+            PeerSyncStatus::Synced { .. } => "Synced",
+            PeerSyncStatus::Unknown => "Unknown",
+            PeerSyncStatus::IrrelevantPeer => "Irrelevant",
         }
     }
 }
+
+impl std::fmt::Display for PeerSyncStatus {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_str())
+    }
+}

beacon_node/eth2_libp2p/src/peer_manager/score.rs

@@ -5,8 +5,10 @@
 //! As the logic develops this documentation will advance.
 //!
 //! The scoring algorithms are currently experimental.
+use crate::behaviour::GOSSIPSUB_GREYLIST_THRESHOLD;
 use serde::Serialize;
 use std::time::Instant;
+use tokio::time::Duration;
 
 lazy_static! {
     static ref HALFLIFE_DECAY: f64 = -(2.0f64.ln()) / SCORE_HALFLIFE;
@@ -18,6 +20,9 @@ pub(crate) const DEFAULT_SCORE: f64 = 0.0;
 const MIN_SCORE_BEFORE_DISCONNECT: f64 = -20.0;
 /// The minimum reputation before a peer is banned.
 const MIN_SCORE_BEFORE_BAN: f64 = -50.0;
+/// If a peer has a lighthouse score below this constant all other score parts will get ignored and
+/// the peer will get banned regardless of the other parts.
+const MIN_LIGHTHOUSE_SCORE_BEFORE_BAN: f64 = -60.0;
 /// The maximum score a peer can obtain.
 const MAX_SCORE: f64 = 100.0;
 /// The minimum score a peer can obtain.
@@ -25,7 +30,13 @@ const MIN_SCORE: f64 = -100.0;
 /// The halflife of a peer's score. I.e the number of seconds it takes for the score to decay to half its value.
 const SCORE_HALFLIFE: f64 = 600.0;
 /// The number of seconds we ban a peer for before their score begins to decay.
-const BANNED_BEFORE_DECAY: u64 = 1800;
+const BANNED_BEFORE_DECAY: Duration = Duration::from_secs(1800);
+
+/// We weight negative gossipsub scores in such a way that they never result in a disconnect by
+/// themselves. This "solves" the problem of non-decaying gossipsub scores for disconnected peers.
+const GOSSIPSUB_NEGATIVE_SCORE_WEIGHT: f64 =
+    (MIN_SCORE_BEFORE_DISCONNECT + 1.0) / GOSSIPSUB_GREYLIST_THRESHOLD;
+const GOSSIPSUB_POSITIVE_SCORE_WEIGHT: f64 = GOSSIPSUB_NEGATIVE_SCORE_WEIGHT;
 
 /// A collection of actions a peer can perform which will adjust its score.
 /// Each variant has an associated score change.
@@ -48,80 +59,12 @@ pub enum PeerAction {
     /// An error occurred with this peer but it is not necessarily malicious.
     /// We have high tolerance for this actions: several occurrences are needed for a peer to get
     /// kicked.
-    /// NOTE: ~15 occurrences will get the peer banned
+    /// NOTE: ~50 occurrences will get the peer banned
     HighToleranceError,
     /// Received an expected message.
     _ValidMessage,
 }
 
-/// The expected state of the peer given the peer's score.
-#[derive(Debug, PartialEq)]
-pub(crate) enum ScoreState {
-    /// We are content with the peers performance. We permit connections and messages.
-    Healthy,
-    /// The peer should be disconnected. We allow re-connections if the peer is persistent.
-    Disconnected,
-    /// The peer is banned. We disallow new connections until it's score has decayed into a
-    /// tolerable threshold.
-    Banned,
-}
-
-/// A peer's score (perceived potential usefulness).
-///
-/// This simplistic version consists of a global score per peer which decays to 0 over time. The
-/// decay rate applies equally to positive and negative scores.
-#[derive(Copy, PartialEq, Clone, Debug, Serialize)]
-pub struct Score {
-    /// The global score.
-    // NOTE: In the future we may separate this into sub-scores involving the RPC, Gossipsub and
-    // lighthouse.
-    score: f64,
-    /// The time the score was last updated to perform time-based adjustments such as score-decay.
-    #[serde(skip)]
-    last_updated: Instant,
-}
-
-impl Default for Score {
-    fn default() -> Self {
-        Score {
-            score: DEFAULT_SCORE,
-            last_updated: Instant::now(),
-        }
-    }
-}
-
-impl Eq for Score {}
-
-impl PartialOrd for Score {
-    fn partial_cmp(&self, other: &Score) -> Option<std::cmp::Ordering> {
-        self.score
-            .partial_cmp(&other.score)
-            .or_else(|| self.last_updated.partial_cmp(&other.last_updated))
-    }
-}
-
-impl Ord for Score {
-    fn cmp(&self, other: &Score) -> std::cmp::Ordering {
-        self.partial_cmp(other)
-            .unwrap_or_else(|| std::cmp::Ordering::Equal)
-    }
-}
-
-impl From<f64> for Score {
-    fn from(f: f64) -> Self {
-        Score {
-            score: f,
-            last_updated: Instant::now(),
-        }
-    }
-}
-
-impl std::fmt::Display for Score {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:.2}", self.score)
-    }
-}
-
 impl std::fmt::Display for PeerAction {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -134,6 +77,18 @@ impl std::fmt::Display for PeerAction {
     }
 }
 
+/// The expected state of the peer given the peer's score.
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub(crate) enum ScoreState {
+    /// We are content with the peers performance. We permit connections and messages.
+    Healthy,
+    /// The peer should be disconnected. We allow re-connections if the peer is persistent.
+    Disconnected,
+    /// The peer is banned. We disallow new connections until it's score has decayed into a
+    /// tolerable threshold.
+    Banned,
+}
+
 impl std::fmt::Display for ScoreState {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -144,16 +99,59 @@ impl std::fmt::Display for ScoreState {
     }
 }
 
-impl Score {
+/// A peer's score (perceived potential usefulness).
+///
+/// This simplistic version consists of a global score per peer which decays to 0 over time. The
+/// decay rate applies equally to positive and negative scores.
+#[derive(PartialEq, Clone, Debug, Serialize)]
+pub struct RealScore {
+    /// The global score.
+    // NOTE: In the future we may separate this into sub-scores involving the RPC, Gossipsub and
+    // lighthouse.
+    lighthouse_score: f64,
+    gossipsub_score: f64,
+    /// We ignore the negative gossipsub scores of some peers to allow decaying without
+    /// disconnecting.
+    ignore_negative_gossipsub_score: bool,
+    score: f64,
+    /// The time the score was last updated to perform time-based adjustments such as score-decay.
+    #[serde(skip)]
+    last_updated: Instant,
+}
+
+impl Default for RealScore {
+    fn default() -> Self {
+        RealScore {
+            lighthouse_score: DEFAULT_SCORE,
+            gossipsub_score: DEFAULT_SCORE,
+            score: DEFAULT_SCORE,
+            last_updated: Instant::now(),
+            ignore_negative_gossipsub_score: false,
+        }
+    }
+}
+
+impl RealScore {
     /// Access to the underlying score.
-    pub fn score(&self) -> f64 {
+    fn recompute_score(&mut self) {
+        self.score = self.lighthouse_score;
+        if self.lighthouse_score <= MIN_LIGHTHOUSE_SCORE_BEFORE_BAN {
+            //ignore all other scores, i.e. do nothing here
+        } else if self.gossipsub_score >= 0.0 {
+            self.score += self.gossipsub_score * GOSSIPSUB_POSITIVE_SCORE_WEIGHT;
+        } else if !self.ignore_negative_gossipsub_score {
+            self.score += self.gossipsub_score * GOSSIPSUB_NEGATIVE_SCORE_WEIGHT;
+        }
+    }
+
+    fn score(&self) -> f64 {
         self.score
     }
 
     /// Modifies the score based on a peer's action.
     pub fn apply_peer_action(&mut self, peer_action: PeerAction) {
         match peer_action {
-            PeerAction::Fatal => self.score = MIN_SCORE, // The worst possible score
+            PeerAction::Fatal => self.set_lighthouse_score(MIN_SCORE), // The worst possible score
             PeerAction::LowToleranceError => self.add(-10.0),
             PeerAction::MidToleranceError => self.add(-5.0),
             PeerAction::HighToleranceError => self.add(-1.0),
@@ -161,18 +159,14 @@ impl Score {
         }
     }
 
-    /// Returns the expected state of the peer given it's score.
-    pub(crate) fn state(&self) -> ScoreState {
-        match self.score {
-            x if x <= MIN_SCORE_BEFORE_BAN => ScoreState::Banned,
-            x if x <= MIN_SCORE_BEFORE_DISCONNECT => ScoreState::Disconnected,
-            _ => ScoreState::Healthy,
-        }
+    fn set_lighthouse_score(&mut self, new_score: f64) {
+        self.lighthouse_score = new_score;
+        self.update_state();
     }
 
     /// Add an f64 to the score abiding by the limits.
-    pub fn add(&mut self, score: f64) {
-        let mut new_score = self.score + score;
+    fn add(&mut self, score: f64) {
+        let mut new_score = self.lighthouse_score + score;
         if new_score > MAX_SCORE {
             new_score = MAX_SCORE;
         }
@@ -180,42 +174,154 @@ impl Score {
             new_score = MIN_SCORE;
         }
 
-        self.score = new_score;
+        self.set_lighthouse_score(new_score);
+    }
+
+    fn update_state(&mut self) {
+        let was_not_banned = self.score > MIN_SCORE_BEFORE_BAN;
+        self.recompute_score();
+        if was_not_banned && self.score <= MIN_SCORE_BEFORE_BAN {
+            //we ban this peer for at least BANNED_BEFORE_DECAY seconds
+            self.last_updated += BANNED_BEFORE_DECAY;
+        }
+    }
+
+    /// Add an f64 to the score abiding by the limits.
+    #[cfg(test)]
+    pub fn test_add(&mut self, score: f64) {
+        self.add(score);
+    }
+
+    #[cfg(test)]
+    // reset the score
+    pub fn test_reset(&mut self) {
+        self.set_lighthouse_score(0f64);
     }
 
     /// Applies time-based logic such as decay rates to the score.
     /// This function should be called periodically.
     pub fn update(&mut self) {
-        // Apply decay logic
-        //
-        // There is two distinct decay processes. One for banned peers and one for all others. If
-        // the score is below the banning threshold and the duration since it was last update is
-        // shorter than the banning threshold, we do nothing.
-        let now = Instant::now();
-        if self.score <= MIN_SCORE_BEFORE_BAN
-            && now
-                .checked_duration_since(self.last_updated)
-                .map(|d| d.as_secs())
-                <= Some(BANNED_BEFORE_DECAY)
-        {
-            // The peer is banned and still within the ban timeout. Do not update it's score.
-            // Update last_updated so that the decay begins correctly when ready.
-            self.last_updated = now;
-            return;
-        }
+        self.update_at(Instant::now())
+    }
 
+    /// Applies time-based logic such as decay rates to the score with the given now value.
+    /// This private sub function is mainly used for testing.
+    fn update_at(&mut self, now: Instant) {
         // Decay the current score
         // Using exponential decay based on a constant half life.
+
+        // It is important that we use here `checked_duration_since` instead of elapsed, since
+        // we set last_updated to the future when banning peers. Therefore `checked_duration_since`
+        // will return None in this case and the score does not get decayed.
         if let Some(secs_since_update) = now
             .checked_duration_since(self.last_updated)
             .map(|d| d.as_secs())
         {
             // e^(-ln(2)/HL*t)
             let decay_factor = (*HALFLIFE_DECAY * secs_since_update as f64).exp();
-            self.score *= decay_factor;
+            self.lighthouse_score *= decay_factor;
             self.last_updated = now;
+            self.update_state();
         }
     }
+
+    pub fn update_gossipsub_score(&mut self, new_score: f64, ignore: bool) {
+        // we only update gossipsub if last_updated is in the past which means either the peer is
+        // not banned or the BANNED_BEFORE_DECAY time is over.
+        if self.last_updated <= Instant::now() {
+            self.gossipsub_score = new_score;
+            self.ignore_negative_gossipsub_score = ignore;
+            self.update_state();
+        }
+    }
+
+    pub fn is_good_gossipsub_peer(&self) -> bool {
+        self.gossipsub_score >= 0.0
+    }
+}
+
+#[derive(PartialEq, Clone, Debug, Serialize)]
+pub enum Score {
+    Max,
+    Real(RealScore),
+}
+
+impl Default for Score {
+    fn default() -> Self {
+        Self::Real(RealScore::default())
+    }
+}
+
+macro_rules! apply {
+    ( $method:ident $(, $param_name: ident: $param_type: ty)*) => {
+        impl Score {
+            pub fn $method(
+                &mut self, $($param_name: $param_type, )*
+            ) {
+                if let Self::Real(score) = self {
+                    score.$method($($param_name, )*);
+                }
+            }
+        }
+    };
+}
+
+apply!(apply_peer_action, peer_action: PeerAction);
+apply!(update);
+apply!(update_gossipsub_score, new_score: f64, ignore: bool);
+#[cfg(test)]
+apply!(test_add, score: f64);
+#[cfg(test)]
+apply!(test_reset);
+
+impl Score {
+    pub fn score(&self) -> f64 {
+        match self {
+            Self::Max => f64::INFINITY,
+            Self::Real(score) => score.score(),
+        }
+    }
+
+    pub fn max_score() -> Self {
+        Self::Max
+    }
+
+    /// Returns the expected state of the peer given it's score.
+    pub(crate) fn state(&self) -> ScoreState {
+        match self.score() {
+            x if x <= MIN_SCORE_BEFORE_BAN => ScoreState::Banned,
+            x if x <= MIN_SCORE_BEFORE_DISCONNECT => ScoreState::Disconnected,
+            _ => ScoreState::Healthy,
+        }
+    }
+
+    pub fn is_good_gossipsub_peer(&self) -> bool {
+        match self {
+            Self::Max => true,
+            Self::Real(score) => score.is_good_gossipsub_peer(),
+        }
+    }
+}
+
+impl Eq for Score {}
+
+impl PartialOrd for Score {
+    fn partial_cmp(&self, other: &Score) -> Option<std::cmp::Ordering> {
+        self.score().partial_cmp(&other.score())
+    }
+}
+
+impl Ord for Score {
+    fn cmp(&self, other: &Score) -> std::cmp::Ordering {
+        self.partial_cmp(other)
+            .unwrap_or_else(|| std::cmp::Ordering::Equal)
+    }
+}
+
+impl std::fmt::Display for Score {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:.2}", self.score())
+    }
 }
 
 #[cfg(test)]
@@ -229,25 +335,60 @@ mod tests {
         // 0 change does not change de reputation
         //
         let change = 0.0;
-        score.add(change);
+        score.test_add(change);
         assert_eq!(score.score(), DEFAULT_SCORE);
 
         // underflowing change is capped
         let mut score = Score::default();
         let change = MIN_SCORE - 50.0;
-        score.add(change);
+        score.test_add(change);
         assert_eq!(score.score(), MIN_SCORE);
 
         // overflowing change is capped
         let mut score = Score::default();
         let change = MAX_SCORE + 50.0;
-        score.add(change);
+        score.test_add(change);
         assert_eq!(score.score(), MAX_SCORE);
 
         // Score adjusts
         let mut score = Score::default();
         let change = 1.32;
-        score.add(change);
+        score.test_add(change);
         assert_eq!(score.score(), DEFAULT_SCORE + change);
     }
+
+    #[test]
+    fn test_ban_time() {
+        let mut score = RealScore::default();
+        let now = Instant::now();
+
+        let change = MIN_SCORE_BEFORE_BAN;
+        score.test_add(change);
+        assert_eq!(score.score(), MIN_SCORE_BEFORE_BAN);
+
+        score.update_at(now + BANNED_BEFORE_DECAY);
+        assert_eq!(score.score(), MIN_SCORE_BEFORE_BAN);
+
+        score.update_at(now + BANNED_BEFORE_DECAY + Duration::from_secs(1));
+        assert!(score.score() > MIN_SCORE_BEFORE_BAN);
+    }
+
+    #[test]
+    fn test_very_negative_gossipsub_score() {
+        let mut score = Score::default();
+        score.update_gossipsub_score(GOSSIPSUB_GREYLIST_THRESHOLD, false);
+        assert!(!score.is_good_gossipsub_peer());
+        assert!(score.score() < 0.0);
+        assert_eq!(score.state(), ScoreState::Healthy);
+        score.test_add(-1.0001);
+        assert_eq!(score.state(), ScoreState::Disconnected);
+    }
+
+    #[test]
+    fn test_ignored_gossipsub_score() {
+        let mut score = Score::default();
+        score.update_gossipsub_score(GOSSIPSUB_GREYLIST_THRESHOLD, true);
+        assert!(!score.is_good_gossipsub_peer());
+        assert_eq!(score.score(), 0.0);
+    }
 }

beacon_node/eth2_libp2p/src/rpc/codec/base.rs

@@ -1,5 +1,6 @@
 //! This handles the various supported encoding mechanism for the Eth 2.0 RPC.
 
+use crate::rpc::methods::ErrorType;
 use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
 use libp2p::bytes::BufMut;
 use libp2p::bytes::BytesMut;
@@ -8,12 +9,12 @@ use tokio_util::codec::{Decoder, Encoder};
 use types::EthSpec;
 
 pub trait OutboundCodec<TItem>: Encoder<TItem> + Decoder {
-    type ErrorType;
+    type CodecErrorType;
 
     fn decode_error(
         &mut self,
         src: &mut BytesMut,
-    ) -> Result<Option<Self::ErrorType>, <Self as Decoder>::Error>;
+    ) -> Result<Option<Self::CodecErrorType>, <Self as Decoder>::Error>;
 }
 
 /* Global Inbound Codec */
@@ -130,8 +131,8 @@ where
 impl<TCodec, TSpec> Decoder for BaseOutboundCodec<TCodec, TSpec>
 where
     TSpec: EthSpec,
-    TCodec:
-        OutboundCodec<RPCRequest<TSpec>, ErrorType = String> + Decoder<Item = RPCResponse<TSpec>>,
+    TCodec: OutboundCodec<RPCRequest<TSpec>, CodecErrorType = ErrorType>
+        + Decoder<Item = RPCResponse<TSpec>>,
 {
     type Item = RPCCodedResponse<TSpec>;
     type Error = <TCodec as Decoder>::Error;
@@ -176,34 +177,153 @@ where
 mod tests {
     use super::super::ssz_snappy::*;
     use super::*;
+    use crate::rpc::methods::StatusMessage;
     use crate::rpc::protocol::*;
+    use snap::write::FrameEncoder;
+    use ssz::Encode;
+    use std::io::Write;
+    use types::{Epoch, Hash256, Slot};
+    use unsigned_varint::codec::Uvi;
+
+    type Spec = types::MainnetEthSpec;
 
     #[test]
     fn test_decode_status_message() {
-        let message = hex::decode("ff060000734e615070590032000006e71e7b54989925efd6c9cbcb8ceb9b5f71216f5137282bf6a1e3b50f64e42d6c7fb347abe07eb0db8200000005029e2800").unwrap();
+        let message = hex::decode("0054ff060000734e615070590032000006e71e7b54989925efd6c9cbcb8ceb9b5f71216f5137282bf6a1e3b50f64e42d6c7fb347abe07eb0db8200000005029e2800").unwrap();
         let mut buf = BytesMut::new();
         buf.extend_from_slice(&message);
 
-        type Spec = types::MainnetEthSpec;
-
         let snappy_protocol_id =
             ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy);
 
         let mut snappy_outbound_codec =
             SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, 1_048_576);
 
+        // remove response code
+        let mut snappy_buf = buf.clone();
+        let _ = snappy_buf.split_to(1);
+
         // decode message just as snappy message
-        let snappy_decoded_message = snappy_outbound_codec.decode(&mut buf.clone());
-        // decode message just a ssz message
+        let snappy_decoded_message = snappy_outbound_codec.decode(&mut snappy_buf).unwrap();
 
         // build codecs for entire chunk
         let mut snappy_base_outbound_codec = BaseOutboundCodec::new(snappy_outbound_codec);
 
         // decode message as ssz snappy chunk
-        let snappy_decoded_chunk = snappy_base_outbound_codec.decode(&mut buf.clone());
-        // decode message just a ssz chunk
+        let snappy_decoded_chunk = snappy_base_outbound_codec.decode(&mut buf).unwrap();
 
-        let _ = dbg!(snappy_decoded_message);
-        let _ = dbg!(snappy_decoded_chunk);
+        dbg!(snappy_decoded_message);
+        dbg!(snappy_decoded_chunk);
+    }
+
+    #[test]
+    fn test_invalid_length_prefix() {
+        let mut uvi_codec: Uvi<u128> = Uvi::default();
+        let mut dst = BytesMut::with_capacity(1024);
+
+        // Smallest > 10 byte varint
+        let len: u128 = 2u128.pow(70);
+
+        // Insert length-prefix
+        uvi_codec.encode(len, &mut dst).unwrap();
+
+        let snappy_protocol_id =
+            ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy);
+        let mut snappy_outbound_codec =
+            SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, 1_048_576);
+
+        let snappy_decoded_message = snappy_outbound_codec.decode(&mut dst).unwrap_err();
+
+        assert_eq!(
+            snappy_decoded_message,
+            RPCError::IoError("input bytes exceed maximum".to_string()),
+            "length-prefix of > 10 bytes is invalid"
+        );
+    }
+
+    #[test]
+    fn test_length_limits() {
+        fn encode_len(len: usize) -> BytesMut {
+            let mut uvi_codec: Uvi<usize> = Uvi::default();
+            let mut dst = BytesMut::with_capacity(1024);
+            uvi_codec.encode(len, &mut dst).unwrap();
+            dst
+        }
+
+        let protocol_id =
+            ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZSnappy);
+
+        // Response limits
+        let limit = protocol_id.rpc_response_limits::<Spec>();
+        let mut max = encode_len(limit.max + 1);
+        let mut codec = SSZSnappyOutboundCodec::<Spec>::new(protocol_id.clone(), 1_048_576);
+        assert_eq!(codec.decode(&mut max).unwrap_err(), RPCError::InvalidData);
+
+        let mut min = encode_len(limit.min - 1);
+        let mut codec = SSZSnappyOutboundCodec::<Spec>::new(protocol_id.clone(), 1_048_576);
+        assert_eq!(codec.decode(&mut min).unwrap_err(), RPCError::InvalidData);
+
+        // Request limits
+        let limit = protocol_id.rpc_request_limits();
+        let mut max = encode_len(limit.max + 1);
+        let mut codec = SSZSnappyOutboundCodec::<Spec>::new(protocol_id.clone(), 1_048_576);
+        assert_eq!(codec.decode(&mut max).unwrap_err(), RPCError::InvalidData);
+
+        let mut min = encode_len(limit.min - 1);
+        let mut codec = SSZSnappyOutboundCodec::<Spec>::new(protocol_id, 1_048_576);
+        assert_eq!(codec.decode(&mut min).unwrap_err(), RPCError::InvalidData);
+    }
+
+    #[test]
+    fn test_decode_malicious_status_message() {
+        // Snappy stream identifier
+        let stream_identifier: &'static [u8] = b"\xFF\x06\x00\x00sNaPpY";
+
+        // byte 0(0xFE) is padding chunk type identifier for snappy messages
+        // byte 1,2,3 are chunk length (little endian)
+        let malicious_padding: &'static [u8] = b"\xFE\x00\x00\x00";
+
+        // Status message is 84 bytes uncompressed. `max_compressed_len` is 130.
+        let status_message_bytes = StatusMessage {
+            fork_digest: [0; 4],
+            finalized_root: Hash256::from_low_u64_be(0),
+            finalized_epoch: Epoch::new(1),
+            head_root: Hash256::from_low_u64_be(0),
+            head_slot: Slot::new(1),
+        }
+        .as_ssz_bytes();
+
+        let mut uvi_codec: Uvi<usize> = Uvi::default();
+        let mut dst = BytesMut::with_capacity(1024);
+
+        // Insert length-prefix
+        uvi_codec
+            .encode(status_message_bytes.len(), &mut dst)
+            .unwrap();
+
+        // Insert snappy stream identifier
+        dst.extend_from_slice(stream_identifier);
+
+        // Insert malicious padding of 80 bytes.
+        for _ in 0..20 {
+            dst.extend_from_slice(malicious_padding);
+        }
+
+        // Insert payload (42 bytes compressed)
+        let mut writer = FrameEncoder::new(Vec::new());
+        writer.write_all(&status_message_bytes).unwrap();
+        writer.flush().unwrap();
+        dst.extend_from_slice(writer.get_ref());
+
+        // 42 + 80 = 132 > max_compressed_len. Hence, decoding should fail with `InvalidData`.
+
+        let snappy_protocol_id =
+            ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy);
+
+        let mut snappy_outbound_codec =
+            SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, 1_048_576);
+
+        let snappy_decoded_message = snappy_outbound_codec.decode(&mut dst.clone()).unwrap_err();
+        assert_eq!(snappy_decoded_message, RPCError::InvalidData);
     }
 }

beacon_node/eth2_libp2p/src/rpc/codec/ssz_snappy.rs

@@ -1,10 +1,7 @@
 use crate::rpc::methods::*;
 use crate::rpc::{
     codec::base::OutboundCodec,
-    protocol::{
-        Encoding, Protocol, ProtocolId, RPCError, Version, BLOCKS_BY_ROOT_REQUEST_MAX,
-        BLOCKS_BY_ROOT_REQUEST_MIN, SIGNED_BEACON_BLOCK_MAX, SIGNED_BEACON_BLOCK_MIN,
-    },
+    protocol::{Encoding, Protocol, ProtocolId, RPCError, Version, ERROR_TYPE_MAX, ERROR_TYPE_MIN},
 };
 use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
 use libp2p::bytes::BytesMut;
@@ -99,6 +96,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
+            // Note: length-prefix of > 10 bytes(uint64) would be a decoding error
             match self.inner.decode(src).map_err(RPCError::from)? {
                 Some(length) => {
                     self.len = Some(length);
@@ -109,79 +107,56 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
 
         let length = self.len.expect("length should be Some");
 
-        // Should not attempt to decode rpc chunks with length > max_packet_size
-        if length > self.max_packet_size {
+        // Should not attempt to decode rpc chunks with `length > max_packet_size` or not within bounds of
+        // packet size for ssz container corresponding to `self.protocol`.
+        let ssz_limits = self.protocol.rpc_request_limits();
+        if length > self.max_packet_size || ssz_limits.is_out_of_bounds(length) {
             return Err(RPCError::InvalidData);
         }
-        let mut reader = FrameDecoder::new(Cursor::new(&src));
+        // Calculate worst case compression length for given uncompressed length
+        let max_compressed_len = snap::raw::max_compress_len(length) as u64;
+
+        // Create a limit reader as a wrapper that reads only upto `max_compressed_len` from `src`.
+        let limit_reader = Cursor::new(src.as_ref()).take(max_compressed_len);
+        let mut reader = FrameDecoder::new(limit_reader);
         let mut decoded_buffer = vec![0; length];
 
         match reader.read_exact(&mut decoded_buffer) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
-                let n = reader.get_ref().position();
+                let n = reader.get_ref().get_ref().position();
                 self.len = None;
                 let _read_bytes = src.split_to(n as usize);
+
+                // We need not check that decoded_buffer.len() is within bounds here
+                // since we have already checked `length` above.
                 match self.protocol.message_name {
                     Protocol::Status => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
+                            &decoded_buffer,
+                        )?))),
                     },
                     Protocol::Goodbye => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <GoodbyeReason as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCRequest::Goodbye(GoodbyeReason::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::Goodbye(
+                            GoodbyeReason::from_ssz_bytes(&decoded_buffer)?,
+                        ))),
                     },
                     Protocol::BlocksByRange => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len()
-                                == <BlocksByRangeRequest as Encode>::ssz_fixed_len()
-                            {
-                                Ok(Some(RPCRequest::BlocksByRange(
-                                    BlocksByRangeRequest::from_ssz_bytes(&decoded_buffer)?,
-                                )))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::BlocksByRange(
+                            BlocksByRangeRequest::from_ssz_bytes(&decoded_buffer)?,
+                        ))),
                     },
                     Protocol::BlocksByRoot => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() >= *BLOCKS_BY_ROOT_REQUEST_MIN
-                                && decoded_buffer.len() <= *BLOCKS_BY_ROOT_REQUEST_MAX
-                            {
-                                Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
-                                    block_roots: VariableList::from_ssz_bytes(&decoded_buffer)?,
-                                })))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
+                            block_roots: VariableList::from_ssz_bytes(&decoded_buffer)?,
+                        }))),
                     },
                     Protocol::Ping => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <Ping as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCRequest::Ping(Ping {
-                                    data: u64::from_ssz_bytes(&decoded_buffer)?,
-                                })))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::Ping(Ping {
+                            data: u64::from_ssz_bytes(&decoded_buffer)?,
+                        }))),
                     },
+                    // This case should be unreachable as `MetaData` requests are handled separately in the `InboundUpgrade`
                     Protocol::MetaData => match self.protocol.version {
                         Version::V1 => {
                             if !decoded_buffer.is_empty() {
@@ -193,12 +168,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
                     },
                 }
             }
-            Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
-                ErrorKind::UnexpectedEof => Ok(None),
-                _ => Err(e).map_err(RPCError::from),
-            },
+            Err(e) => handle_error(e, reader.get_ref().get_ref().position(), max_compressed_len),
         }
     }
 }
@@ -277,6 +247,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
+            // Note: length-prefix of > 10 bytes(uint64) would be a decoding error
             match self.inner.decode(src).map_err(RPCError::from)? {
                 Some(length) => {
                     self.len = Some(length as usize);
@@ -287,96 +258,71 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
 
         let length = self.len.expect("length should be Some");
 
-        // Should not attempt to decode rpc chunks with length > max_packet_size
-        if length > self.max_packet_size {
+        // Should not attempt to decode rpc chunks with `length > max_packet_size` or not within bounds of
+        // packet size for ssz container corresponding to `self.protocol`.
+        let ssz_limits = self.protocol.rpc_response_limits::<TSpec>();
+        if length > self.max_packet_size || ssz_limits.is_out_of_bounds(length) {
             return Err(RPCError::InvalidData);
         }
-        let mut reader = FrameDecoder::new(Cursor::new(&src));
+        // Calculate worst case compression length for given uncompressed length
+        let max_compressed_len = snap::raw::max_compress_len(length) as u64;
+        // Create a limit reader as a wrapper that reads only upto `max_compressed_len` from `src`.
+        let limit_reader = Cursor::new(src.as_ref()).take(max_compressed_len);
+        let mut reader = FrameDecoder::new(limit_reader);
+
         let mut decoded_buffer = vec![0; length];
+
         match reader.read_exact(&mut decoded_buffer) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
-                let n = reader.get_ref().position();
+                let n = reader.get_ref().get_ref().position();
                 self.len = None;
-                let _read_byts = src.split_to(n as usize);
+                let _read_bytes = src.split_to(n as usize);
+
+                // We need not check that decoded_buffer.len() is within bounds here
+                // since we have already checked `length` above.
                 match self.protocol.message_name {
                     Protocol::Status => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCResponse::Status(StatusMessage::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::Status(
+                            StatusMessage::from_ssz_bytes(&decoded_buffer)?,
+                        ))),
                     },
+                    // This case should be unreachable as `Goodbye` has no response.
                     Protocol::Goodbye => Err(RPCError::InvalidData),
                     Protocol::BlocksByRange => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                && decoded_buffer.len() <= *SIGNED_BEACON_BLOCK_MAX
-                            {
-                                Ok(Some(RPCResponse::BlocksByRange(Box::new(
-                                    SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
-                                ))))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::BlocksByRange(Box::new(
+                            SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
+                        )))),
                     },
                     Protocol::BlocksByRoot => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                && decoded_buffer.len() <= *SIGNED_BEACON_BLOCK_MAX
-                            {
-                                Ok(Some(RPCResponse::BlocksByRoot(Box::new(
-                                    SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
-                                ))))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::BlocksByRoot(Box::new(
+                            SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
+                        )))),
                     },
                     Protocol::Ping => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <Ping as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCResponse::Pong(Ping {
-                                    data: u64::from_ssz_bytes(&decoded_buffer)?,
-                                })))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::Pong(Ping {
+                            data: u64::from_ssz_bytes(&decoded_buffer)?,
+                        }))),
                     },
                     Protocol::MetaData => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <MetaData<TSpec> as Encode>::ssz_fixed_len()
-                            {
-                                Ok(Some(RPCResponse::MetaData(MetaData::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::MetaData(MetaData::from_ssz_bytes(
+                            &decoded_buffer,
+                        )?))),
                     },
                 }
             }
-            Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
-                ErrorKind::UnexpectedEof => Ok(None),
-                _ => Err(e).map_err(RPCError::from),
-            },
+            Err(e) => handle_error(e, reader.get_ref().get_ref().position(), max_compressed_len),
         }
     }
 }
 
 impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZSnappyOutboundCodec<TSpec> {
-    type ErrorType = String;
+    type CodecErrorType = ErrorType;
 
-    fn decode_error(&mut self, src: &mut BytesMut) -> Result<Option<Self::ErrorType>, RPCError> {
+    fn decode_error(
+        &mut self,
+        src: &mut BytesMut,
+    ) -> Result<Option<Self::CodecErrorType>, RPCError> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
             match self.inner.decode(src).map_err(RPCError::from)? {
@@ -389,28 +335,52 @@ impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZSnappyOutboundCodec
 
         let length = self.len.expect("length should be Some");
 
-        // Should not attempt to decode rpc chunks with length > max_packet_size
-        if length > self.max_packet_size {
+        // Should not attempt to decode rpc chunks with `length > max_packet_size` or not within bounds of
+        // packet size for ssz container corresponding to `ErrorType`.
+        if length > self.max_packet_size || length > *ERROR_TYPE_MAX || length < *ERROR_TYPE_MIN {
             return Err(RPCError::InvalidData);
         }
-        let mut reader = FrameDecoder::new(Cursor::new(&src));
+
+        // Calculate worst case compression length for given uncompressed length
+        let max_compressed_len = snap::raw::max_compress_len(length) as u64;
+        // // Create a limit reader as a wrapper that reads only upto `max_compressed_len` from `src`.
+        let limit_reader = Cursor::new(src.as_ref()).take(max_compressed_len);
+        let mut reader = FrameDecoder::new(limit_reader);
         let mut decoded_buffer = vec![0; length];
         match reader.read_exact(&mut decoded_buffer) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
-                let n = reader.get_ref().position();
+                let n = reader.get_ref().get_ref().position();
                 self.len = None;
                 let _read_bytes = src.split_to(n as usize);
-                Ok(Some(
-                    String::from_utf8_lossy(&<Vec<u8>>::from_ssz_bytes(&decoded_buffer)?).into(),
-                ))
+                Ok(Some(ErrorType(VariableList::from_ssz_bytes(
+                    &decoded_buffer,
+                )?)))
             }
-            Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
-                ErrorKind::UnexpectedEof => Ok(None),
-                _ => Err(e).map_err(RPCError::from),
-            },
+            Err(e) => handle_error(e, reader.get_ref().get_ref().position(), max_compressed_len),
         }
     }
 }
+
+/// Handle errors that we get from decoding an RPC message from the stream.
+/// `num_bytes_read` is the number of bytes the snappy decoder has read from the underlying stream.
+/// `max_compressed_len` is the maximum compressed size for a given uncompressed size.
+fn handle_error<T>(
+    err: std::io::Error,
+    num_bytes: u64,
+    max_compressed_len: u64,
+) -> Result<Option<T>, RPCError> {
+    match err.kind() {
+        ErrorKind::UnexpectedEof => {
+            // If snappy has read `max_compressed_len` from underlying stream and still can't fill buffer, we have a malicious message.
+            // Report as `InvalidData` so that malicious peer gets banned.
+            if num_bytes >= max_compressed_len {
+                Err(RPCError::InvalidData)
+            } else {
+                // Haven't received enough bytes to decode yet, wait for more
+                Ok(None)
+            }
+        }
+        _ => Err(err).map_err(RPCError::from),
+    }
+}

beacon_node/eth2_libp2p/src/rpc/handler.rs

@@ -14,7 +14,7 @@ use libp2p::swarm::protocols_handler::{
     KeepAlive, ProtocolsHandler, ProtocolsHandlerEvent, ProtocolsHandlerUpgrErr, SubstreamProtocol,
 };
 use libp2p::swarm::NegotiatedSubstream;
-use slog::{crit, debug, warn};
+use slog::{crit, debug, trace, warn};
 use smallvec::SmallVec;
 use std::{
     collections::hash_map::Entry,
@@ -25,8 +25,6 @@ use std::{
 use tokio::time::{delay_queue, delay_until, Delay, DelayQueue, Instant as TInstant};
 use types::EthSpec;
 
-//TODO: Implement check_timeout() on the substream types
-
 /// The time (in seconds) before a substream that is awaiting a response from the user times out.
 pub const RESPONSE_TIMEOUT: u64 = 10;
 
@@ -81,7 +79,7 @@ where
     TSpec: EthSpec,
 {
     /// The upgrade for inbound substreams.
-    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>,
+    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
 
     /// Errors occurring on outbound and inbound connections queued for reporting back.
     pending_errors: Vec<HandlerErr>,
@@ -163,8 +161,6 @@ struct OutboundInfo<TSpec: EthSpec> {
     /// Info over the protocol this substream is handling.
     proto: Protocol,
     /// Number of chunks to be seen from the peer's response.
-    // TODO: removing the option could allow clossing the streams after the number of
-    // expected responses is met for all protocols.
     remaining_chunks: Option<u64>,
     /// `RequestId` as given by the application that sent the request.
     req_id: RequestId,
@@ -180,31 +176,6 @@ enum InboundState<TSpec: EthSpec> {
     Poisoned,
 }
 
-impl<TSpec: EthSpec> InboundState<TSpec> {
-    /// Sends the given items over the underlying substream, if the state allows it, and returns the
-    /// final state.
-    fn send_items(
-        self,
-        pending_items: &mut Vec<RPCCodedResponse<TSpec>>,
-        remaining_chunks: u64,
-    ) -> Self {
-        if let InboundState::Idle(substream) = self {
-            // only send on Idle
-            if !pending_items.is_empty() {
-                // take the items that we need to send
-                let to_send = std::mem::replace(pending_items, vec![]);
-                let fut = process_inbound_substream(substream, remaining_chunks, to_send).boxed();
-                InboundState::Busy(Box::pin(fut))
-            } else {
-                // nothing to do, keep waiting for responses
-                InboundState::Idle(substream)
-            }
-        } else {
-            self
-        }
-    }
-}
-
 /// State of an outbound substream. Either waiting for a response, or in the process of sending.
 pub enum OutboundSubstreamState<TSpec: EthSpec> {
     /// A request has been sent, and we are awaiting a response. This future is driven in the
@@ -225,7 +196,10 @@ impl<TSpec> RPCHandler<TSpec>
 where
     TSpec: EthSpec,
 {
-    pub fn new(listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>, log: &slog::Logger) -> Self {
+    pub fn new(
+        listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
+        log: &slog::Logger,
+    ) -> Self {
         RPCHandler {
             listen_protocol,
             pending_errors: Vec::new(),
@@ -249,7 +223,7 @@ where
     ///
     /// > **Note**: If you modify the protocol, modifications will only applies to future inbound
     /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_ref(&self) -> &SubstreamProtocol<RPCProtocol<TSpec>> {
+    pub fn listen_protocol_ref(&self) -> &SubstreamProtocol<RPCProtocol<TSpec>, ()> {
         &self.listen_protocol
     }
 
@@ -257,14 +231,16 @@ where
     ///
     /// > **Note**: If you modify the protocol, modifications will only apply to future inbound
     /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_mut(&mut self) -> &mut SubstreamProtocol<RPCProtocol<TSpec>> {
+    pub fn listen_protocol_mut(&mut self) -> &mut SubstreamProtocol<RPCProtocol<TSpec>, ()> {
         &mut self.listen_protocol
     }
 
     /// Initiates the handler's shutdown process, sending an optional last message to the peer.
     pub fn shutdown(&mut self, final_msg: Option<(RequestId, RPCRequest<TSpec>)>) {
         if matches!(self.state, HandlerState::Active) {
-            debug!(self.log, "Starting handler shutdown"; "unsent_queued_requests" => self.dial_queue.len());
+            if !self.dial_queue.is_empty() {
+                debug!(self.log, "Starting handler shutdown"; "unsent_queued_requests" => self.dial_queue.len());
+            }
             // we now drive to completion communications already dialed/established
             while let Some((id, req)) = self.dial_queue.pop() {
                 self.pending_errors.push(HandlerErr::Outbound {
@@ -309,8 +285,11 @@ where
         let inbound_info = if let Some(info) = self.inbound_substreams.get_mut(&inbound_id) {
             info
         } else {
-            warn!(self.log, "Stream has expired. Response not sent";
-                "response" => response.to_string(), "id" => inbound_id);
+            if !matches!(response, RPCCodedResponse::StreamTermination(..)) {
+                // the stream is closed after sending the expected number of responses
+                trace!(self.log, "Inbound stream has expired, response not sent";
+                    "response" => %response, "id" => inbound_id);
+            }
             return;
         };
 
@@ -344,14 +323,16 @@ where
     type InboundProtocol = RPCProtocol<TSpec>;
     type OutboundProtocol = RPCRequest<TSpec>;
     type OutboundOpenInfo = (RequestId, RPCRequest<TSpec>); // Keep track of the id and the request
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.listen_protocol.clone()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         substream: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         // only accept new peer requests when active
         if !matches!(self.state, HandlerState::Active) {
@@ -625,69 +606,109 @@ where
         // drive inbound streams that need to be processed
         let mut substreams_to_remove = Vec::new(); // Closed substreams that need to be removed
         for (id, info) in self.inbound_substreams.iter_mut() {
-            match std::mem::replace(&mut info.state, InboundState::Poisoned) {
-                state @ InboundState::Idle(..) if !deactivated => {
-                    info.state = state.send_items(&mut info.pending_items, info.remaining_chunks);
-                }
-                InboundState::Idle(mut substream) => {
-                    // handler is deactivated, close the stream and mark it for removal
-                    match substream.close().poll_unpin(cx) {
-                        // if we can't close right now, put the substream back and try again later
-                        Poll::Pending => info.state = InboundState::Idle(substream),
-                        Poll::Ready(res) => {
-                            substreams_to_remove.push(*id);
-                            if let Some(ref delay_key) = info.delay_key {
-                                self.inbound_substreams_delay.remove(delay_key);
-                            }
-                            if let Err(error) = res {
-                                self.pending_errors.push(HandlerErr::Inbound {
-                                    id: *id,
-                                    error,
-                                    proto: info.protocol,
-                                });
-                            }
-                            if info.pending_items.last().map(|l| l.close_after()) == Some(false) {
-                                // if the request was still active, report back to cancel it
-                                self.pending_errors.push(HandlerErr::Inbound {
-                                    id: *id,
-                                    proto: info.protocol,
-                                    error: RPCError::HandlerRejected,
-                                });
-                            }
+            loop {
+                match std::mem::replace(&mut info.state, InboundState::Poisoned) {
+                    InboundState::Idle(substream) if !deactivated => {
+                        if !info.pending_items.is_empty() {
+                            let to_send = std::mem::replace(&mut info.pending_items, vec![]);
+                            let fut = process_inbound_substream(
+                                substream,
+                                info.remaining_chunks,
+                                to_send,
+                            )
+                            .boxed();
+                            info.state = InboundState::Busy(Box::pin(fut));
+                        } else {
+                            info.state = InboundState::Idle(substream);
+                            break;
                         }
                     }
-                }
-                InboundState::Busy(mut fut) => {
-                    // first check if sending finished
-                    let state = match fut.poll_unpin(cx) {
-                        Poll::Ready((substream, errors, remove, new_remaining_chunks)) => {
-                            info.remaining_chunks = new_remaining_chunks;
-                            // report any error
-                            for error in errors {
-                                self.pending_errors.push(HandlerErr::Inbound {
-                                    id: *id,
-                                    error,
-                                    proto: info.protocol,
-                                })
-                            }
-                            if remove {
+                    InboundState::Idle(mut substream) => {
+                        // handler is deactivated, close the stream and mark it for removal
+                        match substream.close().poll_unpin(cx) {
+                            // if we can't close right now, put the substream back and try again later
+                            Poll::Pending => info.state = InboundState::Idle(substream),
+                            Poll::Ready(res) => {
+                                // The substream closed, we remove it
                                 substreams_to_remove.push(*id);
                                 if let Some(ref delay_key) = info.delay_key {
                                     self.inbound_substreams_delay.remove(delay_key);
                                 }
+                                if let Err(error) = res {
+                                    self.pending_errors.push(HandlerErr::Inbound {
+                                        id: *id,
+                                        error,
+                                        proto: info.protocol,
+                                    });
+                                }
+                                if info.pending_items.last().map(|l| l.close_after()) == Some(false)
+                                {
+                                    // if the request was still active, report back to cancel it
+                                    self.pending_errors.push(HandlerErr::Inbound {
+                                        id: *id,
+                                        proto: info.protocol,
+                                        error: RPCError::HandlerRejected,
+                                    });
+                                }
                             }
-                            InboundState::Idle(substream)
                         }
-                        Poll::Pending => InboundState::Busy(fut),
-                    };
-                    info.state = if !deactivated {
-                        // if the last batch finished, send more.
-                        state.send_items(&mut info.pending_items, info.remaining_chunks)
-                    } else {
-                        state
-                    };
+                        break;
+                    }
+                    InboundState::Busy(mut fut) => {
+                        // first check if sending finished
+                        match fut.poll_unpin(cx) {
+                            Poll::Ready((substream, errors, remove, new_remaining_chunks)) => {
+                                info.remaining_chunks = new_remaining_chunks;
+                                // report any error
+                                for error in errors {
+                                    self.pending_errors.push(HandlerErr::Inbound {
+                                        id: *id,
+                                        error,
+                                        proto: info.protocol,
+                                    })
+                                }
+                                if remove {
+                                    substreams_to_remove.push(*id);
+                                    if let Some(ref delay_key) = info.delay_key {
+                                        self.inbound_substreams_delay.remove(delay_key);
+                                    }
+                                } else {
+                                    // If we are not removing this substream, we reset the timer.
+                                    // Each chunk is allowed RESPONSE_TIMEOUT to be sent.
+                                    if let Some(ref delay_key) = info.delay_key {
+                                        self.inbound_substreams_delay.reset(
+                                            delay_key,
+                                            Duration::from_secs(RESPONSE_TIMEOUT),
+                                        );
+                                    }
+                                }
+
+                                // The stream may be currently idle. Attempt to process more
+                                // elements
+
+                                if !deactivated && !info.pending_items.is_empty() {
+                                    let to_send =
+                                        std::mem::replace(&mut info.pending_items, vec![]);
+                                    let fut = process_inbound_substream(
+                                        substream,
+                                        info.remaining_chunks,
+                                        to_send,
+                                    )
+                                    .boxed();
+                                    info.state = InboundState::Busy(Box::pin(fut));
+                                } else {
+                                    info.state = InboundState::Idle(substream);
+                                    break;
+                                }
+                            }
+                            Poll::Pending => {
+                                info.state = InboundState::Busy(fut);
+                                break;
+                            }
+                        };
+                    }
+                    InboundState::Poisoned => unreachable!("Poisoned inbound substream"),
                 }
-                InboundState::Poisoned => unreachable!("Poisoned inbound substream"),
             }
         }
 
@@ -863,8 +884,7 @@ where
             let (id, req) = self.dial_queue.remove(0);
             self.dial_queue.shrink_to_fit();
             return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                protocol: SubstreamProtocol::new(req.clone()),
-                info: (id, req),
+                protocol: SubstreamProtocol::new(req.clone(), ()).map_info(|()| (id, req)),
             });
         }
         Poll::Pending
@@ -912,6 +932,8 @@ async fn process_inbound_substream<TSpec: EthSpec>(
                     substream_closed = true;
                 }
             }
+        } else if matches!(item, RPCCodedResponse::StreamTermination(_)) {
+            // The sender closed the stream before us, ignore this.
         } else {
             // we have more items after a closed substream, report those as errors
             errors.push(RPCError::InternalError(

beacon_node/eth2_libp2p/src/rpc/methods.rs

@@ -1,6 +1,7 @@
 //! Available RPC methods types and ids.
 
 use crate::types::EnrBitfield;
+use regex::bytes::Regex;
 use serde::Serialize;
 use ssz_derive::{Decode, Encode};
 use ssz_types::{
@@ -15,11 +16,12 @@ pub type MaxRequestBlocks = U1024;
 pub const MAX_REQUEST_BLOCKS: u64 = 1024;
 
 /// Maximum length of error message.
-type MaxErrorLen = U256;
+pub type MaxErrorLen = U256;
+pub const MAX_ERROR_LEN: u64 = 256;
 
 /// Wrapper over SSZ List to represent error message in rpc responses.
 #[derive(Debug, Clone)]
-pub struct ErrorType(VariableList<u8, MaxErrorLen>);
+pub struct ErrorType(pub VariableList<u8, MaxErrorLen>);
 
 impl From<String> for ErrorType {
     fn from(s: String) -> Self {
@@ -42,10 +44,9 @@ impl Deref for ErrorType {
 
 impl ToString for ErrorType {
     fn to_string(&self) -> String {
-        match std::str::from_utf8(self.0.deref()) {
-            Ok(s) => s.to_string(),
-            Err(_) => format!("{:?}", self.0.deref()), // Display raw bytes if not a UTF-8 string
-        }
+        #[allow(clippy::invalid_regex)]
+        let re = Regex::new("\\p{C}").expect("Regex is valid");
+        String::from_utf8_lossy(&re.replace_all(self.0.deref(), &b""[..])).to_string()
     }
 }
 
@@ -256,7 +257,7 @@ pub enum RPCCodedResponse<T: EthSpec> {
 }
 
 /// The code assigned to an erroneous `RPCResponse`.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 pub enum RPCResponseErrorCode {
     RateLimited,
     InvalidRequest,
@@ -276,20 +277,17 @@ impl<T: EthSpec> RPCCodedResponse<T> {
 
     /// Tells the codec whether to decode as an RPCResponse or an error.
     pub fn is_response(response_code: u8) -> bool {
-        match response_code {
-            0 => true,
-            _ => false,
-        }
+        matches!(response_code, 0)
     }
 
     /// Builds an RPCCodedResponse from a response code and an ErrorMessage
-    pub fn from_error(response_code: u8, err: String) -> Self {
+    pub fn from_error(response_code: u8, err: ErrorType) -> Self {
         let code = match response_code {
             1 => RPCResponseErrorCode::InvalidRequest,
             2 => RPCResponseErrorCode::ServerError,
             _ => RPCResponseErrorCode::Unknown,
         };
-        RPCCodedResponse::Error(code, err.into())
+        RPCCodedResponse::Error(code, err)
     }
 
     /// Specifies which response allows for multiple chunks for the stream handler.
@@ -310,10 +308,7 @@ impl<T: EthSpec> RPCCodedResponse<T> {
 
     /// Returns true if this response always terminates the stream.
     pub fn close_after(&self) -> bool {
-        match self {
-            RPCCodedResponse::Success(_) => false,
-            _ => true,
-        }
+        !matches!(self, RPCCodedResponse::Success(_))
     }
 }
 
@@ -397,6 +392,22 @@ impl std::fmt::Display for BlocksByRangeRequest {
     }
 }
 
+impl slog::KV for StatusMessage {
+    fn serialize(
+        &self,
+        record: &slog::Record,
+        serializer: &mut dyn slog::Serializer,
+    ) -> slog::Result {
+        use slog::Value;
+        serializer.emit_str("fork_digest", &format!("{:?}", self.fork_digest))?;
+        Value::serialize(&self.finalized_epoch, record, "finalized_epoch", serializer)?;
+        serializer.emit_str("finalized_root", &self.finalized_root.to_string())?;
+        Value::serialize(&self.head_slot, record, "head_slot", serializer)?;
+        serializer.emit_str("head_root", &self.head_root.to_string())?;
+        slog::Result::Ok(())
+    }
+}
+
 impl slog::Value for RequestId {
     fn serialize(
         &self,

beacon_node/eth2_libp2p/src/rpc/mod.rs

@@ -119,7 +119,7 @@ impl<TSpec: EthSpec> RPC<TSpec> {
                 Duration::from_secs(10),
             )
             .build()
-            .unwrap();
+            .expect("Configuration parameters are valid");
         RPC {
             limiter,
             events: Vec::new(),
@@ -169,9 +169,12 @@ where
 
     fn new_handler(&mut self) -> Self::ProtocolsHandler {
         RPCHandler::new(
-            SubstreamProtocol::new(RPCProtocol {
-                phantom: PhantomData,
-            }),
+            SubstreamProtocol::new(
+                RPCProtocol {
+                    phantom: PhantomData,
+                },
+                (),
+            ),
             &self.log,
         )
     }
@@ -230,9 +233,23 @@ where
                         }))
                 }
                 Err(RateLimitedErr::TooLarge) => {
-                    // we set the batch sizes, so this is a coding/config err
-                    crit!(self.log, "Batch too large to ever be processed";
-                        "protocol" => format!("{}", req.protocol()));
+                    // we set the batch sizes, so this is a coding/config err for most protocols
+                    let protocol = req.protocol();
+                    if matches!(protocol, Protocol::BlocksByRange) {
+                        debug!(self.log, "Blocks by range request will never be processed"; "request" => %req);
+                    } else {
+                        crit!(self.log, "Request size too large to ever be processed"; "protocol" => %protocol);
+                    }
+                    // send an error code to the peer.
+                    // the handler upon receiving the error code will send it back to the behaviour
+                    self.send_response(
+                        peer_id,
+                        (conn_id, *id),
+                        RPCCodedResponse::Error(
+                            RPCResponseErrorCode::RateLimited,
+                            "Rate limited. Request too large".into(),
+                        ),
+                    );
                 }
                 Err(RateLimitedErr::TooSoon(wait_time)) => {
                     debug!(self.log, "Request exceeds the rate limit";

beacon_node/eth2_libp2p/src/rpc/protocol.rs

@@ -5,7 +5,7 @@ use crate::rpc::{
         ssz_snappy::{SSZSnappyInboundCodec, SSZSnappyOutboundCodec},
         InboundCodec, OutboundCodec,
     },
-    methods::ResponseTermination,
+    methods::{MaxErrorLen, ResponseTermination, MAX_ERROR_LEN},
     MaxRequestBlocks, MAX_REQUEST_BLOCKS,
 };
 use futures::future::BoxFuture;
@@ -51,6 +51,19 @@ lazy_static! {
         ])
     .as_ssz_bytes()
     .len();
+    pub static ref ERROR_TYPE_MIN: usize =
+        VariableList::<u8, MaxErrorLen>::from(Vec::<u8>::new())
+    .as_ssz_bytes()
+    .len();
+    pub static ref ERROR_TYPE_MAX: usize =
+        VariableList::<u8, MaxErrorLen>::from(vec![
+            0u8;
+            MAX_ERROR_LEN
+                as usize
+        ])
+    .as_ssz_bytes()
+    .len();
+
 }
 
 /// The maximum bytes that can be sent across the RPC.
@@ -147,6 +160,24 @@ impl<TSpec: EthSpec> UpgradeInfo for RPCProtocol<TSpec> {
     }
 }
 
+/// Represents the ssz length bounds for RPC messages.
+#[derive(Debug, PartialEq)]
+pub struct RpcLimits {
+    pub min: usize,
+    pub max: usize,
+}
+
+impl RpcLimits {
+    pub fn new(min: usize, max: usize) -> Self {
+        Self { min, max }
+    }
+
+    /// Returns true if the given length is out of bounds, false otherwise.
+    pub fn is_out_of_bounds(&self, length: usize) -> bool {
+        length > self.max || length < self.min
+    }
+}
+
 /// Tracks the types in a protocol id.
 #[derive(Clone, Debug)]
 pub struct ProtocolId {
@@ -163,6 +194,59 @@ pub struct ProtocolId {
     protocol_id: String,
 }
 
+impl ProtocolId {
+    /// Returns min and max size for messages of given protocol id requests.
+    pub fn rpc_request_limits(&self) -> RpcLimits {
+        match self.message_name {
+            Protocol::Status => RpcLimits::new(
+                <StatusMessage as Encode>::ssz_fixed_len(),
+                <StatusMessage as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::Goodbye => RpcLimits::new(
+                <GoodbyeReason as Encode>::ssz_fixed_len(),
+                <GoodbyeReason as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::BlocksByRange => RpcLimits::new(
+                <BlocksByRangeRequest as Encode>::ssz_fixed_len(),
+                <BlocksByRangeRequest as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::BlocksByRoot => {
+                RpcLimits::new(*BLOCKS_BY_ROOT_REQUEST_MIN, *BLOCKS_BY_ROOT_REQUEST_MAX)
+            }
+            Protocol::Ping => RpcLimits::new(
+                <Ping as Encode>::ssz_fixed_len(),
+                <Ping as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::MetaData => RpcLimits::new(0, 0), // Metadata requests are empty
+        }
+    }
+
+    /// Returns min and max size for messages of given protocol id responses.
+    pub fn rpc_response_limits<T: EthSpec>(&self) -> RpcLimits {
+        match self.message_name {
+            Protocol::Status => RpcLimits::new(
+                <StatusMessage as Encode>::ssz_fixed_len(),
+                <StatusMessage as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::Goodbye => RpcLimits::new(0, 0), // Goodbye request has no response
+            Protocol::BlocksByRange => {
+                RpcLimits::new(*SIGNED_BEACON_BLOCK_MIN, *SIGNED_BEACON_BLOCK_MAX)
+            }
+            Protocol::BlocksByRoot => {
+                RpcLimits::new(*SIGNED_BEACON_BLOCK_MIN, *SIGNED_BEACON_BLOCK_MAX)
+            }
+            Protocol::Ping => RpcLimits::new(
+                <Ping as Encode>::ssz_fixed_len(),
+                <Ping as Encode>::ssz_fixed_len(),
+            ),
+            Protocol::MetaData => RpcLimits::new(
+                <MetaData<T> as Encode>::ssz_fixed_len(),
+                <MetaData<T> as Encode>::ssz_fixed_len(),
+            ),
+        }
+    }
+}
+
 /// An RPC protocol ID.
 impl ProtocolId {
     pub fn new(message_name: Protocol, version: Version, encoding: Encoding) -> Self {
@@ -233,7 +317,8 @@ where
                     {
                         Err(e) => Err(RPCError::from(e)),
                         Ok((Some(Ok(request)), stream)) => Ok((request, stream)),
-                        Ok((Some(Err(_)), _)) | Ok((None, _)) => Err(RPCError::IncompleteStream),
+                        Ok((Some(Err(e)), _)) => Err(e),
+                        Ok((None, _)) => Err(RPCError::IncompleteStream),
                     }
                 }
             }
@@ -385,7 +470,7 @@ where
 }
 
 /// Error in RPC Encoding/Decoding.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub enum RPCError {
     /// Error when decoding the raw buffer from ssz.
     // NOTE: in the future a ssz::DecodeError should map to an InvalidData error

beacon_node/eth2_libp2p/src/rpc/rate_limiter.rs

@@ -189,7 +189,30 @@ impl RPCRateLimiter {
         request: &RPCRequest<T>,
     ) -> Result<(), RateLimitedErr> {
         let time_since_start = self.init_time.elapsed();
-        let tokens = request.expected_responses().max(1);
+        let mut tokens = request.expected_responses().max(1);
+
+        // Increase the rate limit for blocks by range requests with large step counts.
+        // We count to tokens as a quadratic increase with step size.
+        // Using (step_size/5)^2 + 1 as penalty factor allows step sizes of 1-4 to have no penalty
+        // but step sizes higher than this add a quadratic penalty.
+        // Penalty's go:
+        // Step size | Penalty Factor
+        //     1     |   1
+        //     2     |   1
+        //     3     |   1
+        //     4     |   1
+        //     5     |   2
+        //     6     |   2
+        //     7     |   2
+        //     8     |   3
+        //     9     |   4
+        //     10    |   5
+
+        if let RPCRequest::BlocksByRange(bbr_req) = request {
+            let penalty_factor = (bbr_req.step as f64 / 5.0).powi(2) as u64 + 1;
+            tokens *= penalty_factor;
+        }
+
         let check =
             |limiter: &mut Limiter<PeerId>| limiter.allows(time_since_start, peer_id, tokens);
         let limiter = match request.protocol() {

beacon_node/eth2_libp2p/src/service.rs

@@ -1,31 +1,35 @@
-use crate::behaviour::{Behaviour, BehaviourEvent, PeerRequestId, Request, Response};
+use crate::behaviour::{
+    save_metadata_to_disk, Behaviour, BehaviourEvent, PeerRequestId, Request, Response,
+};
 use crate::discovery::enr;
 use crate::multiaddr::Protocol;
-use crate::rpc::{GoodbyeReason, RPCResponseErrorCode, RequestId};
-use crate::types::{error, GossipKind};
+use crate::rpc::{GoodbyeReason, MetaData, RPCResponseErrorCode, RequestId};
+use crate::types::{error, EnrBitfield, GossipKind};
 use crate::EnrExt;
 use crate::{NetworkConfig, NetworkGlobals, PeerAction};
 use futures::prelude::*;
 use libp2p::core::{
-    identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::boxed::Boxed,
+    identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::Boxed,
 };
 use libp2p::{
     core, noise,
     swarm::{SwarmBuilder, SwarmEvent},
     PeerId, Swarm, Transport,
 };
-use slog::{crit, debug, info, o, trace, warn};
+use slog::{crit, debug, info, o, trace, warn, Logger};
+use ssz::Decode;
 use std::fs::File;
 use std::io::prelude::*;
-use std::io::{Error, ErrorKind};
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
-use types::{EnrForkId, EthSpec};
+use types::{ChainSpec, EnrForkId, EthSpec};
 
 pub const NETWORK_KEY_FILENAME: &str = "key";
 /// The maximum simultaneous libp2p connections per peer.
 const MAX_CONNECTIONS_PER_PEER: usize = 1;
+/// The filename to store our local metadata.
+pub const METADATA_FILENAME: &str = "metadata";
 
 /// The types of events than can be obtained from polling the libp2p service.
 ///
@@ -49,15 +53,16 @@ pub struct Service<TSpec: EthSpec> {
     pub local_peer_id: PeerId,
 
     /// The libp2p logger handle.
-    pub log: slog::Logger,
+    pub log: Logger,
 }
 
 impl<TSpec: EthSpec> Service<TSpec> {
     pub async fn new(
-        executor: environment::TaskExecutor,
+        executor: task_executor::TaskExecutor,
         config: &NetworkConfig,
         enr_fork_id: EnrForkId,
-        log: &slog::Logger,
+        log: &Logger,
+        chain_spec: &ChainSpec,
     ) -> error::Result<(Arc<NetworkGlobals<TSpec>>, Self)> {
         let log = log.new(o!("service"=> "libp2p"));
         trace!(log, "Libp2p Service starting");
@@ -70,11 +75,20 @@ impl<TSpec: EthSpec> Service<TSpec> {
             enr::build_or_load_enr::<TSpec>(local_keypair.clone(), config, enr_fork_id, &log)?;
 
         let local_peer_id = enr.peer_id();
+
+        let meta_data = load_or_build_metadata(&config.network_dir, &log);
+
         // set up a collection of variables accessible outside of the network crate
         let network_globals = Arc::new(NetworkGlobals::new(
             enr.clone(),
             config.libp2p_port,
             config.discovery_port,
+            meta_data,
+            config
+                .trusted_peers
+                .iter()
+                .map(|x| PeerId::from(x.clone()))
+                .collect(),
             &log,
         ));
 
@@ -91,11 +105,17 @@ impl<TSpec: EthSpec> Service<TSpec> {
             let transport = build_transport(local_keypair.clone())
                 .map_err(|e| format!("Failed to build transport: {:?}", e))?;
             // Lighthouse network behaviour
-            let behaviour =
-                Behaviour::new(&local_keypair, config, network_globals.clone(), &log).await?;
+            let behaviour = Behaviour::new(
+                &local_keypair,
+                config,
+                network_globals.clone(),
+                &log,
+                chain_spec,
+            )
+            .await?;
 
             // use the executor for libp2p
-            struct Executor(environment::TaskExecutor);
+            struct Executor(task_executor::TaskExecutor);
             impl libp2p::core::Executor for Executor {
                 fn exec(&self, f: Pin<Box<dyn Future<Output = ()> + Send>>) {
                     self.0.spawn(f, "libp2p");
@@ -186,6 +206,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
         }
 
         let mut subscribed_topics: Vec<GossipKind> = vec![];
+
         for topic_kind in &config.topics {
             if swarm.subscribe_kind(topic_kind.clone()) {
                 subscribed_topics.push(topic_kind.clone());
@@ -193,7 +214,10 @@ impl<TSpec: EthSpec> Service<TSpec> {
                 warn!(log, "Could not subscribe to topic"; "topic" => format!("{}",topic_kind));
             }
         }
-        info!(log, "Subscribed to topics"; "topics" => format!("{:?}", subscribed_topics));
+
+        if !subscribed_topics.is_empty() {
+            info!(log, "Subscribed to topics"; "topics" => format!("{:?}", subscribed_topics));
+        }
 
         let service = Service {
             local_peer_id,
@@ -225,7 +249,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
         self.swarm.report_peer(peer_id, action);
     }
 
-    // Disconnect and ban a peer, providing a reason.
+    /// Disconnect and ban a peer, providing a reason.
     pub fn goodbye_peer(&mut self, peer_id: &PeerId, reason: GoodbyeReason) {
         self.swarm.goodbye_peer(peer_id, reason);
     }
@@ -307,9 +331,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
 
 /// The implementation supports TCP/IP, WebSockets over TCP/IP, noise as the encryption layer, and
 /// mplex as the multiplexing layer.
-fn build_transport(
-    local_private_key: Keypair,
-) -> Result<Boxed<(PeerId, StreamMuxerBox), Error>, Error> {
+fn build_transport(local_private_key: Keypair) -> std::io::Result<Boxed<(PeerId, StreamMuxerBox)>> {
     let transport = libp2p::tcp::TokioTcpConfig::new().nodelay(true);
     let transport = libp2p::dns::DnsConfig::new(transport)?;
     #[cfg(feature = "libp2p-websocket")]
@@ -317,15 +339,21 @@ fn build_transport(
         let trans_clone = transport.clone();
         transport.or_transport(libp2p::websocket::WsConfig::new(trans_clone))
     };
+
+    // mplex config
+    let mut mplex_config = libp2p::mplex::MplexConfig::new();
+    mplex_config.max_buffer_len(256);
+    mplex_config.max_buffer_len_behaviour(libp2p::mplex::MaxBufferBehaviour::Block);
+
     // Authentication
     Ok(transport
         .upgrade(core::upgrade::Version::V1)
         .authenticate(generate_noise_config(&local_private_key))
-        .multiplex(libp2p::mplex::MplexConfig::new())
-        .map(|(peer, muxer), _| (peer, core::muxing::StreamMuxerBox::new(muxer)))
+        .multiplex(core::upgrade::SelectUpgrade::new(
+            libp2p::yamux::Config::default(),
+            mplex_config,
+        ))
         .timeout(Duration::from_secs(10))
-        .timeout(Duration::from_secs(10))
-        .map_err(|err| Error::new(ErrorKind::Other, err))
         .boxed())
 }
 
@@ -357,7 +385,7 @@ fn keypair_from_bytes(mut bytes: Vec<u8>) -> error::Result<Keypair> {
 /// generated and is then saved to disk.
 ///
 /// Currently only secp256k1 keys are allowed, as these are the only keys supported by discv5.
-fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
+pub fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
     // check for key from disk
     let network_key_f = config.network_dir.join(NETWORK_KEY_FILENAME);
     if let Ok(mut network_key_file) = File::open(network_key_f.clone()) {
@@ -420,3 +448,43 @@ fn strip_peer_id(addr: &mut Multiaddr) {
         _ => {}
     }
 }
+
+/// Load metadata from persisted file. Return default metadata if loading fails.
+fn load_or_build_metadata<E: EthSpec>(
+    network_dir: &std::path::PathBuf,
+    log: &slog::Logger,
+) -> MetaData<E> {
+    // Default metadata
+    let mut meta_data = MetaData {
+        seq_number: 0,
+        attnets: EnrBitfield::<E>::default(),
+    };
+    // Read metadata from persisted file if available
+    let metadata_path = network_dir.join(METADATA_FILENAME);
+    if let Ok(mut metadata_file) = File::open(metadata_path) {
+        let mut metadata_ssz = Vec::new();
+        if metadata_file.read_to_end(&mut metadata_ssz).is_ok() {
+            match MetaData::<E>::from_ssz_bytes(&metadata_ssz) {
+                Ok(persisted_metadata) => {
+                    meta_data.seq_number = persisted_metadata.seq_number;
+                    // Increment seq number if persisted attnet is not default
+                    if persisted_metadata.attnets != meta_data.attnets {
+                        meta_data.seq_number += 1;
+                    }
+                    debug!(log, "Loaded metadata from disk");
+                }
+                Err(e) => {
+                    debug!(
+                        log,
+                        "Metadata from file could not be decoded";
+                        "error" => format!("{:?}", e),
+                    );
+                }
+            }
+        }
+    };
+
+    debug!(log, "Metadata sequence number"; "seq_num" => meta_data.seq_number);
+    save_metadata_to_disk(network_dir, meta_data.clone(), &log);
+    meta_data
+}

beacon_node/eth2_libp2p/src/types/globals.rs

@@ -1,5 +1,6 @@
 //! A collection of variables that are accessible outside of the network thread itself.
 use crate::peer_manager::PeerDB;
+use crate::rpc::MetaData;
 use crate::types::SyncState;
 use crate::Client;
 use crate::EnrExt;
@@ -22,6 +23,8 @@ pub struct NetworkGlobals<TSpec: EthSpec> {
     pub listen_port_udp: AtomicU16,
     /// The collection of known peers.
     pub peers: RwLock<PeerDB<TSpec>>,
+    // The local meta data of our node.
+    pub local_metadata: RwLock<MetaData<TSpec>>,
     /// The current gossipsub topic subscriptions.
     pub gossipsub_subscriptions: RwLock<HashSet<GossipTopic>>,
     /// The current sync status of the node.
@@ -29,14 +32,22 @@ pub struct NetworkGlobals<TSpec: EthSpec> {
 }
 
 impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
-    pub fn new(enr: Enr, tcp_port: u16, udp_port: u16, log: &slog::Logger) -> Self {
+    pub fn new(
+        enr: Enr,
+        tcp_port: u16,
+        udp_port: u16,
+        local_metadata: MetaData<TSpec>,
+        trusted_peers: Vec<PeerId>,
+        log: &slog::Logger,
+    ) -> Self {
         NetworkGlobals {
             local_enr: RwLock::new(enr.clone()),
             peer_id: RwLock::new(enr.peer_id()),
             listen_multiaddrs: RwLock::new(Vec::new()),
             listen_port_tcp: AtomicU16::new(tcp_port),
             listen_port_udp: AtomicU16::new(udp_port),
-            peers: RwLock::new(PeerDB::new(log)),
+            local_metadata: RwLock::new(local_metadata),
+            peers: RwLock::new(PeerDB::new(trusted_peers, log)),
             gossipsub_subscriptions: RwLock::new(HashSet::new()),
             sync_state: RwLock::new(SyncState::Stalled),
         }
@@ -99,25 +110,8 @@ impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
 
     /// Updates the syncing state of the node.
     ///
-    /// If there is a new state, the old state and the new states are returned.
-    pub fn update_sync_state(&self) -> Option<(SyncState, SyncState)> {
-        let mut result = None;
-        // if we are in a range sync, nothing changes. Range sync will update this.
-        if !self.is_syncing() {
-            let new_state = self
-                .peers
-                .read()
-                .synced_peers()
-                .next()
-                .map(|_| SyncState::Synced)
-                .unwrap_or_else(|| SyncState::Stalled);
-
-            let mut peer_state = self.sync_state.write();
-            if new_state != *peer_state {
-                result = Some((peer_state.clone(), new_state.clone()));
-            }
-            *peer_state = new_state;
-        }
-        result
+    /// The old state is returned
+    pub fn set_sync_state(&self, new_state: SyncState) -> SyncState {
+        std::mem::replace(&mut *self.sync_state.write(), new_state)
     }
 }

beacon_node/eth2_libp2p/src/types/mod.rs

@@ -13,7 +13,7 @@ pub type EnrBitfield<T: EthSpec> = BitVector<T::SubnetBitfieldLength>;
 pub type Enr = discv5::enr::Enr<discv5::enr::CombinedKey>;
 
 pub use globals::NetworkGlobals;
-pub use pubsub::PubsubMessage;
+pub use pubsub::{MessageData, PubsubMessage};
 pub use subnet::SubnetDiscovery;
 pub use sync_state::SyncState;
-pub use topics::{GossipEncoding, GossipKind, GossipTopic};
+pub use topics::{GossipEncoding, GossipKind, GossipTopic, CORE_TOPICS};

beacon_node/eth2_libp2p/src/types/pubsub.rs

@@ -12,6 +12,33 @@ use types::{
     SignedBeaconBlock, SignedVoluntaryExit,
 };
 
+#[derive(Clone)]
+pub struct MessageData {
+    pub raw: Vec<u8>,
+    pub decompressed: Result<Vec<u8>, String>,
+}
+
+impl AsRef<[u8]> for MessageData {
+    fn as_ref(&self) -> &[u8] {
+        self.raw.as_ref()
+    }
+}
+
+impl Into<Vec<u8>> for MessageData {
+    fn into(self) -> Vec<u8> {
+        self.raw
+    }
+}
+
+impl From<Vec<u8>> for MessageData {
+    fn from(raw: Vec<u8>) -> Self {
+        Self {
+            decompressed: decompress_snappy(raw.as_ref()),
+            raw,
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub enum PubsubMessage<T: EthSpec> {
     /// Gossipsub message providing notification of a new block.
@@ -28,6 +55,24 @@ pub enum PubsubMessage<T: EthSpec> {
     AttesterSlashing(Box<AttesterSlashing<T>>),
 }
 
+fn decompress_snappy(data: &[u8]) -> Result<Vec<u8>, String> {
+    // Exit early if uncompressed data is > GOSSIP_MAX_SIZE
+    match decompress_len(data) {
+        Ok(n) if n > GOSSIP_MAX_SIZE => {
+            return Err("ssz_snappy decoded data > GOSSIP_MAX_SIZE".into());
+        }
+        Ok(_) => {}
+        Err(e) => {
+            return Err(format!("{}", e));
+        }
+    };
+    let mut decoder = Decoder::new();
+    match decoder.decompress_vec(data) {
+        Ok(decompressed_data) => Ok(decompressed_data),
+        Err(e) => Err(format!("{}", e)),
+    }
+}
+
 impl<T: EthSpec> PubsubMessage<T> {
     /// Returns the topics that each pubsub message will be sent across, given a supported
     /// gossipsub encoding and fork version.
@@ -49,94 +94,58 @@ impl<T: EthSpec> PubsubMessage<T> {
         }
     }
 
-    /// This decodes `data` into a `PubsubMessage` given a list of topics.
-    ///
-    /// The topics are checked
-    /// in order and as soon as one topic matches the decoded data, we return the data.
+    /// This decodes `data` into a `PubsubMessage` given a topic.
     /* Note: This is assuming we are not hashing topics. If we choose to hash topics, these will
      * need to be modified.
-     *
-     * Also note that a message can be associated with many topics. As soon as one of the topics is
-     * known we match. If none of the topics are known we return an unknown state.
      */
-    pub fn decode(topics: &[TopicHash], data: &[u8]) -> Result<Self, String> {
-        let mut unknown_topics = Vec::new();
-        for topic in topics {
-            match GossipTopic::decode(topic.as_str()) {
-                Err(_) => {
-                    unknown_topics.push(topic);
-                    continue;
-                }
-                Ok(gossip_topic) => {
-                    let decompressed_data = &(match gossip_topic.encoding() {
-                        GossipEncoding::SSZSnappy => {
-                            // Exit early if uncompressed data is > GOSSIP_MAX_SIZE
-                            match decompress_len(data) {
-                                Ok(n) if n > GOSSIP_MAX_SIZE => {
-                                    return Err("ssz_snappy decoded data > GOSSIP_MAX_SIZE".into());
-                                }
-                                Ok(_) => {}
-                                Err(e) => {
-                                    return Err(format!("{}", e));
-                                }
-                            };
-                            let mut decoder = Decoder::new();
-                            match decoder.decompress_vec(data) {
-                                Ok(decompressed_data) => decompressed_data,
-                                Err(e) => return Err(format!("{}", e)),
-                            }
-                        }
-                    });
-                    // the ssz decoders
-                    match gossip_topic.kind() {
-                        GossipKind::BeaconAggregateAndProof => {
-                            let agg_and_proof =
-                                SignedAggregateAndProof::from_ssz_bytes(decompressed_data)
-                                    .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::AggregateAndProofAttestation(Box::new(
-                                agg_and_proof,
-                            )));
-                        }
-                        GossipKind::Attestation(subnet_id) => {
-                            let attestation = Attestation::from_ssz_bytes(decompressed_data)
+    pub fn decode(topic: &TopicHash, data: &MessageData) -> Result<Self, String> {
+        match GossipTopic::decode(topic.as_str()) {
+            Err(_) => Err(format!("Unknown gossipsub topic: {:?}", topic)),
+            Ok(gossip_topic) => {
+                let decompressed_data = match gossip_topic.encoding() {
+                    GossipEncoding::SSZSnappy => data.decompressed.as_ref()?.as_slice(),
+                };
+                // the ssz decoders
+                match gossip_topic.kind() {
+                    GossipKind::BeaconAggregateAndProof => {
+                        let agg_and_proof =
+                            SignedAggregateAndProof::from_ssz_bytes(decompressed_data)
                                 .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::Attestation(Box::new((
-                                *subnet_id,
-                                attestation,
-                            ))));
-                        }
-                        GossipKind::BeaconBlock => {
-                            let beacon_block = SignedBeaconBlock::from_ssz_bytes(decompressed_data)
-                                .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::BeaconBlock(Box::new(beacon_block)));
-                        }
-                        GossipKind::VoluntaryExit => {
-                            let voluntary_exit =
-                                SignedVoluntaryExit::from_ssz_bytes(decompressed_data)
-                                    .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::VoluntaryExit(Box::new(voluntary_exit)));
-                        }
-                        GossipKind::ProposerSlashing => {
-                            let proposer_slashing =
-                                ProposerSlashing::from_ssz_bytes(decompressed_data)
-                                    .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::ProposerSlashing(Box::new(
-                                proposer_slashing,
-                            )));
-                        }
-                        GossipKind::AttesterSlashing => {
-                            let attester_slashing =
-                                AttesterSlashing::from_ssz_bytes(decompressed_data)
-                                    .map_err(|e| format!("{:?}", e))?;
-                            return Ok(PubsubMessage::AttesterSlashing(Box::new(
-                                attester_slashing,
-                            )));
-                        }
+                        Ok(PubsubMessage::AggregateAndProofAttestation(Box::new(
+                            agg_and_proof,
+                        )))
+                    }
+                    GossipKind::Attestation(subnet_id) => {
+                        let attestation = Attestation::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::Attestation(Box::new((
+                            *subnet_id,
+                            attestation,
+                        ))))
+                    }
+                    GossipKind::BeaconBlock => {
+                        let beacon_block = SignedBeaconBlock::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::BeaconBlock(Box::new(beacon_block)))
+                    }
+                    GossipKind::VoluntaryExit => {
+                        let voluntary_exit = SignedVoluntaryExit::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::VoluntaryExit(Box::new(voluntary_exit)))
+                    }
+                    GossipKind::ProposerSlashing => {
+                        let proposer_slashing = ProposerSlashing::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::ProposerSlashing(Box::new(proposer_slashing)))
+                    }
+                    GossipKind::AttesterSlashing => {
+                        let attester_slashing = AttesterSlashing::from_ssz_bytes(decompressed_data)
+                            .map_err(|e| format!("{:?}", e))?;
+                        Ok(PubsubMessage::AttesterSlashing(Box::new(attester_slashing)))
                     }
                 }
             }
         }
-        Err(format!("Unknown gossipsub topics: {:?}", unknown_topics))
     }
 
     /// Encodes a `PubsubMessage` based on the topic encodings. The first known encoding is used. If
@@ -191,3 +200,20 @@ impl<T: EthSpec> std::fmt::Display for PubsubMessage<T> {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_gossip_max_size() {
+        // Cannot decode more than gossip max size
+        let mut encoder = Encoder::new();
+        let payload = encoder.compress_vec(&[0; GOSSIP_MAX_SIZE + 1]).unwrap();
+        let message_data: MessageData = payload.into();
+        assert_eq!(
+            message_data.decompressed.unwrap_err(),
+            "ssz_snappy decoded data > GOSSIP_MAX_SIZE".to_string()
+        );
+    }
+}

beacon_node/eth2_libp2p/src/types/subnet.rs

@@ -1,28 +1,9 @@
-use std::time::{Duration, Instant};
+use std::time::Instant;
 use types::SubnetId;
 
-const DURATION_DIFFERENCE: Duration = Duration::from_millis(1);
-
 /// A subnet to discover peers on along with the instant after which it's no longer useful.
 #[derive(Debug, Clone)]
 pub struct SubnetDiscovery {
     pub subnet_id: SubnetId,
     pub min_ttl: Option<Instant>,
 }
-
-impl PartialEq for SubnetDiscovery {
-    fn eq(&self, other: &SubnetDiscovery) -> bool {
-        self.subnet_id == other.subnet_id
-            && match (self.min_ttl, other.min_ttl) {
-                (Some(min_ttl_instant), Some(other_min_ttl_instant)) => {
-                    min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
-                        < DURATION_DIFFERENCE
-                        && other_min_ttl_instant.saturating_duration_since(min_ttl_instant)
-                            < DURATION_DIFFERENCE
-                }
-                (None, None) => true,
-                (None, Some(_)) => true,
-                (Some(_), None) => true,
-            }
-    }
-}

beacon_node/eth2_libp2p/src/types/sync_state.rs

@@ -1,19 +1,18 @@
 use serde::{Deserialize, Serialize};
-use types::{Hash256, Slot};
+use types::Slot;
 
 /// The current state of the node.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum SyncState {
     /// The node is performing a long-range (batch) sync over a finalized chain.
     /// In this state, parent lookups are disabled.
-    SyncingFinalized {
-        start_slot: Slot,
-        head_slot: Slot,
-        head_root: Hash256,
-    },
+    SyncingFinalized { start_slot: Slot, target_slot: Slot },
     /// The node is performing a long-range (batch) sync over one or many head chains.
     /// In this state parent lookups are disabled.
-    SyncingHead { start_slot: Slot, head_slot: Slot },
+    SyncingHead { start_slot: Slot, target_slot: Slot },
+    /// The node has identified the need for is sync operations and is transitioning to a syncing
+    /// state.
+    SyncTransition,
     /// The node is up to date with all known peers and is connected to at least one
     /// fully synced peer. In this state, parent lookups are enabled.
     Synced,
@@ -29,6 +28,7 @@ impl PartialEq for SyncState {
             (SyncState::SyncingHead { .. }, SyncState::SyncingHead { .. }) => true,
             (SyncState::Synced, SyncState::Synced) => true,
             (SyncState::Stalled, SyncState::Stalled) => true,
+            (SyncState::SyncTransition, SyncState::SyncTransition) => true,
             _ => false,
         }
     }
@@ -40,6 +40,7 @@ impl SyncState {
         match self {
             SyncState::SyncingFinalized { .. } => true,
             SyncState::SyncingHead { .. } => true,
+            SyncState::SyncTransition => true,
             SyncState::Synced => false,
             SyncState::Stalled => false,
         }
@@ -47,10 +48,7 @@ impl SyncState {
 
     /// Returns true if the node is synced.
     pub fn is_synced(&self) -> bool {
-        match self {
-            SyncState::Synced => true,
-            _ => false,
-        }
+        matches!(self, SyncState::Synced)
     }
 }
 
@@ -61,6 +59,7 @@ impl std::fmt::Display for SyncState {
             SyncState::SyncingHead { .. } => write!(f, "Syncing Head Chain"),
             SyncState::Synced { .. } => write!(f, "Synced"),
             SyncState::Stalled { .. } => write!(f, "Stalled"),
+            SyncState::SyncTransition => write!(f, "Searching syncing peers"),
         }
     }
 }

beacon_node/eth2_libp2p/src/types/topics.rs

@@ -1,4 +1,4 @@
-use libp2p::gossipsub::Topic;
+use libp2p::gossipsub::IdentTopic as Topic;
 use serde_derive::{Deserialize, Serialize};
 use types::SubnetId;
 
@@ -14,6 +14,14 @@ pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit";
 pub const PROPOSER_SLASHING_TOPIC: &str = "proposer_slashing";
 pub const ATTESTER_SLASHING_TOPIC: &str = "attester_slashing";
 
+pub const CORE_TOPICS: [GossipKind; 5] = [
+    GossipKind::BeaconBlock,
+    GossipKind::BeaconAggregateAndProof,
+    GossipKind::VoluntaryExit,
+    GossipKind::ProposerSlashing,
+    GossipKind::AttesterSlashing,
+];
+
 /// A gossipsub topic which encapsulates the type of messages that should be sent and received over
 /// the pubsub protocol and the way the messages should be encoded.
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
@@ -44,15 +52,25 @@ pub enum GossipKind {
     AttesterSlashing,
 }
 
+impl AsRef<str> for GossipKind {
+    fn as_ref(&self) -> &str {
+        use GossipKind::*;
+        match self {
+            BeaconBlock => "beacon_block",
+            BeaconAggregateAndProof => "beacon_aggregate_and_proof",
+            Attestation(_) => "beacon_attestation",
+            VoluntaryExit => "voluntary_exit",
+            ProposerSlashing => "proposer_slashing",
+            AttesterSlashing => "attester_slashing",
+        }
+    }
+}
+
 impl std::fmt::Display for GossipKind {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            GossipKind::BeaconBlock => write!(f, "beacon_block"),
-            GossipKind::BeaconAggregateAndProof => write!(f, "beacon_aggregate_and_proof"),
             GossipKind::Attestation(subnet_id) => write!(f, "beacon_attestation_{}", **subnet_id),
-            GossipKind::VoluntaryExit => write!(f, "voluntary_exit"),
-            GossipKind::ProposerSlashing => write!(f, "proposer_slashing"),
-            GossipKind::AttesterSlashing => write!(f, "attester_slashing"),
+            x => f.write_str(x.as_ref()),
         }
     }
 }
@@ -139,7 +157,7 @@ impl GossipTopic {
 
 impl Into<Topic> for GossipTopic {
     fn into(self) -> Topic {
-        Topic::new(self.into())
+        Topic::new(self)
     }
 }
 
@@ -184,3 +202,91 @@ fn committee_topic_index(topic: &str) -> Option<SubnetId> {
     }
     None
 }
+
+#[cfg(test)]
+mod tests {
+    use super::GossipKind::*;
+    use super::*;
+
+    const GOOD_FORK_DIGEST: &str = "e1925f3b";
+    const BAD_PREFIX: &str = "tezos";
+    const BAD_FORK_DIGEST: &str = "e1925f3b4b";
+    const BAD_ENCODING: &str = "rlp";
+    const BAD_KIND: &str = "blocks";
+
+    fn topics() -> Vec<String> {
+        let mut topics = Vec::new();
+        let fork_digest: [u8; 4] = [1, 2, 3, 4];
+        for encoding in [GossipEncoding::SSZSnappy].iter() {
+            for kind in [
+                BeaconBlock,
+                BeaconAggregateAndProof,
+                Attestation(SubnetId::new(42)),
+                VoluntaryExit,
+                ProposerSlashing,
+                AttesterSlashing,
+            ]
+            .iter()
+            {
+                topics.push(GossipTopic::new(kind.clone(), encoding.clone(), fork_digest).into());
+            }
+        }
+        topics
+    }
+
+    fn create_topic(prefix: &str, fork_digest: &str, kind: &str, encoding: &str) -> String {
+        format!("/{}/{}/{}/{}", prefix, fork_digest, kind, encoding)
+    }
+
+    #[test]
+    fn test_decode() {
+        for topic in topics().iter() {
+            assert!(GossipTopic::decode(topic.as_str()).is_ok());
+        }
+    }
+
+    #[test]
+    fn test_decode_malicious() {
+        let bad_prefix_str = create_topic(
+            BAD_PREFIX,
+            GOOD_FORK_DIGEST,
+            BEACON_BLOCK_TOPIC,
+            SSZ_SNAPPY_ENCODING_POSTFIX,
+        );
+        assert!(GossipTopic::decode(bad_prefix_str.as_str()).is_err());
+
+        let bad_digest_str = create_topic(
+            TOPIC_PREFIX,
+            BAD_FORK_DIGEST,
+            BEACON_BLOCK_TOPIC,
+            SSZ_SNAPPY_ENCODING_POSTFIX,
+        );
+        assert!(GossipTopic::decode(bad_digest_str.as_str()).is_err());
+
+        let bad_kind_str = create_topic(
+            TOPIC_PREFIX,
+            GOOD_FORK_DIGEST,
+            BAD_KIND,
+            SSZ_SNAPPY_ENCODING_POSTFIX,
+        );
+        assert!(GossipTopic::decode(bad_kind_str.as_str()).is_err());
+
+        let bad_encoding_str = create_topic(
+            TOPIC_PREFIX,
+            GOOD_FORK_DIGEST,
+            BEACON_BLOCK_TOPIC,
+            BAD_ENCODING,
+        );
+        assert!(GossipTopic::decode(bad_encoding_str.as_str()).is_err());
+
+        // Extra parts
+        assert!(
+            GossipTopic::decode("/eth2/e1925f3b/beacon_block/ssz_snappy/yolo").is_err(),
+            "should have exactly 5 parts"
+        );
+        // Empty string
+        assert!(GossipTopic::decode("").is_err());
+        // Empty parts
+        assert!(GossipTopic::decode("////").is_err());
+    }
+}