Bump version to v0.2.9 (#1598)

## Issue Addressed

NA

## Proposed Changes

- Bump version tags
- Run `cargo update`

## Additional Info

NA

.github/workflows/test-suite.yml

@@ -123,6 +123,8 @@ jobs:
     - uses: actions/checkout@v1
     - name: Lint code for quality and style with Clippy
       run: make lint
+    - name: Certify Cargo.lock freshness
+      run: git diff --exit-code Cargo.lock
   arbitrary-check:
     name: arbitrary-check
     runs-on: ubuntu-latest

.gitignore

@@ -6,3 +6,4 @@ target/
 flamegraph.svg
 perf.data*
 *.tar.gz
+bin/

Cargo.toml

@@ -28,6 +28,7 @@ members = [
     "common/lighthouse_metrics",
     "common/lighthouse_version",
     "common/logging",
+    "common/lru_cache",
     "common/remote_beacon_node",
     "common/rest_types",
     "common/slot_clock",
@@ -43,6 +44,7 @@ members = [
     "consensus/ssz_derive",
     "consensus/ssz_types",
     "consensus/serde_hex",
+    "consensus/serde_utils",
     "consensus/state_processing",
     "consensus/swap_or_not_shuffle",
     "consensus/tree_hash",

Cross.toml

@@ -0,0 +1,4 @@
+[build.env]
+passthrough = [
+    "RUSTFLAGS",
+]

Makefile

@@ -2,6 +2,13 @@
 
 EF_TESTS = "testing/ef_tests"
 STATE_TRANSITION_VECTORS = "testing/state_transition_vectors"
+GIT_TAG := $(shell git describe --tags --candidates 1)
+BIN_DIR = "bin"
+
+X86_64_TAG = "x86_64-unknown-linux-gnu"
+BUILD_PATH_X86_64 = "target/$(X86_64_TAG)/release"
+AARCH64_TAG = "aarch64-unknown-linux-gnu"
+BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
 
 # Builds the Lighthouse binary in release (optimized).
 #
@@ -21,6 +28,53 @@ else
 	cargo install --path lcli --force --locked
 endif
 
+# The following commands use `cross` to build a cross-compile.
+#
+# These commands require that:
+#
+# - `cross` is installed (`cargo install cross`).
+# - Docker is running.
+# - The current user is in the `docker` group.
+#
+# The resulting binaries will be created in the `target/` directory.
+#
+# The *-portable options compile the blst library *without* the use of some
+# optimized CPU functions that may not be available on some systems. This
+# results in a more portable binary with ~20% slower BLS verification.
+build-x86_64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu
+build-x86_64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features portable
+build-aarch64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu
+build-aarch64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu --features portable
+
+# Create a `.tar.gz` containing a binary for a specific target.
+define tarball_release_binary
+	cp $(1)/lighthouse $(BIN_DIR)/lighthouse
+	cd $(BIN_DIR) && \
+		tar -czf lighthouse-$(GIT_TAG)-$(2)$(3).tar.gz lighthouse && \
+		rm lighthouse
+endef
+
+# Create a series of `.tar.gz` files in the BIN_DIR directory, each containing
+# a `lighthouse` binary for a different target.
+#
+# The current git tag will be used as the version in the output file names. You
+# will likely need to use `git tag` and create a semver tag (e.g., `v0.2.3`).
+build-release-tarballs:
+	[ -d $(BIN_DIR) ] || mkdir -p $(BIN_DIR)
+	$(MAKE) build-x86_64
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"")
+	$(MAKE) build-x86_64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"-portable")
+	$(MAKE) build-aarch64
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"")
+	$(MAKE) build-aarch64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"-portable")
+
+
 # Runs the full workspace tests in **release**, without downloading any additional
 # test vectors.
 test-release:

account_manager/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "account_manager"
-version = "0.2.0"
+version = "0.2.9"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
 edition = "2018"
 

account_manager/src/common.rs

@@ -1,6 +1,15 @@
+use account_utils::PlainText;
+use account_utils::{read_mnemonic_from_user, strip_off_newlines};
 use clap::ArgMatches;
+use eth2_wallet::bip39::{Language, Mnemonic};
+use std::fs;
 use std::fs::create_dir_all;
 use std::path::{Path, PathBuf};
+use std::str::from_utf8;
+use std::thread::sleep;
+use std::time::Duration;
+
+pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
 
 pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> {
     let path = path.as_ref();
@@ -19,3 +28,43 @@ pub fn base_wallet_dir(matches: &ArgMatches, arg: &'static str) -> Result<PathBu
         PathBuf::new().join(".lighthouse").join("wallets"),
     )
 }
+
+pub fn read_mnemonic_from_cli(
+    mnemonic_path: Option<PathBuf>,
+    stdin_password: bool,
+) -> Result<Mnemonic, String> {
+    let mnemonic = match mnemonic_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .and_then(|bytes| {
+                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
+                let phrase = from_utf8(&bytes_no_newlines.as_ref())
+                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
+                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
+                    format!(
+                        "Unable to derive mnemonic from string {:?}: {:?}",
+                        phrase, e
+                    )
+                })
+            })?,
+        None => loop {
+            eprintln!("");
+            eprintln!("{}", MNEMONIC_PROMPT);
+
+            let mnemonic = read_mnemonic_from_user(stdin_password)?;
+
+            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
+                Ok(mnemonic_m) => {
+                    eprintln!("Valid mnemonic provided.");
+                    eprintln!("");
+                    sleep(Duration::from_secs(1));
+                    break mnemonic_m;
+                }
+                Err(_) => {
+                    eprintln!("Invalid mnemonic");
+                }
+            }
+        },
+    };
+    Ok(mnemonic)
+}

account_manager/src/validator/create.rs

@@ -13,7 +13,7 @@ use validator_dir::Builder as ValidatorDirBuilder;
 pub const CMD: &str = "create";
 pub const BASE_DIR_FLAG: &str = "base-dir";
 pub const WALLET_NAME_FLAG: &str = "wallet-name";
-pub const WALLET_PASSPHRASE_FLAG: &str = "wallet-passphrase";
+pub const WALLET_PASSWORD_FLAG: &str = "wallet-password";
 pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
 pub const STORE_WITHDRAW_FLAG: &str = "store-withdrawal-keystore";
 pub const COUNT_FLAG: &str = "count";
@@ -34,8 +34,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .required(true),
         )
         .arg(
-            Arg::with_name(WALLET_PASSPHRASE_FLAG)
-                .long(WALLET_PASSPHRASE_FLAG)
+            Arg::with_name(WALLET_PASSWORD_FLAG)
+                .long(WALLET_PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help("A path to a file containing the password which will unlock the wallet.")
                 .takes_value(true)
@@ -109,8 +109,7 @@ pub fn cli_run<T: EthSpec>(
     let spec = env.core_context().eth2_config.spec;
 
     let name: String = clap_utils::parse_required(matches, WALLET_NAME_FLAG)?;
-    let wallet_password_path: PathBuf =
-        clap_utils::parse_required(matches, WALLET_PASSPHRASE_FLAG)?;
+    let wallet_password_path: PathBuf = clap_utils::parse_required(matches, WALLET_PASSWORD_FLAG)?;
     let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
         matches,
         VALIDATOR_DIR_FLAG,

account_manager/src/validator/import.rs

@@ -6,6 +6,7 @@ use account_utils::{
         recursively_find_voting_keystores, ValidatorDefinition, ValidatorDefinitions,
         CONFIG_FILENAME,
     },
+    ZeroizeString,
 };
 use clap::{App, Arg, ArgMatches};
 use std::fs;
@@ -17,6 +18,7 @@ pub const CMD: &str = "import";
 pub const KEYSTORE_FLAG: &str = "keystore";
 pub const DIR_FLAG: &str = "directory";
 pub const STDIN_PASSWORD_FLAG: &str = "stdin-passwords";
+pub const REUSE_PASSWORD_FLAG: &str = "reuse-password";
 
 pub const PASSWORD_PROMPT: &str = "Enter the keystore password, or press enter to omit it:";
 pub const KEYSTORE_REUSE_WARNING: &str = "DO NOT USE THE ORIGINAL KEYSTORES TO VALIDATE WITH \
@@ -68,6 +70,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .long(STDIN_PASSWORD_FLAG)
                 .help("If present, read passwords from stdin instead of tty."),
         )
+        .arg(
+            Arg::with_name(REUSE_PASSWORD_FLAG)
+                .long(REUSE_PASSWORD_FLAG)
+                .help("If present, the same password will be used for all imported keystores."),
+        )
 }
 
 pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
@@ -79,6 +86,7 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         PathBuf::new().join(".lighthouse").join("validators"),
     )?;
     let stdin_password = matches.is_present(STDIN_PASSWORD_FLAG);
+    let reuse_password = matches.is_present(REUSE_PASSWORD_FLAG);
 
     ensure_dir_exists(&validator_dir)?;
 
@@ -118,7 +126,9 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
     // - Add the keystore to the validator definitions file.
     //
     // Skip keystores that already exist, but exit early if any operation fails.
+    // Reuses the same password for all keystores if the `REUSE_PASSWORD_FLAG` flag is set.
     let mut num_imported_keystores = 0;
+    let mut previous_password: Option<ZeroizeString> = None;
     for src_keystore in &keystore_paths {
         let keystore = Keystore::from_json_file(src_keystore)
             .map_err(|e| format!("Unable to read keystore JSON {:?}: {:?}", src_keystore, e))?;
@@ -136,6 +146,10 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         );
 
         let password_opt = loop {
+            if let Some(password) = previous_password.clone() {
+                eprintln!("Reuse previous password.");
+                break Some(password);
+            }
             eprintln!("");
             eprintln!("{}", PASSWORD_PROMPT);
 
@@ -152,6 +166,9 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
                     eprintln!("Password is correct.");
                     eprintln!("");
                     sleep(Duration::from_secs(1)); // Provides nicer UX.
+                    if reuse_password {
+                        previous_password = Some(password.clone());
+                    }
                     break Some(password);
                 }
                 Err(eth2_keystore::Error::InvalidPassword) => {

account_manager/src/validator/mod.rs

@@ -2,6 +2,7 @@ pub mod create;
 pub mod deposit;
 pub mod import;
 pub mod list;
+pub mod recover;
 
 use crate::common::base_wallet_dir;
 use clap::{App, Arg, ArgMatches};
@@ -24,6 +25,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         .subcommand(deposit::cli_app())
         .subcommand(import::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
 }
 
 pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
@@ -34,6 +36,7 @@ pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<
         (deposit::CMD, Some(matches)) => deposit::cli_run::<T>(matches, env),
         (import::CMD, Some(matches)) => import::cli_run(matches),
         (list::CMD, Some(matches)) => list::cli_run(matches),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/validator/recover.rs

@@ -0,0 +1,156 @@
+use super::create::STORE_WITHDRAW_FLAG;
+use super::import::STDIN_PASSWORD_FLAG;
+use crate::common::{ensure_dir_exists, read_mnemonic_from_cli};
+use crate::validator::create::COUNT_FLAG;
+use crate::{SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG};
+use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
+use account_utils::random_password;
+use clap::{App, Arg, ArgMatches};
+use eth2_wallet::bip39::Seed;
+use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType, ValidatorKeystores};
+use std::path::PathBuf;
+use validator_dir::Builder as ValidatorDirBuilder;
+pub const CMD: &str = "recover";
+pub const FIRST_INDEX_FLAG: &str = "first-index";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Recovers validator private keys given a BIP-39 mnemonic phrase. \
+            If you did not specify a `--first-index` or count `--count`, by default this will \
+            only recover the keys associated with the validator at index 0 for an HD wallet \
+            in accordance with the EIP-2333 spec.")
+        .arg(
+            Arg::with_name(FIRST_INDEX_FLAG)
+                .long(FIRST_INDEX_FLAG)
+                .value_name("FIRST_INDEX")
+                .help("The first of consecutive key indexes you wish to recover.")
+                .takes_value(true)
+                .required(false)
+                .default_value("0"),
+        )
+        .arg(
+            Arg::with_name(COUNT_FLAG)
+                .long(COUNT_FLAG)
+                .value_name("COUNT")
+                .help("The number of validator keys you wish to recover. Counted consecutively from the provided `--first_index`.")
+                .takes_value(true)
+                .required(false)
+                .default_value("1"),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help(
+                    "If present, the mnemonic will be read in from this file.",
+                )
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name(VALIDATOR_DIR_FLAG)
+                .long(VALIDATOR_DIR_FLAG)
+                .value_name("VALIDATOR_DIRECTORY")
+                .help(
+                    "The path where the validator directories will be created. \
+                    Defaults to ~/.lighthouse/validators",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(SECRETS_DIR_FLAG)
+                .long(SECRETS_DIR_FLAG)
+                .value_name("SECRETS_DIR")
+                .help(
+                    "The path where the validator keystore passwords will be stored. \
+                    Defaults to ~/.lighthouse/secrets",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STORE_WITHDRAW_FLAG)
+                .long(STORE_WITHDRAW_FLAG)
+                .help(
+                    "If present, the withdrawal keystore will be stored alongside the voting \
+                    keypair. It is generally recommended to *not* store the withdrawal key and \
+                    instead generate them from the wallet seed when required.",
+                ),
+        )
+        .arg(
+            Arg::with_name(STDIN_PASSWORD_FLAG)
+                .long(STDIN_PASSWORD_FLAG)
+                .help("If present, read passwords from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
+    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
+        matches,
+        VALIDATOR_DIR_FLAG,
+        PathBuf::new().join(".lighthouse").join("validators"),
+    )?;
+    let secrets_dir = clap_utils::parse_path_with_default_in_home_dir(
+        matches,
+        SECRETS_DIR_FLAG,
+        PathBuf::new().join(".lighthouse").join("secrets"),
+    )?;
+    let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
+    let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?;
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_password = matches.is_present(STDIN_PASSWORD_FLAG);
+
+    ensure_dir_exists(&validator_dir)?;
+    ensure_dir_exists(&secrets_dir)?;
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_password)?;
+
+    let seed = Seed::new(&mnemonic, "");
+
+    for index in first_index..first_index + count {
+        let voting_password = random_password();
+        let withdrawal_password = random_password();
+
+        let derive = |key_type: KeyType, password: &[u8]| -> Result<Keystore, String> {
+            let (secret, path) =
+                recover_validator_secret_from_mnemonic(seed.as_bytes(), index, key_type)
+                    .map_err(|e| format!("Unable to recover validator keys: {:?}", e))?;
+
+            let keypair = keypair_from_secret(secret.as_bytes())
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?;
+
+            KeystoreBuilder::new(&keypair, password, format!("{}", path))
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?
+                .build()
+                .map_err(|e| format!("Unable build keystore: {:?}", e))
+        };
+
+        let keystores = ValidatorKeystores {
+            voting: derive(KeyType::Voting, voting_password.as_bytes())?,
+            withdrawal: derive(KeyType::Withdrawal, withdrawal_password.as_bytes())?,
+        };
+
+        let voting_pubkey = keystores.voting.pubkey().to_string();
+
+        ValidatorDirBuilder::new(validator_dir.clone(), secrets_dir.clone())
+            .voting_keystore(keystores.voting, voting_password.as_bytes())
+            .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
+            .store_withdrawal_keystore(matches.is_present(STORE_WITHDRAW_FLAG))
+            .build()
+            .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
+
+        println!(
+            "{}/{}\tIndex: {}\t0x{}",
+            index - first_index,
+            count - first_index,
+            index,
+            voting_pubkey
+        );
+    }
+
+    Ok(())
+}

account_manager/src/wallet/create.rs

@@ -5,7 +5,7 @@ use eth2_wallet::{
     bip39::{Language, Mnemonic, MnemonicType},
     PlainText,
 };
-use eth2_wallet_manager::{WalletManager, WalletType};
+use eth2_wallet_manager::{LockedWallet, WalletManager, WalletType};
 use std::ffi::OsStr;
 use std::fs::{self, File};
 use std::io::prelude::*;
@@ -15,7 +15,7 @@ use std::path::{Path, PathBuf};
 pub const CMD: &str = "create";
 pub const HD_TYPE: &str = "hd";
 pub const NAME_FLAG: &str = "name";
-pub const PASSPHRASE_FLAG: &str = "passphrase-file";
+pub const PASSWORD_FLAG: &str = "password-file";
 pub const TYPE_FLAG: &str = "type";
 pub const MNEMONIC_FLAG: &str = "mnemonic-output-path";
 
@@ -34,8 +34,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .required(true),
         )
         .arg(
-            Arg::with_name(PASSPHRASE_FLAG)
-                .long(PASSPHRASE_FLAG)
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help(
                     "A path to a file containing the password which will unlock the wallet. \
@@ -70,46 +70,14 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 }
 
 pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
-    let name: String = clap_utils::parse_required(matches, NAME_FLAG)?;
-    let wallet_password_path: PathBuf = clap_utils::parse_required(matches, PASSPHRASE_FLAG)?;
     let mnemonic_output_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
-    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
-
-    let wallet_type = match type_field.as_ref() {
-        HD_TYPE => WalletType::Hd,
-        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
-    };
-
-    let mgr = WalletManager::open(&base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
 
     // Create a new random mnemonic.
     //
     // The `tiny-bip39` crate uses `thread_rng()` for this entropy.
     let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English);
 
-    // Create a random password if the file does not exist.
-    if !wallet_password_path.exists() {
-        // To prevent users from accidentally supplying their password to the PASSPHRASE_FLAG and
-        // create a file with that name, we require that the password has a .pass suffix.
-        if wallet_password_path.extension() != Some(&OsStr::new("pass")) {
-            return Err(format!(
-                "Only creates a password file if that file ends in .pass: {:?}",
-                wallet_password_path
-            ));
-        }
-
-        create_with_600_perms(&wallet_password_path, random_password().as_bytes())
-            .map_err(|e| format!("Unable to write to {:?}: {:?}", wallet_password_path, e))?;
-    }
-
-    let wallet_password = fs::read(&wallet_password_path)
-        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
-        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
-
-    let wallet = mgr
-        .create_wallet(name, wallet_type, &mnemonic, wallet_password.as_bytes())
-        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    let wallet = create_wallet_from_mnemonic(matches, &base_dir.as_path(), &mnemonic)?;
 
     if let Some(path) = mnemonic_output_path {
         create_with_600_perms(&path, mnemonic.phrase().as_bytes())
@@ -140,6 +108,48 @@ pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
     Ok(())
 }
 
+pub fn create_wallet_from_mnemonic(
+    matches: &ArgMatches,
+    base_dir: &Path,
+    mnemonic: &Mnemonic,
+) -> Result<LockedWallet, String> {
+    let name: String = clap_utils::parse_required(matches, NAME_FLAG)?;
+    let wallet_password_path: PathBuf = clap_utils::parse_required(matches, PASSWORD_FLAG)?;
+    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
+
+    let wallet_type = match type_field.as_ref() {
+        HD_TYPE => WalletType::Hd,
+        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
+    };
+
+    let mgr = WalletManager::open(&base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+
+    // Create a random password if the file does not exist.
+    if !wallet_password_path.exists() {
+        // To prevent users from accidentally supplying their password to the PASSWORD_FLAG and
+        // create a file with that name, we require that the password has a .pass suffix.
+        if wallet_password_path.extension() != Some(&OsStr::new("pass")) {
+            return Err(format!(
+                "Only creates a password file if that file ends in .pass: {:?}",
+                wallet_password_path
+            ));
+        }
+
+        create_with_600_perms(&wallet_password_path, random_password().as_bytes())
+            .map_err(|e| format!("Unable to write to {:?}: {:?}", wallet_password_path, e))?;
+    }
+
+    let wallet_password = fs::read(&wallet_password_path)
+        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
+        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
+
+    let wallet = mgr
+        .create_wallet(name, wallet_type, &mnemonic, wallet_password.as_bytes())
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    Ok(wallet)
+}
+
 /// Creates a file with `600 (-rw-------)` permissions.
 pub fn create_with_600_perms<P: AsRef<Path>>(path: P, bytes: &[u8]) -> Result<(), String> {
     let path = path.as_ref();

account_manager/src/wallet/mod.rs

@@ -1,5 +1,6 @@
 pub mod create;
 pub mod list;
+pub mod recover;
 
 use crate::{
     common::{base_wallet_dir, ensure_dir_exists},
@@ -21,6 +22,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
         .subcommand(create::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
 }
 
 pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
@@ -30,6 +32,7 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
     match matches.subcommand() {
         (create::CMD, Some(matches)) => create::cli_run(matches, base_dir),
         (list::CMD, Some(_)) => list::cli_run(base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, base_dir),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/wallet/recover.rs

@@ -0,0 +1,87 @@
+use crate::common::read_mnemonic_from_cli;
+use crate::wallet::create::create_wallet_from_mnemonic;
+use crate::wallet::create::{HD_TYPE, NAME_FLAG, PASSWORD_FLAG, TYPE_FLAG};
+use clap::{App, Arg, ArgMatches};
+use std::path::PathBuf;
+
+pub const CMD: &str = "recover";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+pub const STDIN_PASSWORD_FLAG: &str = "stdin-passwords";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Recovers an EIP-2386 wallet from a given a BIP-39 mnemonic phrase.")
+        .arg(
+            Arg::with_name(NAME_FLAG)
+                .long(NAME_FLAG)
+                .value_name("WALLET_NAME")
+                .help(
+                    "The wallet will be created with this name. It is not allowed to \
+                            create two wallets with the same name for the same --base-dir.",
+                )
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
+                .value_name("PASSWORD_FILE_PATH")
+                .help(
+                    "This will be the new password for your recovered wallet. \
+                    A path to a file containing the password which will unlock the wallet. \
+                    If the file does not exist, a random password will be generated and \
+                    saved at that path. To avoid confusion, if the file does not already \
+                    exist it must include a '.pass' suffix.",
+                )
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help("If present, the mnemonic will be read in from this file.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(TYPE_FLAG)
+                .long(TYPE_FLAG)
+                .value_name("WALLET_TYPE")
+                .help(
+                    "The type of wallet to create. Only HD (hierarchical-deterministic) \
+                            wallets are supported presently..",
+                )
+                .takes_value(true)
+                .possible_values(&[HD_TYPE])
+                .default_value(HD_TYPE),
+        )
+        .arg(
+            Arg::with_name(STDIN_PASSWORD_FLAG)
+                .long(STDIN_PASSWORD_FLAG)
+                .help("If present, read passwords from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_password = matches.is_present(STDIN_PASSWORD_FLAG);
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_password)?;
+
+    let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+    println!("Your wallet has been successfully recovered.");
+    println!();
+    println!("Your wallet's UUID is:");
+    println!();
+    println!("\t{}", wallet.wallet().uuid());
+    println!();
+    println!("You do not need to backup your UUID or keep it secret.");
+
+    Ok(())
+}

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "0.2.2"
+version = "0.2.9"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
 edition = "2018"
 

beacon_node/beacon_chain/Cargo.toml

@@ -11,6 +11,7 @@ participation_metrics = []  # Exposes validator participation metrics to Prometh
 
 [dev-dependencies]
 int_to_bytes = { path = "../../consensus/int_to_bytes" }
+maplit = "1.0.2"
 
 [dependencies]
 eth2_config = { path = "../../common/eth2_config" }
@@ -45,6 +46,7 @@ futures = "0.3.5"
 genesis = { path = "../genesis" }
 integer-sqrt = "0.1.3"
 rand = "0.7.3"
+rand_core = "0.5.1"
 proto_array = { path = "../../consensus/proto_array" }
 lru = "0.5.1"
 tempfile = "3.1.0"
@@ -56,3 +58,4 @@ environment = { path = "../../lighthouse/environment" }
 bus = "2.2.3"
 derivative = "2.1.1"
 itertools = "0.9.0"
+regex = "1.3.9"

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -220,6 +220,12 @@ pub enum Error {
     ///
     /// The peer has sent an invalid message.
     Invalid(AttestationValidationError),
+    /// The attestation head block is too far behind the attestation slot, causing many skip slots.
+    /// This is deemed a DoS risk.
+    TooManySkippedSlots {
+        head_block_slot: Slot,
+        attestation_slot: Slot,
+    },
     /// There was an error whilst processing the attestation. It is not known if it is valid or invalid.
     ///
     /// ## Peer scoring
@@ -319,6 +325,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         }?;
 
         // Ensure the block being voted for (attestation.data.beacon_block_root) passes validation.
+        // Don't enforce the skip slot restriction for aggregates.
         //
         // This indirectly checks to see if the `attestation.data.beacon_block_root` is in our fork
         // choice. Any known, non-finalized, processed block should be in fork choice, so this
@@ -327,7 +334,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         //
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
-        verify_head_block_is_known(chain, &attestation)?;
+        verify_head_block_is_known(chain, &attestation, None)?;
 
         // Ensure that the attestation has participants.
         if attestation.aggregation_bits.is_zero() {
@@ -433,7 +440,9 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
-        verify_head_block_is_known(chain, &attestation)?;
+        //
+        // Enforce a maximum skip distance for unaggregated attestations.
+        verify_head_block_is_known(chain, &attestation, chain.config.import_max_skip_slots)?;
 
         let (indexed_attestation, committees_per_slot) =
             obtain_indexed_attestation_and_committees_per_slot(chain, &attestation)?;
@@ -531,12 +540,22 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 fn verify_head_block_is_known<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
+    max_skip_slots: Option<u64>,
 ) -> Result<(), Error> {
-    if chain
+    if let Some(block) = chain
         .fork_choice
         .read()
-        .contains_block(&attestation.data.beacon_block_root)
+        .get_block(&attestation.data.beacon_block_root)
     {
+        // Reject any block that exceeds our limit on skipped slots.
+        if let Some(max_skip_slots) = max_skip_slots {
+            if attestation.data.slot > block.slot + max_skip_slots {
+                return Err(Error::TooManySkippedSlots {
+                    head_block_slot: block.slot,
+                    attestation_slot: attestation.data.slot,
+                });
+            }
+        }
         Ok(())
     } else {
         Err(Error::UnknownHeadBlock {
@@ -775,7 +794,12 @@ where
             metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_READ_TIMES);
 
         let mut state = chain
-            .get_state(&target_block.state_root, Some(target_block.slot))?
+            .store
+            .get_inconsistent_state_for_attestation_verification_only(
+                &target_block.state_root,
+                Some(target_block.slot),
+            )
+            .map_err(BeaconChainError::from)?
             .ok_or_else(|| BeaconChainError::MissingBeaconState(target_block.state_root))?;
 
         metrics::stop_timer(state_read_timer);

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -3,9 +3,11 @@ use crate::attestation_verification::{
     VerifiedUnaggregatedAttestation,
 };
 use crate::block_verification::{
-    check_block_relevancy, get_block_root, signature_verify_chain_segment, BlockError,
-    FullyVerifiedBlock, GossipVerifiedBlock, IntoFullyVerifiedBlock,
+    check_block_is_finalized_descendant, check_block_relevancy, get_block_root,
+    signature_verify_chain_segment, BlockError, FullyVerifiedBlock, GossipVerifiedBlock,
+    IntoFullyVerifiedBlock,
 };
+use crate::chain_config::ChainConfig;
 use crate::errors::{BeaconChainError as Error, BlockProductionError};
 use crate::eth1_chain::{Eth1Chain, Eth1ChainBackend};
 use crate::events::{EventHandler, EventKind};
@@ -29,6 +31,7 @@ use fork_choice::ForkChoice;
 use itertools::process_results;
 use operation_pool::{OperationPool, PersistedOperationPool};
 use parking_lot::RwLock;
+use regex::bytes::Regex;
 use slog::{crit, debug, error, info, trace, warn, Logger};
 use slot_clock::SlotClock;
 use state_processing::{
@@ -70,7 +73,6 @@ pub const ETH1_CACHE_DB_KEY: [u8; 32] = [0; 32];
 pub const FORK_CHOICE_DB_KEY: [u8; 32] = [0; 32];
 
 /// The result of a chain segment processing.
-#[derive(Debug)]
 pub enum ChainSegmentResult<T: EthSpec> {
     /// Processing this chain segment finished successfully.
     Successful { imported_blocks: usize },
@@ -161,6 +163,8 @@ pub trait BeaconChainTypes: Send + Sync + 'static {
 /// operations and chooses a canonical head.
 pub struct BeaconChain<T: BeaconChainTypes> {
     pub spec: ChainSpec,
+    /// Configuration for `BeaconChain` runtime behaviour.
+    pub config: ChainConfig,
     /// Persistent storage for blocks, states, etc. Typically an on-disk store, such as LevelDB.
     pub store: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
     /// Database migrator for running background maintenance on the store.
@@ -423,6 +427,17 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         Ok(iter)
     }
 
+    /// As for `rev_iter_state_roots` but starting from an arbitrary `BeaconState`.
+    pub fn rev_iter_state_roots_from<'a>(
+        &self,
+        state_root: Hash256,
+        state: &'a BeaconState<T::EthSpec>,
+    ) -> impl Iterator<Item = Result<(Hash256, Slot), Error>> + 'a {
+        std::iter::once(Ok((state_root, state.slot)))
+            .chain(StateRootsIterator::new(self.store.clone(), state))
+            .map(|result| result.map_err(Into::into))
+    }
+
     /// Returns the block at the given slot, if any. Only returns blocks in the canonical chain.
     ///
     /// ## Errors
@@ -476,30 +491,36 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// is the state as it was when the head block was received, which could be some slots prior to
     /// now.
     pub fn head(&self) -> Result<BeaconSnapshot<T::EthSpec>, Error> {
-        self.canonical_head
+        self.with_head(|head| Ok(head.clone_with_only_committee_caches()))
+    }
+
+    /// Apply a function to the canonical head without cloning it.
+    pub fn with_head<U>(
+        &self,
+        f: impl FnOnce(&BeaconSnapshot<T::EthSpec>) -> Result<U, Error>,
+    ) -> Result<U, Error> {
+        let head_lock = self
+            .canonical_head
             .try_read_for(HEAD_LOCK_TIMEOUT)
-            .ok_or_else(|| Error::CanonicalHeadLockTimeout)
-            .map(|v| v.clone_with_only_committee_caches())
+            .ok_or_else(|| Error::CanonicalHeadLockTimeout)?;
+        f(&head_lock)
     }
 
     /// Returns info representing the head block and state.
     ///
     /// A summarized version of `Self::head` that involves less cloning.
     pub fn head_info(&self) -> Result<HeadInfo, Error> {
-        let head = self
-            .canonical_head
-            .try_read_for(HEAD_LOCK_TIMEOUT)
-            .ok_or_else(|| Error::CanonicalHeadLockTimeout)?;
-
-        Ok(HeadInfo {
-            slot: head.beacon_block.slot(),
-            block_root: head.beacon_block_root,
-            state_root: head.beacon_state_root,
-            current_justified_checkpoint: head.beacon_state.current_justified_checkpoint,
-            finalized_checkpoint: head.beacon_state.finalized_checkpoint,
-            fork: head.beacon_state.fork,
-            genesis_time: head.beacon_state.genesis_time,
-            genesis_validators_root: head.beacon_state.genesis_validators_root,
+        self.with_head(|head| {
+            Ok(HeadInfo {
+                slot: head.beacon_block.slot(),
+                block_root: head.beacon_block_root,
+                state_root: head.beacon_state_root,
+                current_justified_checkpoint: head.beacon_state.current_justified_checkpoint,
+                finalized_checkpoint: head.beacon_state.finalized_checkpoint,
+                fork: head.beacon_state.fork,
+                genesis_time: head.beacon_state.genesis_time,
+                genesis_validators_root: head.beacon_state.genesis_validators_root,
+            })
         })
     }
 
@@ -832,16 +853,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         if state.slot > slot {
             return Err(Error::CannotAttestToFutureState);
-        } else if state.current_epoch() + 1 < epoch {
+        } else if state.current_epoch() < epoch {
             let mut_state = state.to_mut();
-            while mut_state.current_epoch() + 1 < epoch {
+            while mut_state.current_epoch() < epoch {
                 // Note: here we provide `Hash256::zero()` as the root of the current state. This
                 // has the effect of setting the values of all historic state roots to the zero
                 // hash. This is an optimization, we don't need the state roots so why calculate
                 // them?
                 per_slot_processing(mut_state, Some(Hash256::zero()), &self.spec)?;
             }
-            mut_state.build_committee_cache(RelativeEpoch::Next, &self.spec)?;
+            mut_state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
         }
 
         let committee_len = state.get_beacon_committee(slot, index)?.committee.len();
@@ -1215,6 +1236,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 // However, we will potentially get a `ParentUnknown` on a later block. The sync
                 // protocol will need to ensure this is handled gracefully.
                 Err(BlockError::WouldRevertFinalizedSlot { .. }) => continue,
+                // The block has a known parent that does not descend from the finalized block.
+                // There is no need to process this block or any children.
+                Err(BlockError::NotFinalizedDescendant { block_parent_root }) => {
+                    return ChainSegmentResult::Failed {
+                        imported_blocks,
+                        error: BlockError::NotFinalizedDescendant { block_parent_root },
+                    }
+                }
                 // If there was an error whilst determining if the block was invalid, return that
                 // error.
                 Err(BlockError::BeaconChainError(e)) => {
@@ -1291,8 +1320,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         block: SignedBeaconBlock<T::EthSpec>,
     ) -> Result<GossipVerifiedBlock<T>, BlockError<T::EthSpec>> {
         let slot = block.message.slot;
-        let graffiti_string = String::from_utf8(block.message.body.graffiti[..].to_vec())
-            .unwrap_or_else(|_| format!("{:?}", &block.message.body.graffiti[..]));
+        #[allow(clippy::invalid_regex)]
+        let re = Regex::new("\\p{C}").expect("regex is valid");
+        let graffiti_string =
+            String::from_utf8_lossy(&re.replace_all(&block.message.body.graffiti[..], &b""[..]))
+                .to_string();
 
         match GossipVerifiedBlock::new(block, self) {
             Ok(verified) => {
@@ -1310,7 +1342,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 debug!(
                     self.log,
                     "Rejected gossip block";
-                    "error" => format!("{:?}", e),
+                    "error" => e.to_string(),
                     "graffiti" => graffiti_string,
                     "slot" => slot,
                 );
@@ -1393,11 +1425,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 trace!(
                     self.log,
                     "Beacon block rejected";
-                    "reason" => format!("{:?}", other),
+                    "reason" => other.to_string(),
                 );
 
                 let _ = self.event_handler.register(EventKind::BeaconBlockRejected {
-                    reason: format!("Invalid block: {:?}", other),
+                    reason: format!("Invalid block: {}", other),
                     block: Box::new(block),
                 });
 
@@ -1416,7 +1448,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         fully_verified_block: FullyVerifiedBlock<T>,
     ) -> Result<Hash256, BlockError<T::EthSpec>> {
         let signed_block = fully_verified_block.block;
-        let block = &signed_block.message;
         let block_root = fully_verified_block.block_root;
         let state = fully_verified_block.state;
         let parent_block = fully_verified_block.parent_block;
@@ -1428,7 +1459,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         // Iterate through the attestations in the block and register them as an "observed
         // attestation". This will stop us from propagating them on the gossip network.
-        for a in &block.body.attestations {
+        for a in &signed_block.message.body.attestations {
             match self.observed_attestations.observe_attestation(a, None) {
                 // If the observation was successful or if the slot for the attestation was too
                 // low, continue.
@@ -1478,6 +1509,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         let mut fork_choice = self.fork_choice.write();
 
+        // Do not import a block that doesn't descend from the finalized root.
+        let signed_block =
+            check_block_is_finalized_descendant::<T, _>(signed_block, &fork_choice, &self.store)?;
+        let block = &signed_block.message;
+
         // Register the new block with the fork choice service.
         {
             let _fork_choice_block_timer =
@@ -1565,12 +1601,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         &self,
         randao_reveal: Signature,
         slot: Slot,
+        validator_graffiti: Option<Graffiti>,
     ) -> Result<BeaconBlockAndState<T::EthSpec>, BlockProductionError> {
         let state = self
             .state_at_slot(slot - 1, StateSkipConfig::WithStateRoots)
             .map_err(|_| BlockProductionError::UnableToProduceAtSlot(slot))?;
 
-        self.produce_block_on_state(state, slot, randao_reveal)
+        self.produce_block_on_state(state, slot, randao_reveal, validator_graffiti)
     }
 
     /// Produce a block for some `slot` upon the given `state`.
@@ -1586,6 +1623,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         mut state: BeaconState<T::EthSpec>,
         produce_at_slot: Slot,
         randao_reveal: Signature,
+        validator_graffiti: Option<Graffiti>,
     ) -> Result<BeaconBlockAndState<T::EthSpec>, BlockProductionError> {
         metrics::inc_counter(&metrics::BLOCK_PRODUCTION_REQUESTS);
         let timer = metrics::start_timer(&metrics::BLOCK_PRODUCTION_TIMES);
@@ -1653,6 +1691,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             }
         }
 
+        // Override the beacon node's graffiti with graffiti from the validator, if present.
+        let graffiti = match validator_graffiti {
+            Some(graffiti) => graffiti,
+            None => self.graffiti,
+        };
+
         let mut block = SignedBeaconBlock {
             message: BeaconBlock {
                 slot: state.slot,
@@ -1662,7 +1706,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 body: BeaconBlockBody {
                     randao_reveal,
                     eth1_data,
-                    graffiti: self.graffiti,
+                    graffiti,
                     proposer_slashings: proposer_slashings.into(),
                     attester_slashings: attester_slashings.into(),
                     attestations: self
@@ -1723,7 +1767,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let beacon_block_root = self.fork_choice.write().get_head(self.slot()?)?;
 
         let current_head = self.head_info()?;
-        let old_finalized_root = current_head.finalized_checkpoint.root;
+        let old_finalized_checkpoint = current_head.finalized_checkpoint;
 
         if beacon_block_root == current_head.block_root {
             return Ok(());
@@ -1803,15 +1847,32 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             );
         };
 
-        let old_finalized_epoch = current_head.finalized_checkpoint.epoch;
-        let new_finalized_epoch = new_head.beacon_state.finalized_checkpoint.epoch;
-        let finalized_root = new_head.beacon_state.finalized_checkpoint.root;
+        let new_finalized_checkpoint = new_head.beacon_state.finalized_checkpoint;
+        // State root of the finalized state on the epoch boundary, NOT the state
+        // of the finalized block. We need to use an iterator in case the state is beyond
+        // the reach of the new head's `state_roots` array.
+        let new_finalized_slot = new_finalized_checkpoint
+            .epoch
+            .start_slot(T::EthSpec::slots_per_epoch());
+        let new_finalized_state_root = process_results(
+            StateRootsIterator::new(self.store.clone(), &new_head.beacon_state),
+            |mut iter| {
+                iter.find_map(|(state_root, slot)| {
+                    if slot == new_finalized_slot {
+                        Some(state_root)
+                    } else {
+                        None
+                    }
+                })
+            },
+        )?
+        .ok_or_else(|| Error::MissingFinalizedStateRoot(new_finalized_slot))?;
 
         // It is an error to try to update to a head with a lesser finalized epoch.
-        if new_finalized_epoch < old_finalized_epoch {
+        if new_finalized_checkpoint.epoch < old_finalized_checkpoint.epoch {
             return Err(Error::RevertedFinalizedEpoch {
-                previous_epoch: old_finalized_epoch,
-                new_epoch: new_finalized_epoch,
+                previous_epoch: old_finalized_checkpoint.epoch,
+                new_epoch: new_finalized_checkpoint.epoch,
             });
         }
 
@@ -1850,11 +1911,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 );
             });
 
-        if new_finalized_epoch != old_finalized_epoch {
+        if new_finalized_checkpoint.epoch != old_finalized_checkpoint.epoch {
             self.after_finalization(
-                old_finalized_epoch,
-                finalized_root,
-                old_finalized_root.into(),
+                old_finalized_checkpoint,
+                new_finalized_checkpoint,
+                new_finalized_state_root,
             )?;
         }
 
@@ -1882,68 +1943,53 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Performs pruning and finality-based optimizations.
     fn after_finalization(
         &self,
-        old_finalized_epoch: Epoch,
-        finalized_block_root: Hash256,
-        old_finalized_root: SignedBeaconBlockHash,
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+        new_finalized_state_root: Hash256,
     ) -> Result<(), Error> {
-        let finalized_block = self
-            .store
-            .get_block(&finalized_block_root)?
-            .ok_or_else(|| Error::MissingBeaconBlock(finalized_block_root))?
-            .message;
+        self.fork_choice.write().prune()?;
 
-        let new_finalized_epoch = finalized_block.slot.epoch(T::EthSpec::slots_per_epoch());
+        self.observed_block_producers.prune(
+            new_finalized_checkpoint
+                .epoch
+                .start_slot(T::EthSpec::slots_per_epoch()),
+        );
 
-        if new_finalized_epoch < old_finalized_epoch {
-            Err(Error::RevertedFinalizedEpoch {
-                previous_epoch: old_finalized_epoch,
-                new_epoch: new_finalized_epoch,
+        self.snapshot_cache
+            .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
+            .map(|mut snapshot_cache| {
+                snapshot_cache.prune(new_finalized_checkpoint.epoch);
             })
-        } else {
-            self.fork_choice.write().prune()?;
-
-            self.observed_block_producers
-                .prune(new_finalized_epoch.start_slot(T::EthSpec::slots_per_epoch()));
-
-            self.snapshot_cache
-                .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
-                .map(|mut snapshot_cache| {
-                    snapshot_cache.prune(new_finalized_epoch);
-                })
-                .unwrap_or_else(|| {
-                    error!(
-                        self.log,
-                        "Failed to obtain cache write lock";
-                        "lock" => "snapshot_cache",
-                        "task" => "prune"
-                    );
-                });
-
-            let finalized_state = self
-                .get_state(&finalized_block.state_root, Some(finalized_block.slot))?
-                .ok_or_else(|| Error::MissingBeaconState(finalized_block.state_root))?;
-
-            self.op_pool
-                .prune_all(&finalized_state, self.head_info()?.fork);
-
-            // TODO: configurable max finality distance
-            let max_finality_distance = 0;
-            self.store_migrator.process_finalization(
-                finalized_block.state_root,
-                finalized_state,
-                max_finality_distance,
-                Arc::clone(&self.head_tracker),
-                old_finalized_root,
-                finalized_block_root.into(),
-            );
-
-            let _ = self.event_handler.register(EventKind::BeaconFinalization {
-                epoch: new_finalized_epoch,
-                root: finalized_block_root,
+            .unwrap_or_else(|| {
+                error!(
+                    self.log,
+                    "Failed to obtain cache write lock";
+                    "lock" => "snapshot_cache",
+                    "task" => "prune"
+                );
             });
 
-            Ok(())
-        }
+        let finalized_state = self
+            .get_state(&new_finalized_state_root, None)?
+            .ok_or_else(|| Error::MissingBeaconState(new_finalized_state_root))?;
+
+        self.op_pool
+            .prune_all(&finalized_state, self.head_info()?.fork);
+
+        self.store_migrator.process_finalization(
+            new_finalized_state_root.into(),
+            finalized_state,
+            self.head_tracker.clone(),
+            old_finalized_checkpoint,
+            new_finalized_checkpoint,
+        )?;
+
+        let _ = self.event_handler.register(EventKind::BeaconFinalization {
+            epoch: new_finalized_checkpoint.epoch,
+            root: new_finalized_checkpoint.root,
+        });
+
+        Ok(())
     }
 
     /// Returns `true` if the given block root has not been processed.
@@ -2028,10 +2074,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .beacon_block_root;
         let mut visited: HashSet<Hash256> = HashSet::new();
         let mut finalized_blocks: HashSet<Hash256> = HashSet::new();
+        let mut justified_blocks: HashSet<Hash256> = HashSet::new();
 
         let genesis_block_hash = Hash256::zero();
         writeln!(output, "digraph beacon {{").unwrap();
-        writeln!(output, "\t_{:?}[label=\"genesis\"];", genesis_block_hash).unwrap();
+        writeln!(output, "\t_{:?}[label=\"zero\"];", genesis_block_hash).unwrap();
 
         // Canonical head needs to be processed first as otherwise finalized blocks aren't detected
         // properly.
@@ -2062,6 +2109,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         .unwrap()
                         .unwrap();
                     finalized_blocks.insert(state.finalized_checkpoint.root);
+                    justified_blocks.insert(state.current_justified_checkpoint.root);
+                    justified_blocks.insert(state.previous_justified_checkpoint.root);
                 }
 
                 if block_hash == canonical_head_hash {
@@ -2082,6 +2131,15 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         signed_beacon_block.slot()
                     )
                     .unwrap();
+                } else if justified_blocks.contains(&block_hash) {
+                    writeln!(
+                        output,
+                        "\t_{:?}[label=\"{} ({})\" shape=cds];",
+                        block_hash,
+                        block_hash,
+                        signed_beacon_block.slot()
+                    )
+                    .unwrap();
                 } else {
                     writeln!(
                         output,
@@ -2111,6 +2169,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let mut file = std::fs::File::create(file_name).unwrap();
         self.dump_as_dot(&mut file);
     }
+
+    // Should be used in tests only
+    pub fn set_graffiti(&mut self, graffiti: Graffiti) {
+        self.graffiti = graffiti;
+    }
 }
 
 impl<T: BeaconChainTypes> Drop for BeaconChain<T> {

beacon_node/beacon_chain/src/block_verification.rs

@@ -48,6 +48,7 @@ use crate::{
     },
     metrics, BeaconChain, BeaconChainError, BeaconChainTypes, BeaconSnapshot,
 };
+use fork_choice::{ForkChoice, ForkChoiceStore};
 use parking_lot::RwLockReadGuard;
 use slog::{error, Logger};
 use slot_clock::SlotClock;
@@ -62,7 +63,7 @@ use std::borrow::Cow;
 use std::convert::TryFrom;
 use std::fs;
 use std::io::Write;
-use store::{Error as DBError, HotStateSummary, StoreOp};
+use store::{Error as DBError, HotColdDB, HotStateSummary, StoreOp};
 use tree_hash::TreeHash;
 use types::{
     BeaconBlock, BeaconState, BeaconStateError, ChainSpec, CloneConfig, EthSpec, Hash256,
@@ -91,6 +92,8 @@ pub enum BlockError<T: EthSpec> {
     /// It's unclear if this block is valid, but it cannot be processed without already knowing
     /// its parent.
     ParentUnknown(Box<SignedBeaconBlock<T>>),
+    /// The block skips too many slots and is a DoS risk.
+    TooManySkippedSlots { parent_slot: Slot, block_slot: Slot },
     /// The block slot is greater than the present slot.
     ///
     /// ## Peer scoring
@@ -118,6 +121,13 @@ pub enum BlockError<T: EthSpec> {
         block_slot: Slot,
         finalized_slot: Slot,
     },
+    /// The block conflicts with finalization, no need to propagate.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this block is valid, but it conflicts with finality and shouldn't be
+    /// imported.
+    NotFinalizedDescendant { block_parent_root: Hash256 },
     /// Block is already known, no need to re-import.
     ///
     /// ## Peer scoring
@@ -199,6 +209,17 @@ pub enum BlockError<T: EthSpec> {
     BeaconChainError(BeaconChainError),
 }
 
+impl<T: EthSpec> std::fmt::Display for BlockError<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BlockError::ParentUnknown(block) => {
+                write!(f, "ParentUnknown(parent_root:{})", block.parent_root())
+            }
+            other => write!(f, "{:?}", other),
+        }
+    }
+}
+
 impl<T: EthSpec> From<BlockSignatureVerifierError> for BlockError<T> {
     fn from(e: BlockSignatureVerifierError) -> Self {
         match e {
@@ -371,9 +392,22 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
             });
         }
 
+        let block_root = get_block_root(&block);
+
         // Do not gossip a block from a finalized slot.
         check_block_against_finalized_slot(&block.message, chain)?;
 
+        // Check if the block is already known. We know it is post-finalization, so it is
+        // sufficient to check the fork choice.
+        //
+        // In normal operation this isn't necessary, however it is useful immediately after a
+        // reboot if the `observed_block_producers` cache is empty. In that case, without this
+        // check, we will load the parent and state from disk only to find out later that we
+        // already know this block.
+        if chain.fork_choice.read().contains_block(&block_root) {
+            return Err(BlockError::BlockIsAlreadyKnown);
+        }
+
         // Check that we have not already received a block with a valid signature for this slot.
         if chain
             .observed_block_producers
@@ -386,8 +420,19 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
             });
         }
 
+        // Do not process a block that doesn't descend from the finalized root.
+        //
+        // We check this *before* we load the parent so that we can return a more detailed error.
+        let block = check_block_is_finalized_descendant::<T, _>(
+            block,
+            &chain.fork_choice.read(),
+            &chain.store,
+        )?;
+
         let (mut parent, block) = load_parent(block, chain)?;
-        let block_root = get_block_root(&block);
+
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
 
         let state = cheap_state_advance_to_obtain_committees(
             &mut parent.beacon_state,
@@ -475,6 +520,10 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
         chain: &BeaconChain<T>,
     ) -> Result<Self, BlockError<T::EthSpec>> {
         let (mut parent, block) = load_parent(block, chain)?;
+
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
+
         let block_root = get_block_root(&block);
 
         let state = cheap_state_advance_to_obtain_committees(
@@ -605,6 +654,9 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
             return Err(BlockError::ParentUnknown(Box::new(block)));
         }
 
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
+
         /*
          *  Perform cursory checks to see if the block is even worth processing.
          */
@@ -644,7 +696,10 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
                 let state_root = state.update_tree_hash_cache()?;
 
                 let op = if state.slot % T::EthSpec::slots_per_epoch() == 0 {
-                    StoreOp::PutState(state_root.into(), Cow::Owned(state.clone()))
+                    StoreOp::PutState(
+                        state_root.into(),
+                        Cow::Owned(state.clone_with(CloneConfig::committee_caches_only())),
+                    )
                 } else {
                     StoreOp::PutStateSummary(
                         state_root.into(),
@@ -744,6 +799,30 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
     }
 }
 
+/// Check that the count of skip slots between the block and its parent does not exceed our maximum
+/// value.
+///
+/// Whilst this is not part of the specification, we include this to help prevent us from DoS
+/// attacks. In times of dire network circumstance, the user can configure the
+/// `import_max_skip_slots` value.
+fn check_block_skip_slots<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    parent: &BeaconBlock<T::EthSpec>,
+    block: &BeaconBlock<T::EthSpec>,
+) -> Result<(), BlockError<T::EthSpec>> {
+    // Reject any block that exceeds our limit on skipped slots.
+    if let Some(max_skip_slots) = chain.config.import_max_skip_slots {
+        if block.slot > parent.slot + max_skip_slots {
+            return Err(BlockError::TooManySkippedSlots {
+                parent_slot: parent.slot,
+                block_slot: block.slot,
+            });
+        }
+    }
+
+    Ok(())
+}
+
 /// Returns `Ok(())` if the block is later than the finalized slot on `chain`.
 ///
 /// Returns an error if the block is earlier or equal to the finalized slot, or there was an error
@@ -768,6 +847,36 @@ fn check_block_against_finalized_slot<T: BeaconChainTypes>(
     }
 }
 
+/// Returns `Ok(block)` if the block descends from the finalized root.
+pub fn check_block_is_finalized_descendant<T: BeaconChainTypes, F: ForkChoiceStore<T::EthSpec>>(
+    block: SignedBeaconBlock<T::EthSpec>,
+    fork_choice: &ForkChoice<F, T::EthSpec>,
+    store: &HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>,
+) -> Result<SignedBeaconBlock<T::EthSpec>, BlockError<T::EthSpec>> {
+    if fork_choice.is_descendant_of_finalized(block.parent_root()) {
+        Ok(block)
+    } else {
+        // If fork choice does *not* consider the parent to be a descendant of the finalized block,
+        // then there are two more cases:
+        //
+        // 1. We have the parent stored in our database. Because fork-choice has confirmed the
+        //    parent is *not* in our post-finalization DAG, all other blocks must be either
+        //    pre-finalization or conflicting with finalization.
+        // 2. The parent is unknown to us, we probably want to download it since it might actually
+        //    descend from the finalized root.
+        if store
+            .item_exists::<SignedBeaconBlock<T::EthSpec>>(&block.parent_root())
+            .map_err(|e| BlockError::BeaconChainError(e.into()))?
+        {
+            Err(BlockError::NotFinalizedDescendant {
+                block_parent_root: block.parent_root(),
+            })
+        } else {
+            Err(BlockError::ParentUnknown(Box::new(block)))
+        }
+    }
+}
+
 /// Performs simple, cheap checks to ensure that the block is relevant to be imported.
 ///
 /// `Ok(block_root)` is returned if the block passes these checks and should progress with

beacon_node/beacon_chain/src/builder.rs

@@ -11,6 +11,7 @@ use crate::shuffling_cache::ShufflingCache;
 use crate::snapshot_cache::{SnapshotCache, DEFAULT_SNAPSHOT_CACHE_SIZE};
 use crate::timeout_rw_lock::TimeoutRwLock;
 use crate::validator_pubkey_cache::ValidatorPubkeyCache;
+use crate::ChainConfig;
 use crate::{
     BeaconChain, BeaconChainTypes, BeaconForkChoiceStore, BeaconSnapshot, Eth1Chain,
     Eth1ChainBackend, EventHandler,
@@ -110,6 +111,7 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     pubkey_cache_path: Option<PathBuf>,
     validator_pubkey_cache: Option<ValidatorPubkeyCache>,
     spec: ChainSpec,
+    chain_config: ChainConfig,
     disabled_forks: Vec<String>,
     log: Option<Logger>,
     graffiti: Graffiti,
@@ -157,6 +159,7 @@ where
             disabled_forks: Vec::new(),
             validator_pubkey_cache: None,
             spec: TEthSpec::default_spec(),
+            chain_config: ChainConfig::default(),
             log: None,
             graffiti: Graffiti::default(),
         }
@@ -171,6 +174,15 @@ where
         self
     }
 
+    /// Sets the maximum number of blocks that will be skipped when processing
+    /// some consensus messages.
+    ///
+    /// Set to `None` for no limit.
+    pub fn import_max_skip_slots(mut self, n: Option<u64>) -> Self {
+        self.chain_config.import_max_skip_slots = n;
+        self
+    }
+
     /// Sets the store (database).
     ///
     /// Should generally be called early in the build chain.
@@ -406,6 +418,12 @@ where
         self
     }
 
+    /// Sets the `ChainConfig` that determines `BeaconChain` runtime behaviour.
+    pub fn chain_config(mut self, config: ChainConfig) -> Self {
+        self.chain_config = config;
+        self
+    }
+
     /// Consumes `self`, returning a `BeaconChain` if all required parameters have been supplied.
     ///
     /// An error will be returned at runtime if all required parameters have not been configured.
@@ -489,6 +507,7 @@ where
 
         let beacon_chain = BeaconChain {
             spec: self.spec,
+            config: self.chain_config,
             store,
             store_migrator: self
                 .store_migrator

beacon_node/beacon_chain/src/chain_config.rs

@@ -0,0 +1,21 @@
+use serde_derive::{Deserialize, Serialize};
+
+/// There is a 693 block skip in the current canonical Medalla chain, we use 700 to be safe.
+pub const DEFAULT_IMPORT_BLOCK_MAX_SKIP_SLOTS: u64 = 700;
+
+#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
+pub struct ChainConfig {
+    /// Maximum number of slots to skip when importing a consensus message (e.g., block,
+    /// attestation, etc).
+    ///
+    /// If `None`, there is no limit.
+    pub import_max_skip_slots: Option<u64>,
+}
+
+impl Default for ChainConfig {
+    fn default() -> Self {
+        Self {
+            import_max_skip_slots: Some(DEFAULT_IMPORT_BLOCK_MAX_SKIP_SLOTS),
+        }
+    }
+}

beacon_node/beacon_chain/src/errors.rs

@@ -1,5 +1,6 @@
 use crate::beacon_chain::ForkChoiceError;
 use crate::eth1_chain::Error as Eth1ChainError;
+use crate::migrate::PruningError;
 use crate::naive_aggregation_pool::Error as NaiveAggregationError;
 use crate::observed_attestations::Error as ObservedAttestationsError;
 use crate::observed_attesters::Error as ObservedAttestersError;
@@ -61,6 +62,7 @@ pub enum BeaconChainError {
         requested_slot: Slot,
         max_task_runtime: Duration,
     },
+    MissingFinalizedStateRoot(Slot),
     /// Returned when an internal check fails, indicating corrupt data.
     InvariantViolated(String),
     SszTypesError(SszTypesError),
@@ -79,6 +81,7 @@ pub enum BeaconChainError {
     ObservedAttestationsError(ObservedAttestationsError),
     ObservedAttestersError(ObservedAttestersError),
     ObservedBlockProducersError(ObservedBlockProducersError),
+    PruningError(PruningError),
     ArithError(ArithError),
 }
 
@@ -94,6 +97,7 @@ easy_from_to!(ObservedAttestationsError, BeaconChainError);
 easy_from_to!(ObservedAttestersError, BeaconChainError);
 easy_from_to!(ObservedBlockProducersError, BeaconChainError);
 easy_from_to!(BlockSignatureVerifierError, BeaconChainError);
+easy_from_to!(PruningError, BeaconChainError);
 easy_from_to!(ArithError, BeaconChainError);
 
 #[derive(Debug)]

beacon_node/beacon_chain/src/lib.rs

@@ -2,12 +2,17 @@
 #[macro_use]
 extern crate lazy_static;
 
+#[macro_use]
+extern crate slog;
+extern crate slog_term;
+
 pub mod attestation_verification;
 mod beacon_chain;
 mod beacon_fork_choice_store;
 mod beacon_snapshot;
 mod block_verification;
 pub mod builder;
+pub mod chain_config;
 mod errors;
 pub mod eth1_chain;
 pub mod events;
@@ -32,6 +37,7 @@ pub use self::beacon_chain::{
     ForkChoiceError, StateSkipConfig,
 };
 pub use self::beacon_snapshot::BeaconSnapshot;
+pub use self::chain_config::ChainConfig;
 pub use self::errors::{BeaconChainError, BlockProductionError};
 pub use attestation_verification::Error as AttestationError;
 pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError};

beacon_node/beacon_chain/src/migrate.rs

@@ -7,12 +7,28 @@ use std::mem;
 use std::sync::mpsc;
 use std::sync::Arc;
 use std::thread;
-use store::hot_cold_store::{process_finalization, HotColdDBError};
-use store::iter::{ParentRootBlockIterator, RootsIterator};
+use store::hot_cold_store::{migrate_database, HotColdDBError};
+use store::iter::RootsIterator;
 use store::{Error, ItemStore, StoreOp};
 pub use store::{HotColdDB, MemoryStore};
-use types::*;
-use types::{BeaconState, EthSpec, Hash256, Slot};
+use types::{
+    BeaconState, BeaconStateError, BeaconStateHash, Checkpoint, EthSpec, Hash256,
+    SignedBeaconBlockHash, Slot,
+};
+
+/// Logic errors that can occur during pruning, none of these should ever happen.
+#[derive(Debug)]
+pub enum PruningError {
+    IncorrectFinalizedState {
+        state_slot: Slot,
+        new_finalized_slot: Slot,
+    },
+    MissingInfoForCanonicalChain {
+        slot: Slot,
+    },
+    UnexpectedEqualStateRoots,
+    UnexpectedUnequalStateRoots,
+}
 
 /// Trait for migration processes that update the database upon finalization.
 pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
@@ -22,17 +38,17 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
 
     fn process_finalization(
         &self,
-        _state_root: Hash256,
+        _finalized_state_root: BeaconStateHash,
         _new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
         _head_tracker: Arc<HeadTracker>,
-        _old_finalized_block_hash: SignedBeaconBlockHash,
-        _new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
+        _old_finalized_checkpoint: Checkpoint,
+        _new_finalized_checkpoint: Checkpoint,
+    ) -> Result<(), BeaconChainError> {
+        Ok(())
     }
 
     /// Traverses live heads and prunes blocks and states of chains that we know can't be built
-    /// upon because finalization would prohibit it.  This is an optimisation intended to save disk
+    /// upon because finalization would prohibit it. This is an optimisation intended to save disk
     /// space.
     ///
     /// Assumptions:
@@ -40,37 +56,63 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
     fn prune_abandoned_forks(
         store: Arc<HotColdDB<E, Hot, Cold>>,
         head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_slot: Slot,
+        new_finalized_state_hash: BeaconStateHash,
+        new_finalized_state: &BeaconState<E>,
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+        log: &Logger,
     ) -> Result<(), BeaconChainError> {
         // There will never be any blocks to prune if there is only a single head in the chain.
         if head_tracker.heads().len() == 1 {
             return Ok(());
         }
 
-        let old_finalized_slot = store
-            .get_block(&old_finalized_block_hash.into())?
-            .ok_or_else(|| BeaconChainError::MissingBeaconBlock(old_finalized_block_hash.into()))?
-            .slot();
+        let old_finalized_slot = old_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_slot = new_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_block_hash = new_finalized_checkpoint.root.into();
 
-        // Collect hashes from new_finalized_block back to old_finalized_block (inclusive)
-        let mut found_block = false; // hack for `take_until`
-        let newly_finalized_blocks: HashMap<SignedBeaconBlockHash, Slot> =
-            ParentRootBlockIterator::new(&*store, new_finalized_block_hash.into())
-                .take_while(|result| match result {
-                    Ok((block_hash, _)) => {
-                        if found_block {
-                            false
-                        } else {
-                            found_block |= *block_hash == old_finalized_block_hash.into();
-                            true
-                        }
-                    }
-                    Err(_) => true,
-                })
-                .map(|result| result.map(|(block_hash, block)| (block_hash.into(), block.slot())))
-                .collect::<Result<_, _>>()?;
+        // The finalized state must be for the epoch boundary slot, not the slot of the finalized
+        // block.
+        if new_finalized_state.slot != new_finalized_slot {
+            return Err(PruningError::IncorrectFinalizedState {
+                state_slot: new_finalized_state.slot,
+                new_finalized_slot,
+            }
+            .into());
+        }
+
+        debug!(
+            log,
+            "Starting database pruning";
+            "old_finalized_epoch" => old_finalized_checkpoint.epoch,
+            "old_finalized_root" => format!("{:?}", old_finalized_checkpoint.root),
+            "new_finalized_epoch" => new_finalized_checkpoint.epoch,
+            "new_finalized_root" => format!("{:?}", new_finalized_checkpoint.root),
+        );
+
+        // For each slot between the new finalized checkpoint and the old finalized checkpoint,
+        // collect the beacon block root and state root of the canonical chain.
+        let newly_finalized_chain: HashMap<Slot, (SignedBeaconBlockHash, BeaconStateHash)> =
+            std::iter::once(Ok((
+                new_finalized_slot,
+                (new_finalized_block_hash, new_finalized_state_hash),
+            )))
+            .chain(
+                RootsIterator::new(store.clone(), new_finalized_state).map(|res| {
+                    res.map(|(block_root, state_root, slot)| {
+                        (slot, (block_root.into(), state_root.into()))
+                    })
+                }),
+            )
+            .take_while(|res| {
+                res.as_ref()
+                    .map_or(true, |(slot, _)| *slot >= old_finalized_slot)
+            })
+            .collect::<Result<_, _>>()?;
 
         // We don't know which blocks are shared among abandoned chains, so we buffer and delete
         // everything in one fell swoop.
@@ -79,75 +121,110 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
         let mut abandoned_heads: HashSet<Hash256> = HashSet::new();
 
         for (head_hash, head_slot) in head_tracker.heads() {
-            let mut potentially_abandoned_head: Option<Hash256> = Some(head_hash);
-            let mut potentially_abandoned_blocks: Vec<(
-                Slot,
-                Option<SignedBeaconBlockHash>,
-                Option<BeaconStateHash>,
-            )> = Vec::new();
+            let mut potentially_abandoned_head = Some(head_hash);
+            let mut potentially_abandoned_blocks = vec![];
 
             let head_state_hash = store
                 .get_block(&head_hash)?
                 .ok_or_else(|| BeaconStateError::MissingBeaconBlock(head_hash.into()))?
                 .state_root();
 
+            // Iterate backwards from this head, staging blocks and states for deletion.
             let iter = std::iter::once(Ok((head_hash, head_state_hash, head_slot)))
-                .chain(RootsIterator::from_block(Arc::clone(&store), head_hash)?);
+                .chain(RootsIterator::from_block(store.clone(), head_hash)?);
+
             for maybe_tuple in iter {
-                let (block_hash, state_hash, slot) = maybe_tuple?;
-                if slot < old_finalized_slot {
-                    // We must assume here any candidate chains include old_finalized_block_hash,
-                    // i.e. there aren't any forks starting at a block that is a strict ancestor of
-                    // old_finalized_block_hash.
-                    break;
-                }
-                match newly_finalized_blocks.get(&block_hash.into()).copied() {
-                    // Block is not finalized, mark it and its state for deletion
+                let (block_root, state_root, slot) = maybe_tuple?;
+                let block_root = SignedBeaconBlockHash::from(block_root);
+                let state_root = BeaconStateHash::from(state_root);
+
+                match newly_finalized_chain.get(&slot) {
+                    // If there's no information about a slot on the finalized chain, then
+                    // it should be because it's ahead of the new finalized slot. Stage
+                    // the fork's block and state for possible deletion.
                     None => {
-                        potentially_abandoned_blocks.push((
-                            slot,
-                            Some(block_hash.into()),
-                            Some(state_hash.into()),
-                        ));
+                        if slot > new_finalized_slot {
+                            potentially_abandoned_blocks.push((
+                                slot,
+                                Some(block_root),
+                                Some(state_root),
+                            ));
+                        } else if slot >= old_finalized_slot {
+                            return Err(PruningError::MissingInfoForCanonicalChain { slot }.into());
+                        } else {
+                            // We must assume here any candidate chains include the old finalized
+                            // checkpoint, i.e. there aren't any forks starting at a block that is a
+                            // strict ancestor of old_finalized_checkpoint.
+                            warn!(
+                                log,
+                                "Found a chain that should already have been pruned";
+                                "head_block_root" => format!("{:?}", head_hash),
+                                "head_slot" => head_slot,
+                            );
+                            break;
+                        }
                     }
-                    Some(finalized_slot) => {
-                        // Block root is finalized, and we have reached the slot it was finalized
-                        // at: we've hit a shared part of the chain.
-                        if finalized_slot == slot {
-                            // The first finalized block of a candidate chain lies after (in terms
-                            // of slots order) the newly finalized block.  It's not a candidate for
-                            // prunning.
-                            if finalized_slot == new_finalized_slot {
+                    Some((finalized_block_root, finalized_state_root)) => {
+                        // This fork descends from a newly finalized block, we can stop.
+                        if block_root == *finalized_block_root {
+                            // Sanity check: if the slot and block root match, then the
+                            // state roots should match too.
+                            if state_root != *finalized_state_root {
+                                return Err(PruningError::UnexpectedUnequalStateRoots.into());
+                            }
+
+                            // If the fork descends from the whole finalized chain,
+                            // do not prune it. Otherwise continue to delete all
+                            // of the blocks and states that have been staged for
+                            // deletion so far.
+                            if slot == new_finalized_slot {
                                 potentially_abandoned_blocks.clear();
                                 potentially_abandoned_head.take();
                             }
-
+                            // If there are skipped slots on the fork to be pruned, then
+                            // we will have just staged the common block for deletion.
+                            // Unstage it.
+                            else {
+                                for (_, block_root, _) in
+                                    potentially_abandoned_blocks.iter_mut().rev()
+                                {
+                                    if block_root.as_ref() == Some(finalized_block_root) {
+                                        *block_root = None;
+                                    } else {
+                                        break;
+                                    }
+                                }
+                            }
                             break;
-                        }
-                        // Block root is finalized, but we're at a skip slot: delete the state only.
-                        else {
+                        } else {
+                            if state_root == *finalized_state_root {
+                                return Err(PruningError::UnexpectedEqualStateRoots.into());
+                            }
                             potentially_abandoned_blocks.push((
                                 slot,
-                                None,
-                                Some(state_hash.into()),
+                                Some(block_root),
+                                Some(state_root),
                             ));
                         }
                     }
                 }
             }
 
-            abandoned_heads.extend(potentially_abandoned_head.into_iter());
-            if !potentially_abandoned_blocks.is_empty() {
+            if let Some(abandoned_head) = potentially_abandoned_head {
+                debug!(
+                    log,
+                    "Pruning head";
+                    "head_block_root" => format!("{:?}", abandoned_head),
+                    "head_slot" => head_slot,
+                );
+                abandoned_heads.insert(abandoned_head);
                 abandoned_blocks.extend(
                     potentially_abandoned_blocks
                         .iter()
                         .filter_map(|(_, maybe_block_hash, _)| *maybe_block_hash),
                 );
                 abandoned_states.extend(potentially_abandoned_blocks.iter().filter_map(
-                    |(slot, _, maybe_state_hash)| match maybe_state_hash {
-                        None => None,
-                        Some(state_hash) => Some((*slot, *state_hash)),
-                    },
+                    |(slot, _, maybe_state_hash)| maybe_state_hash.map(|sr| (*slot, sr)),
                 ));
             }
         }
@@ -161,11 +238,14 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
                     .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash, slot)),
             )
             .collect();
+
         store.do_atomically(batch)?;
         for head_hash in abandoned_heads.into_iter() {
             head_tracker.remove_head(head_hash);
         }
 
+        debug!(log, "Database pruning complete");
+
         Ok(())
     }
 }
@@ -174,6 +254,17 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
 pub struct NullMigrator;
 
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold> for NullMigrator {
+    fn process_finalization(
+        &self,
+        _finalized_state_root: BeaconStateHash,
+        _new_finalized_state: BeaconState<E>,
+        _head_tracker: Arc<HeadTracker>,
+        _old_finalized_checkpoint: Checkpoint,
+        _new_finalized_checkpoint: Checkpoint,
+    ) -> Result<(), BeaconChainError> {
+        Ok(())
+    }
+
     fn new(_: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
         NullMigrator
     }
@@ -184,48 +275,59 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold> fo
 /// Mostly useful for tests.
 pub struct BlockingMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
     db: Arc<HotColdDB<E, Hot, Cold>>,
+    log: Logger,
 }
 
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
     for BlockingMigrator<E, Hot, Cold>
 {
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
-        BlockingMigrator { db }
+    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self {
+        BlockingMigrator { db, log }
     }
 
     fn process_finalization(
         &self,
-        state_root: Hash256,
+        finalized_state_root: BeaconStateHash,
         new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
         head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
-        if let Err(e) = process_finalization(self.db.clone(), state_root, &new_finalized_state) {
-            // This migrator is only used for testing, so we just log to stderr without a logger.
-            eprintln!("Migration error: {:?}", e);
-        }
-
-        if let Err(e) = Self::prune_abandoned_forks(
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+    ) -> Result<(), BeaconChainError> {
+        Self::prune_abandoned_forks(
             self.db.clone(),
             head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_state.slot,
+            finalized_state_root,
+            &new_finalized_state,
+            old_finalized_checkpoint,
+            new_finalized_checkpoint,
+            &self.log,
+        )?;
+
+        match migrate_database(
+            self.db.clone(),
+            finalized_state_root.into(),
+            &new_finalized_state,
         ) {
-            eprintln!("Pruning error: {:?}", e);
+            Ok(()) => Ok(()),
+            Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
+                debug!(
+                    self.log,
+                    "Database migration postponed, unaligned finalized block";
+                    "slot" => slot.as_u64()
+                );
+                Ok(())
+            }
+            Err(e) => Err(e.into()),
         }
     }
 }
 
 type MpscSender<E> = mpsc::Sender<(
-    Hash256,
+    BeaconStateHash,
     BeaconState<E>,
     Arc<HeadTracker>,
-    SignedBeaconBlockHash,
-    SignedBeaconBlockHash,
-    Slot,
+    Checkpoint,
+    Checkpoint,
 )>;
 
 /// Migrator that runs a background thread to migrate state from the hot to the cold database.
@@ -243,34 +345,26 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
         Self { db, tx_thread, log }
     }
 
-    /// Perform the freezing operation on the database,
     fn process_finalization(
         &self,
-        finalized_state_root: Hash256,
+        finalized_state_root: BeaconStateHash,
         new_finalized_state: BeaconState<E>,
-        max_finality_distance: u64,
         head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
-        if !self.needs_migration(new_finalized_state.slot, max_finality_distance) {
-            return;
-        }
-
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+    ) -> Result<(), BeaconChainError> {
         let (ref mut tx, ref mut thread) = *self.tx_thread.lock();
 
-        let new_finalized_slot = new_finalized_state.slot;
         if let Err(tx_err) = tx.send((
             finalized_state_root,
             new_finalized_state,
             head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_slot,
+            old_finalized_checkpoint,
+            new_finalized_checkpoint,
         )) {
             let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
 
-            drop(mem::replace(tx, new_tx));
+            *tx = new_tx;
             let old_thread = mem::replace(thread, new_thread);
 
             // Join the old thread, which will probably have panicked, or may have
@@ -286,46 +380,43 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
             // Retry at most once, we could recurse but that would risk overflowing the stack.
             let _ = tx.send(tx_err.0);
         }
+
+        Ok(())
     }
 }
 
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
-    /// Return true if a migration needs to be performed, given a new `finalized_slot`.
-    fn needs_migration(&self, finalized_slot: Slot, max_finality_distance: u64) -> bool {
-        let finality_distance = finalized_slot - self.db.get_split_slot();
-        finality_distance > max_finality_distance
-    }
-
-    #[allow(clippy::type_complexity)]
     /// Spawn a new child thread to run the migration process.
     ///
     /// Return a channel handle for sending new finalized states to the thread.
     fn spawn_thread(
         db: Arc<HotColdDB<E, Hot, Cold>>,
         log: Logger,
-    ) -> (
-        mpsc::Sender<(
-            Hash256,
-            BeaconState<E>,
-            Arc<HeadTracker>,
-            SignedBeaconBlockHash,
-            SignedBeaconBlockHash,
-            Slot,
-        )>,
-        thread::JoinHandle<()>,
-    ) {
+    ) -> (MpscSender<E>, thread::JoinHandle<()>) {
         let (tx, rx) = mpsc::channel();
         let thread = thread::spawn(move || {
             while let Ok((
                 state_root,
                 state,
                 head_tracker,
-                old_finalized_block_hash,
-                new_finalized_block_hash,
-                new_finalized_slot,
+                old_finalized_checkpoint,
+                new_finalized_checkpoint,
             )) = rx.recv()
             {
-                match process_finalization(db.clone(), state_root, &state) {
+                match Self::prune_abandoned_forks(
+                    db.clone(),
+                    head_tracker,
+                    state_root,
+                    &state,
+                    old_finalized_checkpoint,
+                    new_finalized_checkpoint,
+                    &log,
+                ) {
+                    Ok(()) => {}
+                    Err(e) => warn!(log, "Block pruning failed: {:?}", e),
+                }
+
+                match migrate_database(db.clone(), state_root.into(), &state) {
                     Ok(()) => {}
                     Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
                         debug!(
@@ -342,17 +433,6 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                         );
                     }
                 };
-
-                match Self::prune_abandoned_forks(
-                    db.clone(),
-                    head_tracker,
-                    old_finalized_block_hash,
-                    new_finalized_block_hash,
-                    new_finalized_slot,
-                ) {
-                    Ok(()) => {}
-                    Err(e) => warn!(log, "Block pruning failed: {:?}", e),
-                }
             }
         });
 

beacon_node/beacon_chain/tests/attestation_production.rs

@@ -26,7 +26,7 @@ lazy_static! {
 fn produces_attestations() {
     let num_blocks_produced = MainnetEthSpec::slots_per_epoch() * 4;
 
-    let harness = BeaconChainHarness::new(
+    let mut harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[..].to_vec(),
         StoreConfig::default(),

beacon_node/beacon_chain/tests/attestation_verification.rs

@@ -5,7 +5,9 @@ extern crate lazy_static;
 
 use beacon_chain::{
     attestation_verification::Error as AttnError,
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, NullMigratorEphemeralHarnessType,
+    },
     BeaconChain, BeaconChainTypes,
 };
 use int_to_bytes::int_to_bytes32;
@@ -30,7 +32,7 @@ lazy_static! {
 }
 
 /// Returns a beacon chain harness.
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
+fn get_harness(validator_count: usize) -> BeaconChainHarness<NullMigratorEphemeralHarnessType<E>> {
     let harness = BeaconChainHarness::new_with_target_aggregators(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
@@ -184,8 +186,7 @@ fn get_non_aggregator<T: BeaconChainTypes>(
 /// Tests verification of `SignedAggregateAndProof` from the gossip network.
 #[test]
 fn aggregated_gossip_verification() {
-    let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -197,7 +198,7 @@ fn aggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
+    let current_slot = harness.chain.slot().expect("should get slot");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -532,8 +533,7 @@ fn aggregated_gossip_verification() {
 /// Tests the verification conditions for an unaggregated attestation on the gossip network.
 #[test]
 fn unaggregated_gossip_verification() {
-    let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -545,8 +545,8 @@ fn unaggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -772,8 +772,7 @@ fn unaggregated_gossip_verification() {
 /// This also checks that we can do a state lookup if we don't get a hit from the shuffling cache.
 #[test]
 fn attestation_that_skips_epochs() {
-    let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -782,16 +781,18 @@ fn attestation_that_skips_epochs() {
         AttestationStrategy::SomeValidators(vec![]),
     );
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     let earlier_slot = (current_epoch - 2).start_slot(MainnetEthSpec::slots_per_epoch());
-    let earlier_block = chain
+    let earlier_block = harness
+        .chain
         .block_at_slot(earlier_slot)
         .expect("should not error getting block at slot")
         .expect("should find block at slot");
 
-    let mut state = chain
+    let mut state = harness
+        .chain
         .get_state(&earlier_block.state_root(), Some(earlier_slot))
         .expect("should not error getting state")
         .expect("should find state");

beacon_node/beacon_chain/tests/block_verification.rs

@@ -4,7 +4,9 @@
 extern crate lazy_static;
 
 use beacon_chain::{
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, NullMigratorEphemeralHarnessType,
+    },
     BeaconSnapshot, BlockError,
 };
 use store::config::StoreConfig;
@@ -18,8 +20,9 @@ use types::{
 type E = MainnetEthSpec;
 
 // Should ideally be divisible by 3.
-pub const VALIDATOR_COUNT: usize = 24;
-pub const CHAIN_SEGMENT_LENGTH: usize = 64 * 5;
+const VALIDATOR_COUNT: usize = 24;
+const CHAIN_SEGMENT_LENGTH: usize = 64 * 5;
+const BLOCK_INDICES: &[usize] = &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT_LENGTH - 1];
 
 lazy_static! {
     /// A cached set of keys.
@@ -30,7 +33,7 @@ lazy_static! {
 }
 
 fn get_chain_segment() -> Vec<BeaconSnapshot<E>> {
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         CHAIN_SEGMENT_LENGTH,
@@ -47,8 +50,8 @@ fn get_chain_segment() -> Vec<BeaconSnapshot<E>> {
         .collect()
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(validator_count: usize) -> BeaconChainHarness<NullMigratorEphemeralHarnessType<E>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -80,7 +83,7 @@ fn junk_aggregate_signature() -> AggregateSignature {
 
 fn update_proposal_signatures(
     snapshots: &mut [BeaconSnapshot<E>],
-    harness: &BeaconChainHarness<HarnessType<E>>,
+    harness: &BeaconChainHarness<NullMigratorEphemeralHarnessType<E>>,
 ) {
     for snapshot in snapshots {
         let spec = &harness.chain.spec;
@@ -90,7 +93,7 @@ fn update_proposal_signatures(
             .get_beacon_proposer_index(slot, spec)
             .expect("should find proposer index");
         let keypair = harness
-            .keypairs
+            .validators_keypairs
             .get(proposer_index)
             .expect("proposer keypair should be available");
 
@@ -272,17 +275,73 @@ fn chain_segment_non_linear_slots() {
     );
 }
 
+fn assert_invalid_signature(
+    harness: &BeaconChainHarness<NullMigratorEphemeralHarnessType<E>>,
+    block_index: usize,
+    snapshots: &[BeaconSnapshot<E>],
+    item: &str,
+) {
+    let blocks = snapshots
+        .iter()
+        .map(|snapshot| snapshot.beacon_block.clone())
+        .collect();
+
+    // Ensure the block will be rejected if imported in a chain segment.
+    assert!(
+        matches!(
+            harness
+                .chain
+                .process_chain_segment(blocks)
+                .into_block_error(),
+            Err(BlockError::InvalidSignature)
+        ),
+        "should not import chain segment with an invalid {} signature",
+        item
+    );
+
+    // Ensure the block will be rejected if imported on its own (without gossip checking).
+    let ancestor_blocks = CHAIN_SEGMENT
+        .iter()
+        .take(block_index)
+        .map(|snapshot| snapshot.beacon_block.clone())
+        .collect();
+    // We don't care if this fails, we just call this to ensure that all prior blocks have been
+    // imported prior to this test.
+    let _ = harness.chain.process_chain_segment(ancestor_blocks);
+    assert!(
+        matches!(
+            harness
+                .chain
+                .process_block(snapshots[block_index].beacon_block.clone()),
+            Err(BlockError::InvalidSignature)
+        ),
+        "should not import individual block with an invalid {} signature",
+        item
+    );
+
+    // NOTE: we choose not to check gossip verification here. It only checks one signature
+    // (proposal) and that is already tested elsewhere in this file.
+    //
+    // It's not trivial to just check gossip verification since it will start refusing
+    // blocks as soon as it has seen one valid proposal signature for a given (validator,
+    // slot) tuple.
+}
+
+fn get_invalid_sigs_harness() -> BeaconChainHarness<NullMigratorEphemeralHarnessType<E>> {
+    let harness = get_harness(VALIDATOR_COUNT);
+    harness
+        .chain
+        .slot_clock
+        .set_slot(CHAIN_SEGMENT.last().unwrap().beacon_block.slot().as_u64());
+    harness
+}
 #[test]
-fn invalid_signatures() {
-    let mut checked_attestation = false;
-
-    for &block_index in &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT.len() - 1] {
-        let harness = get_harness(VALIDATOR_COUNT);
-        harness
-            .chain
-            .slot_clock
-            .set_slot(CHAIN_SEGMENT.last().unwrap().beacon_block.slot().as_u64());
-
+fn invalid_signature_gossip_block() {
+    for &block_index in BLOCK_INDICES {
+        // Ensure the block will be rejected if imported on its own (without gossip checking).
+        let harness = get_invalid_sigs_harness();
+        let mut snapshots = CHAIN_SEGMENT.clone();
+        snapshots[block_index].beacon_block.signature = junk_signature();
         // Import all the ancestors before the `block_index` block.
         let ancestor_blocks = CHAIN_SEGMENT
             .iter()
@@ -294,75 +353,6 @@ fn invalid_signatures() {
             .process_chain_segment(ancestor_blocks)
             .into_block_error()
             .expect("should import all blocks prior to the one being tested");
-
-        // For the given snapshots, test the following:
-        //
-        // - The `process_chain_segment` function returns `InvalidSignature`.
-        // - The `process_block` function returns `InvalidSignature` when importing the
-        //    `SignedBeaconBlock` directly.
-        // - The `verify_block_for_gossip` function does _not_ return an error.
-        // - The `process_block` function returns `InvalidSignature` when verifying the
-        //    `GossipVerifiedBlock`.
-        let assert_invalid_signature = |snapshots: &[BeaconSnapshot<E>], item: &str| {
-            let blocks = snapshots
-                .iter()
-                .map(|snapshot| snapshot.beacon_block.clone())
-                .collect();
-
-            // Ensure the block will be rejected if imported in a chain segment.
-            assert!(
-                matches!(
-                    harness
-                        .chain
-                        .process_chain_segment(blocks)
-                        .into_block_error(),
-                    Err(BlockError::InvalidSignature)
-                ),
-                "should not import chain segment with an invalid {} signature",
-                item
-            );
-
-            // Ensure the block will be rejected if imported on its own (without gossip checking).
-            assert!(
-                matches!(
-                    harness
-                        .chain
-                        .process_block(snapshots[block_index].beacon_block.clone()),
-                    Err(BlockError::InvalidSignature)
-                ),
-                "should not import individual block with an invalid {} signature",
-                item
-            );
-
-            // NOTE: we choose not to check gossip verification here. It only checks one signature
-            // (proposal) and that is already tested elsewhere in this file.
-            //
-            // It's not trivial to just check gossip verification since it will start refusing
-            // blocks as soon as it has seen one valid proposal signature for a given (validator,
-            // slot) tuple.
-        };
-
-        /*
-         * Block proposal
-         */
-        let mut snapshots = CHAIN_SEGMENT.clone();
-        snapshots[block_index].beacon_block.signature = junk_signature();
-        let blocks = snapshots
-            .iter()
-            .map(|snapshot| snapshot.beacon_block.clone())
-            .collect();
-        // Ensure the block will be rejected if imported in a chain segment.
-        assert!(
-            matches!(
-                harness
-                    .chain
-                    .process_chain_segment(blocks)
-                    .into_block_error(),
-                Err(BlockError::InvalidSignature)
-            ),
-            "should not import chain segment with an invalid gossip signature",
-        );
-        // Ensure the block will be rejected if imported on its own (without gossip checking).
         assert!(
             matches!(
                 harness
@@ -372,10 +362,37 @@ fn invalid_signatures() {
             ),
             "should not import individual block with an invalid gossip signature",
         );
+    }
+}
 
-        /*
-         * Randao reveal
-         */
+#[test]
+fn invalid_signature_block_proposal() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
+        let mut snapshots = CHAIN_SEGMENT.clone();
+        snapshots[block_index].beacon_block.signature = junk_signature();
+        let blocks = snapshots
+            .iter()
+            .map(|snapshot| snapshot.beacon_block.clone())
+            .collect::<Vec<_>>();
+        // Ensure the block will be rejected if imported in a chain segment.
+        assert!(
+            matches!(
+                harness
+                    .chain
+                    .process_chain_segment(blocks)
+                    .into_block_error(),
+                Err(BlockError::InvalidSignature)
+            ),
+            "should not import chain segment with an invalid block signature",
+        );
+    }
+}
+
+#[test]
+fn invalid_signature_randao_reveal() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         snapshots[block_index]
             .beacon_block
@@ -384,11 +401,14 @@ fn invalid_signatures() {
             .randao_reveal = junk_signature();
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "randao");
+        assert_invalid_signature(&harness, block_index, &snapshots, "randao");
+    }
+}
 
-        /*
-         * Proposer slashing
-         */
+#[test]
+fn invalid_signature_proposer_slashing() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let proposer_slashing = ProposerSlashing {
             signed_header_1: SignedBeaconBlockHeader {
@@ -409,11 +429,14 @@ fn invalid_signatures() {
             .expect("should update proposer slashing");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "proposer slashing");
+        assert_invalid_signature(&harness, block_index, &snapshots, "proposer slashing");
+    }
+}
 
-        /*
-         * Attester slashing
-         */
+#[test]
+fn invalid_signature_attester_slashing() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let indexed_attestation = IndexedAttestation {
             attesting_indices: vec![0].into(),
@@ -445,11 +468,16 @@ fn invalid_signatures() {
             .expect("should update attester slashing");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "attester slashing");
+        assert_invalid_signature(&harness, block_index, &snapshots, "attester slashing");
+    }
+}
 
-        /*
-         * Attestation
-         */
+#[test]
+fn invalid_signature_attestation() {
+    let mut checked_attestation = false;
+
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         if let Some(attestation) = snapshots[block_index]
             .beacon_block
@@ -461,15 +489,22 @@ fn invalid_signatures() {
             attestation.signature = junk_aggregate_signature();
             update_parent_roots(&mut snapshots);
             update_proposal_signatures(&mut snapshots, &harness);
-            assert_invalid_signature(&snapshots, "attestation");
+            assert_invalid_signature(&harness, block_index, &snapshots, "attestation");
             checked_attestation = true;
         }
+    }
 
-        /*
-         * Deposit
-         *
-         * Note: an invalid deposit signature is permitted!
-         */
+    assert!(
+        checked_attestation,
+        "the test should check an attestation signature"
+    )
+}
+
+#[test]
+fn invalid_signature_deposit() {
+    for &block_index in BLOCK_INDICES {
+        // Note: an invalid deposit signature is permitted!
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let deposit = Deposit {
             proof: vec![Hash256::zero(); DEPOSIT_TREE_DEPTH + 1].into(),
@@ -503,10 +538,13 @@ fn invalid_signatures() {
             ),
             "should not throw an invalid signature error for a bad deposit signature"
         );
+    }
+}
 
-        /*
-         * Voluntary exit
-         */
+#[test]
+fn invalid_signature_exit() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let epoch = snapshots[block_index].beacon_state.current_epoch();
         snapshots[block_index]
@@ -524,13 +562,8 @@ fn invalid_signatures() {
             .expect("should update deposit");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "voluntary exit");
+        assert_invalid_signature(&harness, block_index, &snapshots, "voluntary exit");
     }
-
-    assert!(
-        checked_attestation,
-        "the test should check an attestation signature"
-    )
 }
 
 fn unwrap_err<T, E>(result: Result<T, E>) -> E {
@@ -641,6 +674,48 @@ fn block_gossip_verification() {
         "should not import a block with an invalid proposal signature"
     );
 
+    /*
+     * This test ensures that:
+     *
+     * Spec v0.12.2
+     *
+     * The block's parent (defined by block.parent_root) passes validation.
+     */
+
+    let mut block = CHAIN_SEGMENT[block_index].beacon_block.clone();
+    let parent_root = Hash256::from_low_u64_be(42);
+    block.message.parent_root = parent_root;
+    assert!(
+        matches!(
+            unwrap_err(harness.chain.verify_block_for_gossip(block)),
+            BlockError::ParentUnknown(block)
+            if block.parent_root() == parent_root
+        ),
+        "should not import a block for an unknown parent"
+    );
+
+    /*
+     * This test ensures that:
+     *
+     * Spec v0.12.2
+     *
+     * The current finalized_checkpoint is an ancestor of block -- i.e. get_ancestor(store,
+     * block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) ==
+     * store.finalized_checkpoint.root
+     */
+
+    let mut block = CHAIN_SEGMENT[block_index].beacon_block.clone();
+    let parent_root = CHAIN_SEGMENT[0].beacon_block_root;
+    block.message.parent_root = parent_root;
+    assert!(
+        matches!(
+            unwrap_err(harness.chain.verify_block_for_gossip(block)),
+            BlockError::NotFinalizedDescendant { block_parent_root }
+            if block_parent_root == parent_root
+        ),
+        "should not import a block that conflicts with finality"
+    );
+
     /*
      * This test ensures that:
      *

beacon_node/beacon_chain/tests/op_verification.rs

@@ -7,7 +7,7 @@ extern crate lazy_static;
 
 use beacon_chain::observed_operations::ObservationOutcome;
 use beacon_chain::test_utils::{
-    AttestationStrategy, BeaconChainHarness, BlockStrategy, DiskHarnessType,
+    AttestationStrategy, BeaconChainHarness, BlockStrategy, BlockingMigratorDiskHarnessType,
 };
 use sloggers::{null::NullLoggerBuilder, Build};
 use std::sync::Arc;
@@ -28,7 +28,7 @@ lazy_static! {
 }
 
 type E = MinimalEthSpec;
-type TestHarness = BeaconChainHarness<DiskHarnessType<E>>;
+type TestHarness = BeaconChainHarness<BlockingMigratorDiskHarnessType<E>>;
 type HotColdDB = store::HotColdDB<E, LevelDB<E>, LevelDB<E>>;
 
 fn get_store(db_path: &TempDir) -> Arc<HotColdDB> {
@@ -57,8 +57,8 @@ fn get_harness(store: Arc<HotColdDB>, validator_count: usize) -> TestHarness {
 fn voluntary_exit() {
     let db_path = tempdir().unwrap();
     let store = get_store(&db_path);
-    let harness = get_harness(store.clone(), VALIDATOR_COUNT);
-    let spec = &harness.chain.spec;
+    let mut harness = get_harness(store.clone(), VALIDATOR_COUNT);
+    let spec = &harness.chain.spec.clone();
 
     harness.extend_chain(
         (E::slots_per_epoch() * (spec.shard_committee_period + 1)) as usize,

beacon_node/beacon_chain/tests/persistence_tests.rs

@@ -44,7 +44,7 @@ fn finalizes_after_resuming_from_db() {
     let db_path = tempdir().unwrap();
     let store = get_store(&db_path);
 
-    let harness = BeaconChainHarness::new_with_disk_store(
+    let mut harness = BeaconChainHarness::new_with_disk_store(
         MinimalEthSpec,
         store.clone(),
         KEYPAIRS[0..validator_count].to_vec(),
@@ -88,7 +88,7 @@ fn finalizes_after_resuming_from_db() {
     let data_dir = harness.data_dir;
     let original_chain = harness.chain;
 
-    let resumed_harness = BeaconChainHarness::resume_from_disk_store(
+    let mut resumed_harness = BeaconChainHarness::resume_from_disk_store(
         MinimalEthSpec,
         store,
         KEYPAIRS[0..validator_count].to_vec(),

beacon_node/beacon_chain/tests/tests.rs

@@ -6,7 +6,8 @@ extern crate lazy_static;
 use beacon_chain::{
     attestation_verification::Error as AttnError,
     test_utils::{
-        AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType, OP_POOL_DB_KEY,
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, NullMigratorEphemeralHarnessType,
+        OP_POOL_DB_KEY,
     },
 };
 use operation_pool::PersistedOperationPool;
@@ -24,8 +25,10 @@ lazy_static! {
     static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<MinimalEthSpec>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(
+    validator_count: usize,
+) -> BeaconChainHarness<NullMigratorEphemeralHarnessType<MinimalEthSpec>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MinimalEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -64,7 +67,7 @@ fn massive_skips() {
 fn iterators() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -139,7 +142,7 @@ fn iterators() {
 
 #[test]
 fn chooses_fork() {
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     let two_thirds = (VALIDATOR_COUNT / 3) * 2;
     let delay = MinimalEthSpec::default_spec().min_attestation_inclusion_delay as usize;
@@ -190,7 +193,7 @@ fn chooses_fork() {
 fn finalizes_with_full_participation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -225,7 +228,7 @@ fn finalizes_with_full_participation() {
 fn finalizes_with_two_thirds_participation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     let two_thirds = (VALIDATOR_COUNT / 3) * 2;
     let attesters = (0..two_thirds).collect();
@@ -268,7 +271,7 @@ fn finalizes_with_two_thirds_participation() {
 fn does_not_finalize_with_less_than_two_thirds_participation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     let two_thirds = (VALIDATOR_COUNT / 3) * 2;
     let less_than_two_thirds = two_thirds - 1;
@@ -305,7 +308,7 @@ fn does_not_finalize_with_less_than_two_thirds_participation() {
 fn does_not_finalize_without_attestation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -338,7 +341,7 @@ fn does_not_finalize_without_attestation() {
 fn roundtrip_operation_pool() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     // Add some attestations
     harness.extend_chain(
@@ -370,7 +373,7 @@ fn roundtrip_operation_pool() {
 fn unaggregated_attestations_added_to_fork_choice_some_none() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() / 2;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -424,7 +427,7 @@ fn unaggregated_attestations_added_to_fork_choice_some_none() {
 fn attestations_with_increasing_slots() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     let mut attestations = vec![];
 
@@ -486,7 +489,7 @@ fn attestations_with_increasing_slots() {
 fn unaggregated_attestations_added_to_fork_choice_all_updated() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -541,7 +544,7 @@ fn unaggregated_attestations_added_to_fork_choice_all_updated() {
 
 fn run_skip_slot_test(skip_slots: u64) {
     let num_validators = 8;
-    let harness_a = get_harness(num_validators);
+    let mut harness_a = get_harness(num_validators);
     let harness_b = get_harness(num_validators);
 
     for _ in 0..skip_slots {

beacon_node/client/Cargo.toml

@@ -31,7 +31,7 @@ slog-async = "2.5.0"
 tokio = "0.2.21"
 dirs = "2.0.2"
 futures = "0.3.5"
-reqwest = "0.10.4"
+reqwest = { version = "0.10.4", features = ["native-tls-vendored"] }
 url = "2.1.1"
 eth1 = { path = "../eth1" }
 genesis = { path = "../genesis" }

beacon_node/client/src/builder.rs

@@ -135,6 +135,7 @@ where
         let eth_spec_instance = self.eth_spec_instance.clone();
         let data_dir = config.data_dir.clone();
         let disabled_forks = config.disabled_forks.clone();
+        let chain_config = config.chain.clone();
         let graffiti = config.graffiti;
 
         let store =
@@ -153,6 +154,7 @@ where
             .store_migrator(store_migrator)
             .data_dir(data_dir)
             .custom_spec(spec.clone())
+            .chain_config(chain_config)
             .disabled_forks(disabled_forks)
             .graffiti(graffiti);
 
@@ -232,8 +234,8 @@ where
         Ok(self)
     }
 
-    /// Immediately starts the networking stack.
-    pub fn network(mut self, config: &NetworkConfig) -> Result<Self, String> {
+    /// Starts the networking stack.
+    pub async fn network(mut self, config: &NetworkConfig) -> Result<Self, String> {
         let beacon_chain = self
             .beacon_chain
             .clone()
@@ -246,6 +248,7 @@ where
 
         let (network_globals, network_send) =
             NetworkService::start(beacon_chain, config, context.executor)
+                .await
                 .map_err(|e| format!("Failed to start network: {:?}", e))?;
 
         self.network_globals = Some(network_globals);
@@ -581,7 +584,7 @@ where
     /// Specifies that the `BeaconChain` should cache eth1 blocks/logs from a remote eth1 node
     /// (e.g., Parity/Geth) and refer to that cache when collecting deposits or eth1 votes during
     /// block production.
-    pub fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
+    pub async fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
         let context = self
             .runtime_context
             .as_ref()
@@ -595,6 +598,17 @@ where
             .clone()
             .ok_or_else(|| "caching_eth1_backend requires a chain spec".to_string())?;
 
+        // Check if the eth1 endpoint we connect to is on the correct network id.
+        let network_id =
+            eth1::http::get_network_id(&config.endpoint, Duration::from_millis(15_000)).await?;
+
+        if network_id != config.network_id {
+            return Err(format!(
+                "Invalid eth1 network id. Expected {:?}, got {:?}",
+                config.network_id, network_id
+            ));
+        }
+
         let backend = if let Some(eth1_service_from_genesis) = self.eth1_service {
             eth1_service_from_genesis.update_config(config)?;
 

beacon_node/client/src/config.rs

@@ -64,6 +64,7 @@ pub struct Config {
     pub store: store::StoreConfig,
     pub network: network::NetworkConfig,
     pub rest_api: rest_api::Config,
+    pub chain: beacon_chain::ChainConfig,
     pub websocket_server: websocket_server::Config,
     pub eth1: eth1::Config,
 }
@@ -78,6 +79,7 @@ impl Default for Config {
             genesis: <_>::default(),
             store: <_>::default(),
             network: NetworkConfig::default(),
+            chain: <_>::default(),
             rest_api: <_>::default(),
             websocket_server: <_>::default(),
             spec_constants: TESTNET_SPEC_CONSTANTS.into(),

beacon_node/client/src/notifier.rs

@@ -122,14 +122,28 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                     head_distance.as_u64(),
                     slot_distance_pretty(head_distance, slot_duration)
                 );
-                info!(
-                    log,
-                    "Syncing";
-                    "peers" => peer_count_pretty(connected_peer_count),
-                    "distance" => distance,
-                    "speed" => sync_speed_pretty(speedo.slots_per_second()),
-                    "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
-                );
+
+                let speed = speedo.slots_per_second();
+                let display_speed = speed.map_or(false, |speed| speed != 0.0);
+
+                if display_speed {
+                    info!(
+                        log,
+                        "Syncing";
+                        "peers" => peer_count_pretty(connected_peer_count),
+                        "distance" => distance,
+                        "speed" => sync_speed_pretty(speed),
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                    );
+                } else {
+                    info!(
+                        log,
+                        "Syncing";
+                        "peers" => peer_count_pretty(connected_peer_count),
+                        "distance" => distance,
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                    );
+                }
             } else if sync_state.is_synced() {
                 let block_info = if current_slot > head_slot {
                     "   …  empty".to_string()
@@ -220,14 +234,37 @@ fn seconds_pretty(secs: f64) -> String {
     let hours = d.whole_hours();
     let minutes = d.whole_minutes();
 
+    let week_string = if weeks == 1 { "week" } else { "weeks" };
+    let day_string = if days == 1 { "day" } else { "days" };
+    let hour_string = if hours == 1 { "hr" } else { "hrs" };
+    let min_string = if minutes == 1 { "min" } else { "mins" };
+
     if weeks > 0 {
-        format!("{:.0} weeks {:.0} days", weeks, days % DAYS_PER_WEEK)
+        format!(
+            "{:.0} {} {:.0} {}",
+            weeks,
+            week_string,
+            days % DAYS_PER_WEEK,
+            day_string
+        )
     } else if days > 0 {
-        format!("{:.0} days {:.0} hrs", days, hours % HOURS_PER_DAY)
+        format!(
+            "{:.0} {} {:.0} {}",
+            days,
+            day_string,
+            hours % HOURS_PER_DAY,
+            hour_string
+        )
     } else if hours > 0 {
-        format!("{:.0} hrs {:.0} mins", hours, minutes % MINUTES_PER_HOUR)
+        format!(
+            "{:.0} {} {:.0} {}",
+            hours,
+            hour_string,
+            minutes % MINUTES_PER_HOUR,
+            min_string
+        )
     } else {
-        format!("{:.0} mins", minutes)
+        format!("{:.0} {}", minutes, min_string)
     }
 }
 

beacon_node/eth1/Cargo.toml

@@ -11,7 +11,7 @@ web3 = "0.11.0"
 sloggers = "1.0.0"
 
 [dependencies]
-reqwest = "0.10.4"
+reqwest = { version = "0.10.4", features = ["native-tls-vendored"] }
 futures = { version = "0.3.5", features = ["compat"] }
 serde_json = "1.0.52"
 serde = { version = "1.0.110", features = ["derive"] }

beacon_node/eth1/src/http.rs

@@ -12,8 +12,10 @@
 
 use futures::future::TryFutureExt;
 use reqwest::{header::CONTENT_TYPE, ClientBuilder, StatusCode};
+use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
 use std::ops::Range;
+use std::str::FromStr;
 use std::time::Duration;
 use types::Hash256;
 
@@ -30,6 +32,40 @@ pub const DEPOSIT_COUNT_RESPONSE_BYTES: usize = 96;
 /// Number of bytes in deposit contract deposit root (value only).
 pub const DEPOSIT_ROOT_BYTES: usize = 32;
 
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub enum Eth1NetworkId {
+    Goerli,
+    Mainnet,
+    Custom(u64),
+}
+
+impl FromStr for Eth1NetworkId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "1" => Ok(Eth1NetworkId::Mainnet),
+            "5" => Ok(Eth1NetworkId::Goerli),
+            custom => {
+                let network_id = u64::from_str_radix(custom, 10)
+                    .map_err(|e| format!("Failed to parse eth1 network id {}", e))?;
+                Ok(Eth1NetworkId::Custom(network_id))
+            }
+        }
+    }
+}
+
+/// Get the eth1 network id of the given endpoint.
+pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result<Eth1NetworkId, String> {
+    let response_body = send_rpc_request(endpoint, "net_version", json!([]), timeout).await?;
+    Eth1NetworkId::from_str(
+        response_result(&response_body)?
+            .ok_or_else(|| "No result was returned for block number".to_string())?
+            .as_str()
+            .ok_or_else(|| "Data was not string")?,
+    )
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct Block {
     pub hash: Hash256,

beacon_node/eth1/src/service.rs

@@ -2,7 +2,9 @@ use crate::metrics;
 use crate::{
     block_cache::{BlockCache, Error as BlockCacheError, Eth1Block},
     deposit_cache::Error as DepositCacheError,
-    http::{get_block, get_block_number, get_deposit_logs_in_range, Log},
+    http::{
+        get_block, get_block_number, get_deposit_logs_in_range, get_network_id, Eth1NetworkId, Log,
+    },
     inner::{DepositUpdater, Inner},
     DepositLog,
 };
@@ -16,6 +18,9 @@ use std::time::{SystemTime, UNIX_EPOCH};
 use tokio::time::{interval_at, Duration, Instant};
 use types::ChainSpec;
 
+/// Indicates the default eth1 network we use for the deposit contract.
+pub const DEFAULT_NETWORK_ID: Eth1NetworkId = Eth1NetworkId::Goerli;
+
 const STANDARD_TIMEOUT_MILLIS: u64 = 15_000;
 
 /// Timeout when doing a eth_blockNumber call.
@@ -76,6 +81,8 @@ pub struct Config {
     pub endpoint: String,
     /// The address the `BlockCache` and `DepositCache` should assume is the canonical deposit contract.
     pub deposit_contract_address: String,
+    /// The eth1 network id where the deposit contract is deployed (Goerli/Mainnet).
+    pub network_id: Eth1NetworkId,
     /// Defines the first block that the `DepositCache` will start searching for deposit logs.
     ///
     /// Setting too high can result in missed logs. Setting too low will result in unnecessary
@@ -105,6 +112,7 @@ impl Default for Config {
         Self {
             endpoint: "http://localhost:8545".into(),
             deposit_contract_address: "0x0000000000000000000000000000000000000000".into(),
+            network_id: DEFAULT_NETWORK_ID,
             deposit_contract_deploy_block: 1,
             lowest_cached_block_number: 1,
             follow_distance: 128,
@@ -350,6 +358,29 @@ impl Service {
     }
 
     async fn do_update(&self, update_interval: Duration) -> Result<(), ()> {
+        let endpoint = self.config().endpoint.clone();
+        let config_network = self.config().network_id.clone();
+        let result =
+            get_network_id(&endpoint, Duration::from_millis(STANDARD_TIMEOUT_MILLIS)).await;
+        match result {
+            Ok(network_id) => {
+                if network_id != config_network {
+                    error!(
+                        self.log,
+                        "Failed to update eth1 cache";
+                        "reason" => "Invalid eth1 network id",
+                        "expected" => format!("{:?}",DEFAULT_NETWORK_ID),
+                        "got" => format!("{:?}",network_id),
+                    );
+                    return Ok(());
+                }
+            }
+            Err(e) => {
+                error!(self.log, "Failed to get eth1 network id"; "error" => e);
+                return Ok(());
+            }
+        }
+
         let update_result = self.update().await;
         match update_result {
             Err(e) => error!(

beacon_node/eth2_libp2p/Cargo.toml

@@ -32,7 +32,7 @@ snap = "1.0.0"
 void = "1.0.2"
 tokio-io-timeout = "0.4.0"
 tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
-discv5 = { version = "0.1.0-alpha.8", features = ["libp2p"] }
+discv5 = { version = "0.1.0-alpha.10", features = ["libp2p"] }
 tiny-keccak = "2.0.2"
 environment = { path = "../../lighthouse/environment" }
 # TODO: Remove rand crate for mainnet
@@ -41,9 +41,9 @@ rand = "0.7.3"
 [dependencies.libp2p]
 #version = "0.23.0"
 git = "https://github.com/sigp/rust-libp2p"
-rev = "3096cb6b89b2883a79ce5ffcb03d41778a09b695"
+rev = "03f998022ce2f566a6c6e6c4206bc0ce4d45109f"
 default-features = false
-features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "tcp-tokio"]
+features = ["websocket", "identify", "mplex", "noise", "gossipsub", "dns", "tcp-tokio"]
 
 [dev-dependencies]
 tokio = { version = "0.2.21", features = ["full"] }

beacon_node/eth2_libp2p/src/behaviour/handler/delegate.rs

@@ -131,8 +131,9 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         let gossip_proto = self.gossip_handler.listen_protocol();
         let rpc_proto = self.rpc_handler.listen_protocol();
         let identify_proto = self.identify_handler.listen_protocol();
@@ -147,24 +148,27 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             SelectUpgrade::new(rpc_proto.into_upgrade().1, identify_proto.into_upgrade().1),
         );
 
-        SubstreamProtocol::new(select).with_timeout(timeout)
+        SubstreamProtocol::new(select, ()).with_timeout(timeout)
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         match out {
             // Gossipsub
-            EitherOutput::First(out) => self.gossip_handler.inject_fully_negotiated_inbound(out),
+            EitherOutput::First(out) => {
+                self.gossip_handler.inject_fully_negotiated_inbound(out, ())
+            }
             // RPC
             EitherOutput::Second(EitherOutput::First(out)) => {
-                self.rpc_handler.inject_fully_negotiated_inbound(out)
+                self.rpc_handler.inject_fully_negotiated_inbound(out, ())
             }
             // Identify
-            EitherOutput::Second(EitherOutput::Second(out)) => {
-                self.identify_handler.inject_fully_negotiated_inbound(out)
-            }
+            EitherOutput::Second(EitherOutput::Second(out)) => self
+                .identify_handler
+                .inject_fully_negotiated_inbound(out, ()),
         }
     }
 
@@ -317,10 +321,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
                     event,
                 )));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(EitherUpgrade::A),
-                    info: EitherOutput::First(info),
+                    protocol: protocol
+                        .map_upgrade(EitherUpgrade::A)
+                        .map_info(EitherOutput::First),
                 });
             }
             Poll::Pending => (),
@@ -333,10 +338,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::RPC(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u))),
-                    info: EitherOutput::Second(EitherOutput::First(info)),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u)))
+                        .map_info(|info| EitherOutput::Second(EitherOutput::First(info))),
                 });
             }
             Poll::Pending => (),
@@ -351,10 +357,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::Identify(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u))),
-                    info: EitherOutput::Second(EitherOutput::Second(())),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u)))
+                        .map_info(|_| EitherOutput::Second(EitherOutput::Second(()))),
                 });
             }
             Poll::Pending => (),

beacon_node/eth2_libp2p/src/behaviour/handler/mod.rs

@@ -54,16 +54,18 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.delegate.listen_protocol()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
-        self.delegate.inject_fully_negotiated_inbound(out)
+        self.delegate.inject_fully_negotiated_inbound(out, ())
     }
 
     fn inject_fully_negotiated_outbound(
@@ -127,11 +129,8 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(err)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(err))
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
-                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol,
-                    info,
-                });
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
+                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol });
             }
             Poll::Pending => (),
         }

beacon_node/eth2_libp2p/src/behaviour/mod.rs

@@ -1,6 +1,6 @@
 use crate::peer_manager::{score::PeerAction, PeerManager, PeerManagerEvent};
 use crate::rpc::*;
-use crate::types::{GossipEncoding, GossipKind, GossipTopic};
+use crate::types::{EnrBitfield, GossipEncoding, GossipKind, GossipTopic, SubnetDiscovery};
 use crate::Eth2Enr;
 use crate::{error, metrics, Enr, NetworkConfig, NetworkGlobals, PubsubMessage, TopicHash};
 use futures::prelude::*;
@@ -11,7 +11,10 @@ use libp2p::{
         identity::Keypair,
         Multiaddr,
     },
-    gossipsub::{Gossipsub, GossipsubEvent, MessageAuthenticity, MessageId},
+    gossipsub::{
+        Gossipsub, GossipsubEvent, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity,
+        MessageId,
+    },
     identify::{Identify, IdentifyEvent},
     swarm::{
         NetworkBehaviour, NetworkBehaviourAction as NBAction, NotifyHandler, PollParameters,
@@ -19,19 +22,23 @@ use libp2p::{
     },
     PeerId,
 };
-use slog::{crit, debug, o, trace};
+use slog::{crit, debug, o, trace, warn};
+use ssz::{Decode, Encode};
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::PathBuf;
 use std::{
     collections::VecDeque,
     marker::PhantomData,
     sync::Arc,
     task::{Context, Poll},
-    time::Instant,
 };
 use types::{EnrForkId, EthSpec, SignedBeaconBlock, SubnetId};
 
 mod handler;
 
 const MAX_IDENTIFY_ADDRESSES: usize = 10;
+const METADATA_FILENAME: &str = "metadata";
 
 /// Builds the network behaviour that manages the core protocols of eth2.
 /// This core behaviour is managed by `Behaviour` which adds peer management to all core
@@ -61,13 +68,15 @@ pub struct Behaviour<TSpec: EthSpec> {
     enr_fork_id: EnrForkId,
     /// The waker for the current thread.
     waker: Option<std::task::Waker>,
+    /// Directory where metadata is stored
+    network_dir: PathBuf,
     /// Logger for behaviour actions.
     log: slog::Logger,
 }
 
 /// Implements the combined behaviour for the libp2p service.
 impl<TSpec: EthSpec> Behaviour<TSpec> {
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         net_conf: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
@@ -86,33 +95,31 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .eth2()
             .expect("Local ENR must have a fork id");
 
-        let attnets = network_globals
-            .local_enr()
-            .bitfield::<TSpec>()
-            .expect("Local ENR must have subnet bitfield");
+        let meta_data = load_or_build_metadata(&net_conf.network_dir, &log);
 
-        let meta_data = MetaData {
-            seq_number: 1,
-            attnets,
-        };
+        let gossipsub = Gossipsub::new(MessageAuthenticity::Anonymous, net_conf.gs_config.clone())
+            .map_err(|e| format!("Could not construct gossipsub: {:?}", e))?;
 
-        // TODO: Until other clients support no author, we will use a 0 peer_id as our author.
-        let message_author = PeerId::from_bytes(vec![0, 1, 0]).expect("Valid peer id");
+        // Temporarily disable scoring until parameters are tested.
+        /*
+        gossipsub
+            .with_peer_score(PeerScoreParams::default(), PeerScoreThresholds::default())
+            .expect("Valid score params and thresholds");
+        */
 
         Ok(Behaviour {
             eth2_rpc: RPC::new(log.clone()),
-            gossipsub: Gossipsub::new(
-                MessageAuthenticity::Author(message_author),
-                net_conf.gs_config.clone(),
-            ),
+            gossipsub,
             identify,
-            peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)?,
+            peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)
+                .await?,
             events: VecDeque::new(),
             peers_to_dc: VecDeque::new(),
             meta_data,
             network_globals,
             enr_fork_id,
             waker: None,
+            network_dir: net_conf.network_dir.clone(),
             log: behaviour_log,
         })
     }
@@ -123,7 +130,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
     ///
     /// All external dials, dial a multiaddr. This is currently unused but kept here in case any
     /// part of lighthouse needs to connect to a peer_id in the future.
-    pub fn _dial(&mut self, peer_id: &PeerId) {
+    pub fn dial(&mut self, peer_id: &PeerId) {
         self.peer_manager.dial_peer(peer_id);
     }
 
@@ -147,6 +154,10 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             GossipEncoding::default(),
             self.enr_fork_id.fork_digest,
         );
+
+        // TODO: Implement scoring
+        // let topic: Topic = gossip_topic.into();
+        // self.gossipsub.set_topic_params(t.hash(), TopicScoreParams::default());
         self.subscribe(gossip_topic)
     }
 
@@ -168,6 +179,12 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             GossipEncoding::default(),
             self.enr_fork_id.fork_digest,
         );
+        // TODO: Implement scoring
+        /*
+        let t: Topic = topic.clone().into();
+        self.gossipsub
+            .set_topic_params(t.hash(), TopicScoreParams::default());
+        */
         self.subscribe(topic)
     }
 
@@ -189,9 +206,18 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .write()
             .insert(topic.clone());
 
-        let topic_str: String = topic.clone().into();
-        debug!(self.log, "Subscribed to topic"; "topic" => topic_str);
-        self.gossipsub.subscribe(topic.into())
+        let topic: Topic = topic.into();
+
+        match self.gossipsub.subscribe(&topic) {
+            Err(_) => {
+                warn!(self.log, "Failed to subscribe to topic"; "topic" => topic.to_string());
+                false
+            }
+            Ok(v) => {
+                debug!(self.log, "Subscribed to topic"; "topic" => topic.to_string());
+                v
+            }
+        }
     }
 
     /// Unsubscribe from a gossipsub topic.
@@ -201,8 +227,20 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .gossipsub_subscriptions
             .write()
             .remove(&topic);
+
         // unsubscribe from the topic
-        self.gossipsub.unsubscribe(topic.into())
+        let topic: Topic = topic.into();
+
+        match self.gossipsub.unsubscribe(&topic) {
+            Err(_) => {
+                warn!(self.log, "Failed to unsubscribe from topic"; "topic" => topic.to_string());
+                false
+            }
+            Ok(v) => {
+                debug!(self.log, "Unsubscribed to topic"; "topic" => topic.to_string());
+                v
+            }
+        }
     }
 
     /// Publishes a list of messages on the pubsub (gossipsub) behaviour, choosing the encoding.
@@ -211,8 +249,28 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) {
                 match message.encode(GossipEncoding::default()) {
                     Ok(message_data) => {
-                        if let Err(e) = self.gossipsub.publish(&topic.into(), message_data) {
+                        if let Err(e) = self.gossipsub.publish(topic.clone().into(), message_data) {
                             slog::warn!(self.log, "Could not publish message"; "error" => format!("{:?}", e));
+
+                            // add to metrics
+                            match topic.kind() {
+                                GossipKind::Attestation(subnet_id) => {
+                                    if let Some(v) = metrics::get_int_gauge(
+                                        &metrics::FAILED_ATTESTATION_PUBLISHES_PER_SUBNET,
+                                        &[&subnet_id.to_string()],
+                                    ) {
+                                        v.inc()
+                                    };
+                                }
+                                kind => {
+                                    if let Some(v) = metrics::get_int_gauge(
+                                        &metrics::FAILED_PUBLISHES_PER_MAIN_TOPIC,
+                                        &[&format!("{:?}", kind)],
+                                    ) {
+                                        v.inc()
+                                    };
+                                }
+                            }
                         }
                     }
                     Err(e) => crit!(self.log, "Could not publish message"; "error" => e),
@@ -221,11 +279,21 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
         }
     }
 
-    /// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated
-    /// once validated by the beacon chain.
-    pub fn validate_message(&mut self, propagation_source: &PeerId, message_id: MessageId) {
-        self.gossipsub
-            .validate_message(&message_id, propagation_source);
+    /// Informs the gossipsub about the result of a message validation.
+    /// If the message is valid it will get propagated by gossipsub.
+    pub fn report_message_validation_result(
+        &mut self,
+        propagation_source: &PeerId,
+        message_id: MessageId,
+        validation_result: MessageAcceptance,
+    ) {
+        if let Err(e) = self.gossipsub.report_message_validation_result(
+            &message_id,
+            propagation_source,
+            validation_result,
+        ) {
+            warn!(self.log, "Failed to report message validation"; "message_id" => message_id.to_string(), "peer_id" => propagation_source.to_string(), "error" => format!("{:?}", e));
+        }
     }
 
     /* Eth2 RPC behaviour functions */
@@ -300,8 +368,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Attempts to discover new peers for a given subnet. The `min_ttl` gives the time at which we
     /// would like to retain the peers for.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
-        self.peer_manager.discover_subnet_peers(subnet_id, min_ttl)
+    pub fn discover_subnet_peers(&mut self, subnet_subscriptions: Vec<SubnetDiscovery>) {
+        self.peer_manager
+            .discover_subnet_peers(subnet_subscriptions)
     }
 
     /// Updates the local ENR's "eth2" field with the latest EnrForkId.
@@ -345,6 +414,8 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .local_enr()
             .bitfield::<TSpec>()
             .expect("Local discovery must have bitfield");
+        // Save the updated metadata to disk
+        save_metadata_to_disk(&self.network_dir, self.meta_data.clone(), &self.log);
     }
 
     /// Sends a Ping request to the peer.
@@ -389,11 +460,25 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     fn on_gossip_event(&mut self, event: GossipsubEvent) {
         match event {
-            GossipsubEvent::Message(propagation_source, id, gs_msg) => {
+            GossipsubEvent::Message {
+                propagation_source,
+                message_id: id,
+                message: gs_msg,
+            } => {
                 // Note: We are keeping track here of the peer that sent us the message, not the
                 // peer that originally published the message.
                 match PubsubMessage::decode(&gs_msg.topics, &gs_msg.data) {
-                    Err(e) => debug!(self.log, "Could not decode gossipsub message"; "error" => e),
+                    Err(e) => {
+                        debug!(self.log, "Could not decode gossipsub message"; "error" => e);
+                        //reject the message
+                        if let Err(e) = self.gossipsub.report_message_validation_result(
+                            &id,
+                            &propagation_source,
+                            MessageAcceptance::Reject,
+                        ) {
+                            warn!(self.log, "Failed to report message validation"; "message_id" => id.to_string(), "peer_id" => propagation_source.to_string(), "error" => format!("{:?}", e));
+                        }
+                    }
                     Ok(msg) => {
                         // Notify the network
                         self.add_event(BehaviourEvent::PubsubMessage {
@@ -406,23 +491,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                 }
             }
             GossipsubEvent::Subscribed { peer_id, topic } => {
-                if let Some(topic_metric) = metrics::get_int_gauge(
-                    &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_COUNT,
-                    &[topic.as_str()],
-                ) {
-                    topic_metric.inc()
-                }
-
                 self.add_event(BehaviourEvent::PeerSubscribed(peer_id, topic));
             }
-            GossipsubEvent::Unsubscribed { peer_id: _, topic } => {
-                if let Some(topic_metric) = metrics::get_int_gauge(
-                    &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_COUNT,
-                    &[topic.as_str()],
-                ) {
-                    topic_metric.dec()
-                }
-            }
+            GossipsubEvent::Unsubscribed { .. } => {}
         }
     }
 
@@ -743,8 +814,8 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
                 .peer_info(peer_id)
                 .map_or(true, |i| !i.has_future_duty())
         {
-            //If we are at our peer limit and we don't need the peer for a future validator
-            //duty, send goodbye with reason TooManyPeers
+            // If we are at our peer limit and we don't need the peer for a future validator
+            // duty, send goodbye with reason TooManyPeers
             Some(GoodbyeReason::TooManyPeers)
         } else {
             None
@@ -1035,3 +1106,60 @@ pub enum BehaviourEvent<TSpec: EthSpec> {
     /// Inform the network to send a Status to this peer.
     StatusPeer(PeerId),
 }
+
+/// Load metadata from persisted file. Return default metadata if loading fails.
+fn load_or_build_metadata<E: EthSpec>(network_dir: &PathBuf, log: &slog::Logger) -> MetaData<E> {
+    // Default metadata
+    let mut meta_data = MetaData {
+        seq_number: 0,
+        attnets: EnrBitfield::<E>::default(),
+    };
+    // Read metadata from persisted file if available
+    let metadata_path = network_dir.join(METADATA_FILENAME);
+    if let Ok(mut metadata_file) = File::open(metadata_path) {
+        let mut metadata_ssz = Vec::new();
+        if metadata_file.read_to_end(&mut metadata_ssz).is_ok() {
+            match MetaData::<E>::from_ssz_bytes(&metadata_ssz) {
+                Ok(persisted_metadata) => {
+                    meta_data.seq_number = persisted_metadata.seq_number;
+                    // Increment seq number if persisted attnet is not default
+                    if persisted_metadata.attnets != meta_data.attnets {
+                        meta_data.seq_number += 1;
+                    }
+                    debug!(log, "Loaded metadata from disk");
+                }
+                Err(e) => {
+                    debug!(
+                        log,
+                        "Metadata from file could not be decoded";
+                        "error" => format!("{:?}", e),
+                    );
+                }
+            }
+        }
+    };
+
+    debug!(log, "Metadata sequence number"; "seq_num" => meta_data.seq_number);
+    save_metadata_to_disk(network_dir, meta_data.clone(), &log);
+    meta_data
+}
+
+/// Persist metadata to disk
+fn save_metadata_to_disk<E: EthSpec>(dir: &PathBuf, metadata: MetaData<E>, log: &slog::Logger) {
+    let _ = std::fs::create_dir_all(&dir);
+    match File::create(dir.join(METADATA_FILENAME))
+        .and_then(|mut f| f.write_all(&metadata.as_ssz_bytes()))
+    {
+        Ok(_) => {
+            debug!(log, "Metadata written to disk");
+        }
+        Err(e) => {
+            warn!(
+                log,
+                "Could not write metadata to disk";
+                "file" => format!("{:?}{:?}",dir, METADATA_FILENAME),
+                "error" => format!("{}", e)
+            );
+        }
+    }
+}

beacon_node/eth2_libp2p/src/config.rs

@@ -50,7 +50,10 @@ pub struct Config {
     pub discv5_config: Discv5Config,
 
     /// List of nodes to initially connect to.
-    pub boot_nodes: Vec<Enr>,
+    pub boot_nodes_enr: Vec<Enr>,
+
+    /// List of nodes to initially connect to, on Multiaddr format.
+    pub boot_nodes_multiaddr: Vec<Multiaddr>,
 
     /// List of libp2p nodes to initially connect to.
     pub libp2p_nodes: Vec<Multiaddr>,
@@ -96,8 +99,8 @@ impl Default for Config {
         let gs_config = GossipsubConfigBuilder::new()
             .max_transmit_size(GOSSIP_MAX_SIZE)
             .heartbeat_interval(Duration::from_millis(700))
-            .mesh_n(6)
-            .mesh_n_low(5)
+            .mesh_n(8)
+            .mesh_n_low(6)
             .mesh_n_high(12)
             .gossip_lazy(6)
             .fanout_ttl(Duration::from_secs(60))
@@ -108,7 +111,8 @@ impl Default for Config {
             // prevent duplicates for 550 heartbeats(700millis * 550) = 385 secs
             .duplicate_cache_time(Duration::from_secs(385))
             .message_id_fn(gossip_message_id)
-            .build();
+            .build()
+            .expect("valid gossipsub configuration");
 
         // discv5 configuration
         let discv5_config = Discv5ConfigBuilder::new()
@@ -136,7 +140,8 @@ impl Default for Config {
             target_peers: 50,
             gs_config,
             discv5_config,
-            boot_nodes: vec![],
+            boot_nodes_enr: vec![],
+            boot_nodes_multiaddr: vec![],
             libp2p_nodes: vec![],
             client_version: lighthouse_version::version_with_platform(),
             disable_discovery: false,

beacon_node/eth2_libp2p/src/discovery/enr.rs

@@ -6,6 +6,7 @@ use super::enr_ext::CombinedKeyExt;
 use super::ENR_FILENAME;
 use crate::types::{Enr, EnrBitfield};
 use crate::NetworkConfig;
+use discv5::enr::EnrKey;
 use libp2p::core::identity::Keypair;
 use slog::{debug, warn};
 use ssz::{Decode, Encode};
@@ -48,6 +49,56 @@ impl Eth2Enr for Enr {
     }
 }
 
+/// Either use the given ENR or load an ENR from file if it exists and matches the current NodeId
+/// and sequence number.
+/// If an ENR exists, with the same NodeId, this function checks to see if the loaded ENR from
+/// disk is suitable to use, otherwise we increment the given ENR's sequence number.
+pub fn use_or_load_enr(
+    enr_key: &CombinedKey,
+    local_enr: &mut Enr,
+    config: &NetworkConfig,
+    log: &slog::Logger,
+) -> Result<(), String> {
+    let enr_f = config.network_dir.join(ENR_FILENAME);
+    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
+        let mut enr_string = String::new();
+        match enr_file.read_to_string(&mut enr_string) {
+            Err(_) => debug!(log, "Could not read ENR from file"),
+            Ok(_) => {
+                match Enr::from_str(&enr_string) {
+                    Ok(disk_enr) => {
+                        // if the same node id, then we may need to update our sequence number
+                        if local_enr.node_id() == disk_enr.node_id() {
+                            if compare_enr(&local_enr, &disk_enr) {
+                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
+                                // the stored ENR has the same configuration, use it
+                                *local_enr = disk_enr;
+                                return Ok(());
+                            }
+
+                            // same node id, different configuration - update the sequence number
+                            // Note: local_enr is generated with default(0) attnets value,
+                            // so a non default value in persisted enr will also update sequence number.
+                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
+                            local_enr.set_seq(new_seq_no, enr_key).map_err(|e| {
+                                format!("Could not update ENR sequence number: {:?}", e)
+                            })?;
+                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
+                        }
+                    }
+                    Err(e) => {
+                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
+                    }
+                }
+            }
+        }
+    }
+
+    save_enr_to_disk(&config.network_dir, &local_enr, log);
+
+    Ok(())
+}
+
 /// Loads an ENR from file if it exists and matches the current NodeId and sequence number. If none
 /// exists, generates a new one.
 ///
@@ -65,49 +116,11 @@ pub fn build_or_load_enr<T: EthSpec>(
     let enr_key = CombinedKey::from_libp2p(&local_key)?;
     let mut local_enr = build_enr::<T>(&enr_key, config, enr_fork_id)?;
 
-    let enr_f = config.network_dir.join(ENR_FILENAME);
-    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
-        let mut enr_string = String::new();
-        match enr_file.read_to_string(&mut enr_string) {
-            Err(_) => debug!(log, "Could not read ENR from file"),
-            Ok(_) => {
-                match Enr::from_str(&enr_string) {
-                    Ok(disk_enr) => {
-                        // if the same node id, then we may need to update our sequence number
-                        if local_enr.node_id() == disk_enr.node_id() {
-                            if compare_enr(&local_enr, &disk_enr) {
-                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
-                                // the stored ENR has the same configuration, use it
-                                return Ok(disk_enr);
-                            }
-
-                            // same node id, different configuration - update the sequence number
-                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
-                            local_enr.set_seq(new_seq_no, &enr_key).map_err(|e| {
-                                format!("Could not update ENR sequence number: {:?}", e)
-                            })?;
-                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
-                        }
-                    }
-                    Err(e) => {
-                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
-                    }
-                }
-            }
-        }
-    }
-
-    save_enr_to_disk(&config.network_dir, &local_enr, log);
-
+    use_or_load_enr(&enr_key, &mut local_enr, config, log)?;
     Ok(local_enr)
 }
 
-/// Builds a lighthouse ENR given a `NetworkConfig`.
-pub fn build_enr<T: EthSpec>(
-    enr_key: &CombinedKey,
-    config: &NetworkConfig,
-    enr_fork_id: EnrForkId,
-) -> Result<Enr, String> {
+pub fn create_enr_builder_from_config<T: EnrKey>(config: &NetworkConfig) -> EnrBuilder<T> {
     let mut builder = EnrBuilder::new("v4");
     if let Some(enr_address) = config.enr_address {
         builder.ip(enr_address);
@@ -118,7 +131,17 @@ pub fn build_enr<T: EthSpec>(
     // we always give it our listening tcp port
     // TODO: Add uPnP support to map udp and tcp ports
     let tcp_port = config.enr_tcp_port.unwrap_or_else(|| config.libp2p_port);
-    builder.tcp(tcp_port);
+    builder.tcp(tcp_port).tcp(config.libp2p_port);
+    builder
+}
+
+/// Builds a lighthouse ENR given a `NetworkConfig`.
+pub fn build_enr<T: EthSpec>(
+    enr_key: &CombinedKey,
+    config: &NetworkConfig,
+    enr_fork_id: EnrForkId,
+) -> Result<Enr, String> {
+    let mut builder = create_enr_builder_from_config(config);
 
     // set the `eth2` field on our ENR
     builder.add_value(ETH2_ENR_KEY.into(), enr_fork_id.as_ssz_bytes());
@@ -129,7 +152,6 @@ pub fn build_enr<T: EthSpec>(
     builder.add_value(BITFIELD_ENR_KEY.into(), bitfield.as_ssz_bytes());
 
     builder
-        .tcp(config.libp2p_port)
         .build(enr_key)
         .map_err(|e| format!("Could not build Local ENR: {:?}", e))
 }

beacon_node/eth2_libp2p/src/discovery/mod.rs

@@ -3,19 +3,19 @@ pub(crate) mod enr;
 pub mod enr_ext;
 
 // Allow external use of the lighthouse ENR builder
-pub use enr::{build_enr, CombinedKey, Eth2Enr};
+pub use enr::{build_enr, create_enr_builder_from_config, use_or_load_enr, CombinedKey, Eth2Enr};
 pub use enr_ext::{CombinedKeyExt, EnrExt};
 pub use libp2p::core::identity::Keypair;
 
 use crate::metrics;
-use crate::{error, Enr, NetworkConfig, NetworkGlobals};
+use crate::{error, Enr, NetworkConfig, NetworkGlobals, SubnetDiscovery};
 use discv5::{enr::NodeId, Discv5, Discv5Event};
 use enr::{BITFIELD_ENR_KEY, ETH2_ENR_KEY};
 use futures::prelude::*;
 use futures::stream::FuturesUnordered;
 use libp2p::core::PeerId;
 use lru::LruCache;
-use slog::{crit, debug, info, warn};
+use slog::{crit, debug, error, info, warn};
 use ssz::{Decode, Encode};
 use ssz_types::BitVector;
 use std::{
@@ -163,7 +163,7 @@ pub struct Discovery<TSpec: EthSpec> {
 
 impl<TSpec: EthSpec> Discovery<TSpec> {
     /// NOTE: Creating discovery requires running within a tokio execution environment.
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         config: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
@@ -189,21 +189,23 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             .map_err(|e| format!("Discv5 service failed. Error: {:?}", e))?;
 
         // Add bootnodes to routing table
-        for bootnode_enr in config.boot_nodes.clone() {
+        for bootnode_enr in config.boot_nodes_enr.clone() {
             debug!(
                 log,
                 "Adding node to routing table";
-                "node_id" => format!("{}", bootnode_enr.node_id()),
-                "peer_id" => format!("{}", bootnode_enr.peer_id()),
+                "node_id" => bootnode_enr.node_id().to_string(),
+                "peer_id" => bootnode_enr.peer_id().to_string(),
                 "ip" => format!("{:?}", bootnode_enr.ip()),
                 "udp" => format!("{:?}", bootnode_enr.udp()),
                 "tcp" => format!("{:?}", bootnode_enr.tcp())
             );
+            let repr = bootnode_enr.to_string();
             let _ = discv5.add_enr(bootnode_enr).map_err(|e| {
-                debug!(
+                error!(
                     log,
                     "Could not add peer to the local routing table";
-                    "error" => e.to_string()
+                    "addr" => repr,
+                    "error" => e.to_string(),
                 )
             });
         }
@@ -217,7 +219,54 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             EventStream::InActive
         };
 
-        // Obtain the event stream
+        if !config.boot_nodes_multiaddr.is_empty() {
+            info!(log, "Contacting Multiaddr boot-nodes for their ENR");
+        }
+
+        // get futures for requesting the Enrs associated to these multiaddr and wait for their
+        // completion
+        let mut fut_coll = config
+            .boot_nodes_multiaddr
+            .iter()
+            .map(|addr| addr.to_string())
+            // request the ENR for this multiaddr and keep the original for logging
+            .map(|addr| {
+                futures::future::join(
+                    discv5.request_enr(addr.clone()),
+                    futures::future::ready(addr),
+                )
+            })
+            .collect::<FuturesUnordered<_>>();
+
+        while let Some((result, original_addr)) = fut_coll.next().await {
+            match result {
+                Ok(Some(enr)) => {
+                    debug!(
+                        log,
+                        "Adding node to routing table";
+                        "node_id" => enr.node_id().to_string(),
+                        "peer_id" => enr.peer_id().to_string(),
+                        "ip" => format!("{:?}", enr.ip()),
+                        "udp" => format!("{:?}", enr.udp()),
+                        "tcp" => format!("{:?}", enr.tcp())
+                    );
+                    let _ = discv5.add_enr(enr).map_err(|e| {
+                        error!(
+                            log,
+                            "Could not add peer to the local routing table";
+                            "addr" => original_addr.to_string(),
+                            "error" => e.to_string(),
+                        )
+                    });
+                }
+                Ok(None) => {
+                    error!(log, "No ENR found for MultiAddr"; "addr" => original_addr.to_string())
+                }
+                Err(e) => {
+                    error!(log, "Error getting mapping to ENR"; "multiaddr" => original_addr.to_string(), "error" => e.to_string())
+                }
+            }
+        }
 
         Ok(Self {
             cached_enrs: LruCache::new(50),
@@ -256,12 +305,19 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
     }
 
     /// Processes a request to search for more peers on a subnet.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
+    pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
         // If the discv5 service isn't running, ignore queries
         if !self.started {
             return;
         }
-        self.add_subnet_query(subnet_id, min_ttl, 0);
+        debug!(
+            self.log,
+            "Making discovery query for subnets";
+            "subnets" => format!("{:?}", subnets_to_discover.iter().map(|s| s.subnet_id).collect::<Vec<_>>())
+        );
+        for subnet in subnets_to_discover {
+            self.add_subnet_query(subnet.subnet_id, subnet.min_ttl, 0);
+        }
     }
 
     /// Add an ENR to the routing table of the discovery mechanism.
@@ -335,6 +391,9 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         // replace the global version
         *self.network_globals.local_enr.write() = self.discv5.local_enr();
+
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
         Ok(())
     }
 
@@ -367,6 +426,9 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         // replace the global version with discovery version
         *self.network_globals.local_enr.write() = self.discv5.local_enr();
+
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
     }
 
     /* Internal Functions */
@@ -459,6 +521,11 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                         // This query is for searching for peers of a particular subnet
                         // Drain subnet_queries so we can re-use it as we continue to process the queue
                         let grouped_queries: Vec<SubnetQuery> = subnet_queries.drain(..).collect();
+                        debug!(
+                            self.log,
+                            "Starting grouped subnet query";
+                            "subnets" => format!("{:?}", grouped_queries.iter().map(|q| q.subnet_id).collect::<Vec<_>>()),
+                        );
                         self.start_subnet_query(grouped_queries);
                     }
                 }
@@ -733,8 +800,8 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                             if enr.eth2() == self.local_enr().eth2() {
                                 trace!(self.log, "Peer found in process of query"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
                             } else {
-                                // this is temporary warning for debugging the DHT
-                                warn!(self.log, "Found peer during discovery not on correct fork"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
+                            // this is temporary warning for debugging the DHT
+                            warn!(self.log, "Found peer during discovery not on correct fork"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
                             }
                             */
                         }

beacon_node/eth2_libp2p/src/discovery/subnet_predicate.rs

@@ -1,5 +1,6 @@
 ///! The subnet predicate used for searching for a particular subnet.
 use super::*;
+use slog::{debug, trace};
 use std::ops::Deref;
 
 /// Returns the predicate for a given subnet.
@@ -30,7 +31,7 @@ where
                 .collect();
 
             if matches.is_empty() {
-                debug!(
+                trace!(
                     log_clone,
                     "Peer found but not on any of the desired subnets";
                     "peer_id" => format!("{}", enr.peer_id())

beacon_node/eth2_libp2p/src/lib.rs

@@ -14,16 +14,16 @@ pub mod rpc;
 mod service;
 pub mod types;
 
-pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage};
+pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage, SubnetDiscovery};
 pub use behaviour::{BehaviourEvent, PeerRequestId, Request, Response};
 pub use config::Config as NetworkConfig;
 pub use discovery::{CombinedKeyExt, EnrExt, Eth2Enr};
 pub use discv5;
-pub use libp2p::gossipsub::{MessageId, Topic, TopicHash};
+pub use libp2p::gossipsub::{Gossipsub, MessageAcceptance, MessageId, Topic, TopicHash};
 pub use libp2p::{core::ConnectedPoint, PeerId, Swarm};
 pub use libp2p::{multiaddr, Multiaddr};
 pub use metrics::scrape_discovery_metrics;
 pub use peer_manager::{
     client::Client, score::PeerAction, PeerDB, PeerInfo, PeerSyncStatus, SyncInfo,
 };
-pub use service::{Libp2pEvent, Service, NETWORK_KEY_FILENAME};
+pub use service::{load_private_key, Libp2pEvent, Service, NETWORK_KEY_FILENAME};

beacon_node/eth2_libp2p/src/metrics.rs

@@ -34,9 +34,20 @@ lazy_static! {
         "Unsolicited discovery requests per ip per second",
         &["Addresses"]
     );
-    pub static ref GOSSIPSUB_SUBSCRIBED_PEERS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
-        "gossipsub_peers_per_topic_count",
-        "Peers subscribed per topic",
+    pub static ref PEERS_PER_CLIENT: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "libp2p_peers_per_client",
+        "The connected peers via client implementation",
+        &["Client"]
+    );
+    pub static ref FAILED_ATTESTATION_PUBLISHES_PER_SUBNET: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "gossipsub_failed_attestation_publishes_per_subnet",
+            "Failed attestation publishes per subnet",
+            &["subnet"]
+        );
+    pub static ref FAILED_PUBLISHES_PER_MAIN_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_failed_publishes_per_main_topic",
+        "Failed gossip publishes",
         &["topic_hash"]
     );
 }

beacon_node/eth2_libp2p/src/peer_manager/client.rs

@@ -20,7 +20,7 @@ pub struct Client {
     pub agent_string: Option<String>,
 }
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, PartialEq)]
 pub enum ClientKind {
     /// A lighthouse node (the best kind).
     Lighthouse,
@@ -30,6 +30,8 @@ pub enum ClientKind {
     Teku,
     /// A Prysm node.
     Prysm,
+    /// A lodestar node.
+    Lodestar,
     /// An unknown client.
     Unknown,
 }
@@ -84,6 +86,7 @@ impl std::fmt::Display for Client {
                 "Prysm: version: {}, os_version: {}",
                 self.version, self.os_version
             ),
+            ClientKind::Lodestar => write!(f, "Lodestar: version: {}", self.version),
             ClientKind::Unknown => {
                 if let Some(agent_string) = &self.agent_string {
                     write!(f, "Unknown: {}", agent_string)
@@ -95,6 +98,12 @@ impl std::fmt::Display for Client {
     }
 }
 
+impl std::fmt::Display for ClientKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
 // helper function to identify clients from their agent_version. Returns the client
 // kind and it's associated version and the OS kind.
 fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String) {
@@ -157,6 +166,18 @@ fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String
             }
             (kind, version, os_version)
         }
+        Some("js-libp2p") => {
+            let kind = ClientKind::Lodestar;
+            let mut version = String::from("unknown");
+            let mut os_version = version.clone();
+            if let Some(agent_version) = agent_split.next() {
+                version = agent_version.into();
+                if let Some(agent_os_version) = agent_split.next() {
+                    os_version = agent_os_version.into();
+                }
+            }
+            (kind, version, os_version)
+        }
         _ => {
             let unknown = String::from("unknown");
             (ClientKind::Unknown, unknown.clone(), unknown)

beacon_node/eth2_libp2p/src/peer_manager/mod.rs

@@ -4,7 +4,7 @@ pub use self::peerdb::*;
 use crate::discovery::{Discovery, DiscoveryEvent};
 use crate::rpc::{GoodbyeReason, MetaData, Protocol, RPCError, RPCResponseErrorCode};
 use crate::{error, metrics};
-use crate::{EnrExt, NetworkConfig, NetworkGlobals, PeerId};
+use crate::{EnrExt, NetworkConfig, NetworkGlobals, PeerId, SubnetDiscovery};
 use futures::prelude::*;
 use futures::Stream;
 use hashset_delay::HashSetDelay;
@@ -19,7 +19,7 @@ use std::{
     task::{Context, Poll},
     time::{Duration, Instant},
 };
-use types::{EthSpec, SubnetId};
+use types::EthSpec;
 
 pub use libp2p::core::{identity::Keypair, Multiaddr};
 
@@ -88,14 +88,14 @@ pub enum PeerManagerEvent {
 
 impl<TSpec: EthSpec> PeerManager<TSpec> {
     // NOTE: Must be run inside a tokio executor.
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         config: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
         log: &slog::Logger,
     ) -> error::Result<Self> {
         // start the discovery service
-        let mut discovery = Discovery::new(local_key, config, network_globals.clone(), log)?;
+        let mut discovery = Discovery::new(local_key, config, network_globals.clone(), log).await?;
 
         // start searching for peers
         discovery.discover_peers();
@@ -213,17 +213,19 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     }
 
     /// A request to find peers on a given subnet.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
+    pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
         // Extend the time to maintain peers if required.
-        if let Some(min_ttl) = min_ttl {
-            self.network_globals
-                .peers
-                .write()
-                .extend_peers_on_subnet(subnet_id, min_ttl);
+        for s in subnets_to_discover.iter() {
+            if let Some(min_ttl) = s.min_ttl {
+                self.network_globals
+                    .peers
+                    .write()
+                    .extend_peers_on_subnet(s.subnet_id, min_ttl);
+            }
         }
 
         // request the subnet query from discovery
-        self.discovery.discover_subnet_peers(subnet_id, min_ttl);
+        self.discovery.discover_subnet_peers(subnets_to_discover);
     }
 
     /// A STATUS message has been received from a peer. This resets the status timer.
@@ -237,6 +239,27 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// This is also called when dialing a peer fails.
     pub fn notify_disconnect(&mut self, peer_id: &PeerId) {
+        // Decrement the PEERS_PER_CLIENT metric
+        if let Some(kind) = self
+            .network_globals
+            .peers
+            .read()
+            .peer_info(peer_id)
+            .and_then(|peer_info| {
+                if let Connected { .. } = peer_info.connection_status {
+                    Some(peer_info.client.kind.clone())
+                } else {
+                    None
+                }
+            })
+        {
+            if let Some(v) =
+                metrics::get_int_gauge(&metrics::PEERS_PER_CLIENT, &[&kind.to_string()])
+            {
+                v.dec()
+            };
+        }
+
         self.network_globals.peers.write().disconnect(peer_id);
 
         // remove the ping and status timer for the peer
@@ -294,8 +317,25 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// Updates `PeerInfo` with `identify` information.
     pub fn identify(&mut self, peer_id: &PeerId, info: &IdentifyInfo) {
         if let Some(peer_info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
+            let previous_kind = peer_info.client.kind.clone();
             peer_info.client = client::Client::from_identify_info(info);
             peer_info.listening_addresses = info.listen_addrs.clone();
+
+            if previous_kind != peer_info.client.kind {
+                // update the peer client kind metric
+                if let Some(v) = metrics::get_int_gauge(
+                    &metrics::PEERS_PER_CLIENT,
+                    &[&peer_info.client.kind.to_string()],
+                ) {
+                    v.inc()
+                };
+                if let Some(v) = metrics::get_int_gauge(
+                    &metrics::PEERS_PER_CLIENT,
+                    &[&previous_kind.to_string()],
+                ) {
+                    v.dec()
+                };
+            }
         } else {
             crit!(self.log, "Received an Identify response from an unknown peer"; "peer_id" => peer_id.to_string());
         }
@@ -539,11 +579,8 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// This is called by `connect_ingoing` and `connect_outgoing`.
     ///
-    /// This informs if the peer was accepted in to the db or not.
+    /// Informs if the peer was accepted in to the db or not.
     fn connect_peer(&mut self, peer_id: &PeerId, connection: ConnectingType) -> bool {
-        // TODO: remove after timed updates
-        //self.update_reputations();
-
         {
             let mut peerdb = self.network_globals.peers.write();
             if peerdb.connection_status(peer_id).map(|c| c.is_banned()) == Some(true) {
@@ -552,7 +589,10 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             }
 
             match connection {
-                ConnectingType::Dialing => peerdb.dialing_peer(peer_id),
+                ConnectingType::Dialing => {
+                    peerdb.dialing_peer(peer_id);
+                    return true;
+                }
                 ConnectingType::IngoingConnected => peerdb.connect_outgoing(peer_id),
                 ConnectingType::OutgoingConnected => peerdb.connect_ingoing(peer_id),
             }
@@ -569,6 +609,21 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             self.network_globals.connected_peers() as i64,
         );
 
+        // Increment the PEERS_PER_CLIENT metric
+        if let Some(kind) = self
+            .network_globals
+            .peers
+            .read()
+            .peer_info(peer_id)
+            .map(|peer_info| peer_info.client.kind.clone())
+        {
+            if let Some(v) =
+                metrics::get_int_gauge(&metrics::PEERS_PER_CLIENT, &[&kind.to_string()])
+            {
+                v.inc()
+            };
+        }
+
         true
     }
 
@@ -690,7 +745,8 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// NOTE: Discovery will only add a new query if one isn't already queued.
     fn heartbeat(&mut self) {
         // TODO: Provide a back-off time for discovery queries. I.e Queue many initially, then only
-        // perform discoveries over a larger fixed interval. Perhaps one every 6 heartbeats
+        // perform discoveries over a larger fixed interval. Perhaps one every 6 heartbeats. This
+        // is achievable with a leaky bucket
         let peer_count = self.network_globals.connected_or_dialing_peers();
         if peer_count < self.target_peers {
             // If we need more peers, queue a discovery lookup.

beacon_node/eth2_libp2p/src/peer_manager/peer_info.rs

@@ -4,9 +4,10 @@ use super::PeerSyncStatus;
 use crate::rpc::MetaData;
 use crate::Multiaddr;
 use serde::{
-    ser::{SerializeStructVariant, Serializer},
+    ser::{SerializeStruct, Serializer},
     Serialize,
 };
+use std::net::IpAddr;
 use std::time::Instant;
 use types::{EthSpec, SubnetId};
 use PeerConnectionStatus::*;
@@ -104,6 +105,8 @@ pub enum PeerConnectionStatus {
     Banned {
         /// moment when the peer was banned.
         since: Instant,
+        /// ip addresses this peer had a the moment of the ban
+        ip_addresses: Vec<IpAddr>,
     },
     /// We are currently dialing this peer.
     Dialing {
@@ -117,29 +120,51 @@ pub enum PeerConnectionStatus {
 /// Serialization for http requests.
 impl Serialize for PeerConnectionStatus {
     fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let mut s = serializer.serialize_struct("connection_status", 5)?;
         match self {
             Connected { n_in, n_out } => {
-                let mut s = serializer.serialize_struct_variant("", 0, "Connected", 2)?;
-                s.serialize_field("in", n_in)?;
-                s.serialize_field("out", n_out)?;
+                s.serialize_field("status", "connected")?;
+                s.serialize_field("connections_in", n_in)?;
+                s.serialize_field("connections_out", n_out)?;
+                s.serialize_field("last_seen", &0)?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
                 s.end()
             }
             Disconnected { since } => {
-                let mut s = serializer.serialize_struct_variant("", 1, "Disconnected", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "disconnected")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
                 s.end()
             }
-            Banned { since } => {
-                let mut s = serializer.serialize_struct_variant("", 2, "Banned", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+            Banned {
+                since,
+                ip_addresses,
+            } => {
+                s.serialize_field("status", "banned")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.serialize_field("banned_ips", &ip_addresses)?;
                 s.end()
             }
             Dialing { since } => {
-                let mut s = serializer.serialize_struct_variant("", 3, "Dialing", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "dialing")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
+                s.end()
+            }
+            Unknown => {
+                s.serialize_field("status", "unknown")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &0)?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
                 s.end()
             }
-            Unknown => serializer.serialize_unit_variant("", 4, "Unknown"),
         }
     }
 }
@@ -218,15 +243,16 @@ impl PeerConnectionStatus {
     }
 
     /// Modifies the status to Banned
-    pub fn ban(&mut self) {
+    pub fn ban(&mut self, ip_addresses: Vec<IpAddr>) {
         *self = Banned {
             since: Instant::now(),
+            ip_addresses,
         };
     }
 
     /// The score system has unbanned the peer. Update the connection status
     pub fn unban(&mut self) {
-        if let PeerConnectionStatus::Banned { since } = self {
+        if let PeerConnectionStatus::Banned { since, .. } = self {
             *self = PeerConnectionStatus::Disconnected { since: *since }
         }
     }

beacon_node/eth2_libp2p/src/peer_manager/peerdb.rs

@@ -1,11 +1,13 @@
 use super::peer_info::{PeerConnectionStatus, PeerInfo};
 use super::peer_sync_status::PeerSyncStatus;
 use super::score::{Score, ScoreState};
+use crate::multiaddr::Protocol;
 use crate::rpc::methods::MetaData;
 use crate::PeerId;
 use rand::seq::SliceRandom;
 use slog::{crit, debug, trace, warn};
 use std::collections::HashMap;
+use std::net::IpAddr;
 use std::time::Instant;
 use types::{EthSpec, SubnetId};
 
@@ -13,6 +15,9 @@ use types::{EthSpec, SubnetId};
 const MAX_DC_PEERS: usize = 500;
 /// The maximum number of banned nodes to remember.
 const MAX_BANNED_PEERS: usize = 1000;
+/// If there are more than `BANNED_PEERS_PER_IP_THRESHOLD` many banned peers with the same IP we ban
+/// the IP.
+const BANNED_PEERS_PER_IP_THRESHOLD: usize = 5;
 
 /// Storage of known peers, their reputation and information
 pub struct PeerDB<TSpec: EthSpec> {
@@ -20,18 +25,72 @@ pub struct PeerDB<TSpec: EthSpec> {
     peers: HashMap<PeerId, PeerInfo<TSpec>>,
     /// The number of disconnected nodes in the database.
     disconnected_peers: usize,
-    /// The number of banned peers in the database.
-    banned_peers: usize,
+    /// Counts banned peers in total and per ip
+    banned_peers_count: BannedPeersCount,
     /// PeerDB's logger
     log: slog::Logger,
 }
 
+pub struct BannedPeersCount {
+    /// The number of banned peers in the database.
+    banned_peers: usize,
+    /// maps ips to number of banned peers with this ip
+    banned_peers_per_ip: HashMap<IpAddr, usize>,
+}
+
+impl BannedPeersCount {
+    /// Removes the peer from the counts if it is banned. Returns true if the peer was banned and
+    /// false otherwise.
+    pub fn remove_banned_peer(&mut self, connection_status: &PeerConnectionStatus) -> bool {
+        match connection_status {
+            PeerConnectionStatus::Banned { ip_addresses, .. } => {
+                self.banned_peers = self.banned_peers.saturating_sub(1);
+                for address in ip_addresses {
+                    if let Some(count) = self.banned_peers_per_ip.get_mut(address) {
+                        *count = count.saturating_sub(1);
+                    }
+                }
+                true
+            }
+            _ => false, //if not banned do nothing
+        }
+    }
+
+    pub fn add_banned_peer(&mut self, connection_status: &PeerConnectionStatus) {
+        if let PeerConnectionStatus::Banned { ip_addresses, .. } = connection_status {
+            self.banned_peers += 1;
+            for address in ip_addresses {
+                *self.banned_peers_per_ip.entry(*address).or_insert(0) += 1;
+            }
+        }
+    }
+
+    pub fn banned_peers(&self) -> usize {
+        self.banned_peers
+    }
+
+    /// An IP is considered banned if more than BANNED_PEERS_PER_IP_THRESHOLD banned peers
+    /// exist with this IP
+    pub fn ip_is_banned(&self, ip: &IpAddr) -> bool {
+        self.banned_peers_per_ip
+            .get(ip)
+            .map_or(false, |count| *count > BANNED_PEERS_PER_IP_THRESHOLD)
+    }
+
+    pub fn new() -> Self {
+        BannedPeersCount {
+            banned_peers: 0,
+            banned_peers_per_ip: HashMap::new(),
+        }
+    }
+}
+
 impl<TSpec: EthSpec> PeerDB<TSpec> {
     pub fn new(log: &slog::Logger) -> Self {
         Self {
             log: log.clone(),
             disconnected_peers: 0,
-            banned_peers: 0,
+            banned_peers_count: BannedPeersCount::new(),
             peers: HashMap::new(),
         }
     }
@@ -99,17 +158,35 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
 
     /// Returns true if the Peer is banned.
     pub fn is_banned(&self, peer_id: &PeerId) -> bool {
-        match self.peers.get(peer_id).map(|info| info.score.state()) {
-            Some(ScoreState::Banned) => true,
-            _ => false,
+        if let Some(peer) = self.peers.get(peer_id) {
+            match peer.score.state() {
+                ScoreState::Banned => true,
+                _ => self.ip_is_banned(peer),
+            }
+        } else {
+            false
         }
     }
 
+    fn ip_is_banned(&self, peer: &PeerInfo<TSpec>) -> bool {
+        peer.listening_addresses.iter().any(|addr| {
+            addr.iter().any(|p| match p {
+                Protocol::Ip4(ip) => self.banned_peers_count.ip_is_banned(&ip.into()),
+                Protocol::Ip6(ip) => self.banned_peers_count.ip_is_banned(&ip.into()),
+                _ => false,
+            })
+        })
+    }
+
     /// Returns true if the Peer is either banned or in the disconnected state.
     pub fn is_banned_or_disconnected(&self, peer_id: &PeerId) -> bool {
-        match self.peers.get(peer_id).map(|info| info.score.state()) {
-            Some(ScoreState::Banned) | Some(ScoreState::Disconnected) => true,
-            _ => false,
+        if let Some(peer) = self.peers.get(peer_id) {
+            match peer.score.state() {
+                ScoreState::Banned | ScoreState::Disconnected => true,
+                _ => self.ip_is_banned(peer),
+            }
+        } else {
+            false
         }
     }
 
@@ -233,9 +310,10 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         if info.connection_status.is_disconnected() {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
-        if info.connection_status.is_banned() {
-            self.banned_peers = self.banned_peers.saturating_sub(1);
-        }
+
+        self.banned_peers_count
+            .remove_banned_peer(&info.connection_status);
+
         info.connection_status = PeerConnectionStatus::Dialing {
             since: Instant::now(),
         };
@@ -284,9 +362,8 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         if info.connection_status.is_disconnected() {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
-        if info.connection_status.is_banned() {
-            self.banned_peers = self.banned_peers.saturating_sub(1);
-        }
+        self.banned_peers_count
+            .remove_banned_peer(&info.connection_status);
         info.connection_status.connect_ingoing();
     }
 
@@ -297,9 +374,8 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         if info.connection_status.is_disconnected() {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
-        if info.connection_status.is_banned() {
-            self.banned_peers = self.banned_peers.saturating_sub(1);
-        }
+        self.banned_peers_count
+            .remove_banned_peer(&info.connection_status);
         info.connection_status.connect_outgoing();
     }
 
@@ -329,8 +405,23 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
         if !info.connection_status.is_banned() {
-            info.connection_status.ban();
-            self.banned_peers += 1;
+            info.connection_status
+                .ban(
+                    info.listening_addresses
+                        .iter()
+                        .fold(Vec::new(), |mut v, a| {
+                            for p in a {
+                                match p {
+                                    Protocol::Ip4(ip) => v.push(ip.into()),
+                                    Protocol::Ip6(ip) => v.push(ip.into()),
+                                    _ => (),
+                                }
+                            }
+                            v
+                        }),
+                );
+            self.banned_peers_count
+                .add_banned_peer(&info.connection_status);
         }
         self.shrink_to_fit();
     }
@@ -345,8 +436,9 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         });
 
         if info.connection_status.is_banned() {
+            self.banned_peers_count
+                .remove_banned_peer(&info.connection_status);
             info.connection_status.unban();
-            self.banned_peers = self.banned_peers.saturating_sub(1);
         }
         self.shrink_to_fit();
     }
@@ -355,9 +447,9 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
     /// Drops the peers with the lowest reputation so that the number of
     /// disconnected peers is less than MAX_DC_PEERS
     pub fn shrink_to_fit(&mut self) {
-        // Remove excess baned peers
-        while self.banned_peers > MAX_BANNED_PEERS {
-            if let Some(to_drop) = self
+        // Remove excess banned peers
+        while self.banned_peers_count.banned_peers() > MAX_BANNED_PEERS {
+            if let Some(to_drop) = if let Some((id, info)) = self
                 .peers
                 .iter()
                 .filter(|(_, info)| info.connection_status.is_banned())
@@ -366,15 +458,23 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
                         .score
                         .partial_cmp(&info_b.score)
                         .unwrap_or(std::cmp::Ordering::Equal)
-                })
-                .map(|(id, _)| id.clone())
-            {
+                }) {
+                self.banned_peers_count
+                    .remove_banned_peer(&info.connection_status);
+                Some(id.clone())
+            } else {
+                // If there is no minimum, this is a coding error.
+                crit!(
+                    self.log,
+                    "banned_peers > MAX_BANNED_PEERS despite no banned peers in db!"
+                );
+                // reset banned_peers this will also exit the loop
+                self.banned_peers_count = BannedPeersCount::new();
+                None
+            } {
                 debug!(self.log, "Removing old banned peer"; "peer_id" => to_drop.to_string());
                 self.peers.remove(&to_drop);
             }
-            // If there is no minimum, this is a coding error. For safety we decrease
-            // the count to avoid a potential infinite loop.
-            self.banned_peers = self.banned_peers.saturating_sub(1);
         }
 
         // Remove excess disconnected peers
@@ -422,8 +522,11 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use libp2p::core::Multiaddr;
     use slog::{o, Drain};
+    use std::net::{Ipv4Addr, Ipv6Addr};
     use types::MinimalEthSpec;
+
     type M = MinimalEthSpec;
 
     pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger {
@@ -503,13 +606,13 @@ mod tests {
             let p = PeerId::random();
             pdb.connect_ingoing(&p);
         }
-        assert_eq!(pdb.banned_peers, 0);
+        assert_eq!(pdb.banned_peers_count.banned_peers(), 0);
 
         for p in pdb.connected_peer_ids().cloned().collect::<Vec<_>>() {
             pdb.ban(&p);
         }
 
-        assert_eq!(pdb.banned_peers, MAX_BANNED_PEERS);
+        assert_eq!(pdb.banned_peers_count.banned_peers(), MAX_BANNED_PEERS);
     }
 
     #[test]
@@ -608,24 +711,39 @@ mod tests {
         pdb.connect_ingoing(&random_peer2);
         pdb.connect_ingoing(&random_peer3);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.connect_ingoing(&random_peer);
         pdb.disconnect(&random_peer1);
         pdb.ban(&random_peer2);
         pdb.connect_ingoing(&random_peer3);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.ban(&random_peer1);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.connect_outgoing(&random_peer2);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.ban(&random_peer3);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.ban(&random_peer3);
         pdb.connect_ingoing(&random_peer1);
@@ -633,15 +751,191 @@ mod tests {
         pdb.ban(&random_peer3);
         pdb.connect_ingoing(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.disconnect(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.disconnect(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.ban(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
     }
+
+    fn connect_peer_with_ips(pdb: &mut PeerDB<M>, ips: Vec<Vec<IpAddr>>) -> PeerId {
+        let p = PeerId::random();
+        pdb.connect_ingoing(&p);
+        pdb.peers.get_mut(&p).unwrap().listening_addresses = ips
+            .into_iter()
+            .map(|ip_addresses| {
+                let mut addr = Multiaddr::empty();
+                for ip_address in ip_addresses {
+                    addr.push(Protocol::from(ip_address));
+                }
+                addr
+            })
+            .collect();
+        p
+    }
+
+    #[test]
+    fn test_ban_address() {
+        let mut pdb = get_db();
+
+        let ip1: IpAddr = Ipv4Addr::new(1, 2, 3, 4).into();
+        let ip2: IpAddr = Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8).into();
+        let ip3: IpAddr = Ipv4Addr::new(1, 2, 3, 5).into();
+        let ip4: IpAddr = Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 9).into();
+        let ip5: IpAddr = Ipv4Addr::new(2, 2, 3, 4).into();
+
+        let mut peers = Vec::new();
+        for i in 0..BANNED_PEERS_PER_IP_THRESHOLD + 2 {
+            peers.push(connect_peer_with_ips(
+                &mut pdb,
+                if i == 0 {
+                    vec![vec![ip1], vec![ip2]]
+                } else {
+                    vec![vec![ip1, ip2], vec![ip3, ip4]]
+                },
+            ));
+        }
+
+        let p1 = connect_peer_with_ips(&mut pdb, vec![vec![ip1]]);
+        let p2 = connect_peer_with_ips(&mut pdb, vec![vec![ip2, ip5]]);
+        let p3 = connect_peer_with_ips(&mut pdb, vec![vec![ip3], vec![ip5]]);
+        let p4 = connect_peer_with_ips(&mut pdb, vec![vec![ip5, ip4]]);
+        let p5 = connect_peer_with_ips(&mut pdb, vec![vec![ip5]]);
+
+        for p in &peers[..BANNED_PEERS_PER_IP_THRESHOLD + 1] {
+            pdb.ban(p);
+        }
+
+        //check that ip1 and ip2 are banned but ip3-5 not
+        assert!(pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+        assert!(!pdb.is_banned(&p3));
+        assert!(!pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+
+        //ban also the last peer in peers
+        pdb.ban(&peers[BANNED_PEERS_PER_IP_THRESHOLD + 1]);
+
+        //check that ip1-ip4 are banned but ip5 not
+        assert!(pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+        assert!(pdb.is_banned(&p3));
+        assert!(pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+
+        //peers[0] gets unbanned
+        pdb.unban(&peers[0]);
+
+        //nothing changed
+        assert!(pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+        assert!(pdb.is_banned(&p3));
+        assert!(pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+
+        //peers[1] gets unbanned
+        pdb.unban(&peers[1]);
+
+        //all ips are unbanned
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+        assert!(!pdb.is_banned(&p3));
+        assert!(!pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+    }
+
+    #[test]
+    fn test_banned_ip_consistent_after_changing_ips() {
+        let mut pdb = get_db();
+
+        let ip1: IpAddr = Ipv4Addr::new(1, 2, 3, 4).into();
+        let ip2: IpAddr = Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8).into();
+
+        let mut peers = Vec::new();
+        for _ in 0..BANNED_PEERS_PER_IP_THRESHOLD + 1 {
+            peers.push(connect_peer_with_ips(&mut pdb, vec![vec![ip1]]));
+        }
+
+        let p1 = connect_peer_with_ips(&mut pdb, vec![vec![ip1]]);
+        let p2 = connect_peer_with_ips(&mut pdb, vec![vec![ip2]]);
+
+        //ban all peers
+        for p in &peers {
+            pdb.ban(p);
+        }
+
+        //check ip is banned
+        assert!(pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //change addresses of banned peers
+        for p in &peers {
+            pdb.peers.get_mut(p).unwrap().listening_addresses =
+                vec![Multiaddr::empty().with(Protocol::from(ip2))];
+        }
+
+        //check still the same ip is banned
+        assert!(pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //unban a peer
+        pdb.unban(&peers[0]);
+
+        //check not banned anymore
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //check still not banned after new ban
+        pdb.ban(&peers[0]);
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //unban and reban all peers
+        for p in &peers {
+            pdb.unban(p);
+            pdb.ban(p);
+        }
+
+        //ip2 is now banned
+        assert!(!pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+
+        //change ips back again
+        for p in &peers {
+            pdb.peers.get_mut(p).unwrap().listening_addresses =
+                vec![Multiaddr::empty().with(Protocol::from(ip1))];
+        }
+
+        //reban every peer except one
+        for p in &peers[1..] {
+            pdb.unban(p);
+            pdb.ban(p);
+        }
+
+        //nothing is banned
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //reban last peer
+        pdb.unban(&peers[0]);
+        pdb.ban(&peers[0]);
+
+        //ip1 is banned
+        assert!(pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+    }
 }

beacon_node/eth2_libp2p/src/rpc/codec/base.rs

@@ -1,5 +1,6 @@
 //! This handles the various supported encoding mechanism for the Eth 2.0 RPC.
 
+use crate::rpc::methods::ErrorType;
 use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
 use libp2p::bytes::BufMut;
 use libp2p::bytes::BytesMut;
@@ -8,12 +9,12 @@ use tokio_util::codec::{Decoder, Encoder};
 use types::EthSpec;
 
 pub trait OutboundCodec<TItem>: Encoder<TItem> + Decoder {
-    type ErrorType;
+    type CodecErrorType;
 
     fn decode_error(
         &mut self,
         src: &mut BytesMut,
-    ) -> Result<Option<Self::ErrorType>, <Self as Decoder>::Error>;
+    ) -> Result<Option<Self::CodecErrorType>, <Self as Decoder>::Error>;
 }
 
 /* Global Inbound Codec */
@@ -130,8 +131,8 @@ where
 impl<TCodec, TSpec> Decoder for BaseOutboundCodec<TCodec, TSpec>
 where
     TSpec: EthSpec,
-    TCodec:
-        OutboundCodec<RPCRequest<TSpec>, ErrorType = String> + Decoder<Item = RPCResponse<TSpec>>,
+    TCodec: OutboundCodec<RPCRequest<TSpec>, CodecErrorType = ErrorType>
+        + Decoder<Item = RPCResponse<TSpec>>,
 {
     type Item = RPCCodedResponse<TSpec>;
     type Error = <TCodec as Decoder>::Error;

beacon_node/eth2_libp2p/src/rpc/codec/ssz_snappy.rs

@@ -374,9 +374,12 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
 }
 
 impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZSnappyOutboundCodec<TSpec> {
-    type ErrorType = String;
+    type CodecErrorType = ErrorType;
 
-    fn decode_error(&mut self, src: &mut BytesMut) -> Result<Option<Self::ErrorType>, RPCError> {
+    fn decode_error(
+        &mut self,
+        src: &mut BytesMut,
+    ) -> Result<Option<Self::CodecErrorType>, RPCError> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
             match self.inner.decode(src).map_err(RPCError::from)? {
@@ -401,9 +404,9 @@ impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZSnappyOutboundCodec
                 let n = reader.get_ref().position();
                 self.len = None;
                 let _read_bytes = src.split_to(n as usize);
-                Ok(Some(
-                    String::from_utf8_lossy(&<Vec<u8>>::from_ssz_bytes(&decoded_buffer)?).into(),
-                ))
+                Ok(Some(ErrorType(VariableList::from_ssz_bytes(
+                    &decoded_buffer,
+                )?)))
             }
             Err(e) => match e.kind() {
                 // Haven't received enough bytes to decode yet

beacon_node/eth2_libp2p/src/rpc/handler.rs

@@ -81,7 +81,7 @@ where
     TSpec: EthSpec,
 {
     /// The upgrade for inbound substreams.
-    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>,
+    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
 
     /// Errors occurring on outbound and inbound connections queued for reporting back.
     pending_errors: Vec<HandlerErr>,
@@ -116,9 +116,6 @@ where
     /// Maximum number of concurrent outbound substreams being opened. Value is never modified.
     max_dial_negotiated: u32,
 
-    /// Value to return from `connection_keep_alive`.
-    keep_alive: KeepAlive,
-
     /// State of the handler.
     state: HandlerState,
 
@@ -228,7 +225,10 @@ impl<TSpec> RPCHandler<TSpec>
 where
     TSpec: EthSpec,
 {
-    pub fn new(listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>, log: &slog::Logger) -> Self {
+    pub fn new(
+        listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
+        log: &slog::Logger,
+    ) -> Self {
         RPCHandler {
             listen_protocol,
             pending_errors: Vec::new(),
@@ -243,7 +243,6 @@ where
             current_outbound_substream_id: SubstreamId(0),
             state: HandlerState::Active,
             max_dial_negotiated: 8,
-            keep_alive: KeepAlive::Yes,
             outbound_io_error_retries: 0,
             log: log.clone(),
         }
@@ -253,7 +252,7 @@ where
     ///
     /// > **Note**: If you modify the protocol, modifications will only applies to future inbound
     /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_ref(&self) -> &SubstreamProtocol<RPCProtocol<TSpec>> {
+    pub fn listen_protocol_ref(&self) -> &SubstreamProtocol<RPCProtocol<TSpec>, ()> {
         &self.listen_protocol
     }
 
@@ -261,7 +260,7 @@ where
     ///
     /// > **Note**: If you modify the protocol, modifications will only apply to future inbound
     /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_mut(&mut self) -> &mut SubstreamProtocol<RPCProtocol<TSpec>> {
+    pub fn listen_protocol_mut(&mut self) -> &mut SubstreamProtocol<RPCProtocol<TSpec>, ()> {
         &mut self.listen_protocol
     }
 
@@ -287,7 +286,6 @@ where
                 TInstant::now() + Duration::from_secs(SHUTDOWN_TIMEOUT_SECS as u64),
             ));
         }
-        self.update_keep_alive();
     }
 
     /// Opens an outbound substream with a request.
@@ -295,7 +293,6 @@ where
         match self.state {
             HandlerState::Active => {
                 self.dial_queue.push((id, req));
-                self.update_keep_alive();
             }
             _ => {
                 self.pending_errors.push(HandlerErr::Outbound {
@@ -338,43 +335,6 @@ where
         }
         inbound_info.pending_items.push(response);
     }
-
-    /// Updates the `KeepAlive` returned by `connection_keep_alive`.
-    ///
-    /// The handler stays alive as long as there are inbound/outbound substreams established and no
-    /// items dialing/to be dialed. Otherwise it is given a grace period of inactivity of
-    /// `self.inactive_timeout`.
-    fn update_keep_alive(&mut self) {
-        // Check that we don't have outbound items pending for dialing, nor dialing, nor
-        // established. Also check that there are no established inbound substreams.
-        // Errors and events need to be reported back, so check those too.
-        let should_shutdown = match self.state {
-            HandlerState::ShuttingDown(_) => {
-                self.dial_queue.is_empty()
-                    && self.outbound_substreams.is_empty()
-                    && self.inbound_substreams.is_empty()
-                    && self.pending_errors.is_empty()
-                    && self.events_out.is_empty()
-                    && self.dial_negotiated == 0
-            }
-            HandlerState::Deactivated => {
-                // Regardless of events, the timeout has expired. Force the disconnect.
-                true
-            }
-            _ => false,
-        };
-
-        match self.keep_alive {
-            KeepAlive::Yes if should_shutdown => self.keep_alive = KeepAlive::No,
-            KeepAlive::Yes => {} // We continue being active
-            KeepAlive::Until(_) if should_shutdown => self.keep_alive = KeepAlive::No, // Already deemed inactive
-            KeepAlive::Until(_) => {
-                // No longer idle
-                self.keep_alive = KeepAlive::Yes;
-            }
-            KeepAlive::No => {} // currently not used
-        }
-    }
 }
 
 impl<TSpec> ProtocolsHandler for RPCHandler<TSpec>
@@ -387,14 +347,16 @@ where
     type InboundProtocol = RPCProtocol<TSpec>;
     type OutboundProtocol = RPCRequest<TSpec>;
     type OutboundOpenInfo = (RequestId, RPCRequest<TSpec>); // Keep track of the id and the request
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.listen_protocol.clone()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         substream: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         // only accept new peer requests when active
         if !matches!(self.state, HandlerState::Active) {
@@ -427,8 +389,6 @@ where
         self.events_out
             .push(RPCReceived::Request(self.current_inbound_substream_id, req));
         self.current_inbound_substream_id.0 += 1;
-
-        self.update_keep_alive();
     }
 
     fn inject_fully_negotiated_outbound(
@@ -486,8 +446,6 @@ where
             }
             self.current_outbound_substream_id.0 += 1;
         }
-
-        self.update_keep_alive();
     }
 
     fn inject_event(&mut self, rpc_event: Self::InEvent) {
@@ -515,7 +473,6 @@ where
 
         // This dialing is now considered failed
         self.dial_negotiated -= 1;
-        self.update_keep_alive();
 
         self.outbound_io_error_retries = 0;
         // map the error
@@ -548,7 +505,29 @@ where
     }
 
     fn connection_keep_alive(&self) -> KeepAlive {
-        self.keep_alive
+        // Check that we don't have outbound items pending for dialing, nor dialing, nor
+        // established. Also check that there are no established inbound substreams.
+        // Errors and events need to be reported back, so check those too.
+        let should_shutdown = match self.state {
+            HandlerState::ShuttingDown(_) => {
+                self.dial_queue.is_empty()
+                    && self.outbound_substreams.is_empty()
+                    && self.inbound_substreams.is_empty()
+                    && self.pending_errors.is_empty()
+                    && self.events_out.is_empty()
+                    && self.dial_negotiated == 0
+            }
+            HandlerState::Deactivated => {
+                // Regardless of events, the timeout has expired. Force the disconnect.
+                true
+            }
+            _ => false,
+        };
+        if should_shutdown {
+            KeepAlive::No
+        } else {
+            KeepAlive::Yes
+        }
     }
 
     fn poll(
@@ -624,8 +603,6 @@ where
                     if let Some(OutboundInfo { proto, req_id, .. }) =
                         self.outbound_substreams.remove(outbound_id.get_ref())
                     {
-                        self.update_keep_alive();
-
                         let outbound_err = HandlerErr::Outbound {
                             id: req_id,
                             proto,
@@ -724,7 +701,6 @@ where
             self.inbound_substreams.remove(&inbound_id);
         }
 
-        self.update_keep_alive();
         // drive outbound streams that need to be processed
         for outbound_id in self.outbound_substreams.keys().copied().collect::<Vec<_>>() {
             // get the state and mark it as poisoned
@@ -813,7 +789,6 @@ where
                         let request_id = entry.get().req_id;
                         self.outbound_substreams_delay.remove(delay_key);
                         entry.remove_entry();
-                        self.update_keep_alive();
                         // notify the application error
                         if request.expected_responses() > 1 {
                             // return an end of stream result
@@ -844,7 +819,6 @@ where
                             error: e,
                         };
                         entry.remove_entry();
-                        self.update_keep_alive();
                         return Poll::Ready(ProtocolsHandlerEvent::Custom(Err(outbound_err)));
                     }
                 },
@@ -857,7 +831,6 @@ where
                             let request_id = entry.get().req_id;
                             self.outbound_substreams_delay.remove(delay_key);
                             entry.remove_entry();
-                            self.update_keep_alive();
 
                             // report the stream termination to the user
                             //
@@ -894,10 +867,8 @@ where
             self.dial_negotiated += 1;
             let (id, req) = self.dial_queue.remove(0);
             self.dial_queue.shrink_to_fit();
-            self.update_keep_alive();
             return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                protocol: SubstreamProtocol::new(req.clone()),
-                info: (id, req),
+                protocol: SubstreamProtocol::new(req.clone(), ()).map_info(|()| (id, req)),
             });
         }
         Poll::Pending

beacon_node/eth2_libp2p/src/rpc/methods.rs

@@ -19,7 +19,7 @@ type MaxErrorLen = U256;
 
 /// Wrapper over SSZ List to represent error message in rpc responses.
 #[derive(Debug, Clone)]
-pub struct ErrorType(VariableList<u8, MaxErrorLen>);
+pub struct ErrorType(pub VariableList<u8, MaxErrorLen>);
 
 impl From<String> for ErrorType {
     fn from(s: String) -> Self {
@@ -283,13 +283,13 @@ impl<T: EthSpec> RPCCodedResponse<T> {
     }
 
     /// Builds an RPCCodedResponse from a response code and an ErrorMessage
-    pub fn from_error(response_code: u8, err: String) -> Self {
+    pub fn from_error(response_code: u8, err: ErrorType) -> Self {
         let code = match response_code {
             1 => RPCResponseErrorCode::InvalidRequest,
             2 => RPCResponseErrorCode::ServerError,
             _ => RPCResponseErrorCode::Unknown,
         };
-        RPCCodedResponse::Error(code, err.into())
+        RPCCodedResponse::Error(code, err)
     }
 
     /// Specifies which response allows for multiple chunks for the stream handler.

beacon_node/eth2_libp2p/src/rpc/mod.rs

@@ -169,9 +169,12 @@ where
 
     fn new_handler(&mut self) -> Self::ProtocolsHandler {
         RPCHandler::new(
-            SubstreamProtocol::new(RPCProtocol {
-                phantom: PhantomData,
-            }),
+            SubstreamProtocol::new(
+                RPCProtocol {
+                    phantom: PhantomData,
+                },
+                (),
+            ),
             &self.log,
         )
     }

beacon_node/eth2_libp2p/src/service.rs

@@ -36,6 +36,8 @@ pub enum Libp2pEvent<TSpec: EthSpec> {
     Behaviour(BehaviourEvent<TSpec>),
     /// A new listening address has been established.
     NewListenAddr(Multiaddr),
+    /// We reached zero listening addresses.
+    ZeroListeners,
 }
 
 /// The configuration and state of the libp2p components for the beacon node.
@@ -51,7 +53,7 @@ pub struct Service<TSpec: EthSpec> {
 }
 
 impl<TSpec: EthSpec> Service<TSpec> {
-    pub fn new(
+    pub async fn new(
         executor: environment::TaskExecutor,
         config: &NetworkConfig,
         enr_fork_id: EnrForkId,
@@ -76,7 +78,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
             &log,
         ));
 
-        info!(log, "Libp2p Service"; "peer_id" => format!("{:?}", enr.peer_id()));
+        info!(log, "Libp2p Service"; "peer_id" => enr.peer_id().to_string());
         let discovery_string = if config.disable_discovery {
             "None".into()
         } else {
@@ -85,11 +87,12 @@ impl<TSpec: EthSpec> Service<TSpec> {
         debug!(log, "Attempting to open listening ports"; "address" => format!("{}", config.listen_address), "tcp_port" => config.libp2p_port, "udp_port" => discovery_string);
 
         let mut swarm = {
-            // Set up the transport - tcp/ws with noise and yamux/mplex
+            // Set up the transport - tcp/ws with noise and mplex
             let transport = build_transport(local_keypair.clone())
                 .map_err(|e| format!("Failed to build transport: {:?}", e))?;
             // Lighthouse network behaviour
-            let behaviour = Behaviour::new(&local_keypair, config, network_globals.clone(), &log)?;
+            let behaviour =
+                Behaviour::new(&local_keypair, config, network_globals.clone(), &log).await?;
 
             // use the executor for libp2p
             struct Executor(environment::TaskExecutor);
@@ -102,6 +105,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
                 .notify_handler_buffer_size(std::num::NonZeroUsize::new(32).expect("Not zero"))
                 .connection_event_buffer_size(64)
                 .incoming_connection_limit(10)
+                .outgoing_connection_limit(config.target_peers * 2)
                 .peer_connection_limit(MAX_CONNECTIONS_PER_PEER)
                 .executor(Box::new(Executor(executor)))
                 .build()
@@ -150,7 +154,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
         }
 
         // attempt to connect to any specified boot-nodes
-        let mut boot_nodes = config.boot_nodes.clone();
+        let mut boot_nodes = config.boot_nodes_enr.clone();
         boot_nodes.dedup();
 
         for bootnode_enr in boot_nodes {
@@ -171,6 +175,16 @@ impl<TSpec: EthSpec> Service<TSpec> {
             }
         }
 
+        for multiaddr in &config.boot_nodes_multiaddr {
+            // check TCP support for dialing
+            if multiaddr
+                .iter()
+                .any(|proto| matches!(proto, Protocol::Tcp(_)))
+            {
+                dial_addr(multiaddr.clone());
+            }
+        }
+
         let mut subscribed_topics: Vec<GossipKind> = vec![];
         for topic_kind in &config.topics {
             if swarm.subscribe_kind(topic_kind.clone()) {
@@ -271,10 +285,17 @@ impl<TSpec: EthSpec> Service<TSpec> {
                     debug!(self.log, "Listen address expired"; "multiaddr" => multiaddr.to_string())
                 }
                 SwarmEvent::ListenerClosed { addresses, reason } => {
-                    crit!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason))
+                    crit!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason));
+                    if Swarm::listeners(&self.swarm).count() == 0 {
+                        return Libp2pEvent::ZeroListeners;
+                    }
                 }
                 SwarmEvent::ListenerError { error } => {
-                    warn!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string()))
+                    // this is non fatal, but we still check
+                    warn!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string()));
+                    if Swarm::listeners(&self.swarm).count() == 0 {
+                        return Libp2pEvent::ZeroListeners;
+                    }
                 }
                 SwarmEvent::Dialing(peer_id) => {
                     debug!(self.log, "Dialing peer"; "peer_id" => peer_id.to_string());
@@ -285,7 +306,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
 }
 
 /// The implementation supports TCP/IP, WebSockets over TCP/IP, noise as the encryption layer, and
-/// yamux or mplex as the multiplexing layer.
+/// mplex as the multiplexing layer.
 fn build_transport(
     local_private_key: Keypair,
 ) -> Result<Boxed<(PeerId, StreamMuxerBox), Error>, Error> {
@@ -300,10 +321,7 @@ fn build_transport(
     Ok(transport
         .upgrade(core::upgrade::Version::V1)
         .authenticate(generate_noise_config(&local_private_key))
-        .multiplex(core::upgrade::SelectUpgrade::new(
-            libp2p::mplex::MplexConfig::new(),
-            libp2p::yamux::Config::default(),
-        ))
+        .multiplex(libp2p::mplex::MplexConfig::new())
         .map(|(peer, muxer), _| (peer, core::muxing::StreamMuxerBox::new(muxer)))
         .timeout(Duration::from_secs(10))
         .timeout(Duration::from_secs(10))
@@ -339,7 +357,7 @@ fn keypair_from_bytes(mut bytes: Vec<u8>) -> error::Result<Keypair> {
 /// generated and is then saved to disk.
 ///
 /// Currently only secp256k1 keys are allowed, as these are the only keys supported by discv5.
-fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
+pub fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
     // check for key from disk
     let network_key_f = config.network_dir.join(NETWORK_KEY_FILENAME);
     if let Ok(mut network_key_file) = File::open(network_key_f.clone()) {

beacon_node/eth2_libp2p/src/types/globals.rs

@@ -1,10 +1,9 @@
 //! A collection of variables that are accessible outside of the network thread itself.
 use crate::peer_manager::PeerDB;
-use crate::rpc::methods::MetaData;
 use crate::types::SyncState;
 use crate::Client;
 use crate::EnrExt;
-use crate::{Enr, Eth2Enr, GossipTopic, Multiaddr, PeerId};
+use crate::{Enr, GossipTopic, Multiaddr, PeerId};
 use parking_lot::RwLock;
 use std::collections::HashSet;
 use std::sync::atomic::{AtomicU16, Ordering};
@@ -13,8 +12,6 @@ use types::EthSpec;
 pub struct NetworkGlobals<TSpec: EthSpec> {
     /// The current local ENR.
     pub local_enr: RwLock<Enr>,
-    /// The current node's meta-data.
-    pub meta_data: RwLock<MetaData<TSpec>>,
     /// The local peer_id.
     pub peer_id: RwLock<PeerId>,
     /// Listening multiaddrs.
@@ -33,17 +30,8 @@ pub struct NetworkGlobals<TSpec: EthSpec> {
 
 impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
     pub fn new(enr: Enr, tcp_port: u16, udp_port: u16, log: &slog::Logger) -> Self {
-        // set up the local meta data of the node
-        let meta_data = RwLock::new(MetaData {
-            seq_number: 0,
-            attnets: enr
-                .bitfield::<TSpec>()
-                .expect("Local ENR must have a bitfield specified"),
-        });
-
         NetworkGlobals {
             local_enr: RwLock::new(enr.clone()),
-            meta_data,
             peer_id: RwLock::new(enr.peer_id()),
             listen_multiaddrs: RwLock::new(Vec::new()),
             listen_port_tcp: AtomicU16::new(tcp_port),

beacon_node/eth2_libp2p/src/types/mod.rs

@@ -1,6 +1,7 @@
 pub mod error;
 mod globals;
 mod pubsub;
+mod subnet;
 mod sync_state;
 mod topics;
 
@@ -13,5 +14,6 @@ pub type Enr = discv5::enr::Enr<discv5::enr::CombinedKey>;
 
 pub use globals::NetworkGlobals;
 pub use pubsub::PubsubMessage;
+pub use subnet::SubnetDiscovery;
 pub use sync_state::SyncState;
 pub use topics::{GossipEncoding, GossipKind, GossipTopic};

beacon_node/eth2_libp2p/src/types/subnet.rs

@@ -0,0 +1,28 @@
+use std::time::{Duration, Instant};
+use types::SubnetId;
+
+const DURATION_DIFFERENCE: Duration = Duration::from_millis(1);
+
+/// A subnet to discover peers on along with the instant after which it's no longer useful.
+#[derive(Debug, Clone)]
+pub struct SubnetDiscovery {
+    pub subnet_id: SubnetId,
+    pub min_ttl: Option<Instant>,
+}
+
+impl PartialEq for SubnetDiscovery {
+    fn eq(&self, other: &SubnetDiscovery) -> bool {
+        self.subnet_id == other.subnet_id
+            && match (self.min_ttl, other.min_ttl) {
+                (Some(min_ttl_instant), Some(other_min_ttl_instant)) => {
+                    min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
+                        < DURATION_DIFFERENCE
+                        && other_min_ttl_instant.saturating_duration_since(min_ttl_instant)
+                            < DURATION_DIFFERENCE
+                }
+                (None, None) => true,
+                (None, Some(_)) => true,
+                (Some(_), None) => true,
+            }
+    }
+}

beacon_node/eth2_libp2p/src/types/topics.rs

@@ -1,4 +1,4 @@
-use libp2p::gossipsub::Topic;
+use libp2p::gossipsub::IdentTopic as Topic;
 use serde_derive::{Deserialize, Serialize};
 use types::SubnetId;
 
@@ -139,7 +139,7 @@ impl GossipTopic {
 
 impl Into<Topic> for GossipTopic {
     fn into(self) -> Topic {
-        Topic::new(self.into())
+        Topic::new(self)
     }
 }
 

beacon_node/eth2_libp2p/tests/common/mod.rs

@@ -10,6 +10,7 @@ use std::time::Duration;
 use types::{EnrForkId, MinimalEthSpec};
 
 type E = MinimalEthSpec;
+use libp2p::gossipsub::GossipsubConfigBuilder;
 use tempdir::TempDir;
 
 pub struct Libp2pInstance(LibP2PService<E>, exit_future::Signal);
@@ -80,24 +81,33 @@ pub fn build_config(port: u16, mut boot_nodes: Vec<Enr>) -> NetworkConfig {
     config.enr_tcp_port = Some(port);
     config.enr_udp_port = Some(port);
     config.enr_address = Some("127.0.0.1".parse().unwrap());
-    config.boot_nodes.append(&mut boot_nodes);
+    config.boot_nodes_enr.append(&mut boot_nodes);
     config.network_dir = path.into_path();
     // Reduce gossipsub heartbeat parameters
-    config.gs_config.heartbeat_initial_delay = Duration::from_millis(500);
-    config.gs_config.heartbeat_interval = Duration::from_millis(500);
+    config.gs_config = GossipsubConfigBuilder::from(config.gs_config)
+        .heartbeat_initial_delay(Duration::from_millis(500))
+        .heartbeat_interval(Duration::from_millis(500))
+        .build()
+        .unwrap();
     config
 }
 
-pub fn build_libp2p_instance(boot_nodes: Vec<Enr>, log: slog::Logger) -> Libp2pInstance {
+pub async fn build_libp2p_instance(boot_nodes: Vec<Enr>, log: slog::Logger) -> Libp2pInstance {
     let port = unused_port("tcp").unwrap();
     let config = build_config(port, boot_nodes);
     // launch libp2p service
 
     let (signal, exit) = exit_future::signal();
-    let executor =
-        environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone());
+    let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+    let executor = environment::TaskExecutor::new(
+        tokio::runtime::Handle::current(),
+        exit,
+        log.clone(),
+        shutdown_tx,
+    );
     Libp2pInstance(
         LibP2PService::new(executor, &config, EnrForkId::default(), &log)
+            .await
             .expect("should build libp2p instance")
             .1,
         signal,
@@ -112,10 +122,11 @@ pub fn get_enr(node: &LibP2PService<E>) -> Enr {
 
 // Returns `n` libp2p peers in fully connected topology.
 #[allow(dead_code)]
-pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
-    let mut nodes: Vec<_> = (0..n)
-        .map(|_| build_libp2p_instance(vec![], log.clone()))
-        .collect();
+pub async fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+    let mut nodes = Vec::with_capacity(n);
+    for _ in 0..n {
+        nodes.push(build_libp2p_instance(vec![], log.clone()).await);
+    }
     let multiaddrs: Vec<Multiaddr> = nodes
         .iter()
         .map(|x| get_enr(&x).multiaddr()[1].clone())
@@ -141,8 +152,8 @@ pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInsta
     let sender_log = log.new(o!("who" => "sender"));
     let receiver_log = log.new(o!("who" => "receiver"));
 
-    let mut sender = build_libp2p_instance(vec![], sender_log);
-    let mut receiver = build_libp2p_instance(vec![], receiver_log);
+    let mut sender = build_libp2p_instance(vec![], sender_log).await;
+    let mut receiver = build_libp2p_instance(vec![], receiver_log).await;
 
     let receiver_multiaddr = receiver.swarm.local_enr().multiaddr()[1].clone();
 
@@ -181,10 +192,12 @@ pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInsta
 
 // Returns `n` peers in a linear topology
 #[allow(dead_code)]
-pub fn build_linear(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
-    let mut nodes: Vec<_> = (0..n)
-        .map(|_| build_libp2p_instance(vec![], log.clone()))
-        .collect();
+pub async fn build_linear(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+    let mut nodes = Vec::with_capacity(n);
+    for _ in 0..n {
+        nodes.push(build_libp2p_instance(vec![], log.clone()).await);
+    }
+
     let multiaddrs: Vec<Multiaddr> = nodes
         .iter()
         .map(|x| get_enr(&x).multiaddr()[1].clone())

beacon_node/network/Cargo.toml

@@ -39,3 +39,5 @@ lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
 environment = { path = "../../lighthouse/environment" }
 itertools = "0.9.0"
+num_cpus = "1.13.0"
+lru_cache = { path = "../../common/lru_cache" }

beacon_node/network/src/attestation_service/mod.rs

@@ -2,7 +2,7 @@
 //! given time. It schedules subscriptions to shard subnets, requests peer discoveries and
 //! determines whether attestations should be aggregated and/or passed to the beacon node.
 
-use std::collections::VecDeque;
+use std::collections::{HashMap, VecDeque};
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
@@ -13,7 +13,7 @@ use rand::seq::SliceRandom;
 use slog::{crit, debug, error, o, trace, warn};
 
 use beacon_chain::{BeaconChain, BeaconChainTypes};
-use eth2_libp2p::{types::GossipKind, NetworkGlobals};
+use eth2_libp2p::{types::GossipKind, NetworkGlobals, SubnetDiscovery};
 use hashset_delay::HashSetDelay;
 use rest_types::ValidatorSubscription;
 use slot_clock::SlotClock;
@@ -25,11 +25,8 @@ mod tests;
 
 /// The minimum number of slots ahead that we attempt to discover peers for a subscription. If the
 /// slot is less than this number, skip the peer discovery process.
-const MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 1;
-/// The number of slots ahead that we attempt to discover peers for a subscription. If the slot to
-/// attest to is greater than this, we queue a discovery request for this many slots prior to
-/// subscribing.
-const TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 6;
+/// Subnet discovery query takes atmost 30 secs, 2 slots take 24s.
+const MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 2;
 /// The time (in slots) before a last seen validator is considered absent and we unsubscribe from the random
 /// gossip topics that we subscribed to due to the validator connection.
 const LAST_SEEN_VALIDATOR_TIMEOUT: u32 = 150;
@@ -39,12 +36,10 @@ const LAST_SEEN_VALIDATOR_TIMEOUT: u32 = 150;
 /// Note: The time is calculated as `time = milliseconds_per_slot / ADVANCE_SUBSCRIPTION_TIME`.
 const ADVANCE_SUBSCRIBE_TIME: u32 = 3;
 /// The default number of slots before items in hash delay sets used by this class should expire.
+///  36s at 12s slot time
 const DEFAULT_EXPIRATION_TIMEOUT: u32 = 3;
-// 36s at 12s slot time
-/// The duration difference between two instance for them to be considered equal.
-const DURATION_DIFFERENCE: Duration = Duration::from_millis(1);
 
-#[derive(Debug, Eq, Clone)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum AttServiceMessage {
     /// Subscribe to the specified subnet id.
     Subscribe(SubnetId),
@@ -54,44 +49,8 @@ pub enum AttServiceMessage {
     EnrAdd(SubnetId),
     /// Remove the `SubnetId` from the ENR bitfield.
     EnrRemove(SubnetId),
-    /// Discover peers for a particular subnet.
-    /// The includes the `Instant` we need the discovered peer until.
-    DiscoverPeers {
-        subnet_id: SubnetId,
-        min_ttl: Option<Instant>,
-    },
-}
-
-impl PartialEq for AttServiceMessage {
-    fn eq(&self, other: &AttServiceMessage) -> bool {
-        match (self, other) {
-            (&AttServiceMessage::Subscribe(a), &AttServiceMessage::Subscribe(b)) => a == b,
-            (&AttServiceMessage::Unsubscribe(a), &AttServiceMessage::Unsubscribe(b)) => a == b,
-            (&AttServiceMessage::EnrAdd(a), &AttServiceMessage::EnrAdd(b)) => a == b,
-            (&AttServiceMessage::EnrRemove(a), &AttServiceMessage::EnrRemove(b)) => a == b,
-            (
-                &AttServiceMessage::DiscoverPeers { subnet_id, min_ttl },
-                &AttServiceMessage::DiscoverPeers {
-                    subnet_id: other_subnet_id,
-                    min_ttl: other_min_ttl,
-                },
-            ) => {
-                subnet_id == other_subnet_id
-                    && match (min_ttl, other_min_ttl) {
-                        (Some(min_ttl_instant), Some(other_min_ttl_instant)) => {
-                            min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
-                                < DURATION_DIFFERENCE
-                                && other_min_ttl_instant.saturating_duration_since(min_ttl_instant)
-                                    < DURATION_DIFFERENCE
-                        }
-                        (None, None) => true,
-                        (None, Some(_)) => true,
-                        (Some(_), None) => true,
-                    }
-            }
-            _ => false,
-        }
-    }
+    /// Discover peers for a list of `SubnetDiscovery`.
+    DiscoverPeers(Vec<SubnetDiscovery>),
 }
 
 /// A particular subnet at a given slot.
@@ -116,9 +75,6 @@ pub struct AttestationService<T: BeaconChainTypes> {
     /// The collection of currently subscribed random subnets mapped to their expiry deadline.
     random_subnets: HashSetDelay<SubnetId>,
 
-    /// A collection of timeouts for when to start searching for peers for a particular shard.
-    discover_peers: HashSetDelay<ExactSubnet>,
-
     /// A collection of timeouts for when to subscribe to a shard subnet.
     subscriptions: HashSetDelay<ExactSubnet>,
 
@@ -172,7 +128,6 @@ impl<T: BeaconChainTypes> AttestationService<T> {
             network_globals,
             beacon_chain,
             random_subnets: HashSetDelay::new(Duration::from_millis(random_subnet_duration_millis)),
-            discover_peers: HashSetDelay::new(default_timeout),
             subscriptions: HashSetDelay::new(default_timeout),
             unsubscriptions: HashSetDelay::new(default_timeout),
             aggregate_validators_on_subnet: HashSetDelay::new(default_timeout),
@@ -198,6 +153,8 @@ impl<T: BeaconChainTypes> AttestationService<T> {
         &mut self,
         subscriptions: Vec<ValidatorSubscription>,
     ) -> Result<(), String> {
+        // Maps each subnet_id subscription to it's highest slot
+        let mut subnets_to_discover: HashMap<SubnetId, Slot> = HashMap::new();
         for subscription in subscriptions {
             metrics::inc_counter(&metrics::SUBNET_SUBSCRIPTION_REQUESTS);
             //NOTE: We assume all subscriptions have been verified before reaching this service
@@ -226,15 +183,20 @@ impl<T: BeaconChainTypes> AttestationService<T> {
                     continue;
                 }
             };
+            // Ensure each subnet_id inserted into the map has the highest slot as it's value.
+            // Higher slot corresponds to higher min_ttl in the `SubnetDiscovery` entry.
+            if let Some(slot) = subnets_to_discover.get(&subnet_id) {
+                if subscription.slot > *slot {
+                    subnets_to_discover.insert(subnet_id, subscription.slot);
+                }
+            } else {
+                subnets_to_discover.insert(subnet_id, subscription.slot);
+            }
 
             let exact_subnet = ExactSubnet {
                 subnet_id,
                 slot: subscription.slot,
             };
-            // determine if we should run a discovery lookup request and request it if required
-            if let Err(e) = self.discover_peers_request(exact_subnet.clone()) {
-                warn!(self.log, "Discovery lookup request error"; "error" => e);
-            }
 
             // determine if the validator is an aggregator. If so, we subscribe to the subnet and
             // if successful add the validator to a mapping of known aggregators for that exact
@@ -264,6 +226,14 @@ impl<T: BeaconChainTypes> AttestationService<T> {
             }
         }
 
+        if let Err(e) = self.discover_peers_request(
+            subnets_to_discover
+                .into_iter()
+                .map(|(subnet_id, slot)| ExactSubnet { subnet_id, slot }),
+        ) {
+            warn!(self.log, "Discovery lookup request error"; "error" => e);
+        };
+
         // pre-emptively wake the thread to check for new events
         if let Some(waker) = &self.waker {
             waker.wake_by_ref();
@@ -290,114 +260,55 @@ impl<T: BeaconChainTypes> AttestationService<T> {
     /// Checks if there are currently queued discovery requests and the time required to make the
     /// request.
     ///
-    /// If there is sufficient time and no other request exists, queues a peer discovery request
-    /// for the required subnet.
-    fn discover_peers_request(&mut self, exact_subnet: ExactSubnet) -> Result<(), &'static str> {
+    /// If there is sufficient time, queues a peer discovery request for all the required subnets.
+    fn discover_peers_request(
+        &mut self,
+        exact_subnets: impl Iterator<Item = ExactSubnet>,
+    ) -> Result<(), &'static str> {
         let current_slot = self
             .beacon_chain
             .slot_clock
             .now()
             .ok_or_else(|| "Could not get the current slot")?;
-        let slot_duration = self.beacon_chain.slot_clock.slot_duration();
 
-        // if there is enough time to perform a discovery lookup
-        if exact_subnet.slot >= current_slot.saturating_add(MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD) {
-            // check if a discovery request already exists
-            if self.discover_peers.get(&exact_subnet).is_some() {
-                // already a request queued, end
-                return Ok(());
-            }
-
-            // if the slot is more than epoch away, add an event to start looking for peers
-            if exact_subnet.slot
-                < current_slot.saturating_add(TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD)
-            {
-                // add one slot to ensure we keep the peer for the subscription slot
-                let min_ttl = self
-                    .beacon_chain
-                    .slot_clock
-                    .duration_to_slot(exact_subnet.slot + 1)
-                    .map(|duration| std::time::Instant::now() + duration);
-
-                self.send_or_update_discovery_event(exact_subnet.subnet_id, min_ttl);
-            } else {
-                // Queue the discovery event to be executed for
-                // TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD
-
-                let duration_to_discover = {
-                    let duration_to_next_slot = self
+        let discovery_subnets: Vec<SubnetDiscovery> = exact_subnets
+            .filter_map(|exact_subnet| {
+                // check if there is enough time to perform a discovery lookup
+                if exact_subnet.slot
+                    >= current_slot.saturating_add(MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD)
+                {
+                    // if the slot is more than epoch away, add an event to start looking for peers
+                    // add one slot to ensure we keep the peer for the subscription slot
+                    let min_ttl = self
                         .beacon_chain
                         .slot_clock
-                        .duration_to_next_slot()
-                        .ok_or_else(|| "Unable to determine duration to next slot")?;
-                    // The -1 is done here to exclude the current slot duration, as we will use
-                    // `duration_to_next_slot`.
-                    let slots_until_discover = exact_subnet
-                        .slot
-                        .saturating_sub(current_slot)
-                        .saturating_sub(1u64)
-                        .saturating_sub(TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD);
+                        .duration_to_slot(exact_subnet.slot + 1)
+                        .map(|duration| std::time::Instant::now() + duration);
+                    Some(SubnetDiscovery {
+                        subnet_id: exact_subnet.subnet_id,
+                        min_ttl,
+                    })
+                } else {
+                    // TODO: Send the time frame needed to have a peer connected, so that we can
+                    // maintain peers for a least this duration.
+                    // We may want to check the global PeerInfo to see estimated timeouts for each
+                    // peer before they can be removed.
+                    warn!(self.log,
+                        "Not enough time for a discovery search";
+                        "subnet_id" => format!("{:?}", exact_subnet)
+                    );
+                    None
+                }
+            })
+            .collect();
 
-                    duration_to_next_slot + slot_duration * (slots_until_discover.as_u64() as u32)
-                };
-
-                self.discover_peers
-                    .insert_at(exact_subnet, duration_to_discover);
-            }
-        } else {
-            // TODO: Send the time frame needed to have a peer connected, so that we can
-            // maintain peers for a least this duration.
-            // We may want to check the global PeerInfo to see estimated timeouts for each
-            // peer before they can be removed.
-            return Err("Not enough time for a discovery search");
+        if !discovery_subnets.is_empty() {
+            self.events
+                .push_back(AttServiceMessage::DiscoverPeers(discovery_subnets));
         }
         Ok(())
     }
 
-    /// Checks if we have a discover peers event already and sends a new event if necessary
-    ///
-    /// If a message exists for the same subnet, compare the `min_ttl` of the current and
-    /// existing messages and extend the existing message as necessary.
-    fn send_or_update_discovery_event(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
-        // track whether this message already exists in the event queue
-        let mut is_duplicate = false;
-
-        self.events.iter_mut().for_each(|event| {
-            if let AttServiceMessage::DiscoverPeers {
-                subnet_id: other_subnet_id,
-                min_ttl: other_min_ttl,
-            } = event
-            {
-                if subnet_id == *other_subnet_id {
-                    let other_min_ttl_clone = *other_min_ttl;
-                    match (min_ttl, other_min_ttl_clone) {
-                        (Some(min_ttl_instant), Some(other_min_ttl_instant)) =>
-                        // only update the min_ttl if it is greater than the existing min_ttl and a DURATION_DIFFERENCE padding
-                        {
-                            if min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
-                                > DURATION_DIFFERENCE
-                            {
-                                *other_min_ttl = min_ttl;
-                            }
-                        }
-                        (None, Some(_)) => {} // Keep the current one as it has an actual min_ttl
-                        (Some(min_ttl), None) => {
-                            // Update the request to include a min_ttl.
-                            *other_min_ttl = Some(min_ttl);
-                        }
-                        (None, None) => {} // Duplicate message, do nothing.
-                    }
-                    is_duplicate = true;
-                    return;
-                }
-            };
-        });
-        if !is_duplicate {
-            self.events
-                .push_back(AttServiceMessage::DiscoverPeers { subnet_id, min_ttl });
-        }
-    }
-
     /// Checks the current random subnets and subscriptions to determine if a new subscription for this
     /// subnet is required for the given slot.
     ///
@@ -547,7 +458,11 @@ impl<T: BeaconChainTypes> AttestationService<T> {
 
             if !already_subscribed {
                 // send a discovery request and a subscription
-                self.send_or_update_discovery_event(subnet_id, None);
+                self.events
+                    .push_back(AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery {
+                        subnet_id,
+                        min_ttl: None,
+                    }]));
                 self.events
                     .push_back(AttServiceMessage::Subscribe(subnet_id));
             }
@@ -558,20 +473,6 @@ impl<T: BeaconChainTypes> AttestationService<T> {
 
     /* A collection of functions that handle the various timeouts */
 
-    /// Request a discovery query to find peers for a particular subnet.
-    fn handle_discover_peers(&mut self, exact_subnet: ExactSubnet) {
-        debug!(self.log, "Searching for peers for subnet"; "subnet" => *exact_subnet.subnet_id, "target_slot" => exact_subnet.slot);
-
-        // add one slot to ensure we keep the peer for the subscription slot
-        let min_ttl = self
-            .beacon_chain
-            .slot_clock
-            .duration_to_slot(exact_subnet.slot + 1)
-            .map(|duration| std::time::Instant::now() + duration);
-
-        self.send_or_update_discovery_event(exact_subnet.subnet_id, min_ttl)
-    }
-
     /// A queued subscription is ready.
     ///
     /// We add subscriptions events even if we are already subscribed to a random subnet (as these
@@ -731,15 +632,6 @@ impl<T: BeaconChainTypes> Stream for AttestationService<T> {
             self.waker = Some(cx.waker().clone());
         }
 
-        // process any peer discovery events
-        match self.discover_peers.poll_next_unpin(cx) {
-            Poll::Ready(Some(Ok(exact_subnet))) => self.handle_discover_peers(exact_subnet),
-            Poll::Ready(Some(Err(e))) => {
-                error!(self.log, "Failed to check for peer discovery requests"; "error"=> e);
-            }
-            Poll::Ready(None) | Poll::Pending => {}
-        }
-
         // process any subscription events
         match self.subscriptions.poll_next_unpin(cx) {
             Poll::Ready(Some(Ok(exact_subnet))) => self.handle_subscriptions(exact_subnet),

beacon_node/network/src/attestation_service/tests/mod.rs

@@ -8,7 +8,9 @@ mod tests {
         migrate::NullMigrator,
     };
     use eth2_libp2p::discovery::{build_enr, Keypair};
-    use eth2_libp2p::{discovery::CombinedKey, CombinedKeyExt, NetworkConfig, NetworkGlobals};
+    use eth2_libp2p::{
+        discovery::CombinedKey, CombinedKeyExt, NetworkConfig, NetworkGlobals, SubnetDiscovery,
+    };
     use futures::Stream;
     use genesis::{generate_deterministic_keypairs, interop_genesis_state};
     use lazy_static::lazy_static;
@@ -120,23 +122,21 @@ mod tests {
         }
     }
 
-    fn _get_subscriptions(
+    fn get_subscriptions(
         validator_count: u64,
         slot: Slot,
         committee_count_at_slot: u64,
     ) -> Vec<ValidatorSubscription> {
-        let mut subscriptions: Vec<ValidatorSubscription> = Vec::new();
-        for validator_index in 0..validator_count {
-            let is_aggregator = true;
-            subscriptions.push(ValidatorSubscription {
-                validator_index,
-                attestation_committee_index: validator_index,
-                slot,
-                committee_count_at_slot,
-                is_aggregator,
-            });
-        }
-        subscriptions
+        (0..validator_count)
+            .map(|validator_index| {
+                get_subscription(
+                    validator_index,
+                    validator_index,
+                    slot,
+                    committee_count_at_slot,
+                )
+            })
+            .collect()
     }
 
     // gets a number of events from the subscription service, or returns none if it times out after a number
@@ -210,14 +210,7 @@ mod tests {
         let events = get_events(attestation_service, no_events_expected, 1).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any1),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any1), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -270,14 +263,7 @@ mod tests {
         let events = get_events(attestation_service, no_events_expected, 2).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any1),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any1), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -330,19 +316,15 @@ mod tests {
             &attestation_service.beacon_chain.spec,
         )
         .unwrap();
-        let expected = vec![AttServiceMessage::DiscoverPeers { subnet_id, min_ttl }];
+        let expected = vec![AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery {
+            subnet_id,
+            min_ttl,
+        }])];
 
         let events = get_events(attestation_service, no_events_expected, 1).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -396,21 +378,14 @@ mod tests {
         )
         .unwrap();
         let expected = vec![
-            AttServiceMessage::DiscoverPeers { subnet_id, min_ttl },
+            AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery { subnet_id, min_ttl }]),
             AttServiceMessage::Subscribe(subnet_id),
         ];
 
         let events = get_events(attestation_service, no_events_expected, 5).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -454,14 +429,7 @@ mod tests {
 
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -517,20 +485,16 @@ mod tests {
 
         // expect discover peers because we will enter TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD range
         let expected: Vec<AttServiceMessage> =
-            vec![AttServiceMessage::DiscoverPeers { subnet_id, min_ttl }];
+            vec![AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery {
+                subnet_id,
+                min_ttl,
+            }])];
 
         let events = get_events(attestation_service, no_events_expected, 5).await;
 
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -553,7 +517,7 @@ mod tests {
             .now()
             .expect("Could not get current slot");
 
-        let subscriptions = _get_subscriptions(
+        let subscriptions = get_subscriptions(
             subscription_count,
             current_slot + subscription_slot,
             committee_count,
@@ -572,10 +536,9 @@ mod tests {
 
         for event in events {
             match event {
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant,
-                } => discover_peer_count = discover_peer_count + 1,
+                AttServiceMessage::DiscoverPeers(_) => {
+                    discover_peer_count = discover_peer_count + 1
+                }
                 AttServiceMessage::Subscribe(_any_subnet) => subscribe_count = subscribe_count + 1,
                 AttServiceMessage::EnrAdd(_any_subnet) => enr_add_count = enr_add_count + 1,
                 _ => unexpected_msg_count = unexpected_msg_count + 1,
@@ -605,7 +568,7 @@ mod tests {
             .now()
             .expect("Could not get current slot");
 
-        let subscriptions = _get_subscriptions(
+        let subscriptions = get_subscriptions(
             subscription_count,
             current_slot + subscription_slot,
             committee_count,
@@ -624,10 +587,9 @@ mod tests {
 
         for event in events {
             match event {
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant,
-                } => discover_peer_count = discover_peer_count + 1,
+                AttServiceMessage::DiscoverPeers(_) => {
+                    discover_peer_count = discover_peer_count + 1
+                }
                 AttServiceMessage::Subscribe(_any_subnet) => subscribe_count = subscribe_count + 1,
                 AttServiceMessage::EnrAdd(_any_subnet) => enr_add_count = enr_add_count + 1,
                 _ => unexpected_msg_count = unexpected_msg_count + 1,
@@ -639,4 +601,40 @@ mod tests {
         assert_eq!(enr_add_count, 64);
         assert_eq!(unexpected_msg_count, 0);
     }
+
+    #[tokio::test]
+    async fn test_discovery_peers_count() {
+        let subscription_slot = 10;
+        let validator_count = 32;
+        let committee_count = 1;
+        let expected_events = 97;
+
+        // create the attestation service and subscriptions
+        let mut attestation_service = get_attestation_service();
+        let current_slot = attestation_service
+            .beacon_chain
+            .slot_clock
+            .now()
+            .expect("Could not get current slot");
+
+        let subscriptions = get_subscriptions(
+            validator_count,
+            current_slot + subscription_slot,
+            committee_count,
+        );
+
+        // submit sthe subscriptions
+        attestation_service
+            .validator_subscriptions(subscriptions)
+            .unwrap();
+
+        let events = get_events(attestation_service, expected_events, 3).await;
+
+        let event = events.get(96);
+        if let Some(AttServiceMessage::DiscoverPeers(d)) = event {
+            assert_eq!(d.len(), validator_count as usize);
+        } else {
+            panic!("Unexpected event {:?}", event);
+        }
+    }
 }

beacon_node/network/src/beacon_processor/chain_segment.rs

@@ -0,0 +1,237 @@
+use crate::metrics;
+use crate::router::processor::FUTURE_SLOT_TOLERANCE;
+use crate::sync::manager::SyncMessage;
+use crate::sync::{BatchProcessResult, ChainId};
+use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, ChainSegmentResult};
+use eth2_libp2p::PeerId;
+use slog::{debug, error, trace, warn};
+use std::sync::Arc;
+use tokio::sync::mpsc;
+use types::{Epoch, EthSpec, Hash256, SignedBeaconBlock};
+
+/// Id associated to a block processing request, either a batch or a single block.
+#[derive(Clone, Debug, PartialEq)]
+pub enum ProcessId {
+    /// Processing Id of a range syncing batch.
+    RangeBatchId(ChainId, Epoch),
+    /// Processing Id of the parent lookup of a block.
+    ParentLookup(PeerId, Hash256),
+}
+
+pub fn handle_chain_segment<T: BeaconChainTypes>(
+    chain: Arc<BeaconChain<T>>,
+    process_id: ProcessId,
+    downloaded_blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
+    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    log: slog::Logger,
+) {
+    match process_id {
+        // this a request from the range sync
+        ProcessId::RangeBatchId(chain_id, epoch) => {
+            let len = downloaded_blocks.len();
+            let start_slot = if len > 0 {
+                downloaded_blocks[0].message.slot.as_u64()
+            } else {
+                0
+            };
+            let end_slot = if len > 0 {
+                downloaded_blocks[len - 1].message.slot.as_u64()
+            } else {
+                0
+            };
+
+            debug!(log, "Processing batch"; "batch_epoch" => epoch, "blocks" => downloaded_blocks.len(),  "first_block_slot" => start_slot, "last_block_slot" => end_slot, "service" => "sync");
+            let result = match process_blocks(chain, downloaded_blocks.iter(), &log) {
+                (_, Ok(_)) => {
+                    debug!(log, "Batch processed"; "batch_epoch" => epoch , "first_block_slot" => start_slot, "last_block_slot" => end_slot, "service"=> "sync");
+                    BatchProcessResult::Success
+                }
+                (imported_blocks, Err(e)) if imported_blocks > 0 => {
+                    debug!(log, "Batch processing failed but imported some blocks";
+                        "batch_epoch" => epoch, "error" => e, "imported_blocks"=> imported_blocks, "service" => "sync");
+                    BatchProcessResult::Partial
+                }
+                (_, Err(e)) => {
+                    debug!(log, "Batch processing failed"; "batch_epoch" => epoch, "error" => e, "service" => "sync");
+                    BatchProcessResult::Failed
+                }
+            };
+
+            let msg = SyncMessage::BatchProcessed {
+                chain_id,
+                epoch,
+                downloaded_blocks,
+                result,
+            };
+            sync_send.send(msg).unwrap_or_else(|_| {
+                debug!(
+                    log,
+                    "Block processor could not inform range sync result. Likely shutting down."
+                );
+            });
+        }
+        // this a parent lookup request from the sync manager
+        ProcessId::ParentLookup(peer_id, chain_head) => {
+            debug!(
+                log, "Processing parent lookup";
+                "last_peer_id" => format!("{}", peer_id),
+                "blocks" => downloaded_blocks.len()
+            );
+            // parent blocks are ordered from highest slot to lowest, so we need to process in
+            // reverse
+            match process_blocks(chain, downloaded_blocks.iter().rev(), &log) {
+                (_, Err(e)) => {
+                    debug!(log, "Parent lookup failed"; "last_peer_id" => format!("{}", peer_id), "error" => e);
+                    sync_send
+                        .send(SyncMessage::ParentLookupFailed{peer_id, chain_head})
+                        .unwrap_or_else(|_| {
+                            // on failure, inform to downvote the peer
+                            debug!(
+                                log,
+                                "Block processor could not inform parent lookup result. Likely shutting down."
+                            );
+                        });
+                }
+                (_, Ok(_)) => {
+                    debug!(log, "Parent lookup processed successfully");
+                }
+            }
+        }
+    }
+}
+
+/// Helper function to process blocks batches which only consumes the chain and blocks to process.
+fn process_blocks<
+    'a,
+    T: BeaconChainTypes,
+    I: Iterator<Item = &'a SignedBeaconBlock<T::EthSpec>>,
+>(
+    chain: Arc<BeaconChain<T>>,
+    downloaded_blocks: I,
+    log: &slog::Logger,
+) -> (usize, Result<(), String>) {
+    let blocks = downloaded_blocks.cloned().collect::<Vec<_>>();
+    match chain.process_chain_segment(blocks) {
+        ChainSegmentResult::Successful { imported_blocks } => {
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_SUCCESS_TOTAL);
+            if imported_blocks == 0 {
+                debug!(log, "All blocks already known");
+            } else {
+                debug!(
+                    log, "Imported blocks from network";
+                    "count" => imported_blocks,
+                );
+                // Batch completed successfully with at least one block, run fork choice.
+                run_fork_choice(chain, log);
+            }
+
+            (imported_blocks, Ok(()))
+        }
+        ChainSegmentResult::Failed {
+            imported_blocks,
+            error,
+        } => {
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_FAILED_TOTAL);
+            let r = handle_failed_chain_segment(error, log);
+            if imported_blocks > 0 {
+                run_fork_choice(chain, log);
+            }
+            (imported_blocks, r)
+        }
+    }
+}
+
+/// Runs fork-choice on a given chain. This is used during block processing after one successful
+/// block import.
+fn run_fork_choice<T: BeaconChainTypes>(chain: Arc<BeaconChain<T>>, log: &slog::Logger) {
+    match chain.fork_choice() {
+        Ok(()) => trace!(
+            log,
+            "Fork choice success";
+            "location" => "batch processing"
+        ),
+        Err(e) => error!(
+            log,
+            "Fork choice failed";
+            "error" => format!("{:?}", e),
+            "location" => "batch import error"
+        ),
+    }
+}
+
+/// Helper function to handle a `BlockError` from `process_chain_segment`
+fn handle_failed_chain_segment<T: EthSpec>(
+    error: BlockError<T>,
+    log: &slog::Logger,
+) -> Result<(), String> {
+    match error {
+        BlockError::ParentUnknown(block) => {
+            // blocks should be sequential and all parents should exist
+
+            Err(format!(
+                "Block has an unknown parent: {}",
+                block.parent_root()
+            ))
+        }
+        BlockError::BlockIsAlreadyKnown => {
+            // This can happen for many reasons. Head sync's can download multiples and parent
+            // lookups can download blocks before range sync
+            Ok(())
+        }
+        BlockError::FutureSlot {
+            present_slot,
+            block_slot,
+        } => {
+            if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot {
+                // The block is too far in the future, drop it.
+                warn!(
+                    log, "Block is ahead of our slot clock";
+                    "msg" => "block for future slot rejected, check your time",
+                    "present_slot" => present_slot,
+                    "block_slot" => block_slot,
+                    "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE,
+                );
+            } else {
+                // The block is in the future, but not too far.
+                debug!(
+                    log, "Block is slightly ahead of our slot clock, ignoring.";
+                    "present_slot" => present_slot,
+                    "block_slot" => block_slot,
+                    "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE,
+                );
+            }
+
+            Err(format!(
+                "Block with slot {} is higher than the current slot {}",
+                block_slot, present_slot
+            ))
+        }
+        BlockError::WouldRevertFinalizedSlot { .. } => {
+            debug!( log, "Finalized or earlier block processed";);
+
+            Ok(())
+        }
+        BlockError::GenesisBlock => {
+            debug!(log, "Genesis block was processed");
+            Ok(())
+        }
+        BlockError::BeaconChainError(e) => {
+            warn!(
+                log, "BlockProcessingFailure";
+                "msg" => "unexpected condition in processing block.",
+                "outcome" => format!("{:?}", e)
+            );
+
+            Err(format!("Internal error whilst processing block: {:?}", e))
+        }
+        other => {
+            debug!(
+                log, "Invalid block received";
+                "msg" => "peer sent invalid block",
+                "outcome" => format!("{:?}", other),
+            );
+
+            Err(format!("Peer sent invalid block. Reason: {:?}", other))
+        }
+    }
+}

beacon_node/network/src/beacon_processor/mod.rs

@@ -0,0 +1,819 @@
+//! Provides the `BeaconProcessor`, a mutli-threaded processor for messages received on the network
+//! that need to be processed by the `BeaconChain`.
+//!
+//! Uses `tokio` tasks (instead of raw threads) to provide the following tasks:
+//!
+//! - A "manager" task, which either spawns worker tasks or enqueues work.
+//! - One or more "worker" tasks which perform time-intensive work on the `BeaconChain`.
+//!
+//! ## Purpose
+//!
+//! The purpose of the `BeaconProcessor` is to provide two things:
+//!
+//! 1. Moving long-running, blocking tasks off the main `tokio` executor.
+//! 2. A fixed-length buffer for consensus messages.
+//!
+//! (1) ensures that we don't clog up the networking stack with long-running tasks, potentially
+//! causing timeouts. (2) means that we can easily and explicitly reject messages when we're
+//! overloaded and also distribute load across time.
+//!
+//! ## Detail
+//!
+//! There is a single "manager" thread who listens to two event channels. These events are either:
+//!
+//! - A new parcel of work (work event).
+//! - Indication that a worker has finished a parcel of work (worker idle).
+//!
+//! Then, there is a maximum of `n` "worker" blocking threads, where `n` is the CPU count.
+//!
+//! Whenever the manager receives a new parcel of work, it either:
+//!
+//! - Provided to a newly-spawned worker tasks (if we are not already at `n` workers).
+//! - Added to a queue.
+//!
+//! Whenever the manager receives a notification that a worker has finished a parcel of work, it
+//! checks the queues to see if there are more parcels of work that can be spawned in a new worker
+//! task.
+
+use crate::{metrics, service::NetworkMessage, sync::SyncMessage};
+use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError};
+use environment::TaskExecutor;
+use eth2_libp2p::{MessageId, NetworkGlobals, PeerId};
+use slog::{crit, debug, error, trace, warn, Logger};
+use std::collections::VecDeque;
+use std::sync::{Arc, Weak};
+use std::time::{Duration, Instant};
+use tokio::sync::{mpsc, oneshot};
+use types::{
+    Attestation, AttesterSlashing, EthSpec, Hash256, ProposerSlashing, SignedAggregateAndProof,
+    SignedBeaconBlock, SignedVoluntaryExit, SubnetId,
+};
+use worker::Worker;
+
+mod chain_segment;
+mod worker;
+
+pub use chain_segment::ProcessId;
+
+/// The maximum size of the channel for work events to the `BeaconProcessor`.
+///
+/// Setting this too low will cause consensus messages to be dropped.
+pub const MAX_WORK_EVENT_QUEUE_LEN: usize = 16_384;
+
+/// The maximum size of the channel for idle events to the `BeaconProcessor`.
+///
+/// Setting this too low will prevent new workers from being spawned. It *should* only need to be
+/// set to the CPU count, but we set it high to be safe.
+const MAX_IDLE_QUEUE_LEN: usize = 16_384;
+
+/// The maximum number of queued `Attestation` objects that will be stored before we start dropping
+/// them.
+const MAX_UNAGGREGATED_ATTESTATION_QUEUE_LEN: usize = 16_384;
+
+/// The maximum number of queued `SignedAggregateAndProof` objects that will be stored before we
+/// start dropping them.
+const MAX_AGGREGATED_ATTESTATION_QUEUE_LEN: usize = 1_024;
+
+/// The maximum number of queued `SignedBeaconBlock` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_BLOCK_QUEUE_LEN: usize = 1_024;
+
+/// The maximum number of queued `SignedVoluntaryExit` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_EXIT_QUEUE_LEN: usize = 4_096;
+
+/// The maximum number of queued `ProposerSlashing` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_PROPOSER_SLASHING_QUEUE_LEN: usize = 4_096;
+
+/// The maximum number of queued `AttesterSlashing` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_ATTESTER_SLASHING_QUEUE_LEN: usize = 4_096;
+
+/// The maximum number of queued `SignedBeaconBlock` objects received from the network RPC that
+/// will be stored before we start dropping them.
+const MAX_RPC_BLOCK_QUEUE_LEN: usize = 1_024;
+
+/// The maximum number of queued `Vec<SignedBeaconBlock>` objects received during syncing that will
+/// be stored before we start dropping them.
+const MAX_CHAIN_SEGMENT_QUEUE_LEN: usize = 64;
+
+/// The name of the manager tokio task.
+const MANAGER_TASK_NAME: &str = "beacon_gossip_processor_manager";
+/// The name of the worker tokio tasks.
+const WORKER_TASK_NAME: &str = "beacon_gossip_processor_worker";
+
+/// The minimum interval between log messages indicating that a queue is full.
+const LOG_DEBOUNCE_INTERVAL: Duration = Duration::from_secs(30);
+
+/// Used to send/receive results from a rpc block import in a blocking task.
+pub type BlockResultSender<E> = oneshot::Sender<Result<Hash256, BlockError<E>>>;
+pub type BlockResultReceiver<E> = oneshot::Receiver<Result<Hash256, BlockError<E>>>;
+
+/// A simple first-in-first-out queue with a maximum length.
+struct FifoQueue<T> {
+    queue: VecDeque<T>,
+    max_length: usize,
+}
+
+impl<T> FifoQueue<T> {
+    /// Create a new, empty queue with the given length.
+    pub fn new(max_length: usize) -> Self {
+        Self {
+            queue: VecDeque::default(),
+            max_length,
+        }
+    }
+
+    /// Add a new item to the queue.
+    ///
+    /// Drops `item` if the queue is full.
+    pub fn push(&mut self, item: T, item_desc: &str, log: &Logger) {
+        if self.queue.len() == self.max_length {
+            error!(
+                log,
+                "Block queue full";
+                "msg" => "the system has insufficient resources for load",
+                "queue_len" => self.max_length,
+                "queue" => item_desc,
+            )
+        } else {
+            self.queue.push_back(item);
+        }
+    }
+
+    /// Remove the next item from the queue.
+    pub fn pop(&mut self) -> Option<T> {
+        self.queue.pop_front()
+    }
+
+    /// Returns the current length of the queue.
+    pub fn len(&self) -> usize {
+        self.queue.len()
+    }
+}
+
+/// A simple last-in-first-out queue with a maximum length.
+struct LifoQueue<T> {
+    queue: VecDeque<T>,
+    max_length: usize,
+}
+
+impl<T> LifoQueue<T> {
+    /// Create a new, empty queue with the given length.
+    pub fn new(max_length: usize) -> Self {
+        Self {
+            queue: VecDeque::default(),
+            max_length,
+        }
+    }
+
+    /// Add a new item to the front of the queue.
+    ///
+    /// If the queue is full, the item at the back of the queue is dropped.
+    pub fn push(&mut self, item: T) {
+        if self.queue.len() == self.max_length {
+            self.queue.pop_back();
+        }
+        self.queue.push_front(item);
+    }
+
+    /// Remove the next item from the queue.
+    pub fn pop(&mut self) -> Option<T> {
+        self.queue.pop_front()
+    }
+
+    /// Returns `true` if the queue is full.
+    pub fn is_full(&self) -> bool {
+        self.queue.len() >= self.max_length
+    }
+
+    /// Returns the current length of the queue.
+    pub fn len(&self) -> usize {
+        self.queue.len()
+    }
+}
+
+/// An event to be processed by the manager task.
+#[derive(Debug)]
+pub struct WorkEvent<E: EthSpec> {
+    drop_during_sync: bool,
+    work: Work<E>,
+}
+
+impl<E: EthSpec> WorkEvent<E> {
+    /// Create a new `Work` event for some unaggregated attestation.
+    pub fn unaggregated_attestation(
+        message_id: MessageId,
+        peer_id: PeerId,
+        attestation: Attestation<E>,
+        subnet_id: SubnetId,
+        should_import: bool,
+    ) -> Self {
+        Self {
+            drop_during_sync: true,
+            work: Work::GossipAttestation {
+                message_id,
+                peer_id,
+                attestation: Box::new(attestation),
+                subnet_id,
+                should_import,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some aggregated attestation.
+    pub fn aggregated_attestation(
+        message_id: MessageId,
+        peer_id: PeerId,
+        aggregate: SignedAggregateAndProof<E>,
+    ) -> Self {
+        Self {
+            drop_during_sync: true,
+            work: Work::GossipAggregate {
+                message_id,
+                peer_id,
+                aggregate: Box::new(aggregate),
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some block.
+    pub fn gossip_beacon_block(
+        message_id: MessageId,
+        peer_id: PeerId,
+        block: Box<SignedBeaconBlock<E>>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipBlock {
+                message_id,
+                peer_id,
+                block,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some exit.
+    pub fn gossip_voluntary_exit(
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: Box<SignedVoluntaryExit>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipVoluntaryExit {
+                message_id,
+                peer_id,
+                voluntary_exit,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some proposer slashing.
+    pub fn gossip_proposer_slashing(
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipProposerSlashing {
+                message_id,
+                peer_id,
+                proposer_slashing,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some attester slashing.
+    pub fn gossip_attester_slashing(
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<E>>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipAttesterSlashing {
+                message_id,
+                peer_id,
+                attester_slashing,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some block, where the result from computation (if any) is
+    /// sent to the other side of `result_tx`.
+    pub fn rpc_beacon_block(block: Box<SignedBeaconBlock<E>>) -> (Self, BlockResultReceiver<E>) {
+        let (result_tx, result_rx) = oneshot::channel();
+        let event = Self {
+            drop_during_sync: false,
+            work: Work::RpcBlock { block, result_tx },
+        };
+        (event, result_rx)
+    }
+
+    /// Create a new work event to import `blocks` as a beacon chain segment.
+    pub fn chain_segment(process_id: ProcessId, blocks: Vec<SignedBeaconBlock<E>>) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::ChainSegment { process_id, blocks },
+        }
+    }
+}
+
+/// A consensus message (or multiple) from the network that requires processing.
+#[derive(Debug)]
+pub enum Work<E: EthSpec> {
+    GossipAttestation {
+        message_id: MessageId,
+        peer_id: PeerId,
+        attestation: Box<Attestation<E>>,
+        subnet_id: SubnetId,
+        should_import: bool,
+    },
+    GossipAggregate {
+        message_id: MessageId,
+        peer_id: PeerId,
+        aggregate: Box<SignedAggregateAndProof<E>>,
+    },
+    GossipBlock {
+        message_id: MessageId,
+        peer_id: PeerId,
+        block: Box<SignedBeaconBlock<E>>,
+    },
+    GossipVoluntaryExit {
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: Box<SignedVoluntaryExit>,
+    },
+    GossipProposerSlashing {
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
+    },
+    GossipAttesterSlashing {
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<E>>,
+    },
+    RpcBlock {
+        block: Box<SignedBeaconBlock<E>>,
+        result_tx: BlockResultSender<E>,
+    },
+    ChainSegment {
+        process_id: ProcessId,
+        blocks: Vec<SignedBeaconBlock<E>>,
+    },
+}
+
+impl<E: EthSpec> Work<E> {
+    /// Provides a `&str` that uniquely identifies each enum variant.
+    fn str_id(&self) -> &'static str {
+        match self {
+            Work::GossipAttestation { .. } => "gossip_attestation",
+            Work::GossipAggregate { .. } => "gossip_aggregate",
+            Work::GossipBlock { .. } => "gossip_block",
+            Work::GossipVoluntaryExit { .. } => "gossip_voluntary_exit",
+            Work::GossipProposerSlashing { .. } => "gossip_proposer_slashing",
+            Work::GossipAttesterSlashing { .. } => "gossip_attester_slashing",
+            Work::RpcBlock { .. } => "rpc_block",
+            Work::ChainSegment { .. } => "chain_segment",
+        }
+    }
+}
+
+/// Provides de-bounce functionality for logging.
+#[derive(Default)]
+struct TimeLatch(Option<Instant>);
+
+impl TimeLatch {
+    /// Only returns true once every `LOG_DEBOUNCE_INTERVAL`.
+    fn elapsed(&mut self) -> bool {
+        let now = Instant::now();
+
+        let is_elapsed = self.0.map_or(false, |elapse_time| now > elapse_time);
+
+        if is_elapsed || self.0.is_none() {
+            self.0 = Some(now + LOG_DEBOUNCE_INTERVAL);
+        }
+
+        is_elapsed
+    }
+}
+
+/// A mutli-threaded processor for messages received on the network
+/// that need to be processed by the `BeaconChain`
+///
+/// See module level documentation for more information.
+pub struct BeaconProcessor<T: BeaconChainTypes> {
+    pub beacon_chain: Weak<BeaconChain<T>>,
+    pub network_tx: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
+    pub sync_tx: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    pub network_globals: Arc<NetworkGlobals<T::EthSpec>>,
+    pub executor: TaskExecutor,
+    pub max_workers: usize,
+    pub current_workers: usize,
+    pub log: Logger,
+}
+
+impl<T: BeaconChainTypes> BeaconProcessor<T> {
+    /// Spawns the "manager" task which checks the receiver end of the returned `Sender` for
+    /// messages which contain some new work which will be:
+    ///
+    /// - Performed immediately, if a worker is available.
+    /// - Queued for later processing, if no worker is currently available.
+    ///
+    /// Only `self.max_workers` will ever be spawned at one time. Each worker is a `tokio` task
+    /// started with `spawn_blocking`.
+    pub fn spawn_manager(mut self, mut event_rx: mpsc::Receiver<WorkEvent<T::EthSpec>>) {
+        let (idle_tx, mut idle_rx) = mpsc::channel::<()>(MAX_IDLE_QUEUE_LEN);
+
+        // Using LIFO queues for attestations since validator profits rely upon getting fresh
+        // attestations into blocks. Additionally, later attestations contain more information than
+        // earlier ones, so we consider them more valuable.
+        let mut aggregate_queue = LifoQueue::new(MAX_AGGREGATED_ATTESTATION_QUEUE_LEN);
+        let mut aggregate_debounce = TimeLatch::default();
+        let mut attestation_queue = LifoQueue::new(MAX_UNAGGREGATED_ATTESTATION_QUEUE_LEN);
+        let mut attestation_debounce = TimeLatch::default();
+
+        // Using a FIFO queue for voluntary exits since it prevents exit censoring. I don't have
+        // a strong feeling about queue type for exits.
+        let mut gossip_voluntary_exit_queue = FifoQueue::new(MAX_GOSSIP_EXIT_QUEUE_LEN);
+
+        // Using a FIFO queue for slashing to prevent people from flushing their slashings from the
+        // queues with lots of junk messages.
+        let mut gossip_proposer_slashing_queue =
+            FifoQueue::new(MAX_GOSSIP_PROPOSER_SLASHING_QUEUE_LEN);
+        let mut gossip_attester_slashing_queue =
+            FifoQueue::new(MAX_GOSSIP_ATTESTER_SLASHING_QUEUE_LEN);
+
+        // Using a FIFO queue since blocks need to be imported sequentially.
+        let mut rpc_block_queue = FifoQueue::new(MAX_RPC_BLOCK_QUEUE_LEN);
+        let mut chain_segment_queue = FifoQueue::new(MAX_CHAIN_SEGMENT_QUEUE_LEN);
+        let mut gossip_block_queue = FifoQueue::new(MAX_GOSSIP_BLOCK_QUEUE_LEN);
+
+        let executor = self.executor.clone();
+
+        // The manager future will run on the core executor and delegate tasks to worker
+        // threads on the blocking executor.
+        let manager_future = async move {
+            loop {
+                // Listen to both the event and idle channels, acting on whichever is ready
+                // first.
+                //
+                // Set `work_event = Some(event)` if there is new work to be done. Otherwise sets
+                // `event = None` if it was a worker becoming idle.
+                let work_event = tokio::select! {
+                    // A worker has finished some work.
+                    new_idle_opt = idle_rx.recv() => {
+                        if new_idle_opt.is_some() {
+                            self.current_workers = self.current_workers.saturating_sub(1);
+                            None
+                        } else {
+                            // Exit if all idle senders have been dropped.
+                            //
+                            // This shouldn't happen since this function holds a sender.
+                            crit!(
+                                self.log,
+                                "Gossip processor stopped";
+                                "msg" => "all idle senders dropped"
+                            );
+                            break
+                        }
+                    },
+                    // There is a new piece of work to be handled.
+                    new_work_event_opt = event_rx.recv() => {
+                        if let Some(new_work_event) = new_work_event_opt {
+                            Some(new_work_event)
+                        } else {
+                            // Exit if all event senders have been dropped.
+                            //
+                            // This should happen when the client shuts down.
+                            debug!(
+                                self.log,
+                                "Gossip processor stopped";
+                                "msg" => "all event senders dropped"
+                            );
+                            break
+                        }
+                    }
+                };
+
+                let _event_timer =
+                    metrics::start_timer(&metrics::BEACON_PROCESSOR_EVENT_HANDLING_SECONDS);
+                if let Some(event) = &work_event {
+                    metrics::inc_counter_vec(
+                        &metrics::BEACON_PROCESSOR_WORK_EVENTS_RX_COUNT,
+                        &[event.work.str_id()],
+                    );
+                } else {
+                    metrics::inc_counter(&metrics::BEACON_PROCESSOR_IDLE_EVENTS_TOTAL);
+                }
+
+                let can_spawn = self.current_workers < self.max_workers;
+                let drop_during_sync = work_event
+                    .as_ref()
+                    .map_or(false, |event| event.drop_during_sync);
+
+                match work_event {
+                    // There is no new work event, but we are able to spawn a new worker.
+                    //
+                    // We don't check the `work.drop_during_sync` here. We assume that if it made
+                    // it into the queue at any point then we should process it.
+                    None if can_spawn => {
+                        // Check for chain segments first, they're the most efficient way to get
+                        // blocks into the system.
+                        if let Some(item) = chain_segment_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check sync blocks before gossip blocks, since we've already explicitly
+                        // requested these blocks.
+                        } else if let Some(item) = rpc_block_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check gossip blocks before gossip attestations, since a block might be
+                        // required to verify some attestations.
+                        } else if let Some(item) = gossip_block_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check the aggregates, *then* the unaggregates
+                        // since we assume that aggregates are more valuable to local validators
+                        // and effectively give us more information with less signature
+                        // verification time.
+                        } else if let Some(item) = aggregate_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        } else if let Some(item) = attestation_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check slashings after all other consensus messages so we prioritize
+                        // following head.
+                        //
+                        // Check attester slashings before proposer slashings since they have the
+                        // potential to slash multiple validators at once.
+                        } else if let Some(item) = gossip_attester_slashing_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        } else if let Some(item) = gossip_proposer_slashing_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check exits last since our validators don't get rewards from them.
+                        } else if let Some(item) = gossip_voluntary_exit_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        }
+                    }
+                    // There is no new work event and we are unable to spawn a new worker.
+                    //
+                    // I cannot see any good reason why this would happen.
+                    None => {
+                        warn!(
+                            self.log,
+                            "Unexpected gossip processor condition";
+                            "msg" => "no new work and cannot spawn worker"
+                        );
+                    }
+                    // The chain is syncing and this event should be dropped during sync.
+                    Some(work_event)
+                        if self.network_globals.sync_state.read().is_syncing()
+                            && drop_during_sync =>
+                    {
+                        let work_id = work_event.work.str_id();
+                        metrics::inc_counter_vec(
+                            &metrics::BEACON_PROCESSOR_WORK_EVENTS_IGNORED_COUNT,
+                            &[work_id],
+                        );
+                        trace!(
+                            self.log,
+                            "Gossip processor skipping work";
+                            "msg" => "chain is syncing",
+                            "work_id" => work_id
+                        );
+                    }
+                    // There is a new work event and the chain is not syncing. Process it.
+                    Some(WorkEvent { work, .. }) => {
+                        let work_id = work.str_id();
+                        match work {
+                            _ if can_spawn => self.spawn_worker(idle_tx.clone(), work),
+                            Work::GossipAttestation { .. } => attestation_queue.push(work),
+                            Work::GossipAggregate { .. } => aggregate_queue.push(work),
+                            Work::GossipBlock { .. } => {
+                                gossip_block_queue.push(work, work_id, &self.log)
+                            }
+                            Work::GossipVoluntaryExit { .. } => {
+                                gossip_voluntary_exit_queue.push(work, work_id, &self.log)
+                            }
+                            Work::GossipProposerSlashing { .. } => {
+                                gossip_proposer_slashing_queue.push(work, work_id, &self.log)
+                            }
+                            Work::GossipAttesterSlashing { .. } => {
+                                gossip_attester_slashing_queue.push(work, work_id, &self.log)
+                            }
+                            Work::RpcBlock { .. } => rpc_block_queue.push(work, work_id, &self.log),
+                            Work::ChainSegment { .. } => {
+                                chain_segment_queue.push(work, work_id, &self.log)
+                            }
+                        }
+                    }
+                }
+
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_WORKERS_ACTIVE_TOTAL,
+                    self.current_workers as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL,
+                    attestation_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL,
+                    aggregate_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL,
+                    gossip_block_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL,
+                    rpc_block_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL,
+                    chain_segment_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_EXIT_QUEUE_TOTAL,
+                    gossip_voluntary_exit_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL,
+                    gossip_proposer_slashing_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL,
+                    gossip_attester_slashing_queue.len() as i64,
+                );
+
+                if aggregate_queue.is_full() && aggregate_debounce.elapsed() {
+                    error!(
+                        self.log,
+                        "Aggregate attestation queue full";
+                        "msg" => "the system has insufficient resources for load",
+                        "queue_len" => aggregate_queue.max_length,
+                    )
+                }
+
+                if attestation_queue.is_full() && attestation_debounce.elapsed() {
+                    error!(
+                        self.log,
+                        "Attestation queue full";
+                        "msg" => "the system has insufficient resources for load",
+                        "queue_len" => attestation_queue.max_length,
+                    )
+                }
+            }
+        };
+
+        // Spawn on the core executor.
+        executor.spawn(manager_future, MANAGER_TASK_NAME);
+    }
+
+    /// Spawns a blocking worker thread to process some `Work`.
+    ///
+    /// Sends an message on `idle_tx` when the work is complete and the task is stopping.
+    fn spawn_worker(&mut self, mut idle_tx: mpsc::Sender<()>, work: Work<T::EthSpec>) {
+        let work_id = work.str_id();
+        let worker_timer =
+            metrics::start_timer_vec(&metrics::BEACON_PROCESSOR_WORKER_TIME, &[work_id]);
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_WORKERS_SPAWNED_TOTAL);
+        metrics::inc_counter_vec(
+            &metrics::BEACON_PROCESSOR_WORK_EVENTS_STARTED_COUNT,
+            &[work.str_id()],
+        );
+
+        let worker_id = self.current_workers;
+        self.current_workers = self.current_workers.saturating_add(1);
+
+        let chain = if let Some(chain) = self.beacon_chain.upgrade() {
+            chain
+        } else {
+            debug!(
+                self.log,
+                "Beacon chain dropped, shutting down";
+            );
+            return;
+        };
+
+        let log = self.log.clone();
+        let executor = self.executor.clone();
+
+        let worker = Worker {
+            chain,
+            network_tx: self.network_tx.clone(),
+            sync_tx: self.sync_tx.clone(),
+            log: self.log.clone(),
+        };
+
+        trace!(
+            self.log,
+            "Spawning beacon processor worker";
+            "work" => work_id,
+            "worker" => worker_id,
+        );
+
+        executor.spawn_blocking(
+            move || {
+                let _worker_timer = worker_timer;
+
+                match work {
+                    /*
+                     * Unaggregated attestation verification.
+                     */
+                    Work::GossipAttestation {
+                        message_id,
+                        peer_id,
+                        attestation,
+                        subnet_id,
+                        should_import,
+                    } => worker.process_gossip_attestation(
+                        message_id,
+                        peer_id,
+                        *attestation,
+                        subnet_id,
+                        should_import,
+                    ),
+                    /*
+                     * Aggregated attestation verification.
+                     */
+                    Work::GossipAggregate {
+                        message_id,
+                        peer_id,
+                        aggregate,
+                    } => worker.process_gossip_aggregate(message_id, peer_id, *aggregate),
+                    /*
+                     * Verification for beacon blocks received on gossip.
+                     */
+                    Work::GossipBlock {
+                        message_id,
+                        peer_id,
+                        block,
+                    } => worker.process_gossip_block(message_id, peer_id, *block),
+                    /*
+                     * Voluntary exits received on gossip.
+                     */
+                    Work::GossipVoluntaryExit {
+                        message_id,
+                        peer_id,
+                        voluntary_exit,
+                    } => worker.process_gossip_voluntary_exit(message_id, peer_id, *voluntary_exit),
+                    /*
+                     * Proposer slashings received on gossip.
+                     */
+                    Work::GossipProposerSlashing {
+                        message_id,
+                        peer_id,
+                        proposer_slashing,
+                    } => worker.process_gossip_proposer_slashing(
+                        message_id,
+                        peer_id,
+                        *proposer_slashing,
+                    ),
+                    /*
+                     * Attester slashings received on gossip.
+                     */
+                    Work::GossipAttesterSlashing {
+                        message_id,
+                        peer_id,
+                        attester_slashing,
+                    } => worker.process_gossip_attester_slashing(
+                        message_id,
+                        peer_id,
+                        *attester_slashing,
+                    ),
+                    /*
+                     * Verification for beacon blocks received during syncing via RPC.
+                     */
+                    Work::RpcBlock { block, result_tx } => {
+                        worker.process_rpc_block(*block, result_tx)
+                    }
+                    /*
+                     * Verification for a chain segment (multiple blocks).
+                     */
+                    Work::ChainSegment { process_id, blocks } => {
+                        worker.process_chain_segment(process_id, blocks)
+                    }
+                };
+
+                trace!(
+                    log,
+                    "Beacon processor worker done";
+                    "work" => work_id,
+                    "worker" => worker_id,
+                );
+
+                idle_tx.try_send(()).unwrap_or_else(|e| {
+                    crit!(
+                        log,
+                        "Unable to free worker";
+                        "msg" => "failed to send idle_tx message",
+                        "error" => e.to_string()
+                    )
+                });
+            },
+            WORKER_TASK_NAME,
+        );
+    }
+}

beacon_node/network/src/beacon_processor/worker.rs

@@ -0,0 +1,850 @@
+use super::{
+    chain_segment::{handle_chain_segment, ProcessId},
+    BlockResultSender,
+};
+use crate::{metrics, service::NetworkMessage, sync::SyncMessage};
+use beacon_chain::{
+    attestation_verification::Error as AttnError, observed_operations::ObservationOutcome,
+    BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError,
+};
+use eth2_libp2p::{MessageAcceptance, MessageId, PeerId};
+use slog::{crit, debug, error, info, trace, warn, Logger};
+use ssz::Encode;
+use std::sync::Arc;
+use tokio::sync::mpsc;
+use types::{
+    Attestation, AttesterSlashing, Hash256, ProposerSlashing, SignedAggregateAndProof,
+    SignedBeaconBlock, SignedVoluntaryExit, SubnetId,
+};
+
+/// Contains the context necessary to import blocks, attestations, etc to the beacon chain.
+pub struct Worker<T: BeaconChainTypes> {
+    pub chain: Arc<BeaconChain<T>>,
+    pub network_tx: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
+    pub sync_tx: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    pub log: Logger,
+}
+
+impl<T: BeaconChainTypes> Worker<T> {
+    /// Process the unaggregated attestation received from the gossip network and:
+    ///
+    /// - If it passes gossip propagation criteria, tell the network thread to forward it.
+    /// - Attempt to apply it to fork choice.
+    /// - Attempt to add it to the naive aggregation pool.
+    ///
+    /// Raises a log if there are errors.
+    pub fn process_gossip_attestation(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attestation: Attestation<T::EthSpec>,
+        subnet_id: SubnetId,
+        should_import: bool,
+    ) {
+        let beacon_block_root = attestation.data.beacon_block_root;
+
+        let attestation = match self
+            .chain
+            .verify_unaggregated_attestation_for_gossip(attestation, subnet_id)
+        {
+            Ok(attestation) => attestation,
+            Err(e) => {
+                self.handle_attestation_verification_failure(
+                    peer_id,
+                    message_id,
+                    beacon_block_root,
+                    "unaggregated",
+                    e,
+                );
+                return;
+            }
+        };
+
+        // Indicate to the `Network` service that this message is valid and can be
+        // propagated on the gossip network.
+        self.propagate_validation_result(message_id, peer_id.clone(), MessageAcceptance::Accept);
+
+        if !should_import {
+            return;
+        }
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL);
+
+        if let Err(e) = self.chain.apply_attestation_to_fork_choice(&attestation) {
+            match e {
+                BeaconChainError::ForkChoiceError(ForkChoiceError::InvalidAttestation(e)) => {
+                    debug!(
+                        self.log,
+                        "Attestation invalid for fork choice";
+                        "reason" => format!("{:?}", e),
+                        "peer" => peer_id.to_string(),
+                        "beacon_block_root" => format!("{:?}", beacon_block_root)
+                    )
+                }
+                e => error!(
+                    self.log,
+                    "Error applying attestation to fork choice";
+                    "reason" => format!("{:?}", e),
+                    "peer" => peer_id.to_string(),
+                    "beacon_block_root" => format!("{:?}", beacon_block_root)
+                ),
+            }
+        }
+
+        if let Err(e) = self.chain.add_to_naive_aggregation_pool(attestation) {
+            debug!(
+                self.log,
+                "Attestation invalid for agg pool";
+                "reason" => format!("{:?}", e),
+                "peer" => peer_id.to_string(),
+                "beacon_block_root" => format!("{:?}", beacon_block_root)
+            )
+        }
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_IMPORTED_TOTAL);
+    }
+
+    /// Process the aggregated attestation received from the gossip network and:
+    ///
+    /// - If it passes gossip propagation criteria, tell the network thread to forward it.
+    /// - Attempt to apply it to fork choice.
+    /// - Attempt to add it to the block inclusion pool.
+    ///
+    /// Raises a log if there are errors.
+    pub fn process_gossip_aggregate(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        aggregate: SignedAggregateAndProof<T::EthSpec>,
+    ) {
+        let beacon_block_root = aggregate.message.aggregate.data.beacon_block_root;
+
+        let aggregate = match self
+            .chain
+            .verify_aggregated_attestation_for_gossip(aggregate)
+        {
+            Ok(aggregate) => aggregate,
+            Err(e) => {
+                // Report the failure to gossipsub
+                self.handle_attestation_verification_failure(
+                    peer_id,
+                    message_id,
+                    beacon_block_root,
+                    "aggregated",
+                    e,
+                );
+                return;
+            }
+        };
+
+        // Indicate to the `Network` service that this message is valid and can be
+        // propagated on the gossip network.
+        self.propagate_validation_result(message_id, peer_id.clone(), MessageAcceptance::Accept);
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL);
+
+        if let Err(e) = self.chain.apply_attestation_to_fork_choice(&aggregate) {
+            match e {
+                BeaconChainError::ForkChoiceError(ForkChoiceError::InvalidAttestation(e)) => {
+                    debug!(
+                        self.log,
+                        "Aggregate invalid for fork choice";
+                        "reason" => format!("{:?}", e),
+                        "peer" => peer_id.to_string(),
+                        "beacon_block_root" => format!("{:?}", beacon_block_root)
+                    )
+                }
+                e => error!(
+                    self.log,
+                    "Error applying aggregate to fork choice";
+                    "reason" => format!("{:?}", e),
+                    "peer" => peer_id.to_string(),
+                    "beacon_block_root" => format!("{:?}", beacon_block_root)
+                ),
+            }
+        }
+
+        if let Err(e) = self.chain.add_to_block_inclusion_pool(aggregate) {
+            debug!(
+                self.log,
+                "Attestation invalid for op pool";
+                "reason" => format!("{:?}", e),
+                "peer" => peer_id.to_string(),
+                "beacon_block_root" => format!("{:?}", beacon_block_root)
+            )
+        }
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_IMPORTED_TOTAL);
+    }
+
+    /// Process the beacon block received from the gossip network and:
+    ///
+    /// - If it passes gossip propagation criteria, tell the network thread to forward it.
+    /// - Attempt to add it to the beacon chain, informing the sync thread if more blocks need to
+    ///   be downloaded.
+    ///
+    /// Raises a log if there are errors.
+    pub fn process_gossip_block(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        block: SignedBeaconBlock<T::EthSpec>,
+    ) {
+        let verified_block = match self.chain.verify_block_for_gossip(block) {
+            Ok(verified_block) => {
+                info!(
+                    self.log,
+                    "New block received";
+                    "slot" => verified_block.block.slot(),
+                    "hash" => verified_block.block_root.to_string()
+                );
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Accept,
+                );
+                verified_block
+            }
+            Err(BlockError::ParentUnknown(block)) => {
+                self.send_sync_message(SyncMessage::UnknownBlock(peer_id, block));
+                return;
+            }
+            Err(e @ BlockError::FutureSlot { .. })
+            | Err(e @ BlockError::WouldRevertFinalizedSlot { .. })
+            | Err(e @ BlockError::BlockIsAlreadyKnown)
+            | Err(e @ BlockError::RepeatProposal { .. })
+            | Err(e @ BlockError::NotFinalizedDescendant { .. })
+            | Err(e @ BlockError::BeaconChainError(_)) => {
+                warn!(self.log, "Could not verify block for gossip, ignoring the block";
+                            "error" => e.to_string());
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            Err(e @ BlockError::StateRootMismatch { .. })
+            | Err(e @ BlockError::IncorrectBlockProposer { .. })
+            | Err(e @ BlockError::BlockSlotLimitReached)
+            | Err(e @ BlockError::ProposalSignatureInvalid)
+            | Err(e @ BlockError::NonLinearSlots)
+            | Err(e @ BlockError::UnknownValidator(_))
+            | Err(e @ BlockError::PerBlockProcessingError(_))
+            | Err(e @ BlockError::NonLinearParentRoots)
+            | Err(e @ BlockError::BlockIsNotLaterThanParent { .. })
+            | Err(e @ BlockError::InvalidSignature)
+            | Err(e @ BlockError::TooManySkippedSlots { .. })
+            | Err(e @ BlockError::GenesisBlock) => {
+                warn!(self.log, "Could not verify block for gossip, rejecting the block";
+                            "error" => e.to_string());
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Reject);
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL);
+
+        let block = Box::new(verified_block.block.clone());
+        match self.chain.process_block(verified_block) {
+            Ok(_block_root) => {
+                metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL);
+
+                trace!(
+                    self.log,
+                    "Gossipsub block processed";
+                    "peer_id" => peer_id.to_string()
+                );
+
+                // TODO: It would be better if we can run this _after_ we publish the block to
+                // reduce block propagation latency.
+                //
+                // The `MessageHandler` would be the place to put this, however it doesn't seem
+                // to have a reference to the `BeaconChain`. I will leave this for future
+                // works.
+                match self.chain.fork_choice() {
+                    Ok(()) => trace!(
+                        self.log,
+                        "Fork choice success";
+                        "location" => "block gossip"
+                    ),
+                    Err(e) => error!(
+                        self.log,
+                        "Fork choice failed";
+                        "error" => format!("{:?}", e),
+                        "location" => "block gossip"
+                    ),
+                }
+            }
+            Err(BlockError::ParentUnknown { .. }) => {
+                // Inform the sync manager to find parents for this block
+                // This should not occur. It should be checked by `should_forward_block`
+                error!(
+                    self.log,
+                    "Block with unknown parent attempted to be processed";
+                    "peer_id" => peer_id.to_string()
+                );
+                self.send_sync_message(SyncMessage::UnknownBlock(peer_id, block));
+            }
+            other => {
+                debug!(
+                    self.log,
+                    "Invalid gossip beacon block";
+                    "outcome" => format!("{:?}", other),
+                    "block root" => format!("{}", block.canonical_root()),
+                    "block slot" => block.slot()
+                );
+                trace!(
+                    self.log,
+                    "Invalid gossip beacon block ssz";
+                    "ssz" => format!("0x{}", hex::encode(block.as_ssz_bytes())),
+                );
+            }
+        };
+    }
+
+    pub fn process_gossip_voluntary_exit(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: SignedVoluntaryExit,
+    ) {
+        let validator_index = voluntary_exit.message.validator_index;
+
+        let exit = match self.chain.verify_voluntary_exit_for_gossip(voluntary_exit) {
+            Ok(ObservationOutcome::New(exit)) => exit,
+            Ok(ObservationOutcome::AlreadyKnown) => {
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Ignore,
+                );
+                debug!(
+                    self.log,
+                    "Dropping exit for already exiting validator";
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string()
+                );
+                return;
+            }
+            Err(e) => {
+                debug!(
+                    self.log,
+                    "Dropping invalid exit";
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string(),
+                    "error" => format!("{:?}", e)
+                );
+                // These errors occur due to a fault in the beacon chain. It is not necessarily
+                // the fault on the peer.
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL);
+
+        self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept);
+
+        self.chain.import_voluntary_exit(exit);
+        debug!(self.log, "Successfully imported voluntary exit");
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_EXIT_IMPORTED_TOTAL);
+    }
+
+    pub fn process_gossip_proposer_slashing(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: ProposerSlashing,
+    ) {
+        let validator_index = proposer_slashing.signed_header_1.message.proposer_index;
+
+        let slashing = match self
+            .chain
+            .verify_proposer_slashing_for_gossip(proposer_slashing)
+        {
+            Ok(ObservationOutcome::New(slashing)) => slashing,
+            Ok(ObservationOutcome::AlreadyKnown) => {
+                debug!(
+                    self.log,
+                    "Dropping proposer slashing";
+                    "reason" => "Already seen a proposer slashing for that validator",
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string()
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            Err(e) => {
+                // This is likely a fault with the beacon chain and not necessarily a
+                // malicious message from the peer.
+                debug!(
+                    self.log,
+                    "Dropping invalid proposer slashing";
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string(),
+                    "error" => format!("{:?}", e)
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL);
+
+        self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept);
+
+        self.chain.import_proposer_slashing(slashing);
+        debug!(self.log, "Successfully imported proposer slashing");
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_IMPORTED_TOTAL);
+    }
+
+    pub fn process_gossip_attester_slashing(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: AttesterSlashing<T::EthSpec>,
+    ) {
+        let slashing = match self
+            .chain
+            .verify_attester_slashing_for_gossip(attester_slashing)
+        {
+            Ok(ObservationOutcome::New(slashing)) => slashing,
+            Ok(ObservationOutcome::AlreadyKnown) => {
+                debug!(
+                    self.log,
+                    "Dropping attester slashing";
+                    "reason" => "Slashings already known for all slashed validators",
+                    "peer" => peer_id.to_string()
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            Err(e) => {
+                debug!(
+                    self.log,
+                    "Dropping invalid attester slashing";
+                    "peer" => peer_id.to_string(),
+                    "error" => format!("{:?}", e)
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL);
+
+        self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept);
+
+        if let Err(e) = self.chain.import_attester_slashing(slashing) {
+            debug!(self.log, "Error importing attester slashing"; "error" => format!("{:?}", e));
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_ERROR_TOTAL);
+        } else {
+            debug!(self.log, "Successfully imported attester slashing");
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_IMPORTED_TOTAL);
+        }
+    }
+
+    /// Attempt to process a block received from a direct RPC request, returning the processing
+    /// result on the `result_tx` channel.
+    ///
+    /// Raises a log if there are errors publishing the result to the channel.
+    pub fn process_rpc_block(
+        self,
+        block: SignedBeaconBlock<T::EthSpec>,
+        result_tx: BlockResultSender<T::EthSpec>,
+    ) {
+        let block_result = self.chain.process_block(block);
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL);
+
+        if result_tx.send(block_result).is_err() {
+            crit!(self.log, "Failed return sync block result");
+        }
+    }
+
+    /// Attempt to import the chain segment (`blocks`) to the beacon chain, informing the sync
+    /// thread if more blocks are needed to process it.
+    pub fn process_chain_segment(
+        self,
+        process_id: ProcessId,
+        blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
+    ) {
+        handle_chain_segment(self.chain, process_id, blocks, self.sync_tx, self.log)
+    }
+
+    /// Send a message on `message_tx` that the `message_id` sent by `peer_id` should be propagated on
+    /// the gossip network.
+    ///
+    /// Creates a log if there is an interal error.
+    /// Propagates the result of the validation fot the given message to the network. If the result
+    /// is valid the message gets forwarded to other peers.
+    fn propagate_validation_result(
+        &self,
+        message_id: MessageId,
+        propagation_source: PeerId,
+        validation_result: MessageAcceptance,
+    ) {
+        self.network_tx
+            .send(NetworkMessage::ValidationResult {
+                propagation_source,
+                message_id,
+                validation_result,
+            })
+            .unwrap_or_else(|_| {
+                warn!(
+                    self.log,
+                    "Could not send propagation request to the network service"
+                )
+            });
+    }
+
+    /// Send a message to `sync_tx`.
+    ///
+    /// Creates a log if there is an interal error.
+    fn send_sync_message(&self, message: SyncMessage<T::EthSpec>) {
+        self.sync_tx
+            .send(message)
+            .unwrap_or_else(|_| error!(self.log, "Could not send message to the sync service"));
+    }
+
+    /// Handle an error whilst verifying an `Attestation` or `SignedAggregateAndProof` from the
+    /// network.
+    pub fn handle_attestation_verification_failure(
+        &self,
+        peer_id: PeerId,
+        message_id: MessageId,
+        beacon_block_root: Hash256,
+        attestation_type: &str,
+        error: AttnError,
+    ) {
+        metrics::register_attestation_error(&error);
+        match &error {
+            AttnError::FutureEpoch { .. }
+            | AttnError::PastEpoch { .. }
+            | AttnError::FutureSlot { .. }
+            | AttnError::PastSlot { .. } => {
+                /*
+                 * These errors can be triggered by a mismatch between our slot and the peer.
+                 *
+                 *
+                 * The peer has published an invalid consensus message, _only_ if we trust our own clock.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::InvalidSelectionProof { .. } | AttnError::InvalidSignature => {
+                /*
+                 * These errors are caused by invalid signatures.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::EmptyAggregationBitfield => {
+                /*
+                 * The aggregate had no signatures and is therefore worthless.
+                 *
+                 * Whilst we don't gossip this attestation, this act is **not** a clear
+                 * violation of the spec nor indication of fault.
+                 *
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::AggregatorPubkeyUnknown(_) => {
+                /*
+                 * The aggregator index was higher than any known validator index. This is
+                 * possible in two cases:
+                 *
+                 * 1. The attestation is malformed
+                 * 2. The attestation attests to a beacon_block_root that we do not know.
+                 *
+                 * It should be impossible to reach (2) without triggering
+                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
+                 * faulty.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::AggregatorNotInCommittee { .. } => {
+                /*
+                 * The aggregator index was higher than any known validator index. This is
+                 * possible in two cases:
+                 *
+                 * 1. The attestation is malformed
+                 * 2. The attestation attests to a beacon_block_root that we do not know.
+                 *
+                 * It should be impossible to reach (2) without triggering
+                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
+                 * faulty.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::AttestationAlreadyKnown { .. } => {
+                /*
+                 * The aggregate attestation has already been observed on the network or in
+                 * a block.
+                 *
+                 * The peer is not necessarily faulty.
+                 */
+                trace!(
+                    self.log,
+                    "Attestation already known";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            AttnError::AggregatorAlreadyKnown(_) => {
+                /*
+                 * There has already been an aggregate attestation seen from this
+                 * aggregator index.
+                 *
+                 * The peer is not necessarily faulty.
+                 */
+                trace!(
+                    self.log,
+                    "Aggregator already known";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            AttnError::PriorAttestationKnown { .. } => {
+                /*
+                 * We have already seen an attestation from this validator for this epoch.
+                 *
+                 * The peer is not necessarily faulty.
+                 */
+                trace!(
+                    self.log,
+                    "Prior attestation known";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            AttnError::ValidatorIndexTooHigh(_) => {
+                /*
+                 * The aggregator index (or similar field) was higher than the maximum
+                 * possible number of validators.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::UnknownHeadBlock { beacon_block_root } => {
+                // Note: its a little bit unclear as to whether or not this block is unknown or
+                // just old. See:
+                //
+                // https://github.com/sigp/lighthouse/issues/1039
+
+                // TODO: Maintain this attestation and re-process once sync completes
+                debug!(
+                    self.log,
+                    "Attestation for unknown block";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root)
+                );
+                // we don't know the block, get the sync manager to handle the block lookup
+                self.sync_tx
+                    .send(SyncMessage::UnknownBlockHash(
+                        peer_id.clone(),
+                        *beacon_block_root,
+                    ))
+                    .unwrap_or_else(|_| {
+                        warn!(
+                            self.log,
+                            "Failed to send to sync service";
+                            "msg" => "UnknownBlockHash"
+                        )
+                    });
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            AttnError::UnknownTargetRoot(_) => {
+                /*
+                 * The block indicated by the target root is not known to us.
+                 *
+                 * We should always get `AttnError::UnknwonHeadBlock` before we get this
+                 * error, so this means we can get this error if:
+                 *
+                 * 1. The target root does not represent a valid block.
+                 * 2. We do not have the target root in our DB.
+                 *
+                 * For (2), we should only be processing attestations when we should have
+                 * all the available information. Note: if we do a weak-subjectivity sync
+                 * it's possible that this situation could occur, but I think it's
+                 * unlikely. For now, we will declare this to be an invalid message>
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::BadTargetEpoch => {
+                /*
+                 * The aggregator index (or similar field) was higher than the maximum
+                 * possible number of validators.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::NoCommitteeForSlotAndIndex { .. } => {
+                /*
+                 * It is not possible to attest this the given committee in the given slot.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::NotExactlyOneAggregationBitSet(_) => {
+                /*
+                 * The unaggregated attestation doesn't have only one signature.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::AttestsToFutureBlock { .. } => {
+                /*
+                 * The beacon_block_root is from a higher slot than the attestation.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+
+            AttnError::InvalidSubnetId { received, expected } => {
+                /*
+                 * The attestation was received on an incorrect subnet id.
+                 */
+                debug!(
+                    self.log,
+                    "Received attestation on incorrect subnet";
+                    "expected" => format!("{:?}", expected),
+                    "received" => format!("{:?}", received),
+                );
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::Invalid(_) => {
+                /*
+                 * The attestation failed the state_processing verification.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::TooManySkippedSlots {
+                head_block_slot,
+                attestation_slot,
+            } => {
+                /*
+                 * The attestation references a head block that is too far behind the attestation slot.
+                 *
+                 * The message is not necessarily invalid, but we choose to ignore it.
+                 */
+                debug!(
+                    self.log,
+                    "Rejected long skip slot attestation";
+                    "head_block_slot" => head_block_slot,
+                    "attestation_slot" => attestation_slot,
+                );
+                // In this case we wish to penalize gossipsub peers that do this to avoid future
+                // attestations that have too many skip slots.
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+            }
+            AttnError::BeaconChainError(e) => {
+                /*
+                 * Lighthouse hit an unexpected error whilst processing the attestation. It
+                 * should be impossible to trigger a `BeaconChainError` from the network,
+                 * so we have a bug.
+                 *
+                 * It's not clear if the message is invalid/malicious.
+                 */
+                error!(
+                    self.log,
+                    "Unable to validate aggregate";
+                    "peer_id" => peer_id.to_string(),
+                    "error" => format!("{:?}", e),
+                );
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Ignore,
+                );
+            }
+        }
+
+        debug!(
+            self.log,
+            "Invalid attestation from network";
+            "reason" => format!("{:?}", error),
+            "block" => format!("{}", beacon_block_root),
+            "peer_id" => peer_id.to_string(),
+            "type" => format!("{:?}", attestation_type),
+        );
+    }
+}

beacon_node/network/src/lib.rs

@@ -6,6 +6,7 @@ pub mod error;
 pub mod service;
 
 mod attestation_service;
+mod beacon_processor;
 mod metrics;
 mod persisted_dht;
 mod router;

beacon_node/network/src/metrics.rs

@@ -1,35 +1,91 @@
+use beacon_chain::attestation_verification::Error as AttnError;
 pub use lighthouse_metrics::*;
 
+lazy_static! {
+
+    /*
+     * Gossip subnets and scoring
+     */
+    pub static ref PEERS_PER_PROTOCOL: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_peers_per_protocol",
+        "Peers via supported protocol",
+        &["protocol"]
+    );
+
+    pub static ref GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_subscribed_subnets",
+        "Subnets currently subscribed to",
+        &["subnet"]
+    );
+
+    pub static ref GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_peers_per_subnet_topic_count",
+        "Peers subscribed per subnet topic",
+        &["subnet"]
+    );
+
+    pub static ref MESH_PEERS_PER_MAIN_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_mesh_peers_per_main_topic",
+        "Mesh peers per main topic",
+        &["topic_hash"]
+    );
+
+    pub static ref MESH_PEERS_PER_SUBNET_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_mesh_peers_per_subnet_topic",
+        "Mesh peers per subnet topic",
+        &["subnet"]
+    );
+
+    pub static ref AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_avg_peer_score_per_topic",
+        "Average peer's score per topic",
+        &["topic_hash"]
+    );
+
+    pub static ref AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_avg_peer_score_per_subnet_topic",
+        "Average peer's score per subnet topic",
+        &["subnet"]
+    );
+
+    pub static ref ATTESTATIONS_PUBLISHED_PER_SUBNET_PER_SLOT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "gossipsub_attestations_published_per_subnet_per_slot",
+        "Failed attestation publishes per subnet",
+        &["subnet"]
+    );
+}
+
 lazy_static! {
     /*
      * Gossip Rx
      */
     pub static ref GOSSIP_BLOCKS_RX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_blocks_rx_total",
+        "gossipsub_blocks_rx_total",
         "Count of gossip blocks received"
     );
     pub static ref GOSSIP_UNAGGREGATED_ATTESTATIONS_RX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_unaggregated_attestations_rx_total",
+        "gossipsub_unaggregated_attestations_rx_total",
         "Count of gossip unaggregated attestations received"
     );
     pub static ref GOSSIP_AGGREGATED_ATTESTATIONS_RX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_aggregated_attestations_rx_total",
+        "gossipsub_aggregated_attestations_rx_total",
         "Count of gossip aggregated attestations received"
     );
 
+
     /*
      * Gossip Tx
      */
     pub static ref GOSSIP_BLOCKS_TX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_blocks_tx_total",
+        "gossipsub_blocks_tx_total",
         "Count of gossip blocks transmitted"
     );
     pub static ref GOSSIP_UNAGGREGATED_ATTESTATIONS_TX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_unaggregated_attestations_tx_total",
+        "gossipsub_unaggregated_attestations_tx_total",
         "Count of gossip unaggregated attestations transmitted"
     );
     pub static ref GOSSIP_AGGREGATED_ATTESTATIONS_TX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_aggregated_attestations_tx_total",
+        "gossipsub_aggregated_attestations_tx_total",
         "Count of gossip aggregated attestations transmitted"
     );
 
@@ -37,11 +93,309 @@ lazy_static! {
      * Attestation subnet subscriptions
      */
     pub static ref SUBNET_SUBSCRIPTION_REQUESTS: Result<IntCounter> = try_create_int_counter(
-        "network_subnet_subscriptions_total",
+        "gossipsub_subnet_subscriptions_total",
         "Count of validator subscription requests."
     );
     pub static ref SUBNET_SUBSCRIPTION_AGGREGATOR_REQUESTS: Result<IntCounter> = try_create_int_counter(
-        "network_subnet_subscriptions_aggregator_total",
+        "gossipsub_subnet_subscriptions_aggregator_total",
         "Count of validator subscription requests where the subscriber is an aggregator."
     );
+
+    /*
+     * Gossip processor
+     */
+    pub static ref BEACON_PROCESSOR_WORK_EVENTS_RX_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_processor_work_events_rx_count",
+        "Count of work events received (but not necessarily processed)",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORK_EVENTS_IGNORED_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_processor_work_events_ignored_count",
+        "Count of work events purposefully ignored",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORK_EVENTS_STARTED_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_processor_work_events_started_count",
+        "Count of work events which have been started by a worker",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORKER_TIME: Result<HistogramVec> = try_create_histogram_vec(
+        "beacon_processor_worker_time",
+        "Time taken for a worker to fully process some parcel of work.",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORKERS_SPAWNED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_workers_spawned_total",
+        "The number of workers ever spawned by the gossip processing pool."
+    );
+    pub static ref BEACON_PROCESSOR_WORKERS_ACTIVE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_workers_active_total",
+        "Count of active workers in the gossip processing pool."
+    );
+    pub static ref BEACON_PROCESSOR_IDLE_EVENTS_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_idle_events_total",
+        "Count of idle events processed by the gossip processor manager."
+    );
+    pub static ref BEACON_PROCESSOR_EVENT_HANDLING_SECONDS: Result<Histogram> = try_create_histogram(
+        "beacon_processor_event_handling_seconds",
+        "Time spent handling a new message and allocating it to a queue or worker."
+    );
+    // Gossip blocks.
+    pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_gossip_block_queue_total",
+        "Count of blocks from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_gossip_block_verified_total",
+        "Total number of gossip blocks verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_gossip_block_imported_total",
+        "Total number of gossip blocks imported to fork choice, etc."
+    );
+    // Gossip Exits.
+    pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_exit_queue_total",
+        "Count of exits from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_exit_verified_total",
+        "Total number of voluntary exits verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_EXIT_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_exit_imported_total",
+        "Total number of voluntary exits imported to the op pool."
+    );
+    // Gossip proposer slashings.
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_proposer_slashing_queue_total",
+        "Count of proposer slashings from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_proposer_slashing_verified_total",
+        "Total number of proposer slashings verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_proposer_slashing_imported_total",
+        "Total number of proposer slashings imported to the op pool."
+    );
+    // Gossip attester slashings.
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_attester_slashing_queue_total",
+        "Count of attester slashings from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_attester_slashing_verified_total",
+        "Total number of attester slashings verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_attester_slashing_imported_total",
+        "Total number of attester slashings imported to the op pool."
+    );
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_ERROR_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_attester_slashing_error_total",
+        "Total number of attester slashings that raised an error during processing."
+    );
+    // Rpc blocks.
+    pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_rpc_block_queue_total",
+        "Count of blocks from the rpc waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_rpc_block_imported_total",
+        "Total number of gossip blocks imported to fork choice, etc."
+    );
+    // Chain segments.
+    pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_chain_segment_queue_total",
+        "Count of chain segments from the rpc waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_SUCCESS_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_chain_segment_success_total",
+        "Total number of chain segments successfully processed."
+    );
+    pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_FAILED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_chain_segment_failed_total",
+        "Total number of chain segments that failed processing."
+    );
+    // Unaggregated attestations.
+    pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_unaggregated_attestation_queue_total",
+        "Count of unagg. attestations waiting to be processed."
+    );
+    pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_unaggregated_attestation_verified_total",
+        "Total number of unaggregated attestations verified for gossip."
+    );
+    pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_unaggregated_attestation_imported_total",
+        "Total number of unaggregated attestations imported to fork choice, etc."
+    );
+    // Aggregated attestations.
+    pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_aggregated_attestation_queue_total",
+        "Count of agg. attestations waiting to be processed."
+    );
+    pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_aggregated_attestation_verified_total",
+        "Total number of aggregated attestations verified for gossip."
+    );
+    pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_aggregated_attestation_imported_total",
+        "Total number of aggregated attestations imported to fork choice, etc."
+    );
+
+    /*
+     * Attestation Errors
+     */
+    pub static ref GOSSIP_ATTESTATION_ERROR_FUTURE_EPOCH: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_future_epoch",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_PAST_EPOCH: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_past_epoch",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_FUTURE_SLOT: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_future_slot",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_PAST_SLOT: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_past_slot",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_SELECTION_PROOF: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_invalid_selection_proof",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_SIGNATURE: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_invalid_signature",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_EMPTY_AGGREGATION_BITFIELD: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_empty_aggregation_bitfield",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_AGGREGATOR_PUBKEY_UNKNOWN: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_aggregator_pubkey_unknown",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_AGGREGATOR_NOT_IN_COMMITTEE: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_aggregator_not_in_committee",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_ATTESTATION_ALREADY_KNOWN: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_attestation_already_known",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_AGGREGATOR_ALREADY_KNOWN: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_aggregator_already_known",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_PRIOR_ATTESTATION_KNOWN: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_prior_attestation_known",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_VALIDATOR_INDEX_TOO_HIGH: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_validator_index_too_high",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_UNKNOWN_HEAD_BLOCK: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_unknown_head_block",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_UNKNOWN_TARGET_ROOT: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_unknown_target_root",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_BAD_TARGET_EPOCH: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_bad_target_epoch",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_NO_COMMITTEE_FOR_SLOT_AND_INDEX: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_no_committee_for_slot_and_index",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_NOT_EXACTLY_ONE_AGGREGATION_BIT_SET: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_not_exactly_one_aggregation_bit_set",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_ATTESTS_TO_FUTURE_BLOCK: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_attests_to_future_block",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_SUBNET_ID: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_invalid_subnet_id",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_STATE_PROCESSING: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_invalid_state_processing",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_TOO_MANY_SKIPPED_SLOTS: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_invalid_too_many_skipped_slots",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_BEACON_CHAIN_ERROR: Result<IntCounter> = try_create_int_counter(
+        "gossipsub_attestation_error_beacon_chain_error",
+        "Count of a specific error type (see metric name)"
+    );
+}
+
+pub fn register_attestation_error(error: &AttnError) {
+    match error {
+        AttnError::FutureEpoch { .. } => inc_counter(&GOSSIP_ATTESTATION_ERROR_FUTURE_EPOCH),
+        AttnError::PastEpoch { .. } => inc_counter(&GOSSIP_ATTESTATION_ERROR_PAST_EPOCH),
+        AttnError::FutureSlot { .. } => inc_counter(&GOSSIP_ATTESTATION_ERROR_FUTURE_SLOT),
+        AttnError::PastSlot { .. } => inc_counter(&GOSSIP_ATTESTATION_ERROR_PAST_SLOT),
+        AttnError::InvalidSelectionProof { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_SELECTION_PROOF)
+        }
+        AttnError::InvalidSignature => inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_SIGNATURE),
+        AttnError::EmptyAggregationBitfield => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_EMPTY_AGGREGATION_BITFIELD)
+        }
+        AttnError::AggregatorPubkeyUnknown(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_AGGREGATOR_PUBKEY_UNKNOWN)
+        }
+        AttnError::AggregatorNotInCommittee { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_AGGREGATOR_NOT_IN_COMMITTEE)
+        }
+        AttnError::AttestationAlreadyKnown { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_ATTESTATION_ALREADY_KNOWN)
+        }
+        AttnError::AggregatorAlreadyKnown(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_AGGREGATOR_ALREADY_KNOWN)
+        }
+        AttnError::PriorAttestationKnown { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_PRIOR_ATTESTATION_KNOWN)
+        }
+        AttnError::ValidatorIndexTooHigh(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_VALIDATOR_INDEX_TOO_HIGH)
+        }
+        AttnError::UnknownHeadBlock { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_UNKNOWN_HEAD_BLOCK)
+        }
+        AttnError::UnknownTargetRoot(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_UNKNOWN_TARGET_ROOT)
+        }
+        AttnError::BadTargetEpoch => inc_counter(&GOSSIP_ATTESTATION_ERROR_BAD_TARGET_EPOCH),
+        AttnError::NoCommitteeForSlotAndIndex { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_NO_COMMITTEE_FOR_SLOT_AND_INDEX)
+        }
+        AttnError::NotExactlyOneAggregationBitSet(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_NOT_EXACTLY_ONE_AGGREGATION_BIT_SET)
+        }
+        AttnError::AttestsToFutureBlock { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_ATTESTS_TO_FUTURE_BLOCK)
+        }
+        AttnError::InvalidSubnetId { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_SUBNET_ID)
+        }
+        AttnError::Invalid(_) => inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_STATE_PROCESSING),
+        AttnError::TooManySkippedSlots { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_TOO_MANY_SKIPPED_SLOTS)
+        }
+        AttnError::BeaconChainError(_) => inc_counter(&GOSSIP_ATTESTATION_ERROR_BEACON_CHAIN_ERROR),
+    }
 }

beacon_node/network/src/router/mod.rs

@@ -9,14 +9,14 @@ pub mod processor;
 
 use crate::error;
 use crate::service::NetworkMessage;
-use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError};
+use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::{
     rpc::{RPCError, RequestId},
     MessageId, NetworkGlobals, PeerId, PeerRequestId, PubsubMessage, Request, Response,
 };
 use futures::prelude::*;
 use processor::Processor;
-use slog::{debug, info, o, trace, warn};
+use slog::{debug, o, trace};
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use types::EthSpec;
@@ -26,8 +26,6 @@ use types::EthSpec;
 /// passing them to the internal message processor. The message processor spawns a syncing thread
 /// which manages which blocks need to be requested and processed.
 pub struct Router<T: BeaconChainTypes> {
-    /// A channel to the network service to allow for gossip propagation.
-    network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
     /// Access to the peer db for logging.
     network_globals: Arc<NetworkGlobals<T::EthSpec>>,
     /// Processes validated and decoded messages from the network. Has direct access to the
@@ -89,13 +87,12 @@ impl<T: BeaconChainTypes> Router<T> {
             executor.clone(),
             beacon_chain,
             network_globals.clone(),
-            network_send.clone(),
+            network_send,
             &log,
         );
 
         // generate the Message handler
         let mut handler = Router {
-            network_send,
             network_globals,
             processor,
             log: message_handler_log,
@@ -215,56 +212,24 @@ impl<T: BeaconChainTypes> Router<T> {
         match gossip_message {
             // Attestations should never reach the router.
             PubsubMessage::AggregateAndProofAttestation(aggregate_and_proof) => {
-                if let Some(gossip_verified) = self
-                    .processor
-                    .verify_aggregated_attestation_for_gossip(peer_id.clone(), *aggregate_and_proof)
-                {
-                    self.propagate_message(id, peer_id.clone());
-                    self.processor
-                        .import_aggregated_attestation(peer_id, gossip_verified);
-                }
+                self.processor
+                    .on_aggregated_attestation_gossip(id, peer_id, *aggregate_and_proof);
             }
             PubsubMessage::Attestation(subnet_attestation) => {
-                if let Some(gossip_verified) =
-                    self.processor.verify_unaggregated_attestation_for_gossip(
-                        peer_id.clone(),
-                        subnet_attestation.1.clone(),
-                        subnet_attestation.0,
-                    )
-                {
-                    self.propagate_message(id, peer_id.clone());
-                    if should_process {
-                        self.processor
-                            .import_unaggregated_attestation(peer_id, gossip_verified);
-                    }
-                }
+                self.processor.on_unaggregated_attestation_gossip(
+                    id,
+                    peer_id,
+                    subnet_attestation.1.clone(),
+                    subnet_attestation.0,
+                    should_process,
+                );
             }
             PubsubMessage::BeaconBlock(block) => {
-                match self.processor.should_forward_block(block) {
-                    Ok(verified_block) => {
-                        info!(self.log, "New block received"; "slot" => verified_block.block.slot(), "hash" => verified_block.block_root.to_string());
-                        self.propagate_message(id, peer_id.clone());
-                        self.processor.on_block_gossip(peer_id, verified_block);
-                    }
-                    Err(BlockError::ParentUnknown(block)) => {
-                        self.processor.on_unknown_parent(peer_id, block);
-                    }
-                    Err(e) => {
-                        // performing a parent lookup
-                        warn!(self.log, "Could not verify block for gossip";
-                            "error" => format!("{:?}", e));
-                    }
-                }
+                self.processor.on_block_gossip(id, peer_id, block);
             }
             PubsubMessage::VoluntaryExit(exit) => {
                 debug!(self.log, "Received a voluntary exit"; "peer_id" => format!("{}", peer_id));
-                if let Some(verified_exit) = self
-                    .processor
-                    .verify_voluntary_exit_for_gossip(&peer_id, *exit)
-                {
-                    self.propagate_message(id, peer_id);
-                    self.processor.import_verified_voluntary_exit(verified_exit);
-                }
+                self.processor.on_voluntary_exit_gossip(id, peer_id, exit);
             }
             PubsubMessage::ProposerSlashing(proposer_slashing) => {
                 debug!(
@@ -272,14 +237,8 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received a proposer slashing";
                     "peer_id" => format!("{}", peer_id)
                 );
-                if let Some(verified_proposer_slashing) = self
-                    .processor
-                    .verify_proposer_slashing_for_gossip(&peer_id, *proposer_slashing)
-                {
-                    self.propagate_message(id, peer_id);
-                    self.processor
-                        .import_verified_proposer_slashing(verified_proposer_slashing);
-                }
+                self.processor
+                    .on_proposer_slashing_gossip(id, peer_id, proposer_slashing);
             }
             PubsubMessage::AttesterSlashing(attester_slashing) => {
                 debug!(
@@ -287,30 +246,9 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received a attester slashing";
                     "peer_id" => format!("{}", peer_id)
                 );
-                if let Some(verified_attester_slashing) = self
-                    .processor
-                    .verify_attester_slashing_for_gossip(&peer_id, *attester_slashing)
-                {
-                    self.propagate_message(id, peer_id);
-                    self.processor
-                        .import_verified_attester_slashing(verified_attester_slashing);
-                }
+                self.processor
+                    .on_attester_slashing_gossip(id, peer_id, attester_slashing);
             }
         }
     }
-
-    /// Informs the network service that the message should be forwarded to other peers (is valid).
-    fn propagate_message(&mut self, message_id: MessageId, propagation_source: PeerId) {
-        self.network_send
-            .send(NetworkMessage::Validate {
-                propagation_source,
-                message_id,
-            })
-            .unwrap_or_else(|_| {
-                warn!(
-                    self.log,
-                    "Could not send propagation request to the network service"
-                )
-            });
-    }
 }

beacon_node/network/src/router/processor.rs

@@ -1,20 +1,16 @@
+use crate::beacon_processor::{
+    BeaconProcessor, WorkEvent as BeaconWorkEvent, MAX_WORK_EVENT_QUEUE_LEN,
+};
 use crate::service::NetworkMessage;
 use crate::sync::{PeerSyncInfo, SyncMessage};
-use beacon_chain::{
-    attestation_verification::{
-        Error as AttnError, SignatureVerifiedAttestation, VerifiedAggregatedAttestation,
-        VerifiedUnaggregatedAttestation,
-    },
-    observed_operations::ObservationOutcome,
-    BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError,
-    GossipVerifiedBlock,
-};
+use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::rpc::*;
-use eth2_libp2p::{NetworkGlobals, PeerAction, PeerId, PeerRequestId, Request, Response};
+use eth2_libp2p::{
+    MessageId, NetworkGlobals, PeerAction, PeerId, PeerRequestId, Request, Response,
+};
 use itertools::process_results;
 use slog::{debug, error, o, trace, warn};
-use ssz::Encode;
-use state_processing::SigVerifiedOp;
+use std::cmp;
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use types::{
@@ -22,8 +18,6 @@ use types::{
     SignedAggregateAndProof, SignedBeaconBlock, SignedVoluntaryExit, Slot, SubnetId,
 };
 
-//TODO: Rate limit requests
-
 /// If a block is more than `FUTURE_SLOT_TOLERANCE` slots ahead of our slot clock, we drop it.
 /// Otherwise we queue it.
 pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1;
@@ -37,6 +31,8 @@ pub struct Processor<T: BeaconChainTypes> {
     sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
     /// A network context to return and handle RPC requests.
     network: HandlerNetworkContext<T::EthSpec>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     /// The `RPCHandler` logger.
     log: slog::Logger,
 }
@@ -51,20 +47,36 @@ impl<T: BeaconChainTypes> Processor<T> {
         log: &slog::Logger,
     ) -> Self {
         let sync_logger = log.new(o!("service"=> "sync"));
+        let (beacon_processor_send, beacon_processor_receive) =
+            mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN);
 
         // spawn the sync thread
         let sync_send = crate::sync::manager::spawn(
-            executor,
+            executor.clone(),
             beacon_chain.clone(),
-            network_globals,
+            network_globals.clone(),
             network_send.clone(),
+            beacon_processor_send.clone(),
             sync_logger,
         );
 
+        BeaconProcessor {
+            beacon_chain: Arc::downgrade(&beacon_chain),
+            network_tx: network_send.clone(),
+            sync_tx: sync_send.clone(),
+            network_globals,
+            executor,
+            max_workers: cmp::max(1, num_cpus::get()),
+            current_workers: 0,
+            log: log.clone(),
+        }
+        .spawn_manager(beacon_processor_receive);
+
         Processor {
             chain: beacon_chain,
             sync_send,
             network: HandlerNetworkContext::new(network_send, log.clone()),
+            beacon_processor_send,
             log: log.clone(),
         }
     }
@@ -499,23 +511,6 @@ impl<T: BeaconChainTypes> Processor<T> {
         }
     }
 
-    /// Template function to be called on a block to determine if the block should be propagated
-    /// across the network.
-    pub fn should_forward_block(
-        &mut self,
-        block: Box<SignedBeaconBlock<T::EthSpec>>,
-    ) -> Result<GossipVerifiedBlock<T>, BlockError<T::EthSpec>> {
-        self.chain.verify_block_for_gossip(*block)
-    }
-
-    pub fn on_unknown_parent(
-        &mut self,
-        peer_id: PeerId,
-        block: Box<SignedBeaconBlock<T::EthSpec>>,
-    ) {
-        self.send_to_sync(SyncMessage::UnknownBlock(peer_id, block));
-    }
-
     /// Process a gossip message declaring a new block.
     ///
     /// Attempts to apply to block to the beacon chain. May queue the block for later processing.
@@ -523,544 +518,134 @@ impl<T: BeaconChainTypes> Processor<T> {
     /// Returns a `bool` which, if `true`, indicates we should forward the block to our peers.
     pub fn on_block_gossip(
         &mut self,
+        message_id: MessageId,
         peer_id: PeerId,
-        verified_block: GossipVerifiedBlock<T>,
-    ) -> bool {
-        let block = Box::new(verified_block.block.clone());
-        match self.chain.process_block(verified_block) {
-            Ok(_block_root) => {
-                trace!(
-                    self.log,
-                    "Gossipsub block processed";
-                    "peer_id" => peer_id.to_string()
-                );
-
-                // TODO: It would be better if we can run this _after_ we publish the block to
-                // reduce block propagation latency.
-                //
-                // The `MessageHandler` would be the place to put this, however it doesn't seem
-                // to have a reference to the `BeaconChain`. I will leave this for future
-                // works.
-                match self.chain.fork_choice() {
-                    Ok(()) => trace!(
-                        self.log,
-                        "Fork choice success";
-                        "location" => "block gossip"
-                    ),
-                    Err(e) => error!(
-                        self.log,
-                        "Fork choice failed";
-                        "error" => format!("{:?}", e),
-                        "location" => "block gossip"
-                    ),
-                }
-            }
-            Err(BlockError::ParentUnknown { .. }) => {
-                // Inform the sync manager to find parents for this block
-                // This should not occur. It should be checked by `should_forward_block`
-                error!(
-                    self.log,
-                    "Block with unknown parent attempted to be processed";
-                    "peer_id" => peer_id.to_string()
-                );
-                self.send_to_sync(SyncMessage::UnknownBlock(peer_id, block));
-            }
-            other => {
-                warn!(
-                    self.log,
-                    "Invalid gossip beacon block";
-                    "outcome" => format!("{:?}", other),
-                    "block root" => format!("{}", block.canonical_root()),
-                    "block slot" => block.slot()
-                );
-                trace!(
-                    self.log,
-                    "Invalid gossip beacon block ssz";
-                    "ssz" => format!("0x{}", hex::encode(block.as_ssz_bytes())),
-                );
-            }
-        }
-        // TODO: Update with correct block gossip checking
-        true
-    }
-
-    /// Handle an error whilst verifying an `Attestation` or `SignedAggregateAndProof` from the
-    /// network.
-    pub fn handle_attestation_verification_failure(
-        &mut self,
-        peer_id: PeerId,
-        beacon_block_root: Hash256,
-        attestation_type: &str,
-        error: AttnError,
+        block: Box<SignedBeaconBlock<T::EthSpec>>,
     ) {
-        debug!(
-            self.log,
-            "Invalid attestation from network";
-            "reason" => format!("{:?}", error),
-            "block" => format!("{}", beacon_block_root),
-            "peer_id" => peer_id.to_string(),
-            "type" => format!("{:?}", attestation_type),
-        );
-
-        match error {
-            AttnError::FutureEpoch { .. }
-            | AttnError::PastEpoch { .. }
-            | AttnError::FutureSlot { .. }
-            | AttnError::PastSlot { .. } => {
-                /*
-                 * These errors can be triggered by a mismatch between our slot and the peer.
-                 *
-                 *
-                 * The peer has published an invalid consensus message, _only_ if we trust our own clock.
-                 */
-            }
-            AttnError::InvalidSelectionProof { .. } | AttnError::InvalidSignature => {
-                /*
-                 * These errors are caused by invalid signatures.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::EmptyAggregationBitfield => {
-                /*
-                 * The aggregate had no signatures and is therefore worthless.
-                 *
-                 * Whilst we don't gossip this attestation, this act is **not** a clear
-                 * violation of the spec nor indication of fault.
-                 *
-                 * This may change soon. Reference:
-                 *
-                 * https://github.com/ethereum/eth2.0-specs/pull/1732
-                 */
-            }
-            AttnError::AggregatorPubkeyUnknown(_) => {
-                /*
-                 * The aggregator index was higher than any known validator index. This is
-                 * possible in two cases:
-                 *
-                 * 1. The attestation is malformed
-                 * 2. The attestation attests to a beacon_block_root that we do not know.
-                 *
-                 * It should be impossible to reach (2) without triggering
-                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
-                 * faulty.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::AggregatorNotInCommittee { .. } => {
-                /*
-                 * The aggregator index was higher than any known validator index. This is
-                 * possible in two cases:
-                 *
-                 * 1. The attestation is malformed
-                 * 2. The attestation attests to a beacon_block_root that we do not know.
-                 *
-                 * It should be impossible to reach (2) without triggering
-                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
-                 * faulty.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::AttestationAlreadyKnown { .. } => {
-                /*
-                 * The aggregate attestation has already been observed on the network or in
-                 * a block.
-                 *
-                 * The peer is not necessarily faulty.
-                 */
-            }
-            AttnError::AggregatorAlreadyKnown(_) => {
-                /*
-                 * There has already been an aggregate attestation seen from this
-                 * aggregator index.
-                 *
-                 * The peer is not necessarily faulty.
-                 */
-            }
-            AttnError::PriorAttestationKnown { .. } => {
-                /*
-                 * We have already seen an attestation from this validator for this epoch.
-                 *
-                 * The peer is not necessarily faulty.
-                 */
-            }
-            AttnError::ValidatorIndexTooHigh(_) => {
-                /*
-                 * The aggregator index (or similar field) was higher than the maximum
-                 * possible number of validators.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::UnknownHeadBlock { beacon_block_root } => {
-                // Note: its a little bit unclear as to whether or not this block is unknown or
-                // just old. See:
-                //
-                // https://github.com/sigp/lighthouse/issues/1039
-
-                // TODO: Maintain this attestation and re-process once sync completes
-                debug!(
-                    self.log,
-                    "Attestation for unknown block";
-                    "peer_id" => peer_id.to_string(),
-                    "block" => format!("{}", beacon_block_root)
-                );
-                // we don't know the block, get the sync manager to handle the block lookup
-                self.send_to_sync(SyncMessage::UnknownBlockHash(peer_id, beacon_block_root));
-            }
-            AttnError::UnknownTargetRoot(_) => {
-                /*
-                 * The block indicated by the target root is not known to us.
-                 *
-                 * We should always get `AttnError::UnknwonHeadBlock` before we get this
-                 * error, so this means we can get this error if:
-                 *
-                 * 1. The target root does not represent a valid block.
-                 * 2. We do not have the target root in our DB.
-                 *
-                 * For (2), we should only be processing attestations when we should have
-                 * all the available information. Note: if we do a weak-subjectivity sync
-                 * it's possible that this situation could occur, but I think it's
-                 * unlikely. For now, we will declare this to be an invalid message>
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::BadTargetEpoch => {
-                /*
-                 * The aggregator index (or similar field) was higher than the maximum
-                 * possible number of validators.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::NoCommitteeForSlotAndIndex { .. } => {
-                /*
-                 * It is not possible to attest this the given committee in the given slot.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::NotExactlyOneAggregationBitSet(_) => {
-                /*
-                 * The unaggregated attestation doesn't have only one signature.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::AttestsToFutureBlock { .. } => {
-                /*
-                 * The beacon_block_root is from a higher slot than the attestation.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-
-            AttnError::InvalidSubnetId { received, expected } => {
-                /*
-                 * The attestation was received on an incorrect subnet id.
-                 */
-                debug!(
-                    self.log,
-                    "Received attestation on incorrect subnet";
-                    "expected" => format!("{:?}", expected),
-                    "received" => format!("{:?}", received),
-                )
-            }
-            AttnError::Invalid(_) => {
-                /*
-                 * The attestation failed the state_processing verification.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::BeaconChainError(e) => {
-                /*
-                 * Lighthouse hit an unexpected error whilst processing the attestation. It
-                 * should be impossible to trigger a `BeaconChainError` from the network,
-                 * so we have a bug.
-                 *
-                 * It's not clear if the message is invalid/malicious.
-                 */
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_beacon_block(
+                message_id, peer_id, block,
+            ))
+            .unwrap_or_else(|e| {
                 error!(
-                    self.log,
-                    "Unable to validate aggregate";
-                    "peer_id" => peer_id.to_string(),
-                    "error" => format!("{:?}", e),
-                );
-            }
-        }
-    }
-
-    pub fn verify_aggregated_attestation_for_gossip(
-        &mut self,
-        peer_id: PeerId,
-        aggregate_and_proof: SignedAggregateAndProof<T::EthSpec>,
-    ) -> Option<VerifiedAggregatedAttestation<T>> {
-        // This is provided to the error handling function to assist with debugging.
-        let beacon_block_root = aggregate_and_proof.message.aggregate.data.beacon_block_root;
-
-        self.chain
-            .verify_aggregated_attestation_for_gossip(aggregate_and_proof)
-            .map_err(|e| {
-                self.handle_attestation_verification_failure(
-                    peer_id,
-                    beacon_block_root,
-                    "aggregated",
-                    e,
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "block gossip",
+                    "error" => e.to_string(),
                 )
             })
-            .ok()
     }
 
-    pub fn import_aggregated_attestation(
-        &mut self,
-        peer_id: PeerId,
-        verified_attestation: VerifiedAggregatedAttestation<T>,
-    ) {
-        // This is provided to the error handling function to assist with debugging.
-        let beacon_block_root = verified_attestation.attestation().data.beacon_block_root;
-
-        self.apply_attestation_to_fork_choice(
-            peer_id.clone(),
-            beacon_block_root,
-            &verified_attestation,
-        );
-
-        if let Err(e) = self.chain.add_to_block_inclusion_pool(verified_attestation) {
-            debug!(
-                self.log,
-                "Attestation invalid for op pool";
-                "reason" => format!("{:?}", e),
-                "peer" => peer_id.to_string(),
-                "beacon_block_root" => format!("{:?}", beacon_block_root)
-            )
-        }
-    }
-
-    pub fn verify_unaggregated_attestation_for_gossip(
+    pub fn on_unaggregated_attestation_gossip(
         &mut self,
+        message_id: MessageId,
         peer_id: PeerId,
         unaggregated_attestation: Attestation<T::EthSpec>,
         subnet_id: SubnetId,
-    ) -> Option<VerifiedUnaggregatedAttestation<T>> {
-        // This is provided to the error handling function to assist with debugging.
-        let beacon_block_root = unaggregated_attestation.data.beacon_block_root;
-
-        self.chain
-            .verify_unaggregated_attestation_for_gossip(unaggregated_attestation, subnet_id)
-            .map_err(|e| {
-                self.handle_attestation_verification_failure(
-                    peer_id,
-                    beacon_block_root,
-                    "unaggregated",
-                    e,
+        should_process: bool,
+    ) {
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::unaggregated_attestation(
+                message_id,
+                peer_id,
+                unaggregated_attestation,
+                subnet_id,
+                should_process,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "unaggregated attestation gossip",
+                    "error" => e.to_string(),
                 )
             })
-            .ok()
     }
 
-    pub fn import_unaggregated_attestation(
+    pub fn on_aggregated_attestation_gossip(
         &mut self,
+        message_id: MessageId,
         peer_id: PeerId,
-        verified_attestation: VerifiedUnaggregatedAttestation<T>,
+        aggregate: SignedAggregateAndProof<T::EthSpec>,
     ) {
-        // This is provided to the error handling function to assist with debugging.
-        let beacon_block_root = verified_attestation.attestation().data.beacon_block_root;
-
-        self.apply_attestation_to_fork_choice(
-            peer_id.clone(),
-            beacon_block_root,
-            &verified_attestation,
-        );
-
-        if let Err(e) = self
-            .chain
-            .add_to_naive_aggregation_pool(verified_attestation)
-        {
-            debug!(
-                self.log,
-                "Attestation invalid for agg pool";
-                "reason" => format!("{:?}", e),
-                "peer" => peer_id.to_string(),
-                "beacon_block_root" => format!("{:?}", beacon_block_root)
-            )
-        }
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::aggregated_attestation(
+                message_id, peer_id, aggregate,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "aggregated attestation gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 
-    /// Apply the attestation to fork choice, suppressing errors.
-    ///
-    /// We suppress the errors when adding an attestation to fork choice since the spec
-    /// permits gossiping attestations that are invalid to be applied to fork choice.
-    ///
-    /// An attestation that is invalid for fork choice can still be included in a block.
-    ///
-    /// Reference:
-    /// https://github.com/ethereum/eth2.0-specs/issues/1408#issuecomment-617599260
-    fn apply_attestation_to_fork_choice<'a>(
-        &self,
+    pub fn on_voluntary_exit_gossip(
+        &mut self,
+        message_id: MessageId,
         peer_id: PeerId,
-        beacon_block_root: Hash256,
-        attestation: &'a impl SignatureVerifiedAttestation<T>,
+        voluntary_exit: Box<SignedVoluntaryExit>,
     ) {
-        if let Err(e) = self.chain.apply_attestation_to_fork_choice(attestation) {
-            match e {
-                BeaconChainError::ForkChoiceError(ForkChoiceError::InvalidAttestation(e)) => {
-                    debug!(
-                        self.log,
-                        "Attestation invalid for fork choice";
-                        "reason" => format!("{:?}", e),
-                        "peer" => peer_id.to_string(),
-                        "beacon_block_root" => format!("{:?}", beacon_block_root)
-                    )
-                }
-                e => error!(
-                    self.log,
-                    "Error applying attestation to fork choice";
-                    "reason" => format!("{:?}", e),
-                    "peer" => peer_id.to_string(),
-                    "beacon_block_root" => format!("{:?}", beacon_block_root)
-                ),
-            }
-        }
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_voluntary_exit(
+                message_id,
+                peer_id,
+                voluntary_exit,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "voluntary exit gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 
-    /// Verify a voluntary exit before gossiping or processing it.
-    ///
-    /// Errors are logged at debug level.
-    pub fn verify_voluntary_exit_for_gossip(
-        &self,
-        peer_id: &PeerId,
-        voluntary_exit: SignedVoluntaryExit,
-    ) -> Option<SigVerifiedOp<SignedVoluntaryExit>> {
-        let validator_index = voluntary_exit.message.validator_index;
-
-        match self.chain.verify_voluntary_exit_for_gossip(voluntary_exit) {
-            Ok(ObservationOutcome::New(sig_verified_exit)) => Some(sig_verified_exit),
-            Ok(ObservationOutcome::AlreadyKnown) => {
-                debug!(
-                    self.log,
-                    "Dropping exit for already exiting validator";
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string()
-                );
-                None
-            }
-            Err(e) => {
-                debug!(
-                    self.log,
-                    "Dropping invalid exit";
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string(),
-                    "error" => format!("{:?}", e)
-                );
-                None
-            }
-        }
-    }
-
-    /// Import a verified exit into the op pool.
-    pub fn import_verified_voluntary_exit(
-        &self,
-        verified_voluntary_exit: SigVerifiedOp<SignedVoluntaryExit>,
+    pub fn on_proposer_slashing_gossip(
+        &mut self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
     ) {
-        self.chain.import_voluntary_exit(verified_voluntary_exit);
-        debug!(self.log, "Successfully imported voluntary exit");
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_proposer_slashing(
+                message_id,
+                peer_id,
+                proposer_slashing,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "proposer slashing gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 
-    /// Verify a proposer slashing before gossiping or processing it.
-    ///
-    /// Errors are logged at debug level.
-    pub fn verify_proposer_slashing_for_gossip(
-        &self,
-        peer_id: &PeerId,
-        proposer_slashing: ProposerSlashing,
-    ) -> Option<SigVerifiedOp<ProposerSlashing>> {
-        let validator_index = proposer_slashing.signed_header_1.message.proposer_index;
-
-        match self
-            .chain
-            .verify_proposer_slashing_for_gossip(proposer_slashing)
-        {
-            Ok(ObservationOutcome::New(verified_slashing)) => Some(verified_slashing),
-            Ok(ObservationOutcome::AlreadyKnown) => {
-                debug!(
-                    self.log,
-                    "Dropping proposer slashing";
-                    "reason" => "Already seen a proposer slashing for that validator",
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string()
-                );
-                None
-            }
-            Err(e) => {
-                debug!(
-                    self.log,
-                    "Dropping invalid proposer slashing";
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string(),
-                    "error" => format!("{:?}", e)
-                );
-                None
-            }
-        }
-    }
-
-    /// Import a verified proposer slashing into the op pool.
-    pub fn import_verified_proposer_slashing(
-        &self,
-        proposer_slashing: SigVerifiedOp<ProposerSlashing>,
+    pub fn on_attester_slashing_gossip(
+        &mut self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<T::EthSpec>>,
     ) {
-        self.chain.import_proposer_slashing(proposer_slashing);
-        debug!(self.log, "Successfully imported proposer slashing");
-    }
-
-    /// Verify an attester slashing before gossiping or processing it.
-    ///
-    /// Errors are logged at debug level.
-    pub fn verify_attester_slashing_for_gossip(
-        &self,
-        peer_id: &PeerId,
-        attester_slashing: AttesterSlashing<T::EthSpec>,
-    ) -> Option<SigVerifiedOp<AttesterSlashing<T::EthSpec>>> {
-        match self
-            .chain
-            .verify_attester_slashing_for_gossip(attester_slashing)
-        {
-            Ok(ObservationOutcome::New(verified_slashing)) => Some(verified_slashing),
-            Ok(ObservationOutcome::AlreadyKnown) => {
-                debug!(
-                    self.log,
-                    "Dropping attester slashing";
-                    "reason" => "Slashings already known for all slashed validators",
-                    "peer" => peer_id.to_string()
-                );
-                None
-            }
-            Err(e) => {
-                debug!(
-                    self.log,
-                    "Dropping invalid attester slashing";
-                    "peer" => peer_id.to_string(),
-                    "error" => format!("{:?}", e)
-                );
-                None
-            }
-        }
-    }
-
-    /// Import a verified attester slashing into the op pool.
-    pub fn import_verified_attester_slashing(
-        &self,
-        attester_slashing: SigVerifiedOp<AttesterSlashing<T::EthSpec>>,
-    ) {
-        if let Err(e) = self.chain.import_attester_slashing(attester_slashing) {
-            debug!(self.log, "Error importing attester slashing"; "error" => format!("{:?}", e));
-        } else {
-            debug!(self.log, "Successfully imported attester slashing");
-        }
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_attester_slashing(
+                message_id,
+                peer_id,
+                attester_slashing,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "attester slashing gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 }
 

beacon_node/network/src/service.rs

@@ -6,17 +6,18 @@ use crate::{
 };
 use crate::{error, metrics};
 use beacon_chain::{BeaconChain, BeaconChainTypes};
-use eth2_libp2p::Service as LibP2PService;
 use eth2_libp2p::{
     rpc::{GoodbyeReason, RPCResponseErrorCode, RequestId},
     Libp2pEvent, PeerAction, PeerRequestId, PubsubMessage, Request, Response,
 };
-use eth2_libp2p::{BehaviourEvent, MessageId, NetworkGlobals, PeerId};
+use eth2_libp2p::{
+    types::GossipKind, BehaviourEvent, GossipTopic, MessageId, NetworkGlobals, PeerId, TopicHash,
+};
+use eth2_libp2p::{MessageAcceptance, Service as LibP2PService};
 use futures::prelude::*;
 use rest_types::ValidatorSubscription;
 use slog::{debug, error, info, o, trace, warn};
-use std::sync::Arc;
-use std::time::Duration;
+use std::{collections::HashMap, sync::Arc, time::Duration};
 use store::HotColdDB;
 use tokio::sync::mpsc;
 use tokio::time::Delay;
@@ -24,6 +25,9 @@ use types::EthSpec;
 
 mod tests;
 
+/// The interval (in seconds) that various network metrics will update.
+const METRIC_UPDATE_INTERVAL: u64 = 1;
+
 /// Types of messages that the network service can receive.
 #[derive(Debug)]
 pub enum NetworkMessage<T: EthSpec> {
@@ -55,11 +59,13 @@ pub enum NetworkMessage<T: EthSpec> {
     /// Publish a list of messages to the gossipsub protocol.
     Publish { messages: Vec<PubsubMessage<T>> },
     /// Validates a received gossipsub message. This will propagate the message on the network.
-    Validate {
+    ValidationResult {
         /// The peer that sent us the message. We don't send back to this peer.
         propagation_source: PeerId,
         /// The id of the message we are validating and propagating.
         message_id: MessageId,
+        /// The result of the validation
+        validation_result: MessageAcceptance,
     },
     /// Reports a peer to the peer manager for performing an action.
     ReportPeer { peer_id: PeerId, action: PeerAction },
@@ -89,13 +95,15 @@ pub struct NetworkService<T: BeaconChainTypes> {
     network_globals: Arc<NetworkGlobals<T::EthSpec>>,
     /// A delay that expires when a new fork takes place.
     next_fork_update: Option<Delay>,
+    /// A timer for updating various network metrics.
+    metrics_update: tokio::time::Interval,
     /// The logger for the network service.
     log: slog::Logger,
 }
 
 impl<T: BeaconChainTypes> NetworkService<T> {
     #[allow(clippy::type_complexity)]
-    pub fn start(
+    pub async fn start(
         beacon_chain: Arc<BeaconChain<T>>,
         config: &NetworkConfig,
         executor: environment::TaskExecutor,
@@ -117,7 +125,7 @@ impl<T: BeaconChainTypes> NetworkService<T> {
 
         // launch libp2p service
         let (network_globals, mut libp2p) =
-            LibP2PService::new(executor.clone(), config, enr_fork_id, &network_log)?;
+            LibP2PService::new(executor.clone(), config, enr_fork_id, &network_log).await?;
 
         // Repopulate the DHT with stored ENR's.
         let enrs_to_load = load_dht::<T::EthSpec, T::HotStore, T::ColdStore>(store.clone());
@@ -126,7 +134,7 @@ impl<T: BeaconChainTypes> NetworkService<T> {
             "Loading peers into the routing table"; "peers" => enrs_to_load.len()
         );
         for enr in enrs_to_load {
-            libp2p.swarm.add_enr(enr.clone());
+            libp2p.swarm.add_enr(enr.clone()); //TODO change?
         }
 
         // launch derived network services
@@ -144,8 +152,11 @@ impl<T: BeaconChainTypes> NetworkService<T> {
         let attestation_service =
             AttestationService::new(beacon_chain.clone(), network_globals.clone(), &network_log);
 
+        // create a timer for updating network metrics
+        let metrics_update = tokio::time::interval(Duration::from_secs(METRIC_UPDATE_INTERVAL));
+
         // create the network service and spawn the task
-        let network_log = network_log.new(o!("service"=> "network"));
+        let network_log = network_log.new(o!("service" => "network"));
         let network_service = NetworkService {
             beacon_chain,
             libp2p,
@@ -155,6 +166,7 @@ impl<T: BeaconChainTypes> NetworkService<T> {
             store,
             network_globals: network_globals.clone(),
             next_fork_update,
+            metrics_update,
             log: network_log,
         };
 
@@ -169,12 +181,12 @@ fn spawn_service<T: BeaconChainTypes>(
     mut service: NetworkService<T>,
 ) -> error::Result<()> {
     let mut exit_rx = executor.exit();
+    let mut shutdown_sender = executor.shutdown_sender();
 
     // spawn on the current executor
     executor.spawn_without_exit(async move {
-        // TODO: there is something with this code that prevents cargo fmt from doing anything at
-        // all. Ok, it is worse, the compiler doesn't show errors over this code beyond ast
-        // checking
+
+        let mut metric_update_counter = 0;
         loop {
             // build the futures to check simultaneously
             tokio::select! {
@@ -203,6 +215,17 @@ fn spawn_service<T: BeaconChainTypes>(
                     info!(service.log, "Network service shutdown");
                     return;
                 }
+                _ = service.metrics_update.next() => {
+                    // update various network metrics
+                    metric_update_counter +=1;
+                    if metric_update_counter* 1000 % T::EthSpec::default_spec().milliseconds_per_slot == 0 {
+                        // if a slot has occurred, reset the metrics
+                        let _ = metrics::ATTESTATIONS_PUBLISHED_PER_SUBNET_PER_SLOT
+                            .as_ref()
+                            .map(|gauge| gauge.reset());
+                    }
+                    update_gossip_metrics::<T::EthSpec>(&service.libp2p.swarm.gs());
+                }
                 // handle a message sent to the network
                 Some(message) = service.network_recv.recv() => {
                     match message {
@@ -215,9 +238,10 @@ fn spawn_service<T: BeaconChainTypes>(
                         NetworkMessage::SendError{ peer_id, error, id, reason } => {
                             service.libp2p.respond_with_error(peer_id, id, error, reason);
                         }
-                        NetworkMessage::Validate {
+                        NetworkMessage::ValidationResult {
                             propagation_source,
                             message_id,
+                            validation_result,
                         } => {
                                 trace!(service.log, "Propagating gossipsub message";
                                     "propagation_peer" => format!("{:?}", propagation_source),
@@ -226,7 +250,9 @@ fn spawn_service<T: BeaconChainTypes>(
                                 service
                                     .libp2p
                                     .swarm
-                                    .validate_message(&propagation_source, message_id);
+                                    .report_message_validation_result(
+                                        &propagation_source, message_id, validation_result
+                                    );
                         }
                         NetworkMessage::Publish { messages } => {
                                 let mut topic_kinds = Vec::new();
@@ -271,8 +297,8 @@ fn spawn_service<T: BeaconChainTypes>(
                         AttServiceMessage::EnrRemove(subnet_id) => {
                             service.libp2p.swarm.update_enr_subnet(subnet_id, false);
                         }
-                        AttServiceMessage::DiscoverPeers{subnet_id, min_ttl} => {
-                            service.libp2p.swarm.discover_subnet_peers(subnet_id, min_ttl);
+                        AttServiceMessage::DiscoverPeers(subnets_to_discover) => {
+                            service.libp2p.swarm.discover_subnet_peers(subnets_to_discover);
                         }
                     }
                 }
@@ -376,6 +402,12 @@ fn spawn_service<T: BeaconChainTypes>(
                         Libp2pEvent::NewListenAddr(multiaddr) => {
                             service.network_globals.listen_multiaddrs.write().push(multiaddr);
                         }
+                        Libp2pEvent::ZeroListeners => {
+                            let _ = shutdown_sender.send("All listeners are closed. Unable to listen").await.map_err(|e| {
+                                warn!(service.log, "failed to send a shutdown signal"; "error" => e.to_string()
+                                )
+                            });
+                        }
                     }
                 }
             }
@@ -412,7 +444,11 @@ fn expose_publish_metrics<T: EthSpec>(messages: &[PubsubMessage<T>]) {
     for message in messages {
         match message {
             PubsubMessage::BeaconBlock(_) => metrics::inc_counter(&metrics::GOSSIP_BLOCKS_TX),
-            PubsubMessage::Attestation(_) => {
+            PubsubMessage::Attestation(subnet_id) => {
+                metrics::inc_counter_vec(
+                    &metrics::ATTESTATIONS_PUBLISHED_PER_SUBNET_PER_SLOT,
+                    &[&subnet_id.0.to_string()],
+                );
                 metrics::inc_counter(&metrics::GOSSIP_UNAGGREGATED_ATTESTATIONS_TX)
             }
             PubsubMessage::AggregateAndProofAttestation(_) => {
@@ -436,3 +472,163 @@ fn expose_receive_metrics<T: EthSpec>(message: &PubsubMessage<T>) {
         _ => {}
     }
 }
+
+fn update_gossip_metrics<T: EthSpec>(gossipsub: &eth2_libp2p::Gossipsub) {
+    // Clear the metrics
+    let _ = metrics::PEERS_PER_PROTOCOL
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = metrics::PEERS_PER_PROTOCOL
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = metrics::MESH_PEERS_PER_MAIN_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+
+    // reset the mesh peers, showing all subnets
+    for subnet_id in 0..T::default_spec().attestation_subnet_count {
+        let _ = metrics::get_int_gauge(
+            &metrics::MESH_PEERS_PER_SUBNET_TOPIC,
+            &[&subnet_id.to_string()],
+        )
+        .map(|v| v.set(0));
+
+        let _ = metrics::get_int_gauge(
+            &metrics::GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
+            &[&subnet_id.to_string()],
+        )
+        .map(|v| v.set(0));
+
+        let _ = metrics::get_int_gauge(
+            &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
+            &[&subnet_id.to_string()],
+        )
+        .map(|v| v.set(0));
+    }
+
+    // Subnet topics subscribed to
+    for topic_hash in gossipsub.topics() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            if let GossipKind::Attestation(subnet_id) = topic.kind() {
+                let _ = metrics::get_int_gauge(
+                    &metrics::GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
+                    &[&subnet_id.to_string()],
+                )
+                .map(|v| v.set(1));
+            }
+        }
+    }
+
+    // Peers per subscribed subnet
+    let mut peers_per_topic: HashMap<TopicHash, usize> = HashMap::new();
+    for (peer_id, topics) in gossipsub.all_peers() {
+        for topic_hash in topics {
+            *peers_per_topic.entry(topic_hash.clone()).or_default() += 1;
+
+            if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+                match topic.kind() {
+                    GossipKind::Attestation(subnet_id) => {
+                        if let Some(v) = metrics::get_int_gauge(
+                            &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
+                            &[&subnet_id.to_string()],
+                        ) {
+                            v.inc()
+                        };
+
+                        // average peer scores
+                        if let Some(score) = gossipsub.peer_score(peer_id) {
+                            if let Some(v) = metrics::get_int_gauge(
+                                &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
+                                &[&subnet_id.to_string()],
+                            ) {
+                                v.add(score as i64)
+                            };
+                        }
+                    }
+                    kind => {
+                        // main topics
+                        if let Some(score) = gossipsub.peer_score(peer_id) {
+                            if let Some(v) = metrics::get_int_gauge(
+                                &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC,
+                                &[&format!("{:?}", kind)],
+                            ) {
+                                v.add(score as i64)
+                            };
+                        }
+                    }
+                }
+            }
+        }
+    }
+    // adjust to average scores by dividing by number of peers
+    for (topic_hash, peers) in peers_per_topic.iter() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::Attestation(subnet_id) => {
+                    // average peer scores
+                    if let Some(v) = metrics::get_int_gauge(
+                        &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
+                        &[&subnet_id.to_string()],
+                    ) {
+                        v.set(v.get() / (*peers as i64))
+                    };
+                }
+                kind => {
+                    // main topics
+                    if let Some(v) = metrics::get_int_gauge(
+                        &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC,
+                        &[&format!("{:?}", kind)],
+                    ) {
+                        v.set(v.get() / (*peers as i64))
+                    };
+                }
+            }
+        }
+    }
+
+    // mesh peers
+    for topic_hash in gossipsub.topics() {
+        let peers = gossipsub.mesh_peers(&topic_hash).count();
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::Attestation(subnet_id) => {
+                    if let Some(v) = metrics::get_int_gauge(
+                        &metrics::MESH_PEERS_PER_SUBNET_TOPIC,
+                        &[&subnet_id.to_string()],
+                    ) {
+                        v.set(peers as i64)
+                    };
+                }
+                kind => {
+                    // main topics
+                    if let Some(v) = metrics::get_int_gauge(
+                        &metrics::MESH_PEERS_PER_MAIN_TOPIC,
+                        &[&format!("{:?}", kind)],
+                    ) {
+                        v.set(peers as i64)
+                    };
+                }
+            }
+        }
+    }
+
+    // protocol peers
+    let mut peers_per_protocol: HashMap<String, i64> = HashMap::new();
+    for (_peer, protocol) in gossipsub.peer_protocol() {
+        *peers_per_protocol.entry(protocol.to_string()).or_default() += 1;
+    }
+
+    for (protocol, peers) in peers_per_protocol.iter() {
+        if let Some(v) =
+            metrics::get_int_gauge(&metrics::PEERS_PER_PROTOCOL, &[&protocol.to_string()])
+        {
+            v.set(*peers)
+        };
+    }
+}

beacon_node/network/src/service/tests.rs

@@ -23,7 +23,7 @@ mod tests {
         let log = get_logger();
 
         let beacon_chain = Arc::new(
-            BeaconChainHarness::new(
+            BeaconChainHarness::new_with_store_config(
                 MinimalEthSpec,
                 generate_deterministic_keypairs(8),
                 StoreConfig::default(),
@@ -40,17 +40,25 @@ mod tests {
         let runtime = Runtime::new().unwrap();
 
         let (signal, exit) = exit_future::signal();
-        let executor = environment::TaskExecutor::new(runtime.handle().clone(), exit, log.clone());
+        let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+        let executor = environment::TaskExecutor::new(
+            runtime.handle().clone(),
+            exit,
+            log.clone(),
+            shutdown_tx,
+        );
 
         let mut config = NetworkConfig::default();
         config.libp2p_port = 21212;
         config.discovery_port = 21212;
-        config.boot_nodes = enrs.clone();
+        config.boot_nodes_enr = enrs.clone();
         runtime.spawn(async move {
             // Create a new network service which implicitly gets dropped at the
             // end of the block.
 
-            let _ = NetworkService::start(beacon_chain.clone(), &config, executor).unwrap();
+            let _ = NetworkService::start(beacon_chain.clone(), &config, executor)
+                .await
+                .unwrap();
             drop(signal);
         });
         runtime.shutdown_timeout(tokio::time::Duration::from_millis(300));

beacon_node/network/src/sync/block_processor.rs

@@ -1,255 +0,0 @@
-use crate::router::processor::FUTURE_SLOT_TOLERANCE;
-use crate::sync::manager::SyncMessage;
-use crate::sync::range_sync::{BatchId, ChainId};
-use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, ChainSegmentResult};
-use eth2_libp2p::PeerId;
-use slog::{debug, error, trace, warn};
-use std::sync::{Arc, Weak};
-use tokio::sync::mpsc;
-use types::{EthSpec, SignedBeaconBlock};
-
-/// Id associated to a block processing request, either a batch or a single block.
-#[derive(Clone, Debug, PartialEq)]
-pub enum ProcessId {
-    /// Processing Id of a range syncing batch.
-    RangeBatchId(ChainId, BatchId),
-    /// Processing Id of the parent lookup of a block
-    ParentLookup(PeerId),
-}
-
-/// The result of a block processing request.
-// TODO: When correct batch error handling occurs, we will include an error type.
-#[derive(Debug)]
-pub enum BatchProcessResult {
-    /// The batch was completed successfully.
-    Success,
-    /// The batch processing failed.
-    Failed,
-    /// The batch processing failed but managed to import at least one block.
-    Partial,
-}
-
-/// Spawns a thread handling the block processing of a request: range syncing or parent lookup.
-pub fn spawn_block_processor<T: BeaconChainTypes>(
-    chain: Weak<BeaconChain<T>>,
-    process_id: ProcessId,
-    downloaded_blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
-    log: slog::Logger,
-) {
-    std::thread::spawn(move || {
-        match process_id {
-            // this a request from the range sync
-            ProcessId::RangeBatchId(chain_id, batch_id) => {
-                let len = downloaded_blocks.len();
-                let start_slot = if len > 0 {
-                    downloaded_blocks[0].message.slot.as_u64()
-                } else {
-                    0
-                };
-                let end_slot = if len > 0 {
-                    downloaded_blocks[len - 1].message.slot.as_u64()
-                } else {
-                    0
-                };
-
-                debug!(log, "Processing batch"; "id" => *batch_id, "blocks" => downloaded_blocks.len(),  "start_slot" => start_slot, "end_slot" => end_slot);
-                let result = match process_blocks(chain, downloaded_blocks.iter(), &log) {
-                    (_, Ok(_)) => {
-                        debug!(log, "Batch processed"; "id" => *batch_id , "start_slot" => start_slot, "end_slot" => end_slot);
-                        BatchProcessResult::Success
-                    }
-                    (imported_blocks, Err(e)) if imported_blocks > 0 => {
-                        debug!(log, "Batch processing failed but imported some blocks";
-                            "id" => *batch_id, "error" => e, "imported_blocks"=> imported_blocks);
-                        BatchProcessResult::Partial
-                    }
-                    (_, Err(e)) => {
-                        debug!(log, "Batch processing failed"; "id" => *batch_id, "error" => e);
-                        BatchProcessResult::Failed
-                    }
-                };
-
-                let msg = SyncMessage::BatchProcessed {
-                    chain_id,
-                    batch_id,
-                    downloaded_blocks,
-                    result,
-                };
-                sync_send.send(msg).unwrap_or_else(|_| {
-                    debug!(
-                        log,
-                        "Block processor could not inform range sync result. Likely shutting down."
-                    );
-                });
-            }
-            // this a parent lookup request from the sync manager
-            ProcessId::ParentLookup(peer_id) => {
-                debug!(
-                    log, "Processing parent lookup";
-                    "last_peer_id" => format!("{}", peer_id),
-                    "blocks" => downloaded_blocks.len()
-                );
-                // parent blocks are ordered from highest slot to lowest, so we need to process in
-                // reverse
-                match process_blocks(chain, downloaded_blocks.iter().rev(), &log) {
-                    (_, Err(e)) => {
-                        warn!(log, "Parent lookup failed"; "last_peer_id" => format!("{}", peer_id), "error" => e);
-                        sync_send
-                        .send(SyncMessage::ParentLookupFailed(peer_id))
-                        .unwrap_or_else(|_| {
-                            // on failure, inform to downvote the peer
-                            debug!(
-                                log,
-                                "Block processor could not inform parent lookup result. Likely shutting down."
-                            );
-                        });
-                    }
-                    (_, Ok(_)) => {
-                        debug!(log, "Parent lookup processed successfully");
-                    }
-                }
-            }
-        }
-    });
-}
-
-/// Helper function to process blocks batches which only consumes the chain and blocks to process.
-fn process_blocks<
-    'a,
-    T: BeaconChainTypes,
-    I: Iterator<Item = &'a SignedBeaconBlock<T::EthSpec>>,
->(
-    chain: Weak<BeaconChain<T>>,
-    downloaded_blocks: I,
-    log: &slog::Logger,
-) -> (usize, Result<(), String>) {
-    if let Some(chain) = chain.upgrade() {
-        let blocks = downloaded_blocks.cloned().collect::<Vec<_>>();
-        let (imported_blocks, r) = match chain.process_chain_segment(blocks) {
-            ChainSegmentResult::Successful { imported_blocks } => {
-                if imported_blocks == 0 {
-                    debug!(log, "All blocks already known");
-                } else {
-                    debug!(
-                        log, "Imported blocks from network";
-                        "count" => imported_blocks,
-                    );
-                    // Batch completed successfully with at least one block, run fork choice.
-                    run_fork_choice(chain, log);
-                }
-
-                (imported_blocks, Ok(()))
-            }
-            ChainSegmentResult::Failed {
-                imported_blocks,
-                error,
-            } => {
-                let r = handle_failed_chain_segment(error, log);
-                if imported_blocks > 0 {
-                    run_fork_choice(chain, log);
-                }
-                (imported_blocks, r)
-            }
-        };
-
-        return (imported_blocks, r);
-    }
-
-    (0, Ok(()))
-}
-
-/// Runs fork-choice on a given chain. This is used during block processing after one successful
-/// block import.
-fn run_fork_choice<T: BeaconChainTypes>(chain: Arc<BeaconChain<T>>, log: &slog::Logger) {
-    match chain.fork_choice() {
-        Ok(()) => trace!(
-            log,
-            "Fork choice success";
-            "location" => "batch processing"
-        ),
-        Err(e) => error!(
-            log,
-            "Fork choice failed";
-            "error" => format!("{:?}", e),
-            "location" => "batch import error"
-        ),
-    }
-}
-
-/// Helper function to handle a `BlockError` from `process_chain_segment`
-fn handle_failed_chain_segment<T: EthSpec>(
-    error: BlockError<T>,
-    log: &slog::Logger,
-) -> Result<(), String> {
-    match error {
-        BlockError::ParentUnknown(block) => {
-            // blocks should be sequential and all parents should exist
-
-            Err(format!(
-                "Block has an unknown parent: {}",
-                block.parent_root()
-            ))
-        }
-        BlockError::BlockIsAlreadyKnown => {
-            // This can happen for many reasons. Head sync's can download multiples and parent
-            // lookups can download blocks before range sync
-            Ok(())
-        }
-        BlockError::FutureSlot {
-            present_slot,
-            block_slot,
-        } => {
-            if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot {
-                // The block is too far in the future, drop it.
-                warn!(
-                    log, "Block is ahead of our slot clock";
-                    "msg" => "block for future slot rejected, check your time",
-                    "present_slot" => present_slot,
-                    "block_slot" => block_slot,
-                    "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE,
-                );
-            } else {
-                // The block is in the future, but not too far.
-                debug!(
-                    log, "Block is slightly ahead of our slot clock, ignoring.";
-                    "present_slot" => present_slot,
-                    "block_slot" => block_slot,
-                    "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE,
-                );
-            }
-
-            Err(format!(
-                "Block with slot {} is higher than the current slot {}",
-                block_slot, present_slot
-            ))
-        }
-        BlockError::WouldRevertFinalizedSlot { .. } => {
-            debug!( log, "Finalized or earlier block processed";);
-
-            Ok(())
-        }
-        BlockError::GenesisBlock => {
-            debug!(log, "Genesis block was processed");
-            Ok(())
-        }
-        BlockError::BeaconChainError(e) => {
-            warn!(
-                log, "BlockProcessingFailure";
-                "msg" => "unexpected condition in processing block.",
-                "outcome" => format!("{:?}", e)
-            );
-
-            Err(format!("Internal error whilst processing block: {:?}", e))
-        }
-        other => {
-            debug!(
-                log, "Invalid block received";
-                "msg" => "peer sent invalid block",
-                "outcome" => format!("{:?}", other),
-            );
-
-            Err(format!("Peer sent invalid block. Reason: {:?}", other))
-        }
-    }
-}

beacon_node/network/src/sync/manager.rs

@@ -33,17 +33,18 @@
 //! if an attestation references an unknown block) this manager can search for the block and
 //! subsequently search for parents if needed.
 
-use super::block_processor::{spawn_block_processor, BatchProcessResult, ProcessId};
 use super::network_context::SyncNetworkContext;
 use super::peer_sync_info::{PeerSyncInfo, PeerSyncType};
-use super::range_sync::{BatchId, ChainId, RangeSync, EPOCHS_PER_BATCH};
+use super::range_sync::{ChainId, RangeSync, EPOCHS_PER_BATCH};
 use super::RequestId;
+use crate::beacon_processor::{ProcessId, WorkEvent as BeaconWorkEvent};
 use crate::service::NetworkMessage;
 use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError};
 use eth2_libp2p::rpc::{methods::MAX_REQUEST_BLOCKS, BlocksByRootRequest, GoodbyeReason};
 use eth2_libp2p::types::NetworkGlobals;
 use eth2_libp2p::{PeerAction, PeerId};
 use fnv::FnvHashMap;
+use lru_cache::LRUCache;
 use slog::{crit, debug, error, info, trace, warn, Logger};
 use smallvec::SmallVec;
 use ssz_types::VariableList;
@@ -51,7 +52,7 @@ use std::boxed::Box;
 use std::ops::Sub;
 use std::sync::Arc;
 use tokio::sync::mpsc;
-use types::{EthSpec, Hash256, SignedBeaconBlock, Slot};
+use types::{Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot};
 
 /// The number of slots ahead of us that is allowed before requesting a long-range (batch)  Sync
 /// from a peer. If a peer is within this tolerance (forwards or backwards), it is treated as a
@@ -100,13 +101,30 @@ pub enum SyncMessage<T: EthSpec> {
     /// A batch has been processed by the block processor thread.
     BatchProcessed {
         chain_id: ChainId,
-        batch_id: BatchId,
+        epoch: Epoch,
         downloaded_blocks: Vec<SignedBeaconBlock<T>>,
         result: BatchProcessResult,
     },
 
-    /// A parent lookup has failed for a block given by this `peer_id`.
-    ParentLookupFailed(PeerId),
+    /// A parent lookup has failed.
+    ParentLookupFailed {
+        /// The head of the chain of blocks that failed to process.
+        chain_head: Hash256,
+        /// The peer that instigated the chain lookup.
+        peer_id: PeerId,
+    },
+}
+
+/// The result of processing a multiple blocks (a chain segment).
+// TODO: When correct batch error handling occurs, we will include an error type.
+#[derive(Debug)]
+pub enum BatchProcessResult {
+    /// The batch was completed successfully.
+    Success,
+    /// The batch processing failed.
+    Failed,
+    /// The batch processing failed but managed to import at least one block.
+    Partial,
 }
 
 /// Maintains a sequential list of parents to lookup and the lookup's current state.
@@ -149,6 +167,9 @@ pub struct SyncManager<T: BeaconChainTypes> {
     /// A collection of parent block lookups.
     parent_queue: SmallVec<[ParentRequests<T::EthSpec>; 3]>,
 
+    /// A cache of failed chain lookups to prevent duplicate searches.
+    failed_chains: LRUCache<Hash256>,
+
     /// A collection of block hashes being searched for and a flag indicating if a result has been
     /// received or not.
     ///
@@ -158,8 +179,8 @@ pub struct SyncManager<T: BeaconChainTypes> {
     /// The logger for the import manager.
     log: Logger,
 
-    /// The sending part of input_channel
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
 }
 
 /// Object representing a single block lookup request.
@@ -187,6 +208,7 @@ pub fn spawn<T: BeaconChainTypes>(
     beacon_chain: Arc<BeaconChain<T>>,
     network_globals: Arc<NetworkGlobals<T::EthSpec>>,
     network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     log: slog::Logger,
 ) -> mpsc::UnboundedSender<SyncMessage<T::EthSpec>> {
     assert!(
@@ -201,7 +223,7 @@ pub fn spawn<T: BeaconChainTypes>(
         range_sync: RangeSync::new(
             beacon_chain.clone(),
             network_globals.clone(),
-            sync_send.clone(),
+            beacon_processor_send.clone(),
             log.clone(),
         ),
         network: SyncNetworkContext::new(network_send, network_globals.clone(), log.clone()),
@@ -209,9 +231,10 @@ pub fn spawn<T: BeaconChainTypes>(
         network_globals,
         input_channel: sync_recv,
         parent_queue: SmallVec::new(),
+        failed_chains: LRUCache::new(500),
         single_block_lookups: FnvHashMap::default(),
         log: log.clone(),
-        sync_send: sync_send.clone(),
+        beacon_processor_send,
     };
 
     // spawn the sync manager thread
@@ -300,7 +323,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
     /// There are two reasons we could have received a BlocksByRoot response
     /// - We requested a single hash and have received a response for the single_block_lookup
     /// - We are looking up parent blocks in parent lookup search
-    fn blocks_by_root_response(
+    async fn blocks_by_root_response(
         &mut self,
         peer_id: PeerId,
         request_id: RequestId,
@@ -318,7 +341,8 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                     single_block_hash = Some(block_request.hash);
                 }
                 if let Some(block_hash) = single_block_hash {
-                    self.single_block_lookup_response(peer_id, block, block_hash);
+                    self.single_block_lookup_response(peer_id, block, block_hash)
+                        .await;
                     return;
                 }
 
@@ -337,10 +361,26 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                         return;
                     }
                 };
+
+                // check if the parent of this block isn't in our failed cache. If it is, this
+                // chain should be dropped and the peer downscored.
+                if self.failed_chains.contains(&block.message.parent_root) {
+                    debug!(self.log, "Parent chain ignored due to past failure"; "block" => format!("{:?}", block.message.parent_root), "slot" => block.message.slot);
+                    if !parent_request.downloaded_blocks.is_empty() {
+                        // Add the root block to failed chains
+                        self.failed_chains
+                            .insert(parent_request.downloaded_blocks[0].canonical_root());
+                    } else {
+                        crit!(self.log, "Parent chain has no blocks");
+                    }
+                    self.network
+                        .report_peer(peer_id, PeerAction::MidToleranceError);
+                    return;
+                }
                 // add the block to response
                 parent_request.downloaded_blocks.push(block);
                 // queue for processing
-                self.process_parent_request(parent_request);
+                self.process_parent_request(parent_request).await;
             }
             None => {
                 // this is a stream termination
@@ -381,10 +421,40 @@ impl<T: BeaconChainTypes> SyncManager<T> {
         }
     }
 
+    async fn process_block_async(
+        &mut self,
+        block: SignedBeaconBlock<T::EthSpec>,
+    ) -> Option<Result<Hash256, BlockError<T::EthSpec>>> {
+        let (event, rx) = BeaconWorkEvent::rpc_beacon_block(Box::new(block));
+        match self.beacon_processor_send.try_send(event) {
+            Ok(_) => {}
+            Err(e) => {
+                error!(
+                    self.log,
+                    "Failed to send sync block to processor";
+                    "error" => format!("{:?}", e)
+                );
+                return None;
+            }
+        }
+
+        match rx.await {
+            Ok(block_result) => Some(block_result),
+            Err(_) => {
+                warn!(
+                    self.log,
+                    "Sync block not processed";
+                    "msg" => "likely due to system resource exhaustion"
+                );
+                None
+            }
+        }
+    }
+
     /// Processes the response obtained from a single block lookup search. If the block is
     /// processed or errors, the search ends. If the blocks parent is unknown, a block parent
     /// lookup search is started.
-    fn single_block_lookup_response(
+    async fn single_block_lookup_response(
         &mut self,
         peer_id: PeerId,
         block: SignedBeaconBlock<T::EthSpec>,
@@ -399,8 +469,13 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             return;
         }
 
+        let block_result = match self.process_block_async(block.clone()).await {
+            Some(block_result) => block_result,
+            None => return,
+        };
+
         // we have the correct block, try and process it
-        match self.chain.process_block(block.clone()) {
+        match block_result {
             Ok(block_root) => {
                 info!(self.log, "Processed block"; "block" => format!("{}", block_root));
 
@@ -461,6 +536,15 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             }
         }
 
+        let block_root = block.canonical_root();
+        // If this block or it's parent is part of a known failed chain, ignore it.
+        if self.failed_chains.contains(&block.message.parent_root)
+            || self.failed_chains.contains(&block_root)
+        {
+            debug!(self.log, "Block is from a past failed chain. Dropping"; "block_root" => format!("{:?}", block_root), "block_slot" => block.message.slot);
+            return;
+        }
+
         // Make sure this block is not already being searched for
         // NOTE: Potentially store a hashset of blocks for O(1) lookups
         for parent_req in self.parent_queue.iter() {
@@ -599,7 +683,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
     // manager
 
     /// A new block has been received for a parent lookup query, process it.
-    fn process_parent_request(&mut self, mut parent_request: ParentRequests<T::EthSpec>) {
+    async fn process_parent_request(&mut self, mut parent_request: ParentRequests<T::EthSpec>) {
         // verify the last added block is the parent of the last requested block
 
         if parent_request.downloaded_blocks.len() < 2 {
@@ -648,11 +732,19 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             // If the last block in the queue has an unknown parent, we continue the parent
             // lookup-search.
 
+            let chain_block_hash = parent_request.downloaded_blocks[0].canonical_root();
+
             let newest_block = parent_request
                 .downloaded_blocks
                 .pop()
                 .expect("There is always at least one block in the queue");
-            match self.chain.process_block(newest_block.clone()) {
+
+            let block_result = match self.process_block_async(newest_block.clone()).await {
+                Some(block_result) => block_result,
+                None => return,
+            };
+
+            match block_result {
                 Err(BlockError::ParentUnknown { .. }) => {
                     // need to keep looking for parents
                     // add the block back to the queue and continue the search
@@ -660,13 +752,25 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                     self.request_parent(parent_request);
                 }
                 Ok(_) | Err(BlockError::BlockIsAlreadyKnown { .. }) => {
-                    spawn_block_processor(
-                        Arc::downgrade(&self.chain),
-                        ProcessId::ParentLookup(parent_request.last_submitted_peer.clone()),
-                        parent_request.downloaded_blocks,
-                        self.sync_send.clone(),
-                        self.log.clone(),
+                    let process_id = ProcessId::ParentLookup(
+                        parent_request.last_submitted_peer.clone(),
+                        chain_block_hash,
                     );
+                    let blocks = parent_request.downloaded_blocks;
+
+                    match self
+                        .beacon_processor_send
+                        .try_send(BeaconWorkEvent::chain_segment(process_id, blocks))
+                    {
+                        Ok(_) => {}
+                        Err(e) => {
+                            error!(
+                                self.log,
+                                "Failed to send chain segment to processor";
+                                "error" => format!("{:?}", e)
+                            );
+                        }
+                    }
                 }
                 Err(outcome) => {
                     // all else we consider the chain a failure and downvote the peer that sent
@@ -677,6 +781,10 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                         "outcome" => format!("{:?}", outcome),
                         "last_peer" => parent_request.last_submitted_peer.to_string(),
                     );
+
+                    // Add this chain to cache of failed chains
+                    self.failed_chains.insert(chain_block_hash);
+
                     // This currently can be a host of errors. We permit this due to the partial
                     // ambiguity.
                     // TODO: Refine the error types and score the peer appropriately.
@@ -699,8 +807,17 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             || parent_request.downloaded_blocks.len() >= PARENT_DEPTH_TOLERANCE
         {
             let error = if parent_request.failed_attempts >= PARENT_FAIL_TOLERANCE {
+                // This is a peer-specific error and the chain could be continued with another
+                // peer. We don't consider this chain a failure and prevent retries with another
+                // peer.
                 "too many failed attempts"
             } else {
+                if !parent_request.downloaded_blocks.is_empty() {
+                    self.failed_chains
+                        .insert(parent_request.downloaded_blocks[0].canonical_root());
+                } else {
+                    crit!(self.log, "Parent lookup has no blocks");
+                }
                 "reached maximum lookup-depth"
             };
 
@@ -709,6 +826,11 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             "ancestors_found" => parent_request.downloaded_blocks.len(),
             "reason" => error
             );
+            // Downscore the peer.
+            self.network.report_peer(
+                parent_request.last_submitted_peer,
+                PeerAction::LowToleranceError,
+            );
             return; // drop the request
         }
 
@@ -760,7 +882,8 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                         request_id,
                         beacon_block,
                     } => {
-                        self.blocks_by_root_response(peer_id, request_id, beacon_block.map(|b| *b));
+                        self.blocks_by_root_response(peer_id, request_id, beacon_block.map(|b| *b))
+                            .await;
                     }
                     SyncMessage::UnknownBlock(peer_id, block) => {
                         self.add_unknown_block(peer_id, *block);
@@ -776,24 +899,25 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                     }
                     SyncMessage::BatchProcessed {
                         chain_id,
-                        batch_id,
+                        epoch,
                         downloaded_blocks,
                         result,
                     } => {
                         self.range_sync.handle_block_process_result(
                             &mut self.network,
                             chain_id,
-                            batch_id,
+                            epoch,
                             downloaded_blocks,
                             result,
                         );
                     }
-                    SyncMessage::ParentLookupFailed(peer_id) => {
+                    SyncMessage::ParentLookupFailed {
+                        chain_head,
+                        peer_id,
+                    } => {
                         // A peer sent an object (block or attestation) that referenced a parent.
-                        // On request for this parent the peer indicated it did not have this
-                        // block.
-                        // This is not fatal. Peer's could prune old blocks so we moderately
-                        // tolerate this behaviour.
+                        // The processing of this chain failed.
+                        self.failed_chains.insert(chain_head);
                         self.network
                             .report_peer(peer_id, PeerAction::MidToleranceError);
                     }

beacon_node/network/src/sync/mod.rs

@@ -1,14 +1,14 @@
 //! Syncing for lighthouse.
 //!
 //! Stores the various syncing methods for the beacon chain.
-mod block_processor;
 pub mod manager;
 mod network_context;
 mod peer_sync_info;
 mod range_sync;
 
-pub use manager::SyncMessage;
+pub use manager::{BatchProcessResult, SyncMessage};
 pub use peer_sync_info::PeerSyncInfo;
+pub use range_sync::ChainId;
 
 /// Type of id of rpc requests sent by sync
 pub type RequestId = usize;

beacon_node/network/src/sync/network_context.rs

@@ -110,7 +110,7 @@ impl<T: EthSpec> SyncNetworkContext<T> {
     }
 
     pub fn report_peer(&mut self, peer_id: PeerId, action: PeerAction) {
-        debug!(self.log, "Sync reporting peer"; "peer_id" => peer_id.to_string(), "action"=> action.to_string());
+        debug!(self.log, "Sync reporting peer"; "peer_id" => peer_id.to_string(), "action" => action.to_string());
         self.network_send
             .send(NetworkMessage::ReportPeer { peer_id, action })
             .unwrap_or_else(|_| {

beacon_node/network/src/sync/peer_sync_info.rs

@@ -86,8 +86,8 @@ impl PeerSyncInfo {
             // that we are within SLOT_IMPORT_TOLERANCE of our two heads
             if (self.head_slot >= remote.head_slot
                 && self.head_slot.sub(remote.head_slot).as_usize() <= SLOT_IMPORT_TOLERANCE)
-                || (self.head_slot < remote.head_slot)
-                    && remote.head_slot.sub(self.head_slot).as_usize() <= SLOT_IMPORT_TOLERANCE
+                || (self.head_slot < remote.head_slot
+                    && remote.head_slot.sub(self.head_slot).as_usize() <= SLOT_IMPORT_TOLERANCE)
             {
                 return true;
             }

beacon_node/network/src/sync/range_sync/batch.rs

@@ -9,37 +9,14 @@ use std::collections::hash_map::Entry;
 use std::collections::{HashMap, HashSet};
 use std::hash::{Hash, Hasher};
 use std::ops::Sub;
-use types::{EthSpec, SignedBeaconBlock, Slot};
-
-#[derive(Copy, Clone, Debug, PartialEq)]
-pub struct BatchId(pub u64);
-
-impl std::ops::Deref for BatchId {
-    type Target = u64;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-impl std::ops::DerefMut for BatchId {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-impl std::convert::From<u64> for BatchId {
-    fn from(id: u64) -> Self {
-        BatchId(id)
-    }
-}
+use types::{Epoch, EthSpec, SignedBeaconBlock, Slot};
 
 /// A collection of sequential blocks that are requested from peers in a single RPC request.
 #[derive(PartialEq, Debug)]
 pub struct Batch<T: EthSpec> {
-    /// The ID of the batch, these are sequential.
-    pub id: BatchId,
-    /// The requested start slot of the batch, inclusive.
-    pub start_slot: Slot,
-    /// The requested end slot of batch, exlcusive.
+    /// The requested start epoch of the batch.
+    pub start_epoch: Epoch,
+    /// The requested end slot of batch, exclusive.
     pub end_slot: Slot,
     /// The `Attempts` that have been made to send us this batch.
     pub attempts: Vec<Attempt>,
@@ -69,10 +46,9 @@ pub struct Attempt {
 impl<T: EthSpec> Eq for Batch<T> {}
 
 impl<T: EthSpec> Batch<T> {
-    pub fn new(id: BatchId, start_slot: Slot, end_slot: Slot, peer_id: PeerId) -> Self {
+    pub fn new(start_epoch: Epoch, end_slot: Slot, peer_id: PeerId) -> Self {
         Batch {
-            id,
-            start_slot,
+            start_epoch,
             end_slot,
             attempts: Vec::new(),
             current_peer: peer_id,
@@ -82,12 +58,21 @@ impl<T: EthSpec> Batch<T> {
         }
     }
 
+    pub fn start_slot(&self) -> Slot {
+        // batches are shifted by 1
+        self.start_epoch.start_slot(T::slots_per_epoch()) + 1
+    }
+
+    pub fn end_slot(&self) -> Slot {
+        self.end_slot
+    }
     pub fn to_blocks_by_range_request(&self) -> BlocksByRangeRequest {
+        let start_slot = self.start_slot();
         BlocksByRangeRequest {
-            start_slot: self.start_slot.into(),
+            start_slot: start_slot.into(),
             count: min(
                 T::slots_per_epoch() * EPOCHS_PER_BATCH,
-                self.end_slot.sub(self.start_slot).into(),
+                self.end_slot.sub(start_slot).into(),
             ),
             step: 1,
         }
@@ -105,7 +90,7 @@ impl<T: EthSpec> Batch<T> {
 
 impl<T: EthSpec> Ord for Batch<T> {
     fn cmp(&self, other: &Self) -> Ordering {
-        self.id.0.cmp(&other.id.0)
+        self.start_epoch.cmp(&other.start_epoch)
     }
 }
 

beacon_node/network/src/sync/range_sync/chain.rs

@@ -1,11 +1,12 @@
-use super::batch::{Batch, BatchId, PendingBatches};
-use crate::sync::block_processor::{spawn_block_processor, BatchProcessResult, ProcessId};
-use crate::sync::network_context::SyncNetworkContext;
-use crate::sync::{RequestId, SyncMessage};
+use super::batch::{Batch, PendingBatches};
+use crate::beacon_processor::ProcessId;
+use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
+use crate::sync::RequestId;
+use crate::sync::{network_context::SyncNetworkContext, BatchProcessResult};
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::{PeerAction, PeerId};
 use rand::prelude::*;
-use slog::{crit, debug, warn};
+use slog::{crit, debug, error, warn};
 use std::collections::HashSet;
 use std::sync::Arc;
 use tokio::sync::mpsc;
@@ -72,11 +73,12 @@ pub struct SyncingChain<T: BeaconChainTypes> {
     /// and thus available to download this chain from.
     pub peer_pool: HashSet<PeerId>,
 
-    /// The next batch_id that needs to be downloaded.
-    to_be_downloaded_id: BatchId,
+    /// Starting epoch of the next batch that needs to be downloaded.
+    to_be_downloaded: Epoch,
 
-    /// The next batch id that needs to be processed.
-    to_be_processed_id: BatchId,
+    /// Starting epoch of the batch that needs to be processed next.
+    /// This is incremented as the chain advances.
+    processing_target: Epoch,
 
     /// The current state of the chain.
     pub state: ChainSyncingState,
@@ -84,14 +86,13 @@ pub struct SyncingChain<T: BeaconChainTypes> {
     /// The current processing batch, if any.
     current_processing_batch: Option<Batch<T::EthSpec>>,
 
-    /// A send channel to the sync manager. This is given to the batch processor thread to report
-    /// back once batch processing has completed.
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
 
     /// A reference to the underlying beacon chain.
     chain: Arc<BeaconChain<T>>,
 
-    /// A reference to the sync logger.
+    /// The chain's log.
     log: slog::Logger,
 }
 
@@ -111,7 +112,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         target_head_slot: Slot,
         target_head_root: Hash256,
         peer_id: PeerId,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
         chain: Arc<BeaconChain<T>>,
         log: slog::Logger,
     ) -> Self {
@@ -127,11 +128,11 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             completed_batches: Vec::new(),
             processed_batches: Vec::new(),
             peer_pool,
-            to_be_downloaded_id: BatchId(1),
-            to_be_processed_id: BatchId(1),
+            to_be_downloaded: start_epoch,
+            processing_target: start_epoch,
             state: ChainSyncingState::Stopped,
             current_processing_batch: None,
-            sync_send,
+            beacon_processor_send,
             chain,
             log,
         }
@@ -139,13 +140,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
 
     /// Returns the latest slot number that has been processed.
     fn current_processed_slot(&self) -> Slot {
-        self.start_epoch
+        // the last slot we processed was included in the previous batch, and corresponds to the
+        // first slot of the current target epoch
+        self.processing_target
             .start_slot(T::EthSpec::slots_per_epoch())
-            .saturating_add(
-                self.to_be_processed_id.saturating_sub(1u64)
-                    * T::EthSpec::slots_per_epoch()
-                    * EPOCHS_PER_BATCH,
-            )
     }
 
     /// A batch of blocks has been received. This function gets run on all chains and should
@@ -182,21 +180,19 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         // An entire batch of blocks has been received. This functions checks to see if it can be processed,
         // remove any batches waiting to be verified and if this chain is syncing, request new
         // blocks for the peer.
-        debug!(self.log, "Completed batch received"; "id"=> *batch.id, "blocks" => &batch.downloaded_blocks.len(), "awaiting_batches" => self.completed_batches.len());
+        debug!(self.log, "Completed batch received"; "epoch" => batch.start_epoch, "blocks" => &batch.downloaded_blocks.len(), "awaiting_batches" => self.completed_batches.len());
 
         // verify the range of received blocks
         // Note that the order of blocks is verified in block processing
         if let Some(last_slot) = batch.downloaded_blocks.last().map(|b| b.slot()) {
             // the batch is non-empty
             let first_slot = batch.downloaded_blocks[0].slot();
-            if batch.start_slot > first_slot || batch.end_slot < last_slot {
+            if batch.start_slot() > first_slot || batch.end_slot() < last_slot {
                 warn!(self.log, "BlocksByRange response returned out of range blocks";
-                          "response_initial_slot" => first_slot,
-                          "requested_initial_slot" => batch.start_slot);
-                // This is a pretty bad error. We don't consider this fatal, but we don't tolerate
-                // this much either.
-                network.report_peer(batch.current_peer, PeerAction::LowToleranceError);
-                self.to_be_processed_id = batch.id; // reset the id back to here, when incrementing, it will check against completed batches
+                    "response_initial_slot" => first_slot,
+                    "requested_initial_slot" => batch.start_slot());
+                // this batch can't be used, so we need to request it again.
+                self.failed_batch(network, batch);
                 return;
             }
         }
@@ -242,7 +238,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
 
         // Check if there is a batch ready to be processed
         if !self.completed_batches.is_empty()
-            && self.completed_batches[0].id == self.to_be_processed_id
+            && self.completed_batches[0].start_epoch == self.processing_target
         {
             let batch = self.completed_batches.remove(0);
 
@@ -255,18 +251,23 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         }
     }
 
-    /// Sends a batch to the batch processor.
+    /// Sends a batch to the beacon processor for async processing in a queue.
     fn process_batch(&mut self, mut batch: Batch<T::EthSpec>) {
-        let downloaded_blocks = std::mem::replace(&mut batch.downloaded_blocks, Vec::new());
-        let process_id = ProcessId::RangeBatchId(self.id, batch.id);
+        let blocks = std::mem::replace(&mut batch.downloaded_blocks, Vec::new());
+        let process_id = ProcessId::RangeBatchId(self.id, batch.start_epoch);
         self.current_processing_batch = Some(batch);
-        spawn_block_processor(
-            Arc::downgrade(&self.chain.clone()),
-            process_id,
-            downloaded_blocks,
-            self.sync_send.clone(),
-            self.log.clone(),
-        );
+
+        if let Err(e) = self
+            .beacon_processor_send
+            .try_send(BeaconWorkEvent::chain_segment(process_id, blocks))
+        {
+            error!(
+                self.log,
+                "Failed to send chain segment to processor";
+                "msg" => "process_batch",
+                "error" => format!("{:?}", e)
+            );
+        }
     }
 
     /// The block processor has completed processing a batch. This function handles the result
@@ -275,7 +276,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         &mut self,
         network: &mut SyncNetworkContext<T::EthSpec>,
         chain_id: ChainId,
-        batch_id: BatchId,
+        batch_start_epoch: Epoch,
         downloaded_blocks: &mut Option<Vec<SignedBeaconBlock<T::EthSpec>>>,
         result: &BatchProcessResult,
     ) -> Option<ProcessingResult> {
@@ -284,14 +285,14 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             return None;
         }
         match &self.current_processing_batch {
-            Some(current_batch) if current_batch.id != batch_id => {
+            Some(current_batch) if current_batch.start_epoch != batch_start_epoch => {
                 debug!(self.log, "Unexpected batch result";
-                    "chain_id" => self.id, "batch_id" => *batch_id, "expected_batch_id" => *current_batch.id);
+                    "batch_epoch" => batch_start_epoch, "expected_batch_epoch" => current_batch.start_epoch);
                 return None;
             }
             None => {
                 debug!(self.log, "Chain was not expecting a batch result";
-                    "chain_id" => self.id, "batch_id" => *batch_id);
+                    "batch_epoch" => batch_start_epoch);
                 return None;
             }
             _ => {
@@ -303,7 +304,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         let downloaded_blocks = downloaded_blocks.take().or_else(|| {
             // if taken by another chain, we are no longer waiting on a result.
             self.current_processing_batch = None;
-            crit!(self.log, "Processed batch taken by another chain"; "chain_id" => self.id);
+            crit!(self.log, "Processed batch taken by another chain");
             None
         })?;
 
@@ -313,16 +314,15 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         batch.downloaded_blocks = downloaded_blocks;
 
         // double check batches are processed in order TODO: Remove for prod
-        if batch.id != self.to_be_processed_id {
+        if batch.start_epoch != self.processing_target {
             crit!(self.log, "Batch processed out of order";
-                "chain_id" => self.id,
-                "processed_batch_id" => *batch.id,
-                "expected_id" => *self.to_be_processed_id);
+                "processed_starting_epoch" => batch.start_epoch,
+                "expected_epoch" => self.processing_target);
         }
 
         let res = match result {
             BatchProcessResult::Success => {
-                *self.to_be_processed_id += 1;
+                self.processing_target += EPOCHS_PER_BATCH;
 
                 // If the processed batch was not empty, we can validate previous invalidated
                 // blocks including the current batch.
@@ -352,7 +352,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             }
             BatchProcessResult::Partial => {
                 warn!(self.log, "Batch processing failed but at least one block was imported";
-                    "chain_id" => self.id, "id" => *batch.id, "peer" => format!("{}", batch.current_peer)
+                    "batch_epoch" => batch.start_epoch, "peer" => batch.current_peer.to_string()
                 );
                 // At least one block was successfully verified and imported, so we can be sure all
                 // previous batches are valid and we only need to download the current failed
@@ -370,7 +370,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                     let action = PeerAction::LowToleranceError;
                     warn!(self.log, "Batch failed to download. Dropping chain scoring peers";
                         "score_adjustment" => action.to_string(),
-                        "chain_id" => self.id, "id"=> *batch.id);
+                        "batch_epoch"=> batch.start_epoch);
                     for peer_id in self.peer_pool.drain() {
                         network.report_peer(peer_id, action);
                     }
@@ -383,7 +383,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             }
             BatchProcessResult::Failed => {
                 debug!(self.log, "Batch processing failed";
-                    "chain_id" => self.id,"id" => *batch.id, "peer" => batch.current_peer.to_string(), "client" => network.client_type(&batch.current_peer).to_string());
+                    "batch_epoch" => batch.start_epoch, "peer" => batch.current_peer.to_string(), "client" => network.client_type(&batch.current_peer).to_string());
                 // The batch processing failed
                 // This could be because this batch is invalid, or a previous invalidated batch
                 // is invalid. We need to find out which and downvote the peer that has sent us
@@ -398,7 +398,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                     let action = PeerAction::LowToleranceError;
                     warn!(self.log, "Batch failed to download. Dropping chain scoring peers";
                         "score_adjustment" => action.to_string(),
-                        "chain_id" => self.id, "id"=> *batch.id);
+                        "batch_epoch" => batch.start_epoch);
                     for peer_id in self.peer_pool.drain() {
                         network.report_peer(peer_id, action);
                     }
@@ -428,11 +428,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
     ) {
         while !self.processed_batches.is_empty() {
             let mut processed_batch = self.processed_batches.remove(0);
-            if *processed_batch.id >= *last_batch.id {
+            if processed_batch.start_epoch >= last_batch.start_epoch {
                 crit!(self.log, "A processed batch had a greater id than the current process id";
-                    "chain_id" => self.id,
-                    "processed_id" => *processed_batch.id,
-                    "current_id" => *last_batch.id);
+                    "processed_start_epoch" => processed_batch.start_epoch,
+                    "current_start_epoch" => last_batch.start_epoch);
             }
 
             // Go through passed attempts and downscore peers that returned invalid batches
@@ -447,11 +446,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                         let action = PeerAction::LowToleranceError;
                         debug!(
                             self.log, "Re-processed batch validated. Scoring original peer";
-                                "chain_id" => self.id,
-                                "batch_id" => *processed_batch.id,
-                                "score_adjustment" => action.to_string(),
-                                "original_peer" => format!("{}",attempt.peer_id),
-                                "new_peer" => format!("{}", processed_batch.current_peer)
+                            "batch_epoch" => processed_batch.start_epoch,
+                            "score_adjustment" => action.to_string(),
+                            "original_peer" => format!("{}",attempt.peer_id),
+                            "new_peer" => format!("{}", processed_batch.current_peer)
                         );
                         network.report_peer(attempt.peer_id, action);
                     } else {
@@ -460,11 +458,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                         let action = PeerAction::MidToleranceError;
                         debug!(
                             self.log, "Re-processed batch validated by the same peer.";
-                                "chain_id" => self.id,
-                                "batch_id" => *processed_batch.id,
-                                "score_adjustment" => action.to_string(),
-                                "original_peer" => format!("{}",attempt.peer_id),
-                                "new_peer" => format!("{}", processed_batch.current_peer)
+                            "batch_epoch" => processed_batch.start_epoch,
+                            "score_adjustment" => action.to_string(),
+                            "original_peer" => format!("{}",attempt.peer_id),
+                            "new_peer" => format!("{}", processed_batch.current_peer)
                         );
                         network.report_peer(attempt.peer_id, action);
                     }
@@ -503,7 +500,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         // Find any pre-processed batches awaiting validation
         while !self.processed_batches.is_empty() {
             let past_batch = self.processed_batches.remove(0);
-            *self.to_be_processed_id = std::cmp::min(*self.to_be_processed_id, *past_batch.id);
+            self.processing_target = std::cmp::min(self.processing_target, past_batch.start_epoch);
             self.reprocess_batch(network, past_batch);
         }
 
@@ -547,11 +544,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         batch.current_peer = new_peer.clone();
 
         debug!(self.log, "Re-requesting batch";
-            "chain_id" => self.id,
-            "start_slot" => batch.start_slot,
+            "start_slot" => batch.start_slot(),
             "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
-            "id" => *batch.id,
-            "peer" => format!("{}", batch.current_peer),
+            "batch_epoch" => batch.start_epoch,
+            "peer" => batch.current_peer.to_string(),
             "retries" => batch.retries,
             "re-processes" =>  batch.reprocess_retries);
         self.send_batch(network, batch);
@@ -587,12 +583,11 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             self.start_epoch = local_finalized_epoch;
 
             debug!(self.log, "Updating chain's progress";
-                "chain_id" => self.id,
                 "prev_completed_slot" => current_processed_slot,
                 "new_completed_slot" => self.current_processed_slot());
             // Re-index batches
-            *self.to_be_downloaded_id = 1;
-            *self.to_be_processed_id = 1;
+            self.to_be_downloaded = local_finalized_epoch;
+            self.processing_target = local_finalized_epoch;
 
             // remove any completed or processed batches
             self.completed_batches.clear();
@@ -616,7 +611,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         // do not request blocks if the chain is not syncing
         if let ChainSyncingState::Stopped = self.state {
             debug!(self.log, "Peer added to a non-syncing chain";
-                "chain_id" => self.id, "peer_id" => format!("{}", peer_id));
+                "peer_id" => format!("{}", peer_id));
             return;
         }
 
@@ -645,8 +640,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
     ) -> Option<ProcessingResult> {
         if let Some(batch) = self.pending_batches.remove(request_id) {
             debug!(self.log, "Batch failed. RPC Error";
-                "chain_id" => self.id,
-                "id" => *batch.id,
+                "batch_epoch" => batch.start_epoch,
                 "retries" => batch.retries,
                 "peer" => format!("{:?}", peer_id));
 
@@ -683,17 +677,23 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
 
             batch.current_peer = new_peer.clone();
             debug!(self.log, "Re-Requesting batch";
-                "chain_id" => self.id,
-                "start_slot" => batch.start_slot,
+                "start_slot" => batch.start_slot(),
                 "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
-
-                "id" => *batch.id,
-                "peer" => format!("{:?}", batch.current_peer));
+                "batch_epoch" => batch.start_epoch,
+                "peer" => batch.current_peer.to_string());
             self.send_batch(network, batch);
             ProcessingResult::KeepChain
         }
     }
 
+    /// Returns true if this chain is currently syncing.
+    pub fn is_syncing(&self) -> bool {
+        match self.state {
+            ChainSyncingState::Syncing => true,
+            ChainSyncingState::Stopped => false,
+        }
+    }
+
     /// Attempts to request the next required batches from the peer pool if the chain is syncing. It will exhaust the peer
     /// pool and left over batches until the batch buffer is reached or all peers are exhausted.
     fn request_batches(&mut self, network: &mut SyncNetworkContext<T::EthSpec>) {
@@ -709,10 +709,9 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         if let Some(peer_id) = self.get_next_peer() {
             if let Some(batch) = self.get_next_batch(peer_id) {
                 debug!(self.log, "Requesting batch";
-                    "chain_id" => self.id,
-                    "start_slot" => batch.start_slot,
-                    "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks 
-                    "id" => *batch.id,
+                    "start_slot" => batch.start_slot(),
+                    "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
+                    "batch_epoch" => batch.start_epoch,
                     "peer" => format!("{}", batch.current_peer));
                 // send the batch
                 self.send_batch(network, batch);
@@ -765,22 +764,15 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             return None;
         }
 
-        // One is added to the start slot to begin one slot after the epoch boundary
-        let batch_start_slot = self
-            .start_epoch
-            .start_slot(slots_per_epoch)
-            .saturating_add(1u64)
-            + self.to_be_downloaded_id.saturating_sub(1) * blocks_per_batch;
-
         // don't request batches beyond the target head slot
-        if batch_start_slot > self.target_head_slot {
+        if self.to_be_downloaded.start_slot(slots_per_epoch) > self.target_head_slot {
             return None;
         }
 
         // truncate the batch to the epoch containing the target head of the chain
         let batch_end_slot = std::cmp::min(
             // request either a batch containing the max number of blocks per batch
-            batch_start_slot + blocks_per_batch,
+            self.to_be_downloaded.start_slot(slots_per_epoch) + blocks_per_batch + 1,
             // or a batch of one epoch of blocks, which contains the `target_head_slot`
             self.target_head_slot
                 .saturating_add(slots_per_epoch)
@@ -788,28 +780,9 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                 .start_slot(slots_per_epoch),
         );
 
-        let batch_id = self.to_be_downloaded_id;
-
-        // Find the next batch id. The largest of the next sequential id, or the next uncompleted
-        // id
-        let max_completed_id = self
-            .completed_batches
-            .iter()
-            .last()
-            .map(|x| x.id.0)
-            .unwrap_or_else(|| 0);
-        // TODO: Check if this is necessary
-        self.to_be_downloaded_id = BatchId(std::cmp::max(
-            self.to_be_downloaded_id.0 + 1,
-            max_completed_id + 1,
-        ));
-
-        Some(Batch::new(
-            batch_id,
-            batch_start_slot,
-            batch_end_slot,
-            peer_id,
-        ))
+        let batch = Some(Batch::new(self.to_be_downloaded, batch_end_slot, peer_id));
+        self.to_be_downloaded += EPOCHS_PER_BATCH;
+        batch
     }
 
     /// Requests the provided batch from the provided peer.
@@ -827,14 +800,13 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             }
             Err(e) => {
                 warn!(self.log, "Batch request failed";
-                "chain_id" => self.id,
-                "start_slot" => batch.start_slot,
-                "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
-                "id" => *batch.id,
-                "peer" => format!("{}", batch.current_peer),
-                "retries" => batch.retries,
-                "error" => e,
-                "re-processes" =>  batch.reprocess_retries);
+                    "start_slot" => batch.start_slot(),
+                    "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
+                    "start_epoch" => batch.start_epoch,
+                    "peer" => batch.current_peer.to_string(),
+                    "retries" => batch.retries,
+                    "error" => e,
+                    "re-processes" =>  batch.reprocess_retries);
                 self.failed_batch(network, batch);
             }
         }

beacon_node/network/src/sync/range_sync/chain_collection.rs

@@ -4,17 +4,20 @@
 //! with this struct to to simplify the logic of the other layers of sync.
 
 use super::chain::{ChainSyncingState, SyncingChain};
-use crate::sync::manager::SyncMessage;
+use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
 use crate::sync::network_context::SyncNetworkContext;
 use crate::sync::PeerSyncInfo;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::{types::SyncState, NetworkGlobals, PeerId};
-use slog::{debug, error, info};
+use slog::{debug, error, info, o};
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use types::EthSpec;
 use types::{Epoch, Hash256, Slot};
 
+/// The number of head syncing chains to sync at a time.
+const PARALLEL_HEAD_CHAINS: usize = 2;
+
 /// The state of the long range/batch sync.
 #[derive(Clone)]
 pub enum RangeSyncState {
@@ -205,8 +208,9 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
     /// Updates the state of the chain collection.
     ///
     /// This removes any out-dated chains, swaps to any higher priority finalized chains and
-    /// updates the state of the collection.
-    pub fn update_finalized(&mut self, network: &mut SyncNetworkContext<T::EthSpec>) {
+    /// updates the state of the collection. This starts head chains syncing if any are required to
+    /// do so.
+    pub fn update(&mut self, network: &mut SyncNetworkContext<T::EthSpec>) {
         let local_epoch = {
             let local = match PeerSyncInfo::from_chain(&self.beacon_chain) {
                 Some(local) => local,
@@ -222,9 +226,25 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
             local.finalized_epoch
         };
 
-        // Remove any outdated finalized chains
+        // Remove any outdated finalized/head chains
         self.purge_outdated_chains(network);
 
+        // Choose the best finalized chain if one needs to be selected.
+        self.update_finalized_chains(network, local_epoch);
+
+        if self.finalized_syncing_index().is_none() {
+            // Handle head syncing chains if there are no finalized chains left.
+            self.update_head_chains(network, local_epoch);
+        }
+    }
+
+    /// This looks at all current finalized chains and decides if a new chain should be prioritised
+    /// or not.
+    fn update_finalized_chains(
+        &mut self,
+        network: &mut SyncNetworkContext<T::EthSpec>,
+        local_epoch: Epoch,
+    ) {
         // Check if any chains become the new syncing chain
         if let Some(index) = self.finalized_syncing_index() {
             // There is a current finalized chain syncing
@@ -269,32 +289,92 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
                 head_root: chain.target_head_root,
             };
             self.state = state;
-        } else {
-            // There are no finalized chains, update the state.
-            if self.head_chains.is_empty() {
-                self.state = RangeSyncState::Idle;
-            } else {
-                // for the syncing API, we find the minimal start_slot and the maximum
-                // target_slot of all head chains to report back.
-
-                let (min_epoch, max_slot) = self.head_chains.iter().fold(
-                    (Epoch::from(0u64), Slot::from(0u64)),
-                    |(min, max), chain| {
-                        (
-                            std::cmp::min(min, chain.start_epoch),
-                            std::cmp::max(max, chain.target_head_slot),
-                        )
-                    },
-                );
-                let head_state = RangeSyncState::Head {
-                    start_slot: min_epoch.start_slot(T::EthSpec::slots_per_epoch()),
-                    head_slot: max_slot,
-                };
-                self.state = head_state;
-            }
         }
     }
 
+    /// Start syncing any head chains if required.
+    fn update_head_chains(
+        &mut self,
+        network: &mut SyncNetworkContext<T::EthSpec>,
+        local_epoch: Epoch,
+    ) {
+        // There are no finalized chains, update the state.
+        if self.head_chains.is_empty() {
+            self.state = RangeSyncState::Idle;
+            return;
+        }
+
+        let mut currently_syncing = self
+            .head_chains
+            .iter()
+            .filter(|chain| chain.is_syncing())
+            .count();
+        let mut not_syncing = self.head_chains.len() - currently_syncing;
+
+        // Find all head chains that are not currently syncing ordered by peer count.
+        while currently_syncing <= PARALLEL_HEAD_CHAINS && not_syncing > 0 {
+            // Find the chain with the most peers and start syncing
+            if let Some((_index, chain)) = self
+                .head_chains
+                .iter_mut()
+                .filter(|chain| !chain.is_syncing())
+                .enumerate()
+                .max_by_key(|(_index, chain)| chain.peer_pool.len())
+            {
+                // start syncing this chain
+                debug!(self.log, "New head chain started syncing"; "new_target_root" => format!("{}", chain.target_head_root), "new_end_slot" => chain.target_head_slot, "new_start_epoch"=> chain.start_epoch);
+                chain.start_syncing(network, local_epoch);
+            }
+
+            // update variables
+            currently_syncing = self
+                .head_chains
+                .iter()
+                .filter(|chain| chain.is_syncing())
+                .count();
+            not_syncing = self.head_chains.len() - currently_syncing;
+        }
+
+        // Start
+        // for the syncing API, we find the minimal start_slot and the maximum
+        // target_slot of all head chains to report back.
+
+        let (min_epoch, max_slot) = self
+            .head_chains
+            .iter()
+            .filter(|chain| chain.is_syncing())
+            .fold(
+                (Epoch::from(0u64), Slot::from(0u64)),
+                |(min, max), chain| {
+                    (
+                        std::cmp::min(min, chain.start_epoch),
+                        std::cmp::max(max, chain.target_head_slot),
+                    )
+                },
+            );
+        let head_state = RangeSyncState::Head {
+            start_slot: min_epoch.start_slot(T::EthSpec::slots_per_epoch()),
+            head_slot: max_slot,
+        };
+        self.state = head_state;
+    }
+
+    /// This is called once a head chain has completed syncing. It removes all non-syncing head
+    /// chains and re-status their peers.
+    pub fn clear_head_chains(&mut self, network: &mut SyncNetworkContext<T::EthSpec>) {
+        let log_ref = &self.log;
+        self.head_chains.retain(|chain| {
+            if !chain.is_syncing()
+                {
+                debug!(log_ref, "Removing old head chain"; "start_epoch" => chain.start_epoch, "end_slot" => chain.target_head_slot);
+                chain.status_peers(network);
+                false
+            } else {
+                true
+            }
+        });
+    }
+
     /// Add a new finalized chain to the collection.
     pub fn new_finalized_chain(
         &mut self,
@@ -302,7 +382,7 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         target_head: Hash256,
         target_slot: Slot,
         peer_id: PeerId,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     ) {
         let chain_id = rand::random();
         self.finalized_chains.push(SyncingChain::new(
@@ -311,9 +391,9 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
             target_slot,
             target_head,
             peer_id,
-            sync_send,
+            beacon_processor_send,
             self.beacon_chain.clone(),
-            self.log.clone(),
+            self.log.new(o!("chain" => chain_id)),
         ));
     }
 
@@ -321,12 +401,11 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
     #[allow(clippy::too_many_arguments)]
     pub fn new_head_chain(
         &mut self,
-        network: &mut SyncNetworkContext<T::EthSpec>,
         remote_finalized_epoch: Epoch,
         target_head: Hash256,
         target_slot: Slot,
         peer_id: PeerId,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     ) {
         // remove the peer from any other head chains
 
@@ -336,18 +415,16 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         self.head_chains.retain(|chain| !chain.peer_pool.is_empty());
 
         let chain_id = rand::random();
-        let mut new_head_chain = SyncingChain::new(
+        let new_head_chain = SyncingChain::new(
             chain_id,
             remote_finalized_epoch,
             target_slot,
             target_head,
             peer_id,
-            sync_send,
+            beacon_processor_send,
             self.beacon_chain.clone(),
             self.log.clone(),
         );
-        // All head chains can sync simultaneously
-        new_head_chain.start_syncing(network, remote_finalized_epoch);
         self.head_chains.push(new_head_chain);
     }
 
@@ -511,7 +588,7 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         debug!(self.log, "Chain was removed"; "start_epoch" => chain.start_epoch, "end_slot" => chain.target_head_slot);
 
         // update the state
-        self.update_finalized(network);
+        self.update(network);
     }
 
     /// Returns the index of finalized chain that is currently syncing. Returns `None` if no

beacon_node/network/src/sync/range_sync/mod.rs

@@ -8,6 +8,5 @@ mod range;
 mod sync_type;
 
 pub use batch::Batch;
-pub use batch::BatchId;
 pub use chain::{ChainId, EPOCHS_PER_BATCH};
 pub use range::RangeSync;

beacon_node/network/src/sync/range_sync/range.rs

@@ -42,10 +42,9 @@
 use super::chain::{ChainId, ProcessingResult};
 use super::chain_collection::{ChainCollection, RangeSyncState};
 use super::sync_type::RangeSyncType;
-use super::BatchId;
-use crate::sync::block_processor::BatchProcessResult;
-use crate::sync::manager::SyncMessage;
+use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
 use crate::sync::network_context::SyncNetworkContext;
+use crate::sync::BatchProcessResult;
 use crate::sync::PeerSyncInfo;
 use crate::sync::RequestId;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
@@ -54,7 +53,7 @@ use slog::{debug, error, trace};
 use std::collections::HashSet;
 use std::sync::Arc;
 use tokio::sync::mpsc;
-use types::{EthSpec, SignedBeaconBlock};
+use types::{Epoch, EthSpec, SignedBeaconBlock};
 
 /// The primary object dealing with long range/batch syncing. This contains all the active and
 /// non-active chains that need to be processed before the syncing is considered complete. This
@@ -69,9 +68,8 @@ pub struct RangeSync<T: BeaconChainTypes> {
     /// finalized chain(s) complete, these peer's get STATUS'ed to update their head slot before
     /// the head chains are formed and downloaded.
     awaiting_head_peers: HashSet<PeerId>,
-    /// The sync manager channel, allowing the batch processor thread to callback the sync task
-    /// once complete.
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     /// The syncing logger.
     log: slog::Logger,
 }
@@ -80,14 +78,14 @@ impl<T: BeaconChainTypes> RangeSync<T> {
     pub fn new(
         beacon_chain: Arc<BeaconChain<T>>,
         network_globals: Arc<NetworkGlobals<T::EthSpec>>,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
         log: slog::Logger,
     ) -> Self {
         RangeSync {
             beacon_chain: beacon_chain.clone(),
             chains: ChainCollection::new(beacon_chain, network_globals, log.clone()),
             awaiting_head_peers: HashSet::new(),
-            sync_send,
+            beacon_processor_send,
             log,
         }
     }
@@ -162,28 +160,28 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                     .chains
                     .get_finalized_mut(remote_info.finalized_root, remote_finalized_slot)
                 {
-                    debug!(self.log, "Finalized chain exists, adding peer"; "peer_id" => format!("{:?}", peer_id), "target_root" => format!("{}", chain.target_head_root), "end_slot" => chain.target_head_slot, "start_epoch"=> chain.start_epoch);
+                    debug!(self.log, "Finalized chain exists, adding peer"; "peer_id" => peer_id.to_string(), "target_root" => chain.target_head_root.to_string(), "targe_slot" => chain.target_head_slot);
 
                     // add the peer to the chain's peer pool
                     chain.add_peer(network, peer_id);
 
                     // check if the new peer's addition will favour a new syncing chain.
-                    self.chains.update_finalized(network);
+                    self.chains.update(network);
                     // update the global sync state if necessary
                     self.chains.update_sync_state();
                 } else {
                     // there is no finalized chain that matches this peer's last finalized target
                     // create a new finalized chain
-                    debug!(self.log, "New finalized chain added to sync"; "peer_id" => format!("{:?}", peer_id), "start_epoch" => local_finalized_slot, "end_slot" => remote_finalized_slot, "finalized_root" => format!("{}", remote_info.finalized_root));
+                    debug!(self.log, "New finalized chain added to sync"; "peer_id" => format!("{:?}", peer_id), "start_slot" => local_finalized_slot, "end_slot" => remote_finalized_slot, "finalized_root" => format!("{}", remote_info.finalized_root));
 
                     self.chains.new_finalized_chain(
                         local_info.finalized_epoch,
                         remote_info.finalized_root,
                         remote_finalized_slot,
                         peer_id,
-                        self.sync_send.clone(),
+                        self.beacon_processor_send.clone(),
                     );
-                    self.chains.update_finalized(network);
+                    self.chains.update(network);
                     // update the global sync state
                     self.chains.update_sync_state();
                 }
@@ -223,15 +221,14 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                     debug!(self.log, "Creating a new syncing head chain"; "head_root" => format!("{}",remote_info.head_root), "start_epoch" => start_epoch, "head_slot" => remote_info.head_slot, "peer_id" => format!("{:?}", peer_id));
 
                     self.chains.new_head_chain(
-                        network,
                         start_epoch,
                         remote_info.head_root,
                         remote_info.head_slot,
                         peer_id,
-                        self.sync_send.clone(),
+                        self.beacon_processor_send.clone(),
                     );
                 }
-                self.chains.update_finalized(network);
+                self.chains.update(network);
                 self.chains.update_sync_state();
             }
         }
@@ -272,7 +269,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         &mut self,
         network: &mut SyncNetworkContext<T::EthSpec>,
         chain_id: ChainId,
-        batch_id: BatchId,
+        epoch: Epoch,
         downloaded_blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
         result: BatchProcessResult,
     ) {
@@ -280,19 +277,13 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         let mut downloaded_blocks = Some(downloaded_blocks);
 
         match self.chains.finalized_request(|chain| {
-            chain.on_batch_process_result(
-                network,
-                chain_id,
-                batch_id,
-                &mut downloaded_blocks,
-                &result,
-            )
+            chain.on_batch_process_result(network, chain_id, epoch, &mut downloaded_blocks, &result)
         }) {
             Some((index, ProcessingResult::RemoveChain)) => {
                 let chain = self.chains.remove_finalized_chain(index);
                 debug!(self.log, "Finalized chain removed"; "start_epoch" => chain.start_epoch, "end_slot" => chain.target_head_slot);
                 // update the state of the collection
-                self.chains.update_finalized(network);
+                self.chains.update(network);
 
                 // the chain is complete, re-status it's peers
                 chain.status_peers(network);
@@ -320,7 +311,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                     chain.on_batch_process_result(
                         network,
                         chain_id,
-                        batch_id,
+                        epoch,
                         &mut downloaded_blocks,
                         &result,
                     )
@@ -331,8 +322,12 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                         // the chain is complete, re-status it's peers and remove it
                         chain.status_peers(network);
 
+                        // Remove non-syncing head chains and re-status the peers
+                        // This removes a build-up of potentially duplicate head chains. Any
+                        // legitimate head chains will be re-established
+                        self.chains.clear_head_chains(network);
                         // update the state of the collection
-                        self.chains.update_finalized(network);
+                        self.chains.update(network);
                         // update the global state and log any change
                         self.chains.update_sync_state();
                     }
@@ -340,7 +335,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                     None => {
                         // This can happen if a chain gets purged due to being out of date whilst a
                         // batch process is in progress.
-                        debug!(self.log, "No chains match the block processing id"; "id" => *batch_id);
+                        debug!(self.log, "No chains match the block processing id"; "batch_epoch" => epoch, "chain_id" => chain_id);
                     }
                 }
             }
@@ -361,7 +356,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         self.remove_peer(network, peer_id);
 
         // update the state of the collection
-        self.chains.update_finalized(network);
+        self.chains.update(network);
         // update the global state and inform the user
         self.chains.update_sync_state();
     }

beacon_node/rest_api/Cargo.toml

@@ -34,7 +34,6 @@ hex = "0.4.2"
 parking_lot = "0.11.0"
 futures = "0.3.5"
 operation_pool = { path = "../operation_pool" }
-rayon = "1.3.0"
 environment = { path = "../../lighthouse/environment" }
 uhttp_sse = "0.5.1"
 bus = "2.2.3"

beacon_node/rest_api/src/advanced.rs

@@ -1,34 +0,0 @@
-use crate::response_builder::ResponseBuilder;
-use crate::ApiResult;
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use hyper::{Body, Request};
-use operation_pool::PersistedOperationPool;
-use std::sync::Arc;
-
-/// Returns the `proto_array` fork choice struct, encoded as JSON.
-///
-/// Useful for debugging or advanced inspection of the chain.
-pub fn get_fork_choice<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(
-        &*beacon_chain
-            .fork_choice
-            .read()
-            .proto_array()
-            .core_proto_array(),
-    )
-}
-
-/// Returns the `PersistedOperationPool` struct.
-///
-/// Useful for debugging or advanced inspection of the stored operations.
-pub fn get_operation_pool<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&PersistedOperationPool::from_operation_pool(
-        &beacon_chain.op_pool,
-    ))
-}

beacon_node/rest_api/src/beacon.rs

@@ -1,14 +1,13 @@
 use crate::helpers::*;
-use crate::response_builder::ResponseBuilder;
 use crate::validator::get_state_for_epoch;
-use crate::{ApiError, ApiResult, UrlQuery};
+use crate::Context;
+use crate::{ApiError, UrlQuery};
 use beacon_chain::{
     observed_operations::ObservationOutcome, BeaconChain, BeaconChainTypes, StateSkipConfig,
 };
-use bus::BusReader;
 use futures::executor::block_on;
 use hyper::body::Bytes;
-use hyper::{Body, Request, Response};
+use hyper::{Body, Request};
 use rest_types::{
     BlockResponse, CanonicalHeadResponse, Committee, HeadBeaconBlock, StateResponse,
     ValidatorRequest, ValidatorResponse,
@@ -16,20 +15,20 @@ use rest_types::{
 use std::io::Write;
 use std::sync::Arc;
 
-use slog::{error, Logger};
+use slog::error;
 use types::{
     AttesterSlashing, BeaconState, EthSpec, Hash256, ProposerSlashing, PublicKeyBytes,
     RelativeEpoch, SignedBeaconBlockHash, Slot,
 };
 
-/// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`.
+/// Returns a summary of the head of the beacon chain.
 pub fn get_head<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    ctx: Arc<Context<T>>,
+) -> Result<CanonicalHeadResponse, ApiError> {
+    let beacon_chain = &ctx.beacon_chain;
     let chain_head = beacon_chain.head()?;
 
-    let head = CanonicalHeadResponse {
+    Ok(CanonicalHeadResponse {
         slot: chain_head.beacon_state.slot,
         block_root: chain_head.beacon_block_root,
         state_root: chain_head.beacon_state_root,
@@ -51,33 +50,27 @@ pub fn get_head<T: BeaconChainTypes>(
             .epoch
             .start_slot(T::EthSpec::slots_per_epoch()),
         previous_justified_block_root: chain_head.beacon_state.previous_justified_checkpoint.root,
-    };
-
-    ResponseBuilder::new(&req)?.body(&head)
+    })
 }
 
-/// HTTP handler to return a list of head BeaconBlocks.
-pub fn get_heads<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let heads = beacon_chain
+/// Return the list of heads of the beacon chain.
+pub fn get_heads<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Vec<HeadBeaconBlock> {
+    ctx.beacon_chain
         .heads()
         .into_iter()
         .map(|(beacon_block_root, beacon_block_slot)| HeadBeaconBlock {
             beacon_block_root,
             beacon_block_slot,
         })
-        .collect::<Vec<_>>();
-
-    ResponseBuilder::new(&req)?.body(&heads)
+        .collect()
 }
 
 /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`.
 pub fn get_block<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<BlockResponse<T::EthSpec>, ApiError> {
+    let beacon_chain = &ctx.beacon_chain;
     let query_params = ["root", "slot"];
     let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?;
 
@@ -85,7 +78,7 @@ pub fn get_block<T: BeaconChainTypes>(
         ("slot", value) => {
             let target = parse_slot(&value)?;
 
-            block_root_at_slot(&beacon_chain, target)?.ok_or_else(|| {
+            block_root_at_slot(beacon_chain, target)?.ok_or_else(|| {
                 ApiError::NotFound(format!(
                     "Unable to find SignedBeaconBlock for slot {:?}",
                     target
@@ -103,30 +96,26 @@ pub fn get_block<T: BeaconChainTypes>(
         ))
     })?;
 
-    let response = BlockResponse {
+    Ok(BlockResponse {
         root: block_root,
         beacon_block: block,
-    };
-
-    ResponseBuilder::new(&req)?.body(&response)
+    })
 }
 
 /// HTTP handler to return a `SignedBeaconBlock` root at a given `slot`.
 pub fn get_block_root<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Hash256, ApiError> {
     let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?;
     let target = parse_slot(&slot_string)?;
 
-    let root = block_root_at_slot(&beacon_chain, target)?.ok_or_else(|| {
+    block_root_at_slot(&ctx.beacon_chain, target)?.ok_or_else(|| {
         ApiError::NotFound(format!(
             "Unable to find SignedBeaconBlock for slot {:?}",
             target
         ))
-    })?;
-
-    ResponseBuilder::new(&req)?.body(&root)
+    })
 }
 
 fn make_sse_response_chunk(new_head_hash: SignedBeaconBlockHash) -> std::io::Result<Bytes> {
@@ -140,43 +129,27 @@ fn make_sse_response_chunk(new_head_hash: SignedBeaconBlockHash) -> std::io::Res
     Ok(bytes)
 }
 
-pub fn stream_forks<T: BeaconChainTypes>(
-    log: Logger,
-    mut events: BusReader<SignedBeaconBlockHash>,
-) -> ApiResult {
+pub fn stream_forks<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Result<Body, ApiError> {
+    let mut events = ctx.events.lock().add_rx();
     let (mut sender, body) = Body::channel();
     std::thread::spawn(move || {
         while let Ok(new_head_hash) = events.recv() {
             let chunk = match make_sse_response_chunk(new_head_hash) {
                 Ok(chunk) => chunk,
                 Err(e) => {
-                    error!(log, "Failed to make SSE chunk"; "error" => e.to_string());
+                    error!(ctx.log, "Failed to make SSE chunk"; "error" => e.to_string());
                     sender.abort();
                     break;
                 }
             };
-            if let Err(bytes) = block_on(sender.send_data(chunk)) {
-                error!(log, "Couldn't stream piece {:?}", bytes);
+            match block_on(sender.send_data(chunk)) {
+                Err(e) if e.is_closed() => break,
+                Err(e) => error!(ctx.log, "Couldn't stream piece {:?}", e),
+                Ok(_) => (),
             }
         }
     });
-    let response = Response::builder()
-        .status(200)
-        .header("Content-Type", "text/event-stream")
-        .header("Connection", "Keep-Alive")
-        .header("Cache-Control", "no-cache")
-        .header("Access-Control-Allow-Origin", "*")
-        .body(body)
-        .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))?;
-    Ok(response)
-}
-
-/// HTTP handler to return the `Fork` of the current head.
-pub fn get_fork<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&beacon_chain.head()?.beacon_state.fork)
+    Ok(body)
 }
 
 /// HTTP handler to which accepts a query string of a list of validator pubkeys and maps it to a
@@ -185,9 +158,9 @@ pub fn get_fork<T: BeaconChainTypes>(
 /// This method is limited to as many `pubkeys` that can fit in a URL. See `post_validators` for
 /// doing bulk requests.
 pub fn get_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let validator_pubkeys = query
@@ -202,17 +175,14 @@ pub fn get_validators<T: BeaconChainTypes>(
         None
     };
 
-    let validators =
-        validator_responses_by_pubkey(beacon_chain, state_root_opt, validator_pubkeys)?;
-
-    ResponseBuilder::new(&req)?.body(&validators)
+    validator_responses_by_pubkey(&ctx.beacon_chain, state_root_opt, validator_pubkeys)
 }
 
 /// HTTP handler to return all validators, each as a `ValidatorResponse`.
 pub fn get_all_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let state_root_opt = if let Some((_key, value)) = query.first_of_opt(&["state_root"]) {
@@ -221,23 +191,21 @@ pub fn get_all_validators<T: BeaconChainTypes>(
         None
     };
 
-    let mut state = get_state_from_root_opt(&beacon_chain, state_root_opt)?;
+    let mut state = get_state_from_root_opt(&ctx.beacon_chain, state_root_opt)?;
     state.update_pubkey_cache()?;
 
-    let validators = state
+    state
         .validators
         .iter()
         .map(|validator| validator_response_by_pubkey(&state, validator.pubkey.clone()))
-        .collect::<Result<Vec<_>, _>>()?;
-
-    ResponseBuilder::new(&req)?.body(&validators)
+        .collect::<Result<Vec<_>, _>>()
 }
 
 /// HTTP handler to return all active validators, each as a `ValidatorResponse`.
 pub fn get_active_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let state_root_opt = if let Some((_key, value)) = query.first_of_opt(&["state_root"]) {
@@ -246,17 +214,15 @@ pub fn get_active_validators<T: BeaconChainTypes>(
         None
     };
 
-    let mut state = get_state_from_root_opt(&beacon_chain, state_root_opt)?;
+    let mut state = get_state_from_root_opt(&ctx.beacon_chain, state_root_opt)?;
     state.update_pubkey_cache()?;
 
-    let validators = state
+    state
         .validators
         .iter()
         .filter(|validator| validator.is_active_at(state.current_epoch()))
         .map(|validator| validator_response_by_pubkey(&state, validator.pubkey.clone()))
-        .collect::<Result<Vec<_>, _>>()?;
-
-    ResponseBuilder::new(&req)?.body(&validators)
+        .collect::<Result<Vec<_>, _>>()
 }
 
 /// HTTP handler to which accepts a `ValidatorRequest` and returns a `ValidatorResponse` for
@@ -264,17 +230,11 @@ pub fn get_active_validators<T: BeaconChainTypes>(
 ///
 /// This method allows for a basically unbounded list of `pubkeys`, where as the `get_validators`
 /// request is limited by the max number of pubkeys you can fit in a URL.
-pub async fn post_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
-    let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-    serde_json::from_slice::<ValidatorRequest>(&chunks)
+pub fn post_validators<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
+    serde_json::from_slice::<ValidatorRequest>(&req.into_body())
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorRequest: {:?}",
@@ -283,12 +243,11 @@ pub async fn post_validators<T: BeaconChainTypes>(
         })
         .and_then(|bulk_request| {
             validator_responses_by_pubkey(
-                beacon_chain,
+                &ctx.beacon_chain,
                 bulk_request.state_root,
                 bulk_request.pubkeys,
             )
         })
-        .and_then(|validators| response_builder?.body(&validators))
 }
 
 /// Returns either the state given by `state_root_opt`, or the canonical head state if it is
@@ -315,11 +274,11 @@ fn get_state_from_root_opt<T: BeaconChainTypes>(
 /// Maps a vec of `validator_pubkey` to a vec of `ValidatorResponse`, using the state at the given
 /// `state_root`. If `state_root.is_none()`, uses the canonial head state.
 fn validator_responses_by_pubkey<T: BeaconChainTypes>(
-    beacon_chain: Arc<BeaconChain<T>>,
+    beacon_chain: &BeaconChain<T>,
     state_root_opt: Option<Hash256>,
     validator_pubkeys: Vec<PublicKeyBytes>,
 ) -> Result<Vec<ValidatorResponse>, ApiError> {
-    let mut state = get_state_from_root_opt(&beacon_chain, state_root_opt)?;
+    let mut state = get_state_from_root_opt(beacon_chain, state_root_opt)?;
     state.update_pubkey_cache()?;
 
     validator_pubkeys
@@ -370,24 +329,25 @@ fn validator_response_by_pubkey<E: EthSpec>(
 
 /// HTTP handler
 pub fn get_committees<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<Committee>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
 
-    let mut state = get_state_for_epoch(&beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
+    let mut state =
+        get_state_for_epoch(&ctx.beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
 
     let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), epoch).map_err(|e| {
         ApiError::ServerError(format!("Failed to get state suitable for epoch: {:?}", e))
     })?;
 
     state
-        .build_committee_cache(relative_epoch, &beacon_chain.spec)
+        .build_committee_cache(relative_epoch, &ctx.beacon_chain.spec)
         .map_err(|e| ApiError::ServerError(format!("Unable to build committee cache: {:?}", e)))?;
 
-    let committees = state
+    Ok(state
         .get_beacon_committees_at_epoch(relative_epoch)
         .map_err(|e| ApiError::ServerError(format!("Unable to get all committees: {:?}", e)))?
         .into_iter()
@@ -396,9 +356,7 @@ pub fn get_committees<T: BeaconChainTypes>(
             index: c.index,
             committee: c.committee.to_vec(),
         })
-        .collect::<Vec<_>>();
-
-    ResponseBuilder::new(&req)?.body(&committees)
+        .collect::<Vec<_>>())
 }
 
 /// HTTP handler to return a `BeaconState` at a given `root` or `slot`.
@@ -406,10 +364,10 @@ pub fn get_committees<T: BeaconChainTypes>(
 /// Will not return a state if the request slot is in the future. Will return states higher than
 /// the current head by skipping slots.
 pub fn get_state<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let head_state = beacon_chain.head()?.beacon_state;
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<StateResponse<T::EthSpec>, ApiError> {
+    let head_state = ctx.beacon_chain.head()?.beacon_state;
 
     let (key, value) = match UrlQuery::from_request(&req) {
         Ok(query) => {
@@ -427,11 +385,12 @@ pub fn get_state<T: BeaconChainTypes>(
     };
 
     let (root, state): (Hash256, BeaconState<T::EthSpec>) = match (key.as_ref(), value) {
-        ("slot", value) => state_at_slot(&beacon_chain, parse_slot(&value)?)?,
+        ("slot", value) => state_at_slot(&ctx.beacon_chain, parse_slot(&value)?)?,
         ("root", value) => {
             let root = &parse_root(&value)?;
 
-            let state = beacon_chain
+            let state = ctx
+                .beacon_chain
                 .store
                 .get_state(root, None)?
                 .ok_or_else(|| ApiError::NotFound(format!("No state for root: {:?}", root)))?;
@@ -441,12 +400,10 @@ pub fn get_state<T: BeaconChainTypes>(
         _ => return Err(ApiError::ServerError("Unexpected query parameter".into())),
     };
 
-    let response = StateResponse {
+    Ok(StateResponse {
         root,
         beacon_state: state,
-    };
-
-    ResponseBuilder::new(&req)?.body(&response)
+    })
 }
 
 /// HTTP handler to return a `BeaconState` root at a given `slot`.
@@ -454,15 +411,13 @@ pub fn get_state<T: BeaconChainTypes>(
 /// Will not return a state if the request slot is in the future. Will return states higher than
 /// the current head by skipping slots.
 pub fn get_state_root<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Hash256, ApiError> {
     let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?;
     let slot = parse_slot(&slot_string)?;
 
-    let root = state_root_at_slot(&beacon_chain, slot, StateSkipConfig::WithStateRoots)?;
-
-    ResponseBuilder::new(&req)?.body(&root)
+    state_root_at_slot(&ctx.beacon_chain, slot, StateSkipConfig::WithStateRoots)
 }
 
 /// HTTP handler to return a `BeaconState` at the genesis block.
@@ -470,50 +425,28 @@ pub fn get_state_root<T: BeaconChainTypes>(
 /// This is an undocumented convenience method used during testing. For production, simply do a
 /// state request at slot 0.
 pub fn get_genesis_state<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?;
-
-    ResponseBuilder::new(&req)?.body(&state)
+    ctx: Arc<Context<T>>,
+) -> Result<BeaconState<T::EthSpec>, ApiError> {
+    state_at_slot(&ctx.beacon_chain, Slot::new(0)).map(|(_root, state)| state)
 }
 
-/// Read the genesis time from the current beacon chain state.
-pub fn get_genesis_time<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&beacon_chain.head_info()?.genesis_time)
-}
-
-/// Read the `genesis_validators_root` from the current beacon chain state.
-pub fn get_genesis_validators_root<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&beacon_chain.head_info()?.genesis_validators_root)
-}
-
-pub async fn proposer_slashing<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn proposer_slashing<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<bool, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice::<ProposerSlashing>(&chunks)
+    serde_json::from_slice::<ProposerSlashing>(&body)
         .map_err(|e| format!("Unable to parse JSON into ProposerSlashing: {:?}", e))
         .and_then(move |proposer_slashing| {
-            if beacon_chain.eth1_chain.is_some() {
-                let obs_outcome = beacon_chain
+            if ctx.beacon_chain.eth1_chain.is_some() {
+                let obs_outcome = ctx
+                    .beacon_chain
                     .verify_proposer_slashing_for_gossip(proposer_slashing)
                     .map_err(|e| format!("Error while verifying proposer slashing: {:?}", e))?;
                 if let ObservationOutcome::New(verified_proposer_slashing) = obs_outcome {
-                    beacon_chain.import_proposer_slashing(verified_proposer_slashing);
+                    ctx.beacon_chain
+                        .import_proposer_slashing(verified_proposer_slashing);
                     Ok(())
                 } else {
                     Err("Proposer slashing for that validator index already known".into())
@@ -522,22 +455,17 @@ pub async fn proposer_slashing<T: BeaconChainTypes>(
                 Err("Cannot insert proposer slashing on node without Eth1 connection.".to_string())
             }
         })
-        .map_err(ApiError::BadRequest)
-        .and_then(|_| response_builder?.body(&true))
+        .map_err(ApiError::BadRequest)?;
+
+    Ok(true)
 }
 
-pub async fn attester_slashing<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn attester_slashing<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<bool, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-
-    serde_json::from_slice::<AttesterSlashing<T::EthSpec>>(&chunks)
+    serde_json::from_slice::<AttesterSlashing<T::EthSpec>>(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into AttesterSlashing: {:?}",
@@ -545,13 +473,13 @@ pub async fn attester_slashing<T: BeaconChainTypes>(
             ))
         })
         .and_then(move |attester_slashing| {
-            if beacon_chain.eth1_chain.is_some() {
-                beacon_chain
+            if ctx.beacon_chain.eth1_chain.is_some() {
+                ctx.beacon_chain
                     .verify_attester_slashing_for_gossip(attester_slashing)
                     .map_err(|e| format!("Error while verifying attester slashing: {:?}", e))
                     .and_then(|outcome| {
                         if let ObservationOutcome::New(verified_attester_slashing) = outcome {
-                            beacon_chain
+                            ctx.beacon_chain
                                 .import_attester_slashing(verified_attester_slashing)
                                 .map_err(|e| {
                                     format!("Error while importing attester slashing: {:?}", e)
@@ -566,6 +494,7 @@ pub async fn attester_slashing<T: BeaconChainTypes>(
                     "Cannot insert attester slashing on node without Eth1 connection.".to_string(),
                 ))
             }
-        })
-        .and_then(|_| response_builder?.body(&true))
+        })?;
+
+    Ok(true)
 }

beacon_node/rest_api/src/consensus.rs

@@ -1,8 +1,7 @@
 use crate::helpers::*;
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult, UrlQuery};
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use hyper::{Body, Request};
+use crate::{ApiError, Context, UrlQuery};
+use beacon_chain::BeaconChainTypes;
+use hyper::Request;
 use rest_types::{IndividualVotesRequest, IndividualVotesResponse};
 use serde::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -50,38 +49,31 @@ impl Into<VoteCount> for TotalBalances {
 
 /// HTTP handler return a `VoteCount` for some given `Epoch`.
 pub fn get_vote_count<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<VoteCount, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
     // This is the last slot of the given epoch (one prior to the first slot of the next epoch).
     let target_slot = (epoch + 1).start_slot(T::EthSpec::slots_per_epoch()) - 1;
 
-    let (_root, state) = state_at_slot(&beacon_chain, target_slot)?;
-    let spec = &beacon_chain.spec;
+    let (_root, state) = state_at_slot(&ctx.beacon_chain, target_slot)?;
+    let spec = &ctx.beacon_chain.spec;
 
     let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
     validator_statuses.process_attestations(&state, spec)?;
 
-    let report: VoteCount = validator_statuses.total_balances.into();
-
-    ResponseBuilder::new(&req)?.body(&report)
+    Ok(validator_statuses.total_balances.into())
 }
 
-pub async fn post_individual_votes<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn post_individual_votes<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<IndividualVotesResponse>, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice::<IndividualVotesRequest>(&chunks)
+    serde_json::from_slice::<IndividualVotesRequest>(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorDutiesRequest: {:?}",
@@ -94,8 +86,8 @@ pub async fn post_individual_votes<T: BeaconChainTypes>(
             // This is the last slot of the given epoch (one prior to the first slot of the next epoch).
             let target_slot = (epoch + 1).start_slot(T::EthSpec::slots_per_epoch()) - 1;
 
-            let (_root, mut state) = state_at_slot(&beacon_chain, target_slot)?;
-            let spec = &beacon_chain.spec;
+            let (_root, mut state) = state_at_slot(&ctx.beacon_chain, target_slot)?;
+            let spec = &ctx.beacon_chain.spec;
 
             let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
             validator_statuses.process_attestations(&state, spec)?;
@@ -135,5 +127,4 @@ pub async fn post_individual_votes<T: BeaconChainTypes>(
                 })
                 .collect::<Result<Vec<_>, _>>()
         })
-        .and_then(|votes| response_builder?.body_no_ssz(&votes))
 }

beacon_node/rest_api/src/helpers.rs

@@ -1,9 +1,7 @@
-use crate::{ApiError, ApiResult, NetworkChannel};
+use crate::{ApiError, NetworkChannel};
 use beacon_chain::{BeaconChain, BeaconChainTypes, StateSkipConfig};
 use bls::PublicKeyBytes;
 use eth2_libp2p::PubsubMessage;
-use http::header;
-use hyper::{Body, Request};
 use itertools::process_results;
 use network::NetworkMessage;
 use ssz::Decode;
@@ -41,21 +39,6 @@ pub fn parse_committee_index(string: &str) -> Result<CommitteeIndex, ApiError> {
         .map_err(|e| ApiError::BadRequest(format!("Unable to parse committee index: {:?}", e)))
 }
 
-/// Checks the provided request to ensure that the `content-type` header.
-///
-/// The content-type header should either be omitted, in which case JSON is assumed, or it should
-/// explicitly specify `application/json`. If anything else is provided, an error is returned.
-pub fn check_content_type_for_json(req: &Request<Body>) -> Result<(), ApiError> {
-    match req.headers().get(header::CONTENT_TYPE) {
-        Some(h) if h == "application/json" => Ok(()),
-        Some(h) => Err(ApiError::BadRequest(format!(
-            "The provided content-type {:?} is not available, this endpoint only supports json.",
-            h
-        ))),
-        _ => Ok(()),
-    }
-}
-
 /// Parse an SSZ object from some hex-encoded bytes.
 ///
 /// E.g., A signature is `"0x0000000000000000000000000000000000000000000000000000000000000000"`
@@ -228,14 +211,8 @@ pub fn state_root_at_slot<T: BeaconChainTypes>(
     }
 }
 
-pub fn implementation_pending_response(_req: Request<Body>) -> ApiResult {
-    Err(ApiError::NotImplemented(
-        "API endpoint has not yet been implemented, but is planned to be soon.".to_owned(),
-    ))
-}
-
 pub fn publish_beacon_block_to_network<T: BeaconChainTypes + 'static>(
-    chan: NetworkChannel<T::EthSpec>,
+    chan: &NetworkChannel<T::EthSpec>,
     block: SignedBeaconBlock<T::EthSpec>,
 ) -> Result<(), ApiError> {
     // send the block via SSZ encoding

beacon_node/rest_api/src/lib.rs

@@ -1,22 +1,15 @@
 #[macro_use]
-mod macros;
-#[macro_use]
 extern crate lazy_static;
+mod router;
 extern crate network as client_network;
 
-mod advanced;
 mod beacon;
 pub mod config;
 mod consensus;
-mod error;
 mod helpers;
 mod lighthouse;
 mod metrics;
-mod network;
 mod node;
-mod response_builder;
-mod router;
-mod spec;
 mod url_query;
 mod validator;
 
@@ -24,7 +17,6 @@ use beacon_chain::{BeaconChain, BeaconChainTypes};
 use bus::Bus;
 use client_network::NetworkMessage;
 pub use config::ApiEncodingFormat;
-use error::{ApiError, ApiResult};
 use eth2_config::Eth2Config;
 use eth2_libp2p::NetworkGlobals;
 use futures::future::TryFutureExt;
@@ -32,6 +24,7 @@ use hyper::server::conn::AddrStream;
 use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Request, Server};
 use parking_lot::Mutex;
+use rest_types::ApiError;
 use slog::{info, warn};
 use std::net::SocketAddr;
 use std::path::PathBuf;
@@ -42,6 +35,7 @@ use url_query::UrlQuery;
 
 pub use crate::helpers::parse_pubkey_bytes;
 pub use config::Config;
+pub use router::Context;
 
 pub type NetworkChannel<T> = mpsc::UnboundedSender<NetworkMessage<T>>;
 
@@ -63,36 +57,28 @@ pub fn start_server<T: BeaconChainTypes>(
     events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
 ) -> Result<SocketAddr, hyper::Error> {
     let log = executor.log();
-    let inner_log = log.clone();
-    let rest_api_config = Arc::new(config.clone());
     let eth2_config = Arc::new(eth2_config);
 
+    let context = Arc::new(Context {
+        executor: executor.clone(),
+        config: config.clone(),
+        beacon_chain,
+        network_globals: network_info.network_globals.clone(),
+        network_chan: network_info.network_chan,
+        eth2_config,
+        log: log.clone(),
+        db_path,
+        freezer_db_path,
+        events,
+    });
+
     // Define the function that will build the request handler.
     let make_service = make_service_fn(move |_socket: &AddrStream| {
-        let beacon_chain = beacon_chain.clone();
-        let log = inner_log.clone();
-        let rest_api_config = rest_api_config.clone();
-        let eth2_config = eth2_config.clone();
-        let network_globals = network_info.network_globals.clone();
-        let network_channel = network_info.network_chan.clone();
-        let db_path = db_path.clone();
-        let freezer_db_path = freezer_db_path.clone();
-        let events = events.clone();
+        let ctx = context.clone();
 
         async move {
             Ok::<_, hyper::Error>(service_fn(move |req: Request<Body>| {
-                router::route(
-                    req,
-                    beacon_chain.clone(),
-                    network_globals.clone(),
-                    network_channel.clone(),
-                    rest_api_config.clone(),
-                    eth2_config.clone(),
-                    log.clone(),
-                    db_path.clone(),
-                    freezer_db_path.clone(),
-                    events.clone(),
-                )
+                router::on_http_request(req, ctx.clone())
             }))
         }
     });

beacon_node/rest_api/src/lighthouse.rs

@@ -1,24 +1,16 @@
 //! This contains a collection of lighthouse specific HTTP endpoints.
 
-use crate::response_builder::ResponseBuilder;
-use crate::ApiResult;
-use eth2_libp2p::{NetworkGlobals, PeerInfo};
-use hyper::{Body, Request};
+use crate::{ApiError, Context};
+use beacon_chain::BeaconChainTypes;
+use eth2_libp2p::PeerInfo;
 use serde::Serialize;
 use std::sync::Arc;
 use types::EthSpec;
 
-/// The syncing state of the beacon node.
-pub fn syncing<T: EthSpec>(
-    req: Request<Body>,
-    network_globals: Arc<NetworkGlobals<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&network_globals.sync_state())
-}
-
 /// Returns all known peers and corresponding information
-pub fn peers<T: EthSpec>(req: Request<Body>, network_globals: Arc<NetworkGlobals<T>>) -> ApiResult {
-    let peers: Vec<Peer<T>> = network_globals
+pub fn peers<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Result<Vec<Peer<T::EthSpec>>, ApiError> {
+    Ok(ctx
+        .network_globals
         .peers
         .read()
         .peers()
@@ -26,16 +18,15 @@ pub fn peers<T: EthSpec>(req: Request<Body>, network_globals: Arc<NetworkGlobals
             peer_id: peer_id.to_string(),
             peer_info: peer_info.clone(),
         })
-        .collect();
-    ResponseBuilder::new(&req)?.body_no_ssz(&peers)
+        .collect())
 }
 
 /// Returns all known connected peers and their corresponding information
-pub fn connected_peers<T: EthSpec>(
-    req: Request<Body>,
-    network_globals: Arc<NetworkGlobals<T>>,
-) -> ApiResult {
-    let peers: Vec<Peer<T>> = network_globals
+pub fn connected_peers<T: BeaconChainTypes>(
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<Peer<T::EthSpec>>, ApiError> {
+    Ok(ctx
+        .network_globals
         .peers
         .read()
         .connected_peers()
@@ -43,14 +34,13 @@ pub fn connected_peers<T: EthSpec>(
             peer_id: peer_id.to_string(),
             peer_info: peer_info.clone(),
         })
-        .collect();
-    ResponseBuilder::new(&req)?.body_no_ssz(&peers)
+        .collect())
 }
 
 /// Information returned by `peers` and `connected_peers`.
 #[derive(Clone, Debug, Serialize)]
 #[serde(bound = "T: EthSpec")]
-struct Peer<T: EthSpec> {
+pub struct Peer<T: EthSpec> {
     /// The Peer's ID
     peer_id: String,
     /// The PeerInfo associated with the peer.

beacon_node/rest_api/src/macros.rs

@@ -1,11 +0,0 @@
-macro_rules! try_future {
-    ($expr:expr) => {
-        match $expr {
-            core::result::Result::Ok(val) => val,
-            core::result::Result::Err(err) => return Err(std::convert::From::from(err)),
-        }
-    };
-    ($expr:expr,) => {
-        $crate::try_future!($expr)
-    };
-}

beacon_node/rest_api/src/metrics.rs

@@ -1,42 +1,38 @@
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult};
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use hyper::{Body, Request};
+use crate::{ApiError, Context};
+use beacon_chain::BeaconChainTypes;
 use lighthouse_metrics::{Encoder, TextEncoder};
 use rest_types::Health;
-use std::path::PathBuf;
 use std::sync::Arc;
 
 pub use lighthouse_metrics::*;
 
 lazy_static! {
+    pub static ref BEACON_HTTP_API_REQUESTS_TOTAL: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "beacon_http_api_requests_total",
+            "Count of HTTP requests received",
+            &["endpoint"]
+        );
+    pub static ref BEACON_HTTP_API_SUCCESS_TOTAL: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "beacon_http_api_success_total",
+            "Count of HTTP requests that returned 200 OK",
+            &["endpoint"]
+        );
+    pub static ref BEACON_HTTP_API_ERROR_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_http_api_error_total",
+        "Count of HTTP that did not return 200 OK",
+        &["endpoint"]
+    );
+    pub static ref BEACON_HTTP_API_TIMES_TOTAL: Result<HistogramVec> = try_create_histogram_vec(
+        "beacon_http_api_times_total",
+        "Duration to process HTTP requests",
+        &["endpoint"]
+    );
     pub static ref REQUEST_RESPONSE_TIME: Result<Histogram> = try_create_histogram(
         "http_server_request_duration_seconds",
         "Time taken to build a response to a HTTP request"
     );
-    pub static ref REQUEST_COUNT: Result<IntCounter> = try_create_int_counter(
-        "http_server_request_total",
-        "Total count of HTTP requests received"
-    );
-    pub static ref SUCCESS_COUNT: Result<IntCounter> = try_create_int_counter(
-        "http_server_success_total",
-        "Total count of HTTP 200 responses sent"
-    );
-    pub static ref VALIDATOR_GET_BLOCK_REQUEST_RESPONSE_TIME: Result<Histogram> =
-        try_create_histogram(
-            "http_server_validator_block_get_request_duration_seconds",
-            "Time taken to respond to GET /validator/block"
-        );
-    pub static ref VALIDATOR_GET_ATTESTATION_REQUEST_RESPONSE_TIME: Result<Histogram> =
-        try_create_histogram(
-            "http_server_validator_attestation_get_request_duration_seconds",
-            "Time taken to respond to GET /validator/attestation"
-        );
-    pub static ref VALIDATOR_GET_DUTIES_REQUEST_RESPONSE_TIME: Result<Histogram> =
-        try_create_histogram(
-            "http_server_validator_duties_get_request_duration_seconds",
-            "Time taken to respond to GET /validator/duties"
-        );
     pub static ref PROCESS_NUM_THREADS: Result<IntGauge> = try_create_int_gauge(
         "process_num_threads",
         "Number of threads used by the current process"
@@ -77,11 +73,8 @@ lazy_static! {
 ///
 /// This is a HTTP handler method.
 pub fn get_prometheus<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    db_path: PathBuf,
-    freezer_db_path: PathBuf,
-) -> ApiResult {
+    ctx: Arc<Context<T>>,
+) -> std::result::Result<String, ApiError> {
     let mut buffer = vec![];
     let encoder = TextEncoder::new();
 
@@ -101,9 +94,9 @@ pub fn get_prometheus<T: BeaconChainTypes>(
     // using `lighthouse_metrics::gather(..)` to collect the global `DEFAULT_REGISTRY` metrics into
     // a string that can be returned via HTTP.
 
-    slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&beacon_chain.slot_clock);
-    store::scrape_for_metrics(&db_path, &freezer_db_path);
-    beacon_chain::scrape_for_metrics(&beacon_chain);
+    slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&ctx.beacon_chain.slot_clock);
+    store::scrape_for_metrics(&ctx.db_path, &ctx.freezer_db_path);
+    beacon_chain::scrape_for_metrics(&ctx.beacon_chain);
     eth2_libp2p::scrape_discovery_metrics();
 
     // This will silently fail if we are unable to observe the health. This is desired behaviour
@@ -133,6 +126,5 @@ pub fn get_prometheus<T: BeaconChainTypes>(
         .unwrap();
 
     String::from_utf8(buffer)
-        .map(|string| ResponseBuilder::new(&req)?.body_text(string))
-        .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e)))?
+        .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e)))
 }

beacon_node/rest_api/src/network.rs

@@ -1,72 +0,0 @@
-use crate::error::ApiResult;
-use crate::response_builder::ResponseBuilder;
-use crate::NetworkGlobals;
-use beacon_chain::BeaconChainTypes;
-use eth2_libp2p::{Multiaddr, PeerId};
-use hyper::{Body, Request};
-use std::sync::Arc;
-
-/// HTTP handler to return the list of libp2p multiaddr the client is listening on.
-///
-/// Returns a list of `Multiaddr`, serialized according to their `serde` impl.
-pub fn get_listen_addresses<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    let multiaddresses: Vec<Multiaddr> = network.listen_multiaddrs();
-    ResponseBuilder::new(&req)?.body_no_ssz(&multiaddresses)
-}
-
-/// HTTP handler to return the network port the client is listening on.
-///
-/// Returns the TCP port number in its plain form (which is also valid JSON serialization)
-pub fn get_listen_port<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&network.listen_port_tcp())
-}
-
-/// HTTP handler to return the Discv5 ENR from the client's libp2p service.
-///
-/// ENR is encoded as base64 string.
-pub fn get_enr<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&network.local_enr().to_base64())
-}
-
-/// HTTP handler to return the `PeerId` from the client's libp2p service.
-///
-/// PeerId is encoded as base58 string.
-pub fn get_peer_id<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&network.local_peer_id().to_base58())
-}
-
-/// HTTP handler to return the number of peers connected in the client's libp2p service.
-pub fn get_peer_count<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&network.connected_peers())
-}
-
-/// HTTP handler to return the list of peers connected to the client's libp2p service.
-///
-/// Peers are presented as a list of `PeerId::to_string()`.
-pub fn get_peer_list<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    let connected_peers: Vec<String> = network
-        .peers
-        .read()
-        .connected_peer_ids()
-        .map(PeerId::to_string)
-        .collect();
-    ResponseBuilder::new(&req)?.body_no_ssz(&connected_peers)
-}

beacon_node/rest_api/src/node.rs

@@ -1,23 +1,19 @@
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult};
-use eth2_libp2p::{types::SyncState, NetworkGlobals};
-use hyper::{Body, Request};
-use lighthouse_version::version_with_platform;
-use rest_types::{Health, SyncingResponse, SyncingStatus};
+use crate::{ApiError, Context};
+use beacon_chain::BeaconChainTypes;
+use eth2_libp2p::types::SyncState;
+use rest_types::{SyncingResponse, SyncingStatus};
 use std::sync::Arc;
-use types::{EthSpec, Slot};
+use types::Slot;
 
-/// Read the version string from the current Lighthouse build.
-pub fn get_version(req: Request<Body>) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&version_with_platform())
-}
+/// Returns a syncing status.
+pub fn syncing<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Result<SyncingResponse, ApiError> {
+    let current_slot = ctx
+        .beacon_chain
+        .head_info()
+        .map_err(|e| ApiError::ServerError(format!("Unable to read head slot: {:?}", e)))?
+        .slot;
 
-pub fn syncing<T: EthSpec>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T>>,
-    current_slot: Slot,
-) -> ApiResult {
-    let (starting_slot, highest_slot) = match network.sync_state() {
+    let (starting_slot, highest_slot) = match ctx.network_globals.sync_state() {
         SyncState::SyncingFinalized {
             start_slot,
             head_slot,
@@ -36,14 +32,8 @@ pub fn syncing<T: EthSpec>(
         highest_slot,
     };
 
-    ResponseBuilder::new(&req)?.body(&SyncingResponse {
-        is_syncing: network.is_syncing(),
+    Ok(SyncingResponse {
+        is_syncing: ctx.network_globals.is_syncing(),
         sync_status,
     })
 }
-
-pub fn get_health(req: Request<Body>) -> ApiResult {
-    let health = Health::observe().map_err(ApiError::ServerError)?;
-
-    ResponseBuilder::new(&req)?.body_no_ssz(&health)
-}

beacon_node/rest_api/src/response_builder.rs

@@ -1,83 +0,0 @@
-use super::{ApiError, ApiResult};
-use crate::config::ApiEncodingFormat;
-use hyper::header;
-use hyper::{Body, Request, Response, StatusCode};
-use serde::Serialize;
-use ssz::Encode;
-
-pub struct ResponseBuilder {
-    encoding: ApiEncodingFormat,
-}
-
-impl ResponseBuilder {
-    pub fn new(req: &Request<Body>) -> Result<Self, ApiError> {
-        let accept_header: String = req
-            .headers()
-            .get(header::ACCEPT)
-            .map_or(Ok(""), |h| h.to_str())
-            .map_err(|e| {
-                ApiError::BadRequest(format!(
-                    "The Accept header contains invalid characters: {:?}",
-                    e
-                ))
-            })
-            .map(String::from)?;
-
-        // JSON is our default encoding, unless something else is requested.
-        let encoding = ApiEncodingFormat::from(accept_header.as_str());
-        Ok(Self { encoding })
-    }
-
-    pub fn body<T: Serialize + Encode>(self, item: &T) -> ApiResult {
-        match self.encoding {
-            ApiEncodingFormat::SSZ => Response::builder()
-                .status(StatusCode::OK)
-                .header("content-type", "application/ssz")
-                .body(Body::from(item.as_ssz_bytes()))
-                .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))),
-            _ => self.body_no_ssz(item),
-        }
-    }
-
-    pub fn body_no_ssz<T: Serialize>(self, item: &T) -> ApiResult {
-        let (body, content_type) = match self.encoding {
-            ApiEncodingFormat::JSON => (
-                Body::from(serde_json::to_string(&item).map_err(|e| {
-                    ApiError::ServerError(format!(
-                        "Unable to serialize response body as JSON: {:?}",
-                        e
-                    ))
-                })?),
-                "application/json",
-            ),
-            ApiEncodingFormat::SSZ => {
-                return Err(ApiError::UnsupportedType(
-                    "Response cannot be encoded as SSZ.".into(),
-                ));
-            }
-            ApiEncodingFormat::YAML => (
-                Body::from(serde_yaml::to_string(&item).map_err(|e| {
-                    ApiError::ServerError(format!(
-                        "Unable to serialize response body as YAML: {:?}",
-                        e
-                    ))
-                })?),
-                "application/yaml",
-            ),
-        };
-
-        Response::builder()
-            .status(StatusCode::OK)
-            .header("content-type", content_type)
-            .body(body)
-            .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))
-    }
-
-    pub fn body_text(self, text: String) -> ApiResult {
-        Response::builder()
-            .status(StatusCode::OK)
-            .header("content-type", "text/plain; charset=utf-8")
-            .body(Body::from(text))
-            .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))
-    }
-}

beacon_node/rest_api/src/router.rs

@@ -1,241 +1,322 @@
 use crate::{
-    advanced, beacon, config::Config, consensus, error::ApiError, helpers, lighthouse, metrics,
-    network, node, spec, validator, NetworkChannel,
+    beacon, config::Config, consensus, lighthouse, metrics, node, validator, NetworkChannel,
 };
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use bus::Bus;
+use environment::TaskExecutor;
 use eth2_config::Eth2Config;
-use eth2_libp2p::NetworkGlobals;
+use eth2_libp2p::{NetworkGlobals, PeerId};
 use hyper::header::HeaderValue;
 use hyper::{Body, Method, Request, Response};
+use lighthouse_version::version_with_platform;
+use operation_pool::PersistedOperationPool;
 use parking_lot::Mutex;
+use rest_types::{ApiError, Handler, Health};
 use slog::debug;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
-use types::{SignedBeaconBlockHash, Slot};
+use types::{EthSpec, SignedBeaconBlockHash};
 
-// Allowing more than 7 arguments.
-#[allow(clippy::too_many_arguments)]
-pub async fn route<T: BeaconChainTypes>(
+pub struct Context<T: BeaconChainTypes> {
+    pub executor: TaskExecutor,
+    pub config: Config,
+    pub beacon_chain: Arc<BeaconChain<T>>,
+    pub network_globals: Arc<NetworkGlobals<T::EthSpec>>,
+    pub network_chan: NetworkChannel<T::EthSpec>,
+    pub eth2_config: Arc<Eth2Config>,
+    pub log: slog::Logger,
+    pub db_path: PathBuf,
+    pub freezer_db_path: PathBuf,
+    pub events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
+}
+
+pub async fn on_http_request<T: BeaconChainTypes>(
     req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    network_globals: Arc<NetworkGlobals<T::EthSpec>>,
-    network_channel: NetworkChannel<T::EthSpec>,
-    rest_api_config: Arc<Config>,
-    eth2_config: Arc<Eth2Config>,
-    local_log: slog::Logger,
-    db_path: PathBuf,
-    freezer_db_path: PathBuf,
-    events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
+    ctx: Arc<Context<T>>,
 ) -> Result<Response<Body>, ApiError> {
-    metrics::inc_counter(&metrics::REQUEST_COUNT);
-    let received_instant = Instant::now();
-
     let path = req.uri().path().to_string();
 
-    let log = local_log.clone();
-    let result = {
-        let _timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME);
+    let _timer = metrics::start_timer_vec(&metrics::BEACON_HTTP_API_TIMES_TOTAL, &[&path]);
+    metrics::inc_counter_vec(&metrics::BEACON_HTTP_API_REQUESTS_TOTAL, &[&path]);
 
-        match (req.method(), path.as_ref()) {
-            // Methods for Client
-            (&Method::GET, "/node/health") => node::get_health(req),
-            (&Method::GET, "/node/version") => node::get_version(req),
-            (&Method::GET, "/node/syncing") => {
-                // inform the current slot, or set to 0
-                let current_slot = beacon_chain
-                    .head_info()
-                    .map(|info| info.slot)
-                    .unwrap_or_else(|_| Slot::from(0u64));
+    let received_instant = Instant::now();
+    let log = ctx.log.clone();
+    let allow_origin = ctx.config.allow_origin.clone();
 
-                node::syncing::<T::EthSpec>(req, network_globals, current_slot)
-            }
-
-            // Methods for Network
-            (&Method::GET, "/network/enr") => network::get_enr::<T>(req, network_globals),
-            (&Method::GET, "/network/peer_count") => {
-                network::get_peer_count::<T>(req, network_globals)
-            }
-            (&Method::GET, "/network/peer_id") => network::get_peer_id::<T>(req, network_globals),
-            (&Method::GET, "/network/peers") => network::get_peer_list::<T>(req, network_globals),
-            (&Method::GET, "/network/listen_port") => {
-                network::get_listen_port::<T>(req, network_globals)
-            }
-            (&Method::GET, "/network/listen_addresses") => {
-                network::get_listen_addresses::<T>(req, network_globals)
-            }
-
-            // Methods for Beacon Node
-            (&Method::GET, "/beacon/head") => beacon::get_head::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/heads") => beacon::get_heads::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/block") => beacon::get_block::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/block_root") => beacon::get_block_root::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/fork") => beacon::get_fork::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/fork/stream") => {
-                let reader = events.lock().add_rx();
-                beacon::stream_forks::<T>(log, reader)
-            }
-            (&Method::GET, "/beacon/genesis_time") => {
-                beacon::get_genesis_time::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/genesis_validators_root") => {
-                beacon::get_genesis_validators_root::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/validators") => beacon::get_validators::<T>(req, beacon_chain),
-            (&Method::POST, "/beacon/validators") => {
-                beacon::post_validators::<T>(req, beacon_chain).await
-            }
-            (&Method::GET, "/beacon/validators/all") => {
-                beacon::get_all_validators::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/validators/active") => {
-                beacon::get_active_validators::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/state") => beacon::get_state::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/state_root") => beacon::get_state_root::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/state/genesis") => {
-                beacon::get_genesis_state::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/committees") => beacon::get_committees::<T>(req, beacon_chain),
-            (&Method::POST, "/beacon/proposer_slashing") => {
-                beacon::proposer_slashing::<T>(req, beacon_chain).await
-            }
-            (&Method::POST, "/beacon/attester_slashing") => {
-                beacon::attester_slashing::<T>(req, beacon_chain).await
-            }
-
-            // Methods for Validator
-            (&Method::POST, "/validator/duties") => {
-                let timer =
-                    metrics::start_timer(&metrics::VALIDATOR_GET_DUTIES_REQUEST_RESPONSE_TIME);
-                let response = validator::post_validator_duties::<T>(req, beacon_chain);
-                drop(timer);
-                response.await
-            }
-            (&Method::POST, "/validator/subscribe") => {
-                validator::post_validator_subscriptions::<T>(req, network_channel).await
-            }
-            (&Method::GET, "/validator/duties/all") => {
-                validator::get_all_validator_duties::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/validator/duties/active") => {
-                validator::get_active_validator_duties::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/validator/block") => {
-                let timer =
-                    metrics::start_timer(&metrics::VALIDATOR_GET_BLOCK_REQUEST_RESPONSE_TIME);
-                let response = validator::get_new_beacon_block::<T>(req, beacon_chain, log);
-                drop(timer);
-                response
-            }
-            (&Method::POST, "/validator/block") => {
-                validator::publish_beacon_block::<T>(req, beacon_chain, network_channel, log).await
-            }
-            (&Method::GET, "/validator/attestation") => {
-                let timer =
-                    metrics::start_timer(&metrics::VALIDATOR_GET_ATTESTATION_REQUEST_RESPONSE_TIME);
-                let response = validator::get_new_attestation::<T>(req, beacon_chain);
-                drop(timer);
-                response
-            }
-            (&Method::GET, "/validator/aggregate_attestation") => {
-                validator::get_aggregate_attestation::<T>(req, beacon_chain)
-            }
-            (&Method::POST, "/validator/attestations") => {
-                validator::publish_attestations::<T>(req, beacon_chain, network_channel, log).await
-            }
-            (&Method::POST, "/validator/aggregate_and_proofs") => {
-                validator::publish_aggregate_and_proofs::<T>(
-                    req,
-                    beacon_chain,
-                    network_channel,
-                    log,
-                )
-                .await
-            }
-
-            // Methods for consensus
-            (&Method::GET, "/consensus/global_votes") => {
-                consensus::get_vote_count::<T>(req, beacon_chain)
-            }
-            (&Method::POST, "/consensus/individual_votes") => {
-                consensus::post_individual_votes::<T>(req, beacon_chain).await
-            }
-
-            // Methods for bootstrap and checking configuration
-            (&Method::GET, "/spec") => spec::get_spec::<T>(req, beacon_chain),
-            (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::<T>(req),
-            (&Method::GET, "/spec/deposit_contract") => {
-                helpers::implementation_pending_response(req)
-            }
-            (&Method::GET, "/spec/eth2_config") => spec::get_eth2_config::<T>(req, eth2_config),
-
-            // Methods for advanced parameters
-            (&Method::GET, "/advanced/fork_choice") => {
-                advanced::get_fork_choice::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/advanced/operation_pool") => {
-                advanced::get_operation_pool::<T>(req, beacon_chain)
-            }
-
-            (&Method::GET, "/metrics") => {
-                metrics::get_prometheus::<T>(req, beacon_chain, db_path, freezer_db_path)
-            }
-
-            // Lighthouse specific
-            (&Method::GET, "/lighthouse/syncing") => {
-                lighthouse::syncing::<T::EthSpec>(req, network_globals)
-            }
-
-            (&Method::GET, "/lighthouse/peers") => {
-                lighthouse::peers::<T::EthSpec>(req, network_globals)
-            }
-
-            (&Method::GET, "/lighthouse/connected_peers") => {
-                lighthouse::connected_peers::<T::EthSpec>(req, network_globals)
-            }
-            _ => Err(ApiError::NotFound(
-                "Request path and/or method not found.".to_owned(),
-            )),
-        }
-    };
-
-    let request_processing_duration = Instant::now().duration_since(received_instant);
-
-    // Map the Rust-friendly `Result` in to a http-friendly response. In effect, this ensures that
-    // any `Err` returned from our response handlers becomes a valid http response to the client
-    // (e.g., a response with a 404 or 500 status).
-
-    match result {
+    match route(req, ctx).await {
         Ok(mut response) => {
-            if rest_api_config.allow_origin != "" {
+            metrics::inc_counter_vec(&metrics::BEACON_HTTP_API_SUCCESS_TOTAL, &[&path]);
+
+            if allow_origin != "" {
                 let headers = response.headers_mut();
                 headers.insert(
                     hyper::header::ACCESS_CONTROL_ALLOW_ORIGIN,
-                    HeaderValue::from_str(&rest_api_config.allow_origin)?,
+                    HeaderValue::from_str(&allow_origin)?,
                 );
                 headers.insert(hyper::header::VARY, HeaderValue::from_static("Origin"));
             }
 
             debug!(
-                local_log,
+                log,
                 "HTTP API request successful";
                 "path" => path,
-                "duration_ms" => request_processing_duration.as_millis()
+                "duration_ms" => Instant::now().duration_since(received_instant).as_millis()
             );
-            metrics::inc_counter(&metrics::SUCCESS_COUNT);
             Ok(response)
         }
 
         Err(error) => {
+            metrics::inc_counter_vec(&metrics::BEACON_HTTP_API_ERROR_TOTAL, &[&path]);
+
             debug!(
-                local_log,
+                log,
                 "HTTP API request failure";
                 "path" => path,
-                "duration_ms" => request_processing_duration.as_millis()
+                "duration_ms" => Instant::now().duration_since(received_instant).as_millis()
             );
             Ok(error.into())
         }
     }
 }
+
+async fn route<T: BeaconChainTypes>(
+    req: Request<Body>,
+    ctx: Arc<Context<T>>,
+) -> Result<Response<Body>, ApiError> {
+    let path = req.uri().path().to_string();
+    let ctx = ctx.clone();
+    let method = req.method().clone();
+    let executor = ctx.executor.clone();
+    let handler = Handler::new(req, ctx, executor)?;
+
+    match (method, path.as_ref()) {
+        (Method::GET, "/node/version") => handler
+            .static_value(version_with_platform())
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/node/health") => handler
+            .static_value(Health::observe().map_err(ApiError::ServerError)?)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/node/syncing") => handler
+            .allow_body()
+            .in_blocking_task(|_, ctx| node::syncing(ctx))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/enr") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.local_enr().to_base64()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/peer_count") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.connected_peers()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/peer_id") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.local_peer_id().to_base58()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/peers") => handler
+            .in_blocking_task(|_, ctx| {
+                Ok(ctx
+                    .network_globals
+                    .peers
+                    .read()
+                    .connected_peer_ids()
+                    .map(PeerId::to_string)
+                    .collect::<Vec<_>>())
+            })
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/listen_port") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.listen_port_tcp()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/listen_addresses") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.network_globals.listen_multiaddrs()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/beacon/head") => handler
+            .in_blocking_task(|_, ctx| beacon::get_head(ctx))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/heads") => handler
+            .in_blocking_task(|_, ctx| Ok(beacon::get_heads(ctx)))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/block") => handler
+            .in_blocking_task(beacon::get_block)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/block_root") => handler
+            .in_blocking_task(beacon::get_block_root)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/fork") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.head_info()?.fork))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/fork/stream") => {
+            handler.sse_stream(|_, ctx| beacon::stream_forks(ctx)).await
+        }
+        (Method::GET, "/beacon/genesis_time") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.head_info()?.genesis_time))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/genesis_validators_root") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.head_info()?.genesis_validators_root))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/validators") => handler
+            .in_blocking_task(beacon::get_validators)
+            .await?
+            .all_encodings(),
+        (Method::POST, "/beacon/validators") => handler
+            .allow_body()
+            .in_blocking_task(beacon::post_validators)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/validators/all") => handler
+            .in_blocking_task(beacon::get_all_validators)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/validators/active") => handler
+            .in_blocking_task(beacon::get_active_validators)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/state") => handler
+            .in_blocking_task(beacon::get_state)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/state_root") => handler
+            .in_blocking_task(beacon::get_state_root)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/state/genesis") => handler
+            .in_blocking_task(|_, ctx| beacon::get_genesis_state(ctx))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/committees") => handler
+            .in_blocking_task(beacon::get_committees)
+            .await?
+            .all_encodings(),
+        (Method::POST, "/beacon/proposer_slashing") => handler
+            .allow_body()
+            .in_blocking_task(beacon::proposer_slashing)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/beacon/attester_slashing") => handler
+            .allow_body()
+            .in_blocking_task(beacon::attester_slashing)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/duties") => handler
+            .allow_body()
+            .in_blocking_task(validator::post_validator_duties)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/subscribe") => handler
+            .allow_body()
+            .in_blocking_task(validator::post_validator_subscriptions)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/duties/all") => handler
+            .in_blocking_task(validator::get_all_validator_duties)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/duties/active") => handler
+            .in_blocking_task(validator::get_active_validator_duties)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/block") => handler
+            .in_blocking_task(validator::get_new_beacon_block)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/block") => handler
+            .allow_body()
+            .in_blocking_task(validator::publish_beacon_block)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/attestation") => handler
+            .in_blocking_task(validator::get_new_attestation)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/aggregate_attestation") => handler
+            .in_blocking_task(validator::get_aggregate_attestation)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/attestations") => handler
+            .allow_body()
+            .in_blocking_task(validator::publish_attestations)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/aggregate_and_proofs") => handler
+            .allow_body()
+            .in_blocking_task(validator::publish_aggregate_and_proofs)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/consensus/global_votes") => handler
+            .allow_body()
+            .in_blocking_task(consensus::get_vote_count)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/consensus/individual_votes") => handler
+            .allow_body()
+            .in_blocking_task(consensus::post_individual_votes)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/spec") => handler
+            // TODO: this clone is not ideal.
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.spec.clone()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/spec/slots_per_epoch") => handler
+            .static_value(T::EthSpec::slots_per_epoch())
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/spec/eth2_config") => handler
+            // TODO: this clone is not ideal.
+            .in_blocking_task(|_, ctx| Ok(ctx.eth2_config.as_ref().clone()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/advanced/fork_choice") => handler
+            .in_blocking_task(|_, ctx| {
+                Ok(ctx
+                    .beacon_chain
+                    .fork_choice
+                    .read()
+                    .proto_array()
+                    .core_proto_array()
+                    .clone())
+            })
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/advanced/operation_pool") => handler
+            .in_blocking_task(|_, ctx| {
+                Ok(PersistedOperationPool::from_operation_pool(
+                    &ctx.beacon_chain.op_pool,
+                ))
+            })
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/metrics") => handler
+            .in_blocking_task(|_, ctx| metrics::get_prometheus(ctx))
+            .await?
+            .text_encoding(),
+        (Method::GET, "/lighthouse/syncing") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.network_globals.sync_state()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/lighthouse/peers") => handler
+            .in_blocking_task(|_, ctx| lighthouse::peers(ctx))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/lighthouse/connected_peers") => handler
+            .in_blocking_task(|_, ctx| lighthouse::connected_peers(ctx))
+            .await?
+            .serde_encodings(),
+        _ => Err(ApiError::NotFound(
+            "Request path and/or method not found.".to_owned(),
+        )),
+    }
+}