Add safe_sum and use it in state_processing (#1620)

## Issue Addressed

Closes #1098

## Proposed Changes

Add a `SafeArithIter` trait with a `safe_sum` method, and use it in `state_processing`. This seems to be the only place in `consensus` where it is relevant -- i.e. where we were using `sum` and the integer_arith lint is enabled.

## Additional Info

This PR doesn't include any Clippy linting to prevent `sum` from being called. It seems there is no existing Clippy lint that suits our purpose, but I'm going to look into that and maybe schedule writing one as a lower-priority task.

This theoretically _is_ a consensus breaking change, but it shouldn't impact Medalla (or any other testnet) because `slashings` shouldn't overflow!

.github/workflows/test-suite.yml

@@ -123,6 +123,8 @@ jobs:
     - uses: actions/checkout@v1
     - name: Lint code for quality and style with Clippy
       run: make lint
+    - name: Certify Cargo.lock freshness
+      run: git diff --exit-code Cargo.lock
   arbitrary-check:
     name: arbitrary-check
     runs-on: ubuntu-latest

Cargo.toml

@@ -44,6 +44,7 @@ members = [
     "consensus/ssz_derive",
     "consensus/ssz_types",
     "consensus/serde_hex",
+    "consensus/serde_utils",
     "consensus/state_processing",
     "consensus/swap_or_not_shuffle",
     "consensus/tree_hash",
@@ -78,4 +79,3 @@ eth2_ssz = { path = "consensus/ssz" }
 eth2_ssz_derive = { path = "consensus/ssz_derive" }
 eth2_ssz_types = { path = "consensus/ssz_types" }
 eth2_hashing = { path = "crypto/eth2_hashing" }
-leveldb-sys = { git = "https://github.com/michaelsproul/leveldb-sys", branch = "v2.0.6-cmake" }

README.md

@@ -2,7 +2,7 @@
 
 An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prime.
 
-[![Build Status]][Build Link] [![Book Status]][Book Link] [![RustDoc Status]][RustDoc Link] [![Chat Badge]][Chat Link]
+[![Build Status]][Build Link] [![Book Status]][Book Link] [![Chat Badge]][Chat Link]
 
 [Build Status]: https://github.com/sigp/lighthouse/workflows/test-suite/badge.svg?branch=master
 [Build Link]: https://github.com/sigp/lighthouse/actions
@@ -10,8 +10,6 @@ An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prim
 [Chat Link]: https://discord.gg/cyAszAh
 [Book Status]:https://img.shields.io/badge/user--docs-master-informational
 [Book Link]: http://lighthouse-book.sigmaprime.io/
-[RustDoc Status]:https://img.shields.io/badge/code--docs-master-orange
-[RustDoc Link]: http://lighthouse-docs.sigmaprime.io/
 
 [Documentation](http://lighthouse-book.sigmaprime.io/)
 
@@ -59,9 +57,6 @@ Current development overview:
 The [Lighthouse Book](http://lighthouse-book.sigmaprime.io/) contains information
 for testnet users and developers.
 
-Code documentation is generated via `cargo doc` and hosted at
-[lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/).
-
 If you'd like some background on Sigma Prime, please see the [Lighthouse Update
 \#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or
 [sigmaprime.io](https://sigmaprime.io).

account_manager/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "account_manager"
-version = "0.2.5"
+version = "0.2.11"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
 edition = "2018"
 

account_manager/src/common.rs

@@ -1,6 +1,15 @@
+use account_utils::PlainText;
+use account_utils::{read_mnemonic_from_user, strip_off_newlines};
 use clap::ArgMatches;
+use eth2_wallet::bip39::{Language, Mnemonic};
+use std::fs;
 use std::fs::create_dir_all;
 use std::path::{Path, PathBuf};
+use std::str::from_utf8;
+use std::thread::sleep;
+use std::time::Duration;
+
+pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
 
 pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> {
     let path = path.as_ref();
@@ -19,3 +28,43 @@ pub fn base_wallet_dir(matches: &ArgMatches, arg: &'static str) -> Result<PathBu
         PathBuf::new().join(".lighthouse").join("wallets"),
     )
 }
+
+pub fn read_mnemonic_from_cli(
+    mnemonic_path: Option<PathBuf>,
+    stdin_password: bool,
+) -> Result<Mnemonic, String> {
+    let mnemonic = match mnemonic_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .and_then(|bytes| {
+                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
+                let phrase = from_utf8(&bytes_no_newlines.as_ref())
+                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
+                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
+                    format!(
+                        "Unable to derive mnemonic from string {:?}: {:?}",
+                        phrase, e
+                    )
+                })
+            })?,
+        None => loop {
+            eprintln!("");
+            eprintln!("{}", MNEMONIC_PROMPT);
+
+            let mnemonic = read_mnemonic_from_user(stdin_password)?;
+
+            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
+                Ok(mnemonic_m) => {
+                    eprintln!("Valid mnemonic provided.");
+                    eprintln!("");
+                    sleep(Duration::from_secs(1));
+                    break mnemonic_m;
+                }
+                Err(_) => {
+                    eprintln!("Invalid mnemonic");
+                }
+            }
+        },
+    };
+    Ok(mnemonic)
+}

account_manager/src/validator/create.rs

@@ -13,7 +13,7 @@ use validator_dir::Builder as ValidatorDirBuilder;
 pub const CMD: &str = "create";
 pub const BASE_DIR_FLAG: &str = "base-dir";
 pub const WALLET_NAME_FLAG: &str = "wallet-name";
-pub const WALLET_PASSPHRASE_FLAG: &str = "wallet-passphrase";
+pub const WALLET_PASSWORD_FLAG: &str = "wallet-password";
 pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
 pub const STORE_WITHDRAW_FLAG: &str = "store-withdrawal-keystore";
 pub const COUNT_FLAG: &str = "count";
@@ -34,8 +34,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .required(true),
         )
         .arg(
-            Arg::with_name(WALLET_PASSPHRASE_FLAG)
-                .long(WALLET_PASSPHRASE_FLAG)
+            Arg::with_name(WALLET_PASSWORD_FLAG)
+                .long(WALLET_PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help("A path to a file containing the password which will unlock the wallet.")
                 .takes_value(true)
@@ -109,8 +109,7 @@ pub fn cli_run<T: EthSpec>(
     let spec = env.core_context().eth2_config.spec;
 
     let name: String = clap_utils::parse_required(matches, WALLET_NAME_FLAG)?;
-    let wallet_password_path: PathBuf =
-        clap_utils::parse_required(matches, WALLET_PASSPHRASE_FLAG)?;
+    let wallet_password_path: PathBuf = clap_utils::parse_required(matches, WALLET_PASSWORD_FLAG)?;
     let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
         matches,
         VALIDATOR_DIR_FLAG,

account_manager/src/validator/import.rs

@@ -6,6 +6,7 @@ use account_utils::{
         recursively_find_voting_keystores, ValidatorDefinition, ValidatorDefinitions,
         CONFIG_FILENAME,
     },
+    ZeroizeString,
 };
 use clap::{App, Arg, ArgMatches};
 use std::fs;
@@ -17,6 +18,7 @@ pub const CMD: &str = "import";
 pub const KEYSTORE_FLAG: &str = "keystore";
 pub const DIR_FLAG: &str = "directory";
 pub const STDIN_PASSWORD_FLAG: &str = "stdin-passwords";
+pub const REUSE_PASSWORD_FLAG: &str = "reuse-password";
 
 pub const PASSWORD_PROMPT: &str = "Enter the keystore password, or press enter to omit it:";
 pub const KEYSTORE_REUSE_WARNING: &str = "DO NOT USE THE ORIGINAL KEYSTORES TO VALIDATE WITH \
@@ -68,6 +70,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .long(STDIN_PASSWORD_FLAG)
                 .help("If present, read passwords from stdin instead of tty."),
         )
+        .arg(
+            Arg::with_name(REUSE_PASSWORD_FLAG)
+                .long(REUSE_PASSWORD_FLAG)
+                .help("If present, the same password will be used for all imported keystores."),
+        )
 }
 
 pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
@@ -79,6 +86,7 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         PathBuf::new().join(".lighthouse").join("validators"),
     )?;
     let stdin_password = matches.is_present(STDIN_PASSWORD_FLAG);
+    let reuse_password = matches.is_present(REUSE_PASSWORD_FLAG);
 
     ensure_dir_exists(&validator_dir)?;
 
@@ -118,7 +126,9 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
     // - Add the keystore to the validator definitions file.
     //
     // Skip keystores that already exist, but exit early if any operation fails.
+    // Reuses the same password for all keystores if the `REUSE_PASSWORD_FLAG` flag is set.
     let mut num_imported_keystores = 0;
+    let mut previous_password: Option<ZeroizeString> = None;
     for src_keystore in &keystore_paths {
         let keystore = Keystore::from_json_file(src_keystore)
             .map_err(|e| format!("Unable to read keystore JSON {:?}: {:?}", src_keystore, e))?;
@@ -136,6 +146,10 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         );
 
         let password_opt = loop {
+            if let Some(password) = previous_password.clone() {
+                eprintln!("Reuse previous password.");
+                break Some(password);
+            }
             eprintln!("");
             eprintln!("{}", PASSWORD_PROMPT);
 
@@ -152,6 +166,9 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
                     eprintln!("Password is correct.");
                     eprintln!("");
                     sleep(Duration::from_secs(1)); // Provides nicer UX.
+                    if reuse_password {
+                        previous_password = Some(password.clone());
+                    }
                     break Some(password);
                 }
                 Err(eth2_keystore::Error::InvalidPassword) => {

account_manager/src/validator/mod.rs

@@ -2,6 +2,7 @@ pub mod create;
 pub mod deposit;
 pub mod import;
 pub mod list;
+pub mod recover;
 
 use crate::common::base_wallet_dir;
 use clap::{App, Arg, ArgMatches};
@@ -24,6 +25,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         .subcommand(deposit::cli_app())
         .subcommand(import::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
 }
 
 pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
@@ -34,6 +36,7 @@ pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<
         (deposit::CMD, Some(matches)) => deposit::cli_run::<T>(matches, env),
         (import::CMD, Some(matches)) => import::cli_run(matches),
         (list::CMD, Some(matches)) => list::cli_run(matches),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/validator/recover.rs

@@ -0,0 +1,156 @@
+use super::create::STORE_WITHDRAW_FLAG;
+use super::import::STDIN_PASSWORD_FLAG;
+use crate::common::{ensure_dir_exists, read_mnemonic_from_cli};
+use crate::validator::create::COUNT_FLAG;
+use crate::{SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG};
+use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
+use account_utils::random_password;
+use clap::{App, Arg, ArgMatches};
+use eth2_wallet::bip39::Seed;
+use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType, ValidatorKeystores};
+use std::path::PathBuf;
+use validator_dir::Builder as ValidatorDirBuilder;
+pub const CMD: &str = "recover";
+pub const FIRST_INDEX_FLAG: &str = "first-index";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Recovers validator private keys given a BIP-39 mnemonic phrase. \
+            If you did not specify a `--first-index` or count `--count`, by default this will \
+            only recover the keys associated with the validator at index 0 for an HD wallet \
+            in accordance with the EIP-2333 spec.")
+        .arg(
+            Arg::with_name(FIRST_INDEX_FLAG)
+                .long(FIRST_INDEX_FLAG)
+                .value_name("FIRST_INDEX")
+                .help("The first of consecutive key indexes you wish to recover.")
+                .takes_value(true)
+                .required(false)
+                .default_value("0"),
+        )
+        .arg(
+            Arg::with_name(COUNT_FLAG)
+                .long(COUNT_FLAG)
+                .value_name("COUNT")
+                .help("The number of validator keys you wish to recover. Counted consecutively from the provided `--first_index`.")
+                .takes_value(true)
+                .required(false)
+                .default_value("1"),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help(
+                    "If present, the mnemonic will be read in from this file.",
+                )
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name(VALIDATOR_DIR_FLAG)
+                .long(VALIDATOR_DIR_FLAG)
+                .value_name("VALIDATOR_DIRECTORY")
+                .help(
+                    "The path where the validator directories will be created. \
+                    Defaults to ~/.lighthouse/validators",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(SECRETS_DIR_FLAG)
+                .long(SECRETS_DIR_FLAG)
+                .value_name("SECRETS_DIR")
+                .help(
+                    "The path where the validator keystore passwords will be stored. \
+                    Defaults to ~/.lighthouse/secrets",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STORE_WITHDRAW_FLAG)
+                .long(STORE_WITHDRAW_FLAG)
+                .help(
+                    "If present, the withdrawal keystore will be stored alongside the voting \
+                    keypair. It is generally recommended to *not* store the withdrawal key and \
+                    instead generate them from the wallet seed when required.",
+                ),
+        )
+        .arg(
+            Arg::with_name(STDIN_PASSWORD_FLAG)
+                .long(STDIN_PASSWORD_FLAG)
+                .help("If present, read passwords from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
+    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
+        matches,
+        VALIDATOR_DIR_FLAG,
+        PathBuf::new().join(".lighthouse").join("validators"),
+    )?;
+    let secrets_dir = clap_utils::parse_path_with_default_in_home_dir(
+        matches,
+        SECRETS_DIR_FLAG,
+        PathBuf::new().join(".lighthouse").join("secrets"),
+    )?;
+    let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
+    let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?;
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_password = matches.is_present(STDIN_PASSWORD_FLAG);
+
+    ensure_dir_exists(&validator_dir)?;
+    ensure_dir_exists(&secrets_dir)?;
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_password)?;
+
+    let seed = Seed::new(&mnemonic, "");
+
+    for index in first_index..first_index + count {
+        let voting_password = random_password();
+        let withdrawal_password = random_password();
+
+        let derive = |key_type: KeyType, password: &[u8]| -> Result<Keystore, String> {
+            let (secret, path) =
+                recover_validator_secret_from_mnemonic(seed.as_bytes(), index, key_type)
+                    .map_err(|e| format!("Unable to recover validator keys: {:?}", e))?;
+
+            let keypair = keypair_from_secret(secret.as_bytes())
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?;
+
+            KeystoreBuilder::new(&keypair, password, format!("{}", path))
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?
+                .build()
+                .map_err(|e| format!("Unable build keystore: {:?}", e))
+        };
+
+        let keystores = ValidatorKeystores {
+            voting: derive(KeyType::Voting, voting_password.as_bytes())?,
+            withdrawal: derive(KeyType::Withdrawal, withdrawal_password.as_bytes())?,
+        };
+
+        let voting_pubkey = keystores.voting.pubkey().to_string();
+
+        ValidatorDirBuilder::new(validator_dir.clone(), secrets_dir.clone())
+            .voting_keystore(keystores.voting, voting_password.as_bytes())
+            .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
+            .store_withdrawal_keystore(matches.is_present(STORE_WITHDRAW_FLAG))
+            .build()
+            .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
+
+        println!(
+            "{}/{}\tIndex: {}\t0x{}",
+            index - first_index,
+            count - first_index,
+            index,
+            voting_pubkey
+        );
+    }
+
+    Ok(())
+}

account_manager/src/wallet/create.rs

@@ -5,7 +5,7 @@ use eth2_wallet::{
     bip39::{Language, Mnemonic, MnemonicType},
     PlainText,
 };
-use eth2_wallet_manager::{WalletManager, WalletType};
+use eth2_wallet_manager::{LockedWallet, WalletManager, WalletType};
 use std::ffi::OsStr;
 use std::fs::{self, File};
 use std::io::prelude::*;
@@ -15,7 +15,7 @@ use std::path::{Path, PathBuf};
 pub const CMD: &str = "create";
 pub const HD_TYPE: &str = "hd";
 pub const NAME_FLAG: &str = "name";
-pub const PASSPHRASE_FLAG: &str = "passphrase-file";
+pub const PASSWORD_FLAG: &str = "password-file";
 pub const TYPE_FLAG: &str = "type";
 pub const MNEMONIC_FLAG: &str = "mnemonic-output-path";
 
@@ -34,8 +34,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .required(true),
         )
         .arg(
-            Arg::with_name(PASSPHRASE_FLAG)
-                .long(PASSPHRASE_FLAG)
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help(
                     "A path to a file containing the password which will unlock the wallet. \
@@ -70,46 +70,14 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 }
 
 pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
-    let name: String = clap_utils::parse_required(matches, NAME_FLAG)?;
-    let wallet_password_path: PathBuf = clap_utils::parse_required(matches, PASSPHRASE_FLAG)?;
     let mnemonic_output_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
-    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
-
-    let wallet_type = match type_field.as_ref() {
-        HD_TYPE => WalletType::Hd,
-        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
-    };
-
-    let mgr = WalletManager::open(&base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
 
     // Create a new random mnemonic.
     //
     // The `tiny-bip39` crate uses `thread_rng()` for this entropy.
     let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English);
 
-    // Create a random password if the file does not exist.
-    if !wallet_password_path.exists() {
-        // To prevent users from accidentally supplying their password to the PASSPHRASE_FLAG and
-        // create a file with that name, we require that the password has a .pass suffix.
-        if wallet_password_path.extension() != Some(&OsStr::new("pass")) {
-            return Err(format!(
-                "Only creates a password file if that file ends in .pass: {:?}",
-                wallet_password_path
-            ));
-        }
-
-        create_with_600_perms(&wallet_password_path, random_password().as_bytes())
-            .map_err(|e| format!("Unable to write to {:?}: {:?}", wallet_password_path, e))?;
-    }
-
-    let wallet_password = fs::read(&wallet_password_path)
-        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
-        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
-
-    let wallet = mgr
-        .create_wallet(name, wallet_type, &mnemonic, wallet_password.as_bytes())
-        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    let wallet = create_wallet_from_mnemonic(matches, &base_dir.as_path(), &mnemonic)?;
 
     if let Some(path) = mnemonic_output_path {
         create_with_600_perms(&path, mnemonic.phrase().as_bytes())
@@ -140,6 +108,48 @@ pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
     Ok(())
 }
 
+pub fn create_wallet_from_mnemonic(
+    matches: &ArgMatches,
+    base_dir: &Path,
+    mnemonic: &Mnemonic,
+) -> Result<LockedWallet, String> {
+    let name: String = clap_utils::parse_required(matches, NAME_FLAG)?;
+    let wallet_password_path: PathBuf = clap_utils::parse_required(matches, PASSWORD_FLAG)?;
+    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
+
+    let wallet_type = match type_field.as_ref() {
+        HD_TYPE => WalletType::Hd,
+        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
+    };
+
+    let mgr = WalletManager::open(&base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+
+    // Create a random password if the file does not exist.
+    if !wallet_password_path.exists() {
+        // To prevent users from accidentally supplying their password to the PASSWORD_FLAG and
+        // create a file with that name, we require that the password has a .pass suffix.
+        if wallet_password_path.extension() != Some(&OsStr::new("pass")) {
+            return Err(format!(
+                "Only creates a password file if that file ends in .pass: {:?}",
+                wallet_password_path
+            ));
+        }
+
+        create_with_600_perms(&wallet_password_path, random_password().as_bytes())
+            .map_err(|e| format!("Unable to write to {:?}: {:?}", wallet_password_path, e))?;
+    }
+
+    let wallet_password = fs::read(&wallet_password_path)
+        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
+        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
+
+    let wallet = mgr
+        .create_wallet(name, wallet_type, &mnemonic, wallet_password.as_bytes())
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    Ok(wallet)
+}
+
 /// Creates a file with `600 (-rw-------)` permissions.
 pub fn create_with_600_perms<P: AsRef<Path>>(path: P, bytes: &[u8]) -> Result<(), String> {
     let path = path.as_ref();

account_manager/src/wallet/mod.rs

@@ -1,5 +1,6 @@
 pub mod create;
 pub mod list;
+pub mod recover;
 
 use crate::{
     common::{base_wallet_dir, ensure_dir_exists},
@@ -21,6 +22,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
         .subcommand(create::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
 }
 
 pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
@@ -30,6 +32,7 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
     match matches.subcommand() {
         (create::CMD, Some(matches)) => create::cli_run(matches, base_dir),
         (list::CMD, Some(_)) => list::cli_run(base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, base_dir),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/wallet/recover.rs

@@ -0,0 +1,87 @@
+use crate::common::read_mnemonic_from_cli;
+use crate::wallet::create::create_wallet_from_mnemonic;
+use crate::wallet::create::{HD_TYPE, NAME_FLAG, PASSWORD_FLAG, TYPE_FLAG};
+use clap::{App, Arg, ArgMatches};
+use std::path::PathBuf;
+
+pub const CMD: &str = "recover";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+pub const STDIN_PASSWORD_FLAG: &str = "stdin-passwords";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Recovers an EIP-2386 wallet from a given a BIP-39 mnemonic phrase.")
+        .arg(
+            Arg::with_name(NAME_FLAG)
+                .long(NAME_FLAG)
+                .value_name("WALLET_NAME")
+                .help(
+                    "The wallet will be created with this name. It is not allowed to \
+                            create two wallets with the same name for the same --base-dir.",
+                )
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
+                .value_name("PASSWORD_FILE_PATH")
+                .help(
+                    "This will be the new password for your recovered wallet. \
+                    A path to a file containing the password which will unlock the wallet. \
+                    If the file does not exist, a random password will be generated and \
+                    saved at that path. To avoid confusion, if the file does not already \
+                    exist it must include a '.pass' suffix.",
+                )
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help("If present, the mnemonic will be read in from this file.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(TYPE_FLAG)
+                .long(TYPE_FLAG)
+                .value_name("WALLET_TYPE")
+                .help(
+                    "The type of wallet to create. Only HD (hierarchical-deterministic) \
+                            wallets are supported presently..",
+                )
+                .takes_value(true)
+                .possible_values(&[HD_TYPE])
+                .default_value(HD_TYPE),
+        )
+        .arg(
+            Arg::with_name(STDIN_PASSWORD_FLAG)
+                .long(STDIN_PASSWORD_FLAG)
+                .help("If present, read passwords from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_password = matches.is_present(STDIN_PASSWORD_FLAG);
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_password)?;
+
+    let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+    println!("Your wallet has been successfully recovered.");
+    println!();
+    println!("Your wallet's UUID is:");
+    println!();
+    println!("\t{}", wallet.wallet().uuid());
+    println!();
+    println!("You do not need to backup your UUID or keep it secret.");
+
+    Ok(())
+}

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "0.2.5"
+version = "0.2.11"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
 edition = "2018"
 

beacon_node/beacon_chain/Cargo.toml

@@ -11,6 +11,7 @@ participation_metrics = []  # Exposes validator participation metrics to Prometh
 
 [dev-dependencies]
 int_to_bytes = { path = "../../consensus/int_to_bytes" }
+maplit = "1.0.2"
 
 [dependencies]
 eth2_config = { path = "../../common/eth2_config" }
@@ -45,6 +46,7 @@ futures = "0.3.5"
 genesis = { path = "../genesis" }
 integer-sqrt = "0.1.3"
 rand = "0.7.3"
+rand_core = "0.5.1"
 proto_array = { path = "../../consensus/proto_array" }
 lru = "0.5.1"
 tempfile = "3.1.0"
@@ -56,3 +58,4 @@ environment = { path = "../../lighthouse/environment" }
 bus = "2.2.3"
 derivative = "2.1.1"
 itertools = "0.9.0"
+regex = "1.3.9"

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -31,6 +31,7 @@ use fork_choice::ForkChoice;
 use itertools::process_results;
 use operation_pool::{OperationPool, PersistedOperationPool};
 use parking_lot::RwLock;
+use regex::bytes::Regex;
 use slog::{crit, debug, error, info, trace, warn, Logger};
 use slot_clock::SlotClock;
 use state_processing::{
@@ -426,6 +427,17 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         Ok(iter)
     }
 
+    /// As for `rev_iter_state_roots` but starting from an arbitrary `BeaconState`.
+    pub fn rev_iter_state_roots_from<'a>(
+        &self,
+        state_root: Hash256,
+        state: &'a BeaconState<T::EthSpec>,
+    ) -> impl Iterator<Item = Result<(Hash256, Slot), Error>> + 'a {
+        std::iter::once(Ok((state_root, state.slot)))
+            .chain(StateRootsIterator::new(self.store.clone(), state))
+            .map(|result| result.map_err(Into::into))
+    }
+
     /// Returns the block at the given slot, if any. Only returns blocks in the canonical chain.
     ///
     /// ## Errors
@@ -479,30 +491,36 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// is the state as it was when the head block was received, which could be some slots prior to
     /// now.
     pub fn head(&self) -> Result<BeaconSnapshot<T::EthSpec>, Error> {
-        self.canonical_head
+        self.with_head(|head| Ok(head.clone_with_only_committee_caches()))
+    }
+
+    /// Apply a function to the canonical head without cloning it.
+    pub fn with_head<U>(
+        &self,
+        f: impl FnOnce(&BeaconSnapshot<T::EthSpec>) -> Result<U, Error>,
+    ) -> Result<U, Error> {
+        let head_lock = self
+            .canonical_head
             .try_read_for(HEAD_LOCK_TIMEOUT)
-            .ok_or_else(|| Error::CanonicalHeadLockTimeout)
-            .map(|v| v.clone_with_only_committee_caches())
+            .ok_or_else(|| Error::CanonicalHeadLockTimeout)?;
+        f(&head_lock)
     }
 
     /// Returns info representing the head block and state.
     ///
     /// A summarized version of `Self::head` that involves less cloning.
     pub fn head_info(&self) -> Result<HeadInfo, Error> {
-        let head = self
-            .canonical_head
-            .try_read_for(HEAD_LOCK_TIMEOUT)
-            .ok_or_else(|| Error::CanonicalHeadLockTimeout)?;
-
-        Ok(HeadInfo {
-            slot: head.beacon_block.slot(),
-            block_root: head.beacon_block_root,
-            state_root: head.beacon_state_root,
-            current_justified_checkpoint: head.beacon_state.current_justified_checkpoint,
-            finalized_checkpoint: head.beacon_state.finalized_checkpoint,
-            fork: head.beacon_state.fork,
-            genesis_time: head.beacon_state.genesis_time,
-            genesis_validators_root: head.beacon_state.genesis_validators_root,
+        self.with_head(|head| {
+            Ok(HeadInfo {
+                slot: head.beacon_block.slot(),
+                block_root: head.beacon_block_root,
+                state_root: head.beacon_state_root,
+                current_justified_checkpoint: head.beacon_state.current_justified_checkpoint,
+                finalized_checkpoint: head.beacon_state.finalized_checkpoint,
+                fork: head.beacon_state.fork,
+                genesis_time: head.beacon_state.genesis_time,
+                genesis_validators_root: head.beacon_state.genesis_validators_root,
+            })
         })
     }
 
@@ -835,16 +853,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         if state.slot > slot {
             return Err(Error::CannotAttestToFutureState);
-        } else if state.current_epoch() + 1 < epoch {
+        } else if state.current_epoch() < epoch {
             let mut_state = state.to_mut();
-            while mut_state.current_epoch() + 1 < epoch {
+            while mut_state.current_epoch() < epoch {
                 // Note: here we provide `Hash256::zero()` as the root of the current state. This
                 // has the effect of setting the values of all historic state roots to the zero
                 // hash. This is an optimization, we don't need the state roots so why calculate
                 // them?
                 per_slot_processing(mut_state, Some(Hash256::zero()), &self.spec)?;
             }
-            mut_state.build_committee_cache(RelativeEpoch::Next, &self.spec)?;
+            mut_state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
         }
 
         let committee_len = state.get_beacon_committee(slot, index)?.committee.len();
@@ -1302,8 +1320,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         block: SignedBeaconBlock<T::EthSpec>,
     ) -> Result<GossipVerifiedBlock<T>, BlockError<T::EthSpec>> {
         let slot = block.message.slot;
-        let graffiti_string = String::from_utf8(block.message.body.graffiti[..].to_vec())
-            .unwrap_or_else(|_| format!("{:?}", &block.message.body.graffiti[..]));
+        #[allow(clippy::invalid_regex)]
+        let re = Regex::new("\\p{C}").expect("regex is valid");
+        let graffiti_string =
+            String::from_utf8_lossy(&re.replace_all(&block.message.body.graffiti[..], &b""[..]))
+                .to_string();
 
         match GossipVerifiedBlock::new(block, self) {
             Ok(verified) => {
@@ -1746,7 +1767,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let beacon_block_root = self.fork_choice.write().get_head(self.slot()?)?;
 
         let current_head = self.head_info()?;
-        let old_finalized_root = current_head.finalized_checkpoint.root;
+        let old_finalized_checkpoint = current_head.finalized_checkpoint;
 
         if beacon_block_root == current_head.block_root {
             return Ok(());
@@ -1826,15 +1847,32 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             );
         };
 
-        let old_finalized_epoch = current_head.finalized_checkpoint.epoch;
-        let new_finalized_epoch = new_head.beacon_state.finalized_checkpoint.epoch;
-        let finalized_root = new_head.beacon_state.finalized_checkpoint.root;
+        let new_finalized_checkpoint = new_head.beacon_state.finalized_checkpoint;
+        // State root of the finalized state on the epoch boundary, NOT the state
+        // of the finalized block. We need to use an iterator in case the state is beyond
+        // the reach of the new head's `state_roots` array.
+        let new_finalized_slot = new_finalized_checkpoint
+            .epoch
+            .start_slot(T::EthSpec::slots_per_epoch());
+        let new_finalized_state_root = process_results(
+            StateRootsIterator::new(self.store.clone(), &new_head.beacon_state),
+            |mut iter| {
+                iter.find_map(|(state_root, slot)| {
+                    if slot == new_finalized_slot {
+                        Some(state_root)
+                    } else {
+                        None
+                    }
+                })
+            },
+        )?
+        .ok_or_else(|| Error::MissingFinalizedStateRoot(new_finalized_slot))?;
 
         // It is an error to try to update to a head with a lesser finalized epoch.
-        if new_finalized_epoch < old_finalized_epoch {
+        if new_finalized_checkpoint.epoch < old_finalized_checkpoint.epoch {
             return Err(Error::RevertedFinalizedEpoch {
-                previous_epoch: old_finalized_epoch,
-                new_epoch: new_finalized_epoch,
+                previous_epoch: old_finalized_checkpoint.epoch,
+                new_epoch: new_finalized_checkpoint.epoch,
             });
         }
 
@@ -1873,11 +1911,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 );
             });
 
-        if new_finalized_epoch != old_finalized_epoch {
+        if new_finalized_checkpoint.epoch != old_finalized_checkpoint.epoch {
             self.after_finalization(
-                old_finalized_epoch,
-                finalized_root,
-                old_finalized_root.into(),
+                old_finalized_checkpoint,
+                new_finalized_checkpoint,
+                new_finalized_state_root,
             )?;
         }
 
@@ -1905,68 +1943,53 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Performs pruning and finality-based optimizations.
     fn after_finalization(
         &self,
-        old_finalized_epoch: Epoch,
-        finalized_block_root: Hash256,
-        old_finalized_root: SignedBeaconBlockHash,
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+        new_finalized_state_root: Hash256,
     ) -> Result<(), Error> {
-        let finalized_block = self
-            .store
-            .get_block(&finalized_block_root)?
-            .ok_or_else(|| Error::MissingBeaconBlock(finalized_block_root))?
-            .message;
+        self.fork_choice.write().prune()?;
 
-        let new_finalized_epoch = finalized_block.slot.epoch(T::EthSpec::slots_per_epoch());
+        self.observed_block_producers.prune(
+            new_finalized_checkpoint
+                .epoch
+                .start_slot(T::EthSpec::slots_per_epoch()),
+        );
 
-        if new_finalized_epoch < old_finalized_epoch {
-            Err(Error::RevertedFinalizedEpoch {
-                previous_epoch: old_finalized_epoch,
-                new_epoch: new_finalized_epoch,
+        self.snapshot_cache
+            .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
+            .map(|mut snapshot_cache| {
+                snapshot_cache.prune(new_finalized_checkpoint.epoch);
             })
-        } else {
-            self.fork_choice.write().prune()?;
-
-            self.observed_block_producers
-                .prune(new_finalized_epoch.start_slot(T::EthSpec::slots_per_epoch()));
-
-            self.snapshot_cache
-                .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
-                .map(|mut snapshot_cache| {
-                    snapshot_cache.prune(new_finalized_epoch);
-                })
-                .unwrap_or_else(|| {
-                    error!(
-                        self.log,
-                        "Failed to obtain cache write lock";
-                        "lock" => "snapshot_cache",
-                        "task" => "prune"
-                    );
-                });
-
-            let finalized_state = self
-                .get_state(&finalized_block.state_root, Some(finalized_block.slot))?
-                .ok_or_else(|| Error::MissingBeaconState(finalized_block.state_root))?;
-
-            self.op_pool
-                .prune_all(&finalized_state, self.head_info()?.fork);
-
-            // TODO: configurable max finality distance
-            let max_finality_distance = 0;
-            self.store_migrator.process_finalization(
-                finalized_block.state_root,
-                finalized_state,
-                max_finality_distance,
-                Arc::clone(&self.head_tracker),
-                old_finalized_root,
-                finalized_block_root.into(),
-            );
-
-            let _ = self.event_handler.register(EventKind::BeaconFinalization {
-                epoch: new_finalized_epoch,
-                root: finalized_block_root,
+            .unwrap_or_else(|| {
+                error!(
+                    self.log,
+                    "Failed to obtain cache write lock";
+                    "lock" => "snapshot_cache",
+                    "task" => "prune"
+                );
             });
 
-            Ok(())
-        }
+        let finalized_state = self
+            .get_state(&new_finalized_state_root, None)?
+            .ok_or_else(|| Error::MissingBeaconState(new_finalized_state_root))?;
+
+        self.op_pool
+            .prune_all(&finalized_state, self.head_info()?.fork);
+
+        self.store_migrator.process_finalization(
+            new_finalized_state_root.into(),
+            finalized_state,
+            self.head_tracker.clone(),
+            old_finalized_checkpoint,
+            new_finalized_checkpoint,
+        )?;
+
+        let _ = self.event_handler.register(EventKind::BeaconFinalization {
+            epoch: new_finalized_checkpoint.epoch,
+            root: new_finalized_checkpoint.root,
+        });
+
+        Ok(())
     }
 
     /// Returns `true` if the given block root has not been processed.
@@ -2051,10 +2074,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .beacon_block_root;
         let mut visited: HashSet<Hash256> = HashSet::new();
         let mut finalized_blocks: HashSet<Hash256> = HashSet::new();
+        let mut justified_blocks: HashSet<Hash256> = HashSet::new();
 
         let genesis_block_hash = Hash256::zero();
         writeln!(output, "digraph beacon {{").unwrap();
-        writeln!(output, "\t_{:?}[label=\"genesis\"];", genesis_block_hash).unwrap();
+        writeln!(output, "\t_{:?}[label=\"zero\"];", genesis_block_hash).unwrap();
 
         // Canonical head needs to be processed first as otherwise finalized blocks aren't detected
         // properly.
@@ -2085,6 +2109,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         .unwrap()
                         .unwrap();
                     finalized_blocks.insert(state.finalized_checkpoint.root);
+                    justified_blocks.insert(state.current_justified_checkpoint.root);
+                    justified_blocks.insert(state.previous_justified_checkpoint.root);
                 }
 
                 if block_hash == canonical_head_hash {
@@ -2105,6 +2131,15 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         signed_beacon_block.slot()
                     )
                     .unwrap();
+                } else if justified_blocks.contains(&block_hash) {
+                    writeln!(
+                        output,
+                        "\t_{:?}[label=\"{} ({})\" shape=cds];",
+                        block_hash,
+                        block_hash,
+                        signed_beacon_block.slot()
+                    )
+                    .unwrap();
                 } else {
                     writeln!(
                         output,
@@ -2134,6 +2169,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let mut file = std::fs::File::create(file_name).unwrap();
         self.dump_as_dot(&mut file);
     }
+
+    // Should be used in tests only
+    pub fn set_graffiti(&mut self, graffiti: Graffiti) {
+        self.graffiti = graffiti;
+    }
 }
 
 impl<T: BeaconChainTypes> Drop for BeaconChain<T> {

beacon_node/beacon_chain/src/block_verification.rs

@@ -980,14 +980,17 @@ fn load_parent<T: BeaconChainTypes>(
         // exist in fork choice but not in the database yet. In such a case we simply
         // indicate that we don't yet know the parent.
         let root = block.parent_root();
-        let parent_block = if let Some(block) = chain
+        let parent_block = chain
             .get_block(&block.parent_root())
             .map_err(BlockError::BeaconChainError)?
-        {
-            block
-        } else {
-            return Err(BlockError::ParentUnknown(Box::new(block)));
-        };
+            .ok_or_else(|| {
+                // Return a `MissingBeaconBlock` error instead of a `ParentUnknown` error since
+                // we've already checked fork choice for this block.
+                //
+                // It's an internal error if the block exists in fork choice but not in the
+                // database.
+                BlockError::from(BeaconChainError::MissingBeaconBlock(block.parent_root()))
+            })?;
 
         // Load the parent blocks state from the database, returning an error if it is not found.
         // It is an error because if we know the parent block we should also know the parent state.

beacon_node/beacon_chain/src/builder.rs

@@ -97,11 +97,12 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     #[allow(clippy::type_complexity)]
     store: Option<Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>>,
     store_migrator: Option<T::StoreMigrator>,
-    canonical_head: Option<BeaconSnapshot<T::EthSpec>>,
-    /// The finalized checkpoint to anchor the chain. May be genesis or a higher
-    /// checkpoint.
-    pub finalized_snapshot: Option<BeaconSnapshot<T::EthSpec>>,
+    pub genesis_time: Option<u64>,
     genesis_block_root: Option<Hash256>,
+    #[allow(clippy::type_complexity)]
+    fork_choice: Option<
+        ForkChoice<BeaconForkChoiceStore<T::EthSpec, T::HotStore, T::ColdStore>, T::EthSpec>,
+    >,
     op_pool: Option<OperationPool<T::EthSpec>>,
     eth1_chain: Option<Eth1Chain<T::Eth1Chain, T::EthSpec>>,
     event_handler: Option<T::EventHandler>,
@@ -146,9 +147,9 @@ where
         Self {
             store: None,
             store_migrator: None,
-            canonical_head: None,
-            finalized_snapshot: None,
+            genesis_time: None,
             genesis_block_root: None,
+            fork_choice: None,
             op_pool: None,
             eth1_chain: None,
             event_handler: None,
@@ -278,22 +279,31 @@ where
                     .to_string()
             })?;
 
-        self.genesis_block_root = Some(chain.genesis_block_root);
-        self.head_tracker = Some(
-            HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
-                .map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
-        );
+        let persisted_fork_choice = store
+            .get_item::<PersistedForkChoice>(&Hash256::from_slice(&FORK_CHOICE_DB_KEY))
+            .map_err(|e| format!("DB error when reading persisted fork choice: {:?}", e))?
+            .ok_or_else(|| "No persisted fork choice present in database.".to_string())?;
 
-        let head_block_root = chain.canonical_head_block_root;
-        let head_block = store
-            .get_item::<SignedBeaconBlock<TEthSpec>>(&head_block_root)
-            .map_err(|e| format!("DB error when reading head block: {:?}", e))?
-            .ok_or_else(|| "Head block not found in store".to_string())?;
-        let head_state_root = head_block.state_root();
-        let head_state = store
-            .get_state(&head_state_root, Some(head_block.slot()))
-            .map_err(|e| format!("DB error when reading head state: {:?}", e))?
-            .ok_or_else(|| "Head state not found in store".to_string())?;
+        let fc_store = BeaconForkChoiceStore::from_persisted(
+            persisted_fork_choice.fork_choice_store,
+            store.clone(),
+        )
+        .map_err(|e| format!("Unable to load ForkChoiceStore: {:?}", e))?;
+
+        let fork_choice =
+            ForkChoice::from_persisted(persisted_fork_choice.fork_choice, fc_store)
+                .map_err(|e| format!("Unable to parse persisted fork choice from disk: {:?}", e))?;
+
+        let genesis_block = store
+            .get_item::<SignedBeaconBlock<TEthSpec>>(&chain.genesis_block_root)
+            .map_err(|e| format!("DB error when reading genesis block: {:?}", e))?
+            .ok_or_else(|| "Genesis block not found in store".to_string())?;
+        let genesis_state = store
+            .get_state(&genesis_block.state_root(), Some(genesis_block.slot()))
+            .map_err(|e| format!("DB error when reading genesis state: {:?}", e))?
+            .ok_or_else(|| "Genesis block not found in store".to_string())?;
+
+        self.genesis_time = Some(genesis_state.genesis_time);
 
         self.op_pool = Some(
             store
@@ -303,35 +313,16 @@ where
                 .unwrap_or_else(OperationPool::new),
         );
 
-        let finalized_block_root = head_state.finalized_checkpoint.root;
-        let finalized_block = store
-            .get_item::<SignedBeaconBlock<TEthSpec>>(&finalized_block_root)
-            .map_err(|e| format!("DB error when reading finalized block: {:?}", e))?
-            .ok_or_else(|| "Finalized block not found in store".to_string())?;
-        let finalized_state_root = finalized_block.state_root();
-        let finalized_state = store
-            .get_state(&finalized_state_root, Some(finalized_block.slot()))
-            .map_err(|e| format!("DB error when reading finalized state: {:?}", e))?
-            .ok_or_else(|| "Finalized state not found in store".to_string())?;
-
-        self.finalized_snapshot = Some(BeaconSnapshot {
-            beacon_block_root: finalized_block_root,
-            beacon_block: finalized_block,
-            beacon_state_root: finalized_state_root,
-            beacon_state: finalized_state,
-        });
-
-        self.canonical_head = Some(BeaconSnapshot {
-            beacon_block_root: head_block_root,
-            beacon_block: head_block,
-            beacon_state_root: head_state_root,
-            beacon_state: head_state,
-        });
-
         let pubkey_cache = ValidatorPubkeyCache::load_from_file(pubkey_cache_path)
             .map_err(|e| format!("Unable to open persisted pubkey cache: {:?}", e))?;
 
+        self.genesis_block_root = Some(chain.genesis_block_root);
+        self.head_tracker = Some(
+            HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
+                .map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
+        );
         self.validator_pubkey_cache = Some(pubkey_cache);
+        self.fork_choice = Some(fork_choice);
 
         Ok(self)
     }
@@ -374,12 +365,20 @@ where
                 )
             })?;
 
-        self.finalized_snapshot = Some(BeaconSnapshot {
+        let genesis = BeaconSnapshot {
             beacon_block_root,
             beacon_block,
             beacon_state_root,
             beacon_state,
-        });
+        };
+
+        let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis);
+
+        let fork_choice = ForkChoice::from_genesis(fc_store, &genesis.beacon_block.message)
+            .map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?;
+
+        self.fork_choice = Some(fork_choice);
+        self.genesis_time = Some(genesis.beacon_state.genesis_time);
 
         Ok(self.empty_op_pool())
     }
@@ -457,23 +456,73 @@ where
             .store
             .clone()
             .ok_or_else(|| "Cannot build without a store.".to_string())?;
+        let mut fork_choice = self
+            .fork_choice
+            .ok_or_else(|| "Cannot build without fork choice.".to_string())?;
+        let genesis_block_root = self
+            .genesis_block_root
+            .ok_or_else(|| "Cannot build without a genesis block root".to_string())?;
 
-        // If this beacon chain is being loaded from disk, use the stored head. Otherwise, just use
-        // the finalized checkpoint (which is probably genesis).
-        let mut canonical_head = if let Some(head) = self.canonical_head {
-            head
+        let current_slot = if slot_clock
+            .is_prior_to_genesis()
+            .ok_or_else(|| "Unable to read slot clock".to_string())?
+        {
+            self.spec.genesis_slot
         } else {
-            self.finalized_snapshot
-                .ok_or_else(|| "Cannot build without a state".to_string())?
+            slot_clock
+                .now()
+                .ok_or_else(|| "Unable to read slot".to_string())?
         };
 
+        let head_block_root = fork_choice
+            .get_head(current_slot)
+            .map_err(|e| format!("Unable to get fork choice head: {:?}", e))?;
+
+        let head_block = store
+            .get_item::<SignedBeaconBlock<TEthSpec>>(&head_block_root)
+            .map_err(|e| format!("DB error when reading head block: {:?}", e))?
+            .ok_or_else(|| "Head block not found in store".to_string())?;
+        let head_state_root = head_block.state_root();
+        let head_state = store
+            .get_state(&head_state_root, Some(head_block.slot()))
+            .map_err(|e| format!("DB error when reading head state: {:?}", e))?
+            .ok_or_else(|| "Head state not found in store".to_string())?;
+
+        let mut canonical_head = BeaconSnapshot {
+            beacon_block_root: head_block_root,
+            beacon_block: head_block,
+            beacon_state_root: head_state_root,
+            beacon_state: head_state,
+        };
+
+        if canonical_head.beacon_block.state_root() != canonical_head.beacon_state_root {
+            return Err("beacon_block.state_root != beacon_state".to_string());
+        }
+
         canonical_head
             .beacon_state
             .build_all_caches(&self.spec)
             .map_err(|e| format!("Failed to build state caches: {:?}", e))?;
 
-        if canonical_head.beacon_block.state_root() != canonical_head.beacon_state_root {
-            return Err("beacon_block.state_root != beacon_state".to_string());
+        // Perform a check to ensure that the finalization points of the head and fork choice are
+        // consistent.
+        //
+        // This is a sanity check to detect database corruption.
+        let fc_finalized = fork_choice.finalized_checkpoint();
+        let head_finalized = canonical_head.beacon_state.finalized_checkpoint;
+        if fc_finalized != head_finalized {
+            if head_finalized.root == Hash256::zero()
+                && head_finalized.epoch == fc_finalized.epoch
+                && fc_finalized.root == genesis_block_root
+            {
+                // This is a legal edge-case encountered during genesis.
+            } else {
+                return Err(format!(
+                    "Database corrupt: fork choice is finalized at {:?} whilst head is finalized at \
+                    {:?}",
+                    fc_finalized, head_finalized
+                ));
+            }
         }
 
         let pubkey_cache_path = self
@@ -485,26 +534,6 @@ where
                 .map_err(|e| format!("Unable to init validator pubkey cache: {:?}", e))
         })?;
 
-        let persisted_fork_choice = store
-            .get_item::<PersistedForkChoice>(&Hash256::from_slice(&FORK_CHOICE_DB_KEY))
-            .map_err(|e| format!("DB error when reading persisted fork choice: {:?}", e))?;
-
-        let fork_choice = if let Some(persisted) = persisted_fork_choice {
-            let fc_store =
-                BeaconForkChoiceStore::from_persisted(persisted.fork_choice_store, store.clone())
-                    .map_err(|e| format!("Unable to load ForkChoiceStore: {:?}", e))?;
-
-            ForkChoice::from_persisted(persisted.fork_choice, fc_store)
-                .map_err(|e| format!("Unable to parse persisted fork choice from disk: {:?}", e))?
-        } else {
-            let genesis = &canonical_head;
-
-            let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), genesis);
-
-            ForkChoice::from_genesis(fc_store, &genesis.beacon_block.message)
-                .map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?
-        };
-
         let beacon_chain = BeaconChain {
             spec: self.spec,
             config: self.chain_config,
@@ -533,9 +562,7 @@ where
             eth1_chain: self.eth1_chain,
             genesis_validators_root: canonical_head.beacon_state.genesis_validators_root,
             canonical_head: TimeoutRwLock::new(canonical_head.clone()),
-            genesis_block_root: self
-                .genesis_block_root
-                .ok_or_else(|| "Cannot build without a genesis block root".to_string())?,
+            genesis_block_root,
             fork_choice: RwLock::new(fork_choice),
             event_handler: self
                 .event_handler
@@ -634,11 +661,8 @@ where
     /// Requires the state to be initialized.
     pub fn testing_slot_clock(self, slot_duration: Duration) -> Result<Self, String> {
         let genesis_time = self
-            .finalized_snapshot
-            .as_ref()
-            .ok_or_else(|| "testing_slot_clock requires an initialized state")?
-            .beacon_state
-            .genesis_time;
+            .genesis_time
+            .ok_or_else(|| "testing_slot_clock requires an initialized state")?;
 
         let slot_clock = TestingSlotClock::new(
             Slot::new(0),

beacon_node/beacon_chain/src/errors.rs

@@ -1,5 +1,6 @@
 use crate::beacon_chain::ForkChoiceError;
 use crate::eth1_chain::Error as Eth1ChainError;
+use crate::migrate::PruningError;
 use crate::naive_aggregation_pool::Error as NaiveAggregationError;
 use crate::observed_attestations::Error as ObservedAttestationsError;
 use crate::observed_attesters::Error as ObservedAttestersError;
@@ -61,6 +62,7 @@ pub enum BeaconChainError {
         requested_slot: Slot,
         max_task_runtime: Duration,
     },
+    MissingFinalizedStateRoot(Slot),
     /// Returned when an internal check fails, indicating corrupt data.
     InvariantViolated(String),
     SszTypesError(SszTypesError),
@@ -79,6 +81,7 @@ pub enum BeaconChainError {
     ObservedAttestationsError(ObservedAttestationsError),
     ObservedAttestersError(ObservedAttestersError),
     ObservedBlockProducersError(ObservedBlockProducersError),
+    PruningError(PruningError),
     ArithError(ArithError),
 }
 
@@ -94,6 +97,7 @@ easy_from_to!(ObservedAttestationsError, BeaconChainError);
 easy_from_to!(ObservedAttestersError, BeaconChainError);
 easy_from_to!(ObservedBlockProducersError, BeaconChainError);
 easy_from_to!(BlockSignatureVerifierError, BeaconChainError);
+easy_from_to!(PruningError, BeaconChainError);
 easy_from_to!(ArithError, BeaconChainError);
 
 #[derive(Debug)]

beacon_node/beacon_chain/src/lib.rs

@@ -2,6 +2,10 @@
 #[macro_use]
 extern crate lazy_static;
 
+#[macro_use]
+extern crate slog;
+extern crate slog_term;
+
 pub mod attestation_verification;
 mod beacon_chain;
 mod beacon_fork_choice_store;

beacon_node/beacon_chain/src/migrate.rs

@@ -7,12 +7,28 @@ use std::mem;
 use std::sync::mpsc;
 use std::sync::Arc;
 use std::thread;
-use store::hot_cold_store::{process_finalization, HotColdDBError};
-use store::iter::{ParentRootBlockIterator, RootsIterator};
+use store::hot_cold_store::{migrate_database, HotColdDBError};
+use store::iter::RootsIterator;
 use store::{Error, ItemStore, StoreOp};
 pub use store::{HotColdDB, MemoryStore};
-use types::*;
-use types::{BeaconState, EthSpec, Hash256, Slot};
+use types::{
+    BeaconState, BeaconStateError, BeaconStateHash, Checkpoint, EthSpec, Hash256,
+    SignedBeaconBlockHash, Slot,
+};
+
+/// Logic errors that can occur during pruning, none of these should ever happen.
+#[derive(Debug)]
+pub enum PruningError {
+    IncorrectFinalizedState {
+        state_slot: Slot,
+        new_finalized_slot: Slot,
+    },
+    MissingInfoForCanonicalChain {
+        slot: Slot,
+    },
+    UnexpectedEqualStateRoots,
+    UnexpectedUnequalStateRoots,
+}
 
 /// Trait for migration processes that update the database upon finalization.
 pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
@@ -22,17 +38,17 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
 
     fn process_finalization(
         &self,
-        _state_root: Hash256,
+        _finalized_state_root: BeaconStateHash,
         _new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
         _head_tracker: Arc<HeadTracker>,
-        _old_finalized_block_hash: SignedBeaconBlockHash,
-        _new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
+        _old_finalized_checkpoint: Checkpoint,
+        _new_finalized_checkpoint: Checkpoint,
+    ) -> Result<(), BeaconChainError> {
+        Ok(())
     }
 
     /// Traverses live heads and prunes blocks and states of chains that we know can't be built
-    /// upon because finalization would prohibit it.  This is an optimisation intended to save disk
+    /// upon because finalization would prohibit it. This is an optimisation intended to save disk
     /// space.
     ///
     /// Assumptions:
@@ -40,37 +56,63 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
     fn prune_abandoned_forks(
         store: Arc<HotColdDB<E, Hot, Cold>>,
         head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_slot: Slot,
+        new_finalized_state_hash: BeaconStateHash,
+        new_finalized_state: &BeaconState<E>,
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+        log: &Logger,
     ) -> Result<(), BeaconChainError> {
         // There will never be any blocks to prune if there is only a single head in the chain.
         if head_tracker.heads().len() == 1 {
             return Ok(());
         }
 
-        let old_finalized_slot = store
-            .get_block(&old_finalized_block_hash.into())?
-            .ok_or_else(|| BeaconChainError::MissingBeaconBlock(old_finalized_block_hash.into()))?
-            .slot();
+        let old_finalized_slot = old_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_slot = new_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_block_hash = new_finalized_checkpoint.root.into();
 
-        // Collect hashes from new_finalized_block back to old_finalized_block (inclusive)
-        let mut found_block = false; // hack for `take_until`
-        let newly_finalized_blocks: HashMap<SignedBeaconBlockHash, Slot> =
-            ParentRootBlockIterator::new(&*store, new_finalized_block_hash.into())
-                .take_while(|result| match result {
-                    Ok((block_hash, _)) => {
-                        if found_block {
-                            false
-                        } else {
-                            found_block |= *block_hash == old_finalized_block_hash.into();
-                            true
-                        }
-                    }
-                    Err(_) => true,
-                })
-                .map(|result| result.map(|(block_hash, block)| (block_hash.into(), block.slot())))
-                .collect::<Result<_, _>>()?;
+        // The finalized state must be for the epoch boundary slot, not the slot of the finalized
+        // block.
+        if new_finalized_state.slot != new_finalized_slot {
+            return Err(PruningError::IncorrectFinalizedState {
+                state_slot: new_finalized_state.slot,
+                new_finalized_slot,
+            }
+            .into());
+        }
+
+        debug!(
+            log,
+            "Starting database pruning";
+            "old_finalized_epoch" => old_finalized_checkpoint.epoch,
+            "old_finalized_root" => format!("{:?}", old_finalized_checkpoint.root),
+            "new_finalized_epoch" => new_finalized_checkpoint.epoch,
+            "new_finalized_root" => format!("{:?}", new_finalized_checkpoint.root),
+        );
+
+        // For each slot between the new finalized checkpoint and the old finalized checkpoint,
+        // collect the beacon block root and state root of the canonical chain.
+        let newly_finalized_chain: HashMap<Slot, (SignedBeaconBlockHash, BeaconStateHash)> =
+            std::iter::once(Ok((
+                new_finalized_slot,
+                (new_finalized_block_hash, new_finalized_state_hash),
+            )))
+            .chain(
+                RootsIterator::new(store.clone(), new_finalized_state).map(|res| {
+                    res.map(|(block_root, state_root, slot)| {
+                        (slot, (block_root.into(), state_root.into()))
+                    })
+                }),
+            )
+            .take_while(|res| {
+                res.as_ref()
+                    .map_or(true, |(slot, _)| *slot >= old_finalized_slot)
+            })
+            .collect::<Result<_, _>>()?;
 
         // We don't know which blocks are shared among abandoned chains, so we buffer and delete
         // everything in one fell swoop.
@@ -79,75 +121,110 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
         let mut abandoned_heads: HashSet<Hash256> = HashSet::new();
 
         for (head_hash, head_slot) in head_tracker.heads() {
-            let mut potentially_abandoned_head: Option<Hash256> = Some(head_hash);
-            let mut potentially_abandoned_blocks: Vec<(
-                Slot,
-                Option<SignedBeaconBlockHash>,
-                Option<BeaconStateHash>,
-            )> = Vec::new();
+            let mut potentially_abandoned_head = Some(head_hash);
+            let mut potentially_abandoned_blocks = vec![];
 
             let head_state_hash = store
                 .get_block(&head_hash)?
                 .ok_or_else(|| BeaconStateError::MissingBeaconBlock(head_hash.into()))?
                 .state_root();
 
+            // Iterate backwards from this head, staging blocks and states for deletion.
             let iter = std::iter::once(Ok((head_hash, head_state_hash, head_slot)))
-                .chain(RootsIterator::from_block(Arc::clone(&store), head_hash)?);
+                .chain(RootsIterator::from_block(store.clone(), head_hash)?);
+
             for maybe_tuple in iter {
-                let (block_hash, state_hash, slot) = maybe_tuple?;
-                if slot < old_finalized_slot {
-                    // We must assume here any candidate chains include old_finalized_block_hash,
-                    // i.e. there aren't any forks starting at a block that is a strict ancestor of
-                    // old_finalized_block_hash.
-                    break;
-                }
-                match newly_finalized_blocks.get(&block_hash.into()).copied() {
-                    // Block is not finalized, mark it and its state for deletion
+                let (block_root, state_root, slot) = maybe_tuple?;
+                let block_root = SignedBeaconBlockHash::from(block_root);
+                let state_root = BeaconStateHash::from(state_root);
+
+                match newly_finalized_chain.get(&slot) {
+                    // If there's no information about a slot on the finalized chain, then
+                    // it should be because it's ahead of the new finalized slot. Stage
+                    // the fork's block and state for possible deletion.
                     None => {
-                        potentially_abandoned_blocks.push((
-                            slot,
-                            Some(block_hash.into()),
-                            Some(state_hash.into()),
-                        ));
+                        if slot > new_finalized_slot {
+                            potentially_abandoned_blocks.push((
+                                slot,
+                                Some(block_root),
+                                Some(state_root),
+                            ));
+                        } else if slot >= old_finalized_slot {
+                            return Err(PruningError::MissingInfoForCanonicalChain { slot }.into());
+                        } else {
+                            // We must assume here any candidate chains include the old finalized
+                            // checkpoint, i.e. there aren't any forks starting at a block that is a
+                            // strict ancestor of old_finalized_checkpoint.
+                            warn!(
+                                log,
+                                "Found a chain that should already have been pruned";
+                                "head_block_root" => format!("{:?}", head_hash),
+                                "head_slot" => head_slot,
+                            );
+                            break;
+                        }
                     }
-                    Some(finalized_slot) => {
-                        // Block root is finalized, and we have reached the slot it was finalized
-                        // at: we've hit a shared part of the chain.
-                        if finalized_slot == slot {
-                            // The first finalized block of a candidate chain lies after (in terms
-                            // of slots order) the newly finalized block.  It's not a candidate for
-                            // prunning.
-                            if finalized_slot == new_finalized_slot {
+                    Some((finalized_block_root, finalized_state_root)) => {
+                        // This fork descends from a newly finalized block, we can stop.
+                        if block_root == *finalized_block_root {
+                            // Sanity check: if the slot and block root match, then the
+                            // state roots should match too.
+                            if state_root != *finalized_state_root {
+                                return Err(PruningError::UnexpectedUnequalStateRoots.into());
+                            }
+
+                            // If the fork descends from the whole finalized chain,
+                            // do not prune it. Otherwise continue to delete all
+                            // of the blocks and states that have been staged for
+                            // deletion so far.
+                            if slot == new_finalized_slot {
                                 potentially_abandoned_blocks.clear();
                                 potentially_abandoned_head.take();
                             }
-
+                            // If there are skipped slots on the fork to be pruned, then
+                            // we will have just staged the common block for deletion.
+                            // Unstage it.
+                            else {
+                                for (_, block_root, _) in
+                                    potentially_abandoned_blocks.iter_mut().rev()
+                                {
+                                    if block_root.as_ref() == Some(finalized_block_root) {
+                                        *block_root = None;
+                                    } else {
+                                        break;
+                                    }
+                                }
+                            }
                             break;
-                        }
-                        // Block root is finalized, but we're at a skip slot: delete the state only.
-                        else {
+                        } else {
+                            if state_root == *finalized_state_root {
+                                return Err(PruningError::UnexpectedEqualStateRoots.into());
+                            }
                             potentially_abandoned_blocks.push((
                                 slot,
-                                None,
-                                Some(state_hash.into()),
+                                Some(block_root),
+                                Some(state_root),
                             ));
                         }
                     }
                 }
             }
 
-            abandoned_heads.extend(potentially_abandoned_head.into_iter());
-            if !potentially_abandoned_blocks.is_empty() {
+            if let Some(abandoned_head) = potentially_abandoned_head {
+                debug!(
+                    log,
+                    "Pruning head";
+                    "head_block_root" => format!("{:?}", abandoned_head),
+                    "head_slot" => head_slot,
+                );
+                abandoned_heads.insert(abandoned_head);
                 abandoned_blocks.extend(
                     potentially_abandoned_blocks
                         .iter()
                         .filter_map(|(_, maybe_block_hash, _)| *maybe_block_hash),
                 );
                 abandoned_states.extend(potentially_abandoned_blocks.iter().filter_map(
-                    |(slot, _, maybe_state_hash)| match maybe_state_hash {
-                        None => None,
-                        Some(state_hash) => Some((*slot, *state_hash)),
-                    },
+                    |(slot, _, maybe_state_hash)| maybe_state_hash.map(|sr| (*slot, sr)),
                 ));
             }
         }
@@ -161,11 +238,14 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
                     .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash, slot)),
             )
             .collect();
+
         store.do_atomically(batch)?;
         for head_hash in abandoned_heads.into_iter() {
             head_tracker.remove_head(head_hash);
         }
 
+        debug!(log, "Database pruning complete");
+
         Ok(())
     }
 }
@@ -174,6 +254,17 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
 pub struct NullMigrator;
 
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold> for NullMigrator {
+    fn process_finalization(
+        &self,
+        _finalized_state_root: BeaconStateHash,
+        _new_finalized_state: BeaconState<E>,
+        _head_tracker: Arc<HeadTracker>,
+        _old_finalized_checkpoint: Checkpoint,
+        _new_finalized_checkpoint: Checkpoint,
+    ) -> Result<(), BeaconChainError> {
+        Ok(())
+    }
+
     fn new(_: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
         NullMigrator
     }
@@ -184,48 +275,59 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold> fo
 /// Mostly useful for tests.
 pub struct BlockingMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
     db: Arc<HotColdDB<E, Hot, Cold>>,
+    log: Logger,
 }
 
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
     for BlockingMigrator<E, Hot, Cold>
 {
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
-        BlockingMigrator { db }
+    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self {
+        BlockingMigrator { db, log }
     }
 
     fn process_finalization(
         &self,
-        state_root: Hash256,
+        finalized_state_root: BeaconStateHash,
         new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
         head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
-        if let Err(e) = Self::prune_abandoned_forks(
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+    ) -> Result<(), BeaconChainError> {
+        Self::prune_abandoned_forks(
             self.db.clone(),
             head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_state.slot,
-        ) {
-            eprintln!("Pruning error: {:?}", e);
-        }
+            finalized_state_root,
+            &new_finalized_state,
+            old_finalized_checkpoint,
+            new_finalized_checkpoint,
+            &self.log,
+        )?;
 
-        if let Err(e) = process_finalization(self.db.clone(), state_root, &new_finalized_state) {
-            // This migrator is only used for testing, so we just log to stderr without a logger.
-            eprintln!("Migration error: {:?}", e);
+        match migrate_database(
+            self.db.clone(),
+            finalized_state_root.into(),
+            &new_finalized_state,
+        ) {
+            Ok(()) => Ok(()),
+            Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
+                debug!(
+                    self.log,
+                    "Database migration postponed, unaligned finalized block";
+                    "slot" => slot.as_u64()
+                );
+                Ok(())
+            }
+            Err(e) => Err(e.into()),
         }
     }
 }
 
 type MpscSender<E> = mpsc::Sender<(
-    Hash256,
+    BeaconStateHash,
     BeaconState<E>,
     Arc<HeadTracker>,
-    SignedBeaconBlockHash,
-    SignedBeaconBlockHash,
-    Slot,
+    Checkpoint,
+    Checkpoint,
 )>;
 
 /// Migrator that runs a background thread to migrate state from the hot to the cold database.
@@ -243,34 +345,26 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
         Self { db, tx_thread, log }
     }
 
-    /// Perform the freezing operation on the database,
     fn process_finalization(
         &self,
-        finalized_state_root: Hash256,
+        finalized_state_root: BeaconStateHash,
         new_finalized_state: BeaconState<E>,
-        max_finality_distance: u64,
         head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
-        if !self.needs_migration(new_finalized_state.slot, max_finality_distance) {
-            return;
-        }
-
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+    ) -> Result<(), BeaconChainError> {
         let (ref mut tx, ref mut thread) = *self.tx_thread.lock();
 
-        let new_finalized_slot = new_finalized_state.slot;
         if let Err(tx_err) = tx.send((
             finalized_state_root,
             new_finalized_state,
             head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_slot,
+            old_finalized_checkpoint,
+            new_finalized_checkpoint,
         )) {
             let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
 
-            drop(mem::replace(tx, new_tx));
+            *tx = new_tx;
             let old_thread = mem::replace(thread, new_thread);
 
             // Join the old thread, which will probably have panicked, or may have
@@ -286,57 +380,43 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
             // Retry at most once, we could recurse but that would risk overflowing the stack.
             let _ = tx.send(tx_err.0);
         }
+
+        Ok(())
     }
 }
 
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
-    /// Return true if a migration needs to be performed, given a new `finalized_slot`.
-    fn needs_migration(&self, finalized_slot: Slot, max_finality_distance: u64) -> bool {
-        let finality_distance = finalized_slot - self.db.get_split_slot();
-        finality_distance > max_finality_distance
-    }
-
-    #[allow(clippy::type_complexity)]
     /// Spawn a new child thread to run the migration process.
     ///
     /// Return a channel handle for sending new finalized states to the thread.
     fn spawn_thread(
         db: Arc<HotColdDB<E, Hot, Cold>>,
         log: Logger,
-    ) -> (
-        mpsc::Sender<(
-            Hash256,
-            BeaconState<E>,
-            Arc<HeadTracker>,
-            SignedBeaconBlockHash,
-            SignedBeaconBlockHash,
-            Slot,
-        )>,
-        thread::JoinHandle<()>,
-    ) {
+    ) -> (MpscSender<E>, thread::JoinHandle<()>) {
         let (tx, rx) = mpsc::channel();
         let thread = thread::spawn(move || {
             while let Ok((
                 state_root,
                 state,
                 head_tracker,
-                old_finalized_block_hash,
-                new_finalized_block_hash,
-                new_finalized_slot,
+                old_finalized_checkpoint,
+                new_finalized_checkpoint,
             )) = rx.recv()
             {
                 match Self::prune_abandoned_forks(
                     db.clone(),
                     head_tracker,
-                    old_finalized_block_hash,
-                    new_finalized_block_hash,
-                    new_finalized_slot,
+                    state_root,
+                    &state,
+                    old_finalized_checkpoint,
+                    new_finalized_checkpoint,
+                    &log,
                 ) {
                     Ok(()) => {}
                     Err(e) => warn!(log, "Block pruning failed: {:?}", e),
                 }
 
-                match process_finalization(db.clone(), state_root, &state) {
+                match migrate_database(db.clone(), state_root.into(), &state) {
                     Ok(()) => {}
                     Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
                         debug!(

beacon_node/beacon_chain/src/persisted_beacon_chain.rs

@@ -6,6 +6,13 @@ use types::Hash256;
 
 #[derive(Clone, Encode, Decode)]
 pub struct PersistedBeaconChain {
+    /// This value is ignored to resolve the issue described here:
+    ///
+    /// https://github.com/sigp/lighthouse/pull/1639
+    ///
+    /// The following PR will clean-up and remove this field:
+    ///
+    /// https://github.com/sigp/lighthouse/pull/1638
     pub canonical_head_block_root: Hash256,
     pub genesis_block_root: Hash256,
     pub ssz_head_tracker: SszHeadTracker,

beacon_node/beacon_chain/tests/attestation_production.rs

@@ -26,7 +26,7 @@ lazy_static! {
 fn produces_attestations() {
     let num_blocks_produced = MainnetEthSpec::slots_per_epoch() * 4;
 
-    let harness = BeaconChainHarness::new(
+    let mut harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[..].to_vec(),
         StoreConfig::default(),

beacon_node/beacon_chain/tests/attestation_verification.rs

@@ -5,7 +5,9 @@ extern crate lazy_static;
 
 use beacon_chain::{
     attestation_verification::Error as AttnError,
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, NullMigratorEphemeralHarnessType,
+    },
     BeaconChain, BeaconChainTypes,
 };
 use int_to_bytes::int_to_bytes32;
@@ -30,7 +32,7 @@ lazy_static! {
 }
 
 /// Returns a beacon chain harness.
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
+fn get_harness(validator_count: usize) -> BeaconChainHarness<NullMigratorEphemeralHarnessType<E>> {
     let harness = BeaconChainHarness::new_with_target_aggregators(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
@@ -184,8 +186,7 @@ fn get_non_aggregator<T: BeaconChainTypes>(
 /// Tests verification of `SignedAggregateAndProof` from the gossip network.
 #[test]
 fn aggregated_gossip_verification() {
-    let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -197,7 +198,7 @@ fn aggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
+    let current_slot = harness.chain.slot().expect("should get slot");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -532,8 +533,7 @@ fn aggregated_gossip_verification() {
 /// Tests the verification conditions for an unaggregated attestation on the gossip network.
 #[test]
 fn unaggregated_gossip_verification() {
-    let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -545,8 +545,8 @@ fn unaggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -772,8 +772,7 @@ fn unaggregated_gossip_verification() {
 /// This also checks that we can do a state lookup if we don't get a hit from the shuffling cache.
 #[test]
 fn attestation_that_skips_epochs() {
-    let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -782,16 +781,18 @@ fn attestation_that_skips_epochs() {
         AttestationStrategy::SomeValidators(vec![]),
     );
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     let earlier_slot = (current_epoch - 2).start_slot(MainnetEthSpec::slots_per_epoch());
-    let earlier_block = chain
+    let earlier_block = harness
+        .chain
         .block_at_slot(earlier_slot)
         .expect("should not error getting block at slot")
         .expect("should find block at slot");
 
-    let mut state = chain
+    let mut state = harness
+        .chain
         .get_state(&earlier_block.state_root(), Some(earlier_slot))
         .expect("should not error getting state")
         .expect("should find state");

beacon_node/beacon_chain/tests/block_verification.rs

@@ -4,7 +4,9 @@
 extern crate lazy_static;
 
 use beacon_chain::{
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, NullMigratorEphemeralHarnessType,
+    },
     BeaconSnapshot, BlockError,
 };
 use store::config::StoreConfig;
@@ -31,7 +33,7 @@ lazy_static! {
 }
 
 fn get_chain_segment() -> Vec<BeaconSnapshot<E>> {
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         CHAIN_SEGMENT_LENGTH,
@@ -48,8 +50,8 @@ fn get_chain_segment() -> Vec<BeaconSnapshot<E>> {
         .collect()
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(validator_count: usize) -> BeaconChainHarness<NullMigratorEphemeralHarnessType<E>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -81,7 +83,7 @@ fn junk_aggregate_signature() -> AggregateSignature {
 
 fn update_proposal_signatures(
     snapshots: &mut [BeaconSnapshot<E>],
-    harness: &BeaconChainHarness<HarnessType<E>>,
+    harness: &BeaconChainHarness<NullMigratorEphemeralHarnessType<E>>,
 ) {
     for snapshot in snapshots {
         let spec = &harness.chain.spec;
@@ -91,7 +93,7 @@ fn update_proposal_signatures(
             .get_beacon_proposer_index(slot, spec)
             .expect("should find proposer index");
         let keypair = harness
-            .keypairs
+            .validators_keypairs
             .get(proposer_index)
             .expect("proposer keypair should be available");
 
@@ -274,7 +276,7 @@ fn chain_segment_non_linear_slots() {
 }
 
 fn assert_invalid_signature(
-    harness: &BeaconChainHarness<HarnessType<E>>,
+    harness: &BeaconChainHarness<NullMigratorEphemeralHarnessType<E>>,
     block_index: usize,
     snapshots: &[BeaconSnapshot<E>],
     item: &str,
@@ -325,7 +327,7 @@ fn assert_invalid_signature(
     // slot) tuple.
 }
 
-fn get_invalid_sigs_harness() -> BeaconChainHarness<HarnessType<E>> {
+fn get_invalid_sigs_harness() -> BeaconChainHarness<NullMigratorEphemeralHarnessType<E>> {
     let harness = get_harness(VALIDATOR_COUNT);
     harness
         .chain

beacon_node/beacon_chain/tests/op_verification.rs

@@ -7,7 +7,7 @@ extern crate lazy_static;
 
 use beacon_chain::observed_operations::ObservationOutcome;
 use beacon_chain::test_utils::{
-    AttestationStrategy, BeaconChainHarness, BlockStrategy, DiskHarnessType,
+    AttestationStrategy, BeaconChainHarness, BlockStrategy, BlockingMigratorDiskHarnessType,
 };
 use sloggers::{null::NullLoggerBuilder, Build};
 use std::sync::Arc;
@@ -28,7 +28,7 @@ lazy_static! {
 }
 
 type E = MinimalEthSpec;
-type TestHarness = BeaconChainHarness<DiskHarnessType<E>>;
+type TestHarness = BeaconChainHarness<BlockingMigratorDiskHarnessType<E>>;
 type HotColdDB = store::HotColdDB<E, LevelDB<E>, LevelDB<E>>;
 
 fn get_store(db_path: &TempDir) -> Arc<HotColdDB> {
@@ -57,8 +57,8 @@ fn get_harness(store: Arc<HotColdDB>, validator_count: usize) -> TestHarness {
 fn voluntary_exit() {
     let db_path = tempdir().unwrap();
     let store = get_store(&db_path);
-    let harness = get_harness(store.clone(), VALIDATOR_COUNT);
-    let spec = &harness.chain.spec;
+    let mut harness = get_harness(store.clone(), VALIDATOR_COUNT);
+    let spec = &harness.chain.spec.clone();
 
     harness.extend_chain(
         (E::slots_per_epoch() * (spec.shard_committee_period + 1)) as usize,

beacon_node/beacon_chain/tests/persistence_tests.rs

@@ -44,7 +44,7 @@ fn finalizes_after_resuming_from_db() {
     let db_path = tempdir().unwrap();
     let store = get_store(&db_path);
 
-    let harness = BeaconChainHarness::new_with_disk_store(
+    let mut harness = BeaconChainHarness::new_with_disk_store(
         MinimalEthSpec,
         store.clone(),
         KEYPAIRS[0..validator_count].to_vec(),
@@ -88,7 +88,7 @@ fn finalizes_after_resuming_from_db() {
     let data_dir = harness.data_dir;
     let original_chain = harness.chain;
 
-    let resumed_harness = BeaconChainHarness::resume_from_disk_store(
+    let mut resumed_harness = BeaconChainHarness::resume_from_disk_store(
         MinimalEthSpec,
         store,
         KEYPAIRS[0..validator_count].to_vec(),
@@ -153,8 +153,11 @@ fn assert_chains_pretty_much_the_same<T: BeaconChainTypes>(a: &BeaconChain<T>, b
         a.genesis_block_root, b.genesis_block_root,
         "genesis_block_root should be equal"
     );
+
+    let slot = a.slot().unwrap();
     assert!(
-        *a.fork_choice.read() == *b.fork_choice.read(),
-        "fork_choice should be equal"
+        a.fork_choice.write().get_head(slot).unwrap()
+            == b.fork_choice.write().get_head(slot).unwrap(),
+        "fork_choice heads should be equal"
     );
 }

beacon_node/beacon_chain/tests/tests.rs

@@ -6,7 +6,8 @@ extern crate lazy_static;
 use beacon_chain::{
     attestation_verification::Error as AttnError,
     test_utils::{
-        AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType, OP_POOL_DB_KEY,
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, NullMigratorEphemeralHarnessType,
+        OP_POOL_DB_KEY,
     },
 };
 use operation_pool::PersistedOperationPool;
@@ -24,8 +25,10 @@ lazy_static! {
     static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<MinimalEthSpec>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(
+    validator_count: usize,
+) -> BeaconChainHarness<NullMigratorEphemeralHarnessType<MinimalEthSpec>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MinimalEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -64,7 +67,7 @@ fn massive_skips() {
 fn iterators() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -139,7 +142,7 @@ fn iterators() {
 
 #[test]
 fn chooses_fork() {
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     let two_thirds = (VALIDATOR_COUNT / 3) * 2;
     let delay = MinimalEthSpec::default_spec().min_attestation_inclusion_delay as usize;
@@ -190,7 +193,7 @@ fn chooses_fork() {
 fn finalizes_with_full_participation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -225,7 +228,7 @@ fn finalizes_with_full_participation() {
 fn finalizes_with_two_thirds_participation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     let two_thirds = (VALIDATOR_COUNT / 3) * 2;
     let attesters = (0..two_thirds).collect();
@@ -268,7 +271,7 @@ fn finalizes_with_two_thirds_participation() {
 fn does_not_finalize_with_less_than_two_thirds_participation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     let two_thirds = (VALIDATOR_COUNT / 3) * 2;
     let less_than_two_thirds = two_thirds - 1;
@@ -305,7 +308,7 @@ fn does_not_finalize_with_less_than_two_thirds_participation() {
 fn does_not_finalize_without_attestation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -338,7 +341,7 @@ fn does_not_finalize_without_attestation() {
 fn roundtrip_operation_pool() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     // Add some attestations
     harness.extend_chain(
@@ -370,7 +373,7 @@ fn roundtrip_operation_pool() {
 fn unaggregated_attestations_added_to_fork_choice_some_none() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() / 2;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -424,7 +427,7 @@ fn unaggregated_attestations_added_to_fork_choice_some_none() {
 fn attestations_with_increasing_slots() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     let mut attestations = vec![];
 
@@ -486,7 +489,7 @@ fn attestations_with_increasing_slots() {
 fn unaggregated_attestations_added_to_fork_choice_all_updated() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1;
 
-    let harness = get_harness(VALIDATOR_COUNT);
+    let mut harness = get_harness(VALIDATOR_COUNT);
 
     harness.extend_chain(
         num_blocks_produced as usize,
@@ -541,7 +544,7 @@ fn unaggregated_attestations_added_to_fork_choice_all_updated() {
 
 fn run_skip_slot_test(skip_slots: u64) {
     let num_validators = 8;
-    let harness_a = get_harness(num_validators);
+    let mut harness_a = get_harness(num_validators);
     let harness_b = get_harness(num_validators);
 
     for _ in 0..skip_slots {

beacon_node/client/src/builder.rs

@@ -584,7 +584,7 @@ where
     /// Specifies that the `BeaconChain` should cache eth1 blocks/logs from a remote eth1 node
     /// (e.g., Parity/Geth) and refer to that cache when collecting deposits or eth1 votes during
     /// block production.
-    pub fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
+    pub async fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
         let context = self
             .runtime_context
             .as_ref()
@@ -598,6 +598,17 @@ where
             .clone()
             .ok_or_else(|| "caching_eth1_backend requires a chain spec".to_string())?;
 
+        // Check if the eth1 endpoint we connect to is on the correct network id.
+        let network_id =
+            eth1::http::get_network_id(&config.endpoint, Duration::from_millis(15_000)).await?;
+
+        if network_id != config.network_id {
+            return Err(format!(
+                "Invalid eth1 network id. Expected {:?}, got {:?}",
+                config.network_id, network_id
+            ));
+        }
+
         let backend = if let Some(eth1_service_from_genesis) = self.eth1_service {
             eth1_service_from_genesis.update_config(config)?;
 
@@ -702,11 +713,8 @@ where
             .ok_or_else(|| "system_time_slot_clock requires a beacon_chain_builder")?;
 
         let genesis_time = beacon_chain_builder
-            .finalized_snapshot
-            .as_ref()
-            .ok_or_else(|| "system_time_slot_clock requires an initialized beacon state")?
-            .beacon_state
-            .genesis_time;
+            .genesis_time
+            .ok_or_else(|| "system_time_slot_clock requires an initialized beacon state")?;
 
         let spec = self
             .chain_spec

beacon_node/eth1/src/http.rs

@@ -12,8 +12,10 @@
 
 use futures::future::TryFutureExt;
 use reqwest::{header::CONTENT_TYPE, ClientBuilder, StatusCode};
+use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
 use std::ops::Range;
+use std::str::FromStr;
 use std::time::Duration;
 use types::Hash256;
 
@@ -30,6 +32,40 @@ pub const DEPOSIT_COUNT_RESPONSE_BYTES: usize = 96;
 /// Number of bytes in deposit contract deposit root (value only).
 pub const DEPOSIT_ROOT_BYTES: usize = 32;
 
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub enum Eth1NetworkId {
+    Goerli,
+    Mainnet,
+    Custom(u64),
+}
+
+impl FromStr for Eth1NetworkId {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "1" => Ok(Eth1NetworkId::Mainnet),
+            "5" => Ok(Eth1NetworkId::Goerli),
+            custom => {
+                let network_id = u64::from_str_radix(custom, 10)
+                    .map_err(|e| format!("Failed to parse eth1 network id {}", e))?;
+                Ok(Eth1NetworkId::Custom(network_id))
+            }
+        }
+    }
+}
+
+/// Get the eth1 network id of the given endpoint.
+pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result<Eth1NetworkId, String> {
+    let response_body = send_rpc_request(endpoint, "net_version", json!([]), timeout).await?;
+    Eth1NetworkId::from_str(
+        response_result(&response_body)?
+            .ok_or_else(|| "No result was returned for block number".to_string())?
+            .as_str()
+            .ok_or_else(|| "Data was not string")?,
+    )
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct Block {
     pub hash: Hash256,

beacon_node/eth1/src/service.rs

@@ -2,7 +2,9 @@ use crate::metrics;
 use crate::{
     block_cache::{BlockCache, Error as BlockCacheError, Eth1Block},
     deposit_cache::Error as DepositCacheError,
-    http::{get_block, get_block_number, get_deposit_logs_in_range, Log},
+    http::{
+        get_block, get_block_number, get_deposit_logs_in_range, get_network_id, Eth1NetworkId, Log,
+    },
     inner::{DepositUpdater, Inner},
     DepositLog,
 };
@@ -16,6 +18,9 @@ use std::time::{SystemTime, UNIX_EPOCH};
 use tokio::time::{interval_at, Duration, Instant};
 use types::ChainSpec;
 
+/// Indicates the default eth1 network we use for the deposit contract.
+pub const DEFAULT_NETWORK_ID: Eth1NetworkId = Eth1NetworkId::Goerli;
+
 const STANDARD_TIMEOUT_MILLIS: u64 = 15_000;
 
 /// Timeout when doing a eth_blockNumber call.
@@ -76,6 +81,8 @@ pub struct Config {
     pub endpoint: String,
     /// The address the `BlockCache` and `DepositCache` should assume is the canonical deposit contract.
     pub deposit_contract_address: String,
+    /// The eth1 network id where the deposit contract is deployed (Goerli/Mainnet).
+    pub network_id: Eth1NetworkId,
     /// Defines the first block that the `DepositCache` will start searching for deposit logs.
     ///
     /// Setting too high can result in missed logs. Setting too low will result in unnecessary
@@ -105,6 +112,7 @@ impl Default for Config {
         Self {
             endpoint: "http://localhost:8545".into(),
             deposit_contract_address: "0x0000000000000000000000000000000000000000".into(),
+            network_id: DEFAULT_NETWORK_ID,
             deposit_contract_deploy_block: 1,
             lowest_cached_block_number: 1,
             follow_distance: 128,
@@ -350,6 +358,29 @@ impl Service {
     }
 
     async fn do_update(&self, update_interval: Duration) -> Result<(), ()> {
+        let endpoint = self.config().endpoint.clone();
+        let config_network = self.config().network_id.clone();
+        let result =
+            get_network_id(&endpoint, Duration::from_millis(STANDARD_TIMEOUT_MILLIS)).await;
+        match result {
+            Ok(network_id) => {
+                if network_id != config_network {
+                    error!(
+                        self.log,
+                        "Failed to update eth1 cache";
+                        "reason" => "Invalid eth1 network id",
+                        "expected" => format!("{:?}",DEFAULT_NETWORK_ID),
+                        "got" => format!("{:?}",network_id),
+                    );
+                    return Ok(());
+                }
+            }
+            Err(e) => {
+                error!(self.log, "Failed to get eth1 network id"; "error" => e);
+                return Ok(());
+            }
+        }
+
         let update_result = self.update().await;
         match update_result {
             Err(e) => error!(

beacon_node/eth2_libp2p/Cargo.toml

@@ -32,16 +32,17 @@ snap = "1.0.0"
 void = "1.0.2"
 tokio-io-timeout = "0.4.0"
 tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
-discv5 = { version = "0.1.0-alpha.8", features = ["libp2p", "openssl-vendored"] }
+discv5 = { version = "0.1.0-alpha.10", features = ["libp2p"] }
 tiny-keccak = "2.0.2"
 environment = { path = "../../lighthouse/environment" }
 # TODO: Remove rand crate for mainnet
 rand = "0.7.3"
+regex = "1.3.9"
 
 [dependencies.libp2p]
 #version = "0.23.0"
 git = "https://github.com/sigp/rust-libp2p"
-rev = "bbf0cfbaff2f733b3ae7bfed3caba8b7ee542803"
+rev = "03f998022ce2f566a6c6e6c4206bc0ce4d45109f"
 default-features = false
 features = ["websocket", "identify", "mplex", "noise", "gossipsub", "dns", "tcp-tokio"]
 

beacon_node/eth2_libp2p/src/behaviour/handler/delegate.rs

@@ -131,8 +131,9 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         let gossip_proto = self.gossip_handler.listen_protocol();
         let rpc_proto = self.rpc_handler.listen_protocol();
         let identify_proto = self.identify_handler.listen_protocol();
@@ -147,24 +148,27 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             SelectUpgrade::new(rpc_proto.into_upgrade().1, identify_proto.into_upgrade().1),
         );
 
-        SubstreamProtocol::new(select).with_timeout(timeout)
+        SubstreamProtocol::new(select, ()).with_timeout(timeout)
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         match out {
             // Gossipsub
-            EitherOutput::First(out) => self.gossip_handler.inject_fully_negotiated_inbound(out),
+            EitherOutput::First(out) => {
+                self.gossip_handler.inject_fully_negotiated_inbound(out, ())
+            }
             // RPC
             EitherOutput::Second(EitherOutput::First(out)) => {
-                self.rpc_handler.inject_fully_negotiated_inbound(out)
+                self.rpc_handler.inject_fully_negotiated_inbound(out, ())
             }
             // Identify
-            EitherOutput::Second(EitherOutput::Second(out)) => {
-                self.identify_handler.inject_fully_negotiated_inbound(out)
-            }
+            EitherOutput::Second(EitherOutput::Second(out)) => self
+                .identify_handler
+                .inject_fully_negotiated_inbound(out, ()),
         }
     }
 
@@ -317,10 +321,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
                     event,
                 )));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(EitherUpgrade::A),
-                    info: EitherOutput::First(info),
+                    protocol: protocol
+                        .map_upgrade(EitherUpgrade::A)
+                        .map_info(EitherOutput::First),
                 });
             }
             Poll::Pending => (),
@@ -333,10 +338,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::RPC(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u))),
-                    info: EitherOutput::Second(EitherOutput::First(info)),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u)))
+                        .map_info(|info| EitherOutput::Second(EitherOutput::First(info))),
                 });
             }
             Poll::Pending => (),
@@ -351,10 +357,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::Identify(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u))),
-                    info: EitherOutput::Second(EitherOutput::Second(())),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u)))
+                        .map_info(|_| EitherOutput::Second(EitherOutput::Second(()))),
                 });
             }
             Poll::Pending => (),

beacon_node/eth2_libp2p/src/behaviour/handler/mod.rs

@@ -54,16 +54,18 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.delegate.listen_protocol()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
-        self.delegate.inject_fully_negotiated_inbound(out)
+        self.delegate.inject_fully_negotiated_inbound(out, ())
     }
 
     fn inject_fully_negotiated_outbound(
@@ -127,11 +129,8 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(err)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(err))
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
-                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol,
-                    info,
-                });
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
+                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol });
             }
             Poll::Pending => (),
         }

beacon_node/eth2_libp2p/src/behaviour/mod.rs

@@ -1,6 +1,7 @@
 use crate::peer_manager::{score::PeerAction, PeerManager, PeerManagerEvent};
 use crate::rpc::*;
-use crate::types::{EnrBitfield, GossipEncoding, GossipKind, GossipTopic};
+use crate::service::METADATA_FILENAME;
+use crate::types::{GossipEncoding, GossipKind, GossipTopic, SubnetDiscovery};
 use crate::Eth2Enr;
 use crate::{error, metrics, Enr, NetworkConfig, NetworkGlobals, PubsubMessage, TopicHash};
 use futures::prelude::*;
@@ -11,7 +12,10 @@ use libp2p::{
         identity::Keypair,
         Multiaddr,
     },
-    gossipsub::{Gossipsub, GossipsubEvent, MessageAuthenticity, MessageId},
+    gossipsub::{
+        Gossipsub, GossipsubEvent, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity,
+        MessageId,
+    },
     identify::{Identify, IdentifyEvent},
     swarm::{
         NetworkBehaviour, NetworkBehaviourAction as NBAction, NotifyHandler, PollParameters,
@@ -20,23 +24,21 @@ use libp2p::{
     PeerId,
 };
 use slog::{crit, debug, o, trace, warn};
-use ssz::{Decode, Encode};
+use ssz::Encode;
 use std::fs::File;
-use std::io::{Read, Write};
+use std::io::Write;
 use std::path::PathBuf;
 use std::{
     collections::VecDeque,
     marker::PhantomData,
     sync::Arc,
     task::{Context, Poll},
-    time::Instant,
 };
 use types::{EnrForkId, EthSpec, SignedBeaconBlock, SubnetId};
 
 mod handler;
 
 const MAX_IDENTIFY_ADDRESSES: usize = 10;
-const METADATA_FILENAME: &str = "metadata";
 
 /// Builds the network behaviour that manages the core protocols of eth2.
 /// This core behaviour is managed by `Behaviour` which adds peer management to all core
@@ -56,8 +58,6 @@ pub struct Behaviour<TSpec: EthSpec> {
     events: VecDeque<BehaviourEvent<TSpec>>,
     /// Queue of peers to disconnect and an optional reason for the disconnection.
     peers_to_dc: VecDeque<(PeerId, Option<GoodbyeReason>)>,
-    /// The current meta data of the node, so respond to pings and get metadata
-    meta_data: MetaData<TSpec>,
     /// A collections of variables accessible outside the network service.
     network_globals: Arc<NetworkGlobals<TSpec>>,
     /// Keeps track of the current EnrForkId for upgrading gossipsub topics.
@@ -93,23 +93,24 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .eth2()
             .expect("Local ENR must have a fork id");
 
-        let meta_data = load_or_build_metadata(&net_conf.network_dir, &log);
+        let gossipsub = Gossipsub::new(MessageAuthenticity::Anonymous, net_conf.gs_config.clone())
+            .map_err(|e| format!("Could not construct gossipsub: {:?}", e))?;
 
-        // TODO: Until other clients support no author, we will use a 0 peer_id as our author.
-        let message_author = PeerId::from_bytes(vec![0, 1, 0]).expect("Valid peer id");
+        // Temporarily disable scoring until parameters are tested.
+        /*
+        gossipsub
+            .with_peer_score(PeerScoreParams::default(), PeerScoreThresholds::default())
+            .expect("Valid score params and thresholds");
+        */
 
         Ok(Behaviour {
             eth2_rpc: RPC::new(log.clone()),
-            gossipsub: Gossipsub::new(
-                MessageAuthenticity::Author(message_author),
-                net_conf.gs_config.clone(),
-            ),
+            gossipsub,
             identify,
             peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)
                 .await?,
             events: VecDeque::new(),
             peers_to_dc: VecDeque::new(),
-            meta_data,
             network_globals,
             enr_fork_id,
             waker: None,
@@ -148,6 +149,10 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             GossipEncoding::default(),
             self.enr_fork_id.fork_digest,
         );
+
+        // TODO: Implement scoring
+        // let topic: Topic = gossip_topic.into();
+        // self.gossipsub.set_topic_params(t.hash(), TopicScoreParams::default());
         self.subscribe(gossip_topic)
     }
 
@@ -169,6 +174,12 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             GossipEncoding::default(),
             self.enr_fork_id.fork_digest,
         );
+        // TODO: Implement scoring
+        /*
+        let t: Topic = topic.clone().into();
+        self.gossipsub
+            .set_topic_params(t.hash(), TopicScoreParams::default());
+        */
         self.subscribe(topic)
     }
 
@@ -190,9 +201,18 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .write()
             .insert(topic.clone());
 
-        let topic_str: String = topic.clone().into();
-        debug!(self.log, "Subscribed to topic"; "topic" => topic_str);
-        self.gossipsub.subscribe(topic.into())
+        let topic: Topic = topic.into();
+
+        match self.gossipsub.subscribe(&topic) {
+            Err(_) => {
+                warn!(self.log, "Failed to subscribe to topic"; "topic" => topic.to_string());
+                false
+            }
+            Ok(v) => {
+                debug!(self.log, "Subscribed to topic"; "topic" => topic.to_string());
+                v
+            }
+        }
     }
 
     /// Unsubscribe from a gossipsub topic.
@@ -202,8 +222,20 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .gossipsub_subscriptions
             .write()
             .remove(&topic);
+
         // unsubscribe from the topic
-        self.gossipsub.unsubscribe(topic.into())
+        let topic: Topic = topic.into();
+
+        match self.gossipsub.unsubscribe(&topic) {
+            Err(_) => {
+                warn!(self.log, "Failed to unsubscribe from topic"; "topic" => topic.to_string());
+                false
+            }
+            Ok(v) => {
+                debug!(self.log, "Unsubscribed to topic"; "topic" => topic.to_string());
+                v
+            }
+        }
     }
 
     /// Publishes a list of messages on the pubsub (gossipsub) behaviour, choosing the encoding.
@@ -212,8 +244,28 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) {
                 match message.encode(GossipEncoding::default()) {
                     Ok(message_data) => {
-                        if let Err(e) = self.gossipsub.publish(&topic.into(), message_data) {
+                        if let Err(e) = self.gossipsub.publish(topic.clone().into(), message_data) {
                             slog::warn!(self.log, "Could not publish message"; "error" => format!("{:?}", e));
+
+                            // add to metrics
+                            match topic.kind() {
+                                GossipKind::Attestation(subnet_id) => {
+                                    if let Some(v) = metrics::get_int_gauge(
+                                        &metrics::FAILED_ATTESTATION_PUBLISHES_PER_SUBNET,
+                                        &[&subnet_id.to_string()],
+                                    ) {
+                                        v.inc()
+                                    };
+                                }
+                                kind => {
+                                    if let Some(v) = metrics::get_int_gauge(
+                                        &metrics::FAILED_PUBLISHES_PER_MAIN_TOPIC,
+                                        &[&format!("{:?}", kind)],
+                                    ) {
+                                        v.inc()
+                                    };
+                                }
+                            }
                         }
                     }
                     Err(e) => crit!(self.log, "Could not publish message"; "error" => e),
@@ -222,11 +274,21 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
         }
     }
 
-    /// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated
-    /// once validated by the beacon chain.
-    pub fn validate_message(&mut self, propagation_source: &PeerId, message_id: MessageId) {
-        self.gossipsub
-            .validate_message(&message_id, propagation_source);
+    /// Informs the gossipsub about the result of a message validation.
+    /// If the message is valid it will get propagated by gossipsub.
+    pub fn report_message_validation_result(
+        &mut self,
+        propagation_source: &PeerId,
+        message_id: MessageId,
+        validation_result: MessageAcceptance,
+    ) {
+        if let Err(e) = self.gossipsub.report_message_validation_result(
+            &message_id,
+            propagation_source,
+            validation_result,
+        ) {
+            warn!(self.log, "Failed to report message validation"; "message_id" => message_id.to_string(), "peer_id" => propagation_source.to_string(), "error" => format!("{:?}", e));
+        }
     }
 
     /* Eth2 RPC behaviour functions */
@@ -301,8 +363,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Attempts to discover new peers for a given subnet. The `min_ttl` gives the time at which we
     /// would like to retain the peers for.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
-        self.peer_manager.discover_subnet_peers(subnet_id, min_ttl)
+    pub fn discover_subnet_peers(&mut self, subnet_subscriptions: Vec<SubnetDiscovery>) {
+        self.peer_manager
+            .discover_subnet_peers(subnet_subscriptions)
     }
 
     /// Updates the local ENR's "eth2" field with the latest EnrForkId.
@@ -339,21 +402,31 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Updates the current meta data of the node to match the local ENR.
     fn update_metadata(&mut self) {
-        self.meta_data.seq_number += 1;
-        self.meta_data.attnets = self
+        let local_attnets = self
             .peer_manager
             .discovery()
             .local_enr()
             .bitfield::<TSpec>()
             .expect("Local discovery must have bitfield");
+
+        {
+            // write lock scope
+            let mut meta_data = self.network_globals.local_metadata.write();
+            meta_data.seq_number += 1;
+            meta_data.attnets = local_attnets;
+        }
         // Save the updated metadata to disk
-        save_metadata_to_disk(&self.network_dir, self.meta_data.clone(), &self.log);
+        save_metadata_to_disk(
+            &self.network_dir,
+            self.network_globals.local_metadata.read().clone(),
+            &self.log,
+        );
     }
 
     /// Sends a Ping request to the peer.
     fn ping(&mut self, id: RequestId, peer_id: PeerId) {
         let ping = crate::rpc::Ping {
-            data: self.meta_data.seq_number,
+            data: self.network_globals.local_metadata.read().seq_number,
         };
         trace!(self.log, "Sending Ping"; "request_id" => id, "peer_id" => peer_id.to_string());
 
@@ -364,7 +437,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
     /// Sends a Pong response to the peer.
     fn pong(&mut self, id: PeerRequestId, peer_id: PeerId) {
         let ping = crate::rpc::Ping {
-            data: self.meta_data.seq_number,
+            data: self.network_globals.local_metadata.read().seq_number,
         };
         trace!(self.log, "Sending Pong"; "request_id" => id.1, "peer_id" => peer_id.to_string());
         let event = RPCCodedResponse::Success(RPCResponse::Pong(ping));
@@ -380,7 +453,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Sends a METADATA response to a peer.
     fn send_meta_data_response(&mut self, id: PeerRequestId, peer_id: PeerId) {
-        let event = RPCCodedResponse::Success(RPCResponse::MetaData(self.meta_data.clone()));
+        let event = RPCCodedResponse::Success(RPCResponse::MetaData(
+            self.network_globals.local_metadata.read().clone(),
+        ));
         self.eth2_rpc.send_response(peer_id, id, event);
     }
 
@@ -392,11 +467,25 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     fn on_gossip_event(&mut self, event: GossipsubEvent) {
         match event {
-            GossipsubEvent::Message(propagation_source, id, gs_msg) => {
+            GossipsubEvent::Message {
+                propagation_source,
+                message_id: id,
+                message: gs_msg,
+            } => {
                 // Note: We are keeping track here of the peer that sent us the message, not the
                 // peer that originally published the message.
                 match PubsubMessage::decode(&gs_msg.topics, &gs_msg.data) {
-                    Err(e) => debug!(self.log, "Could not decode gossipsub message"; "error" => e),
+                    Err(e) => {
+                        debug!(self.log, "Could not decode gossipsub message"; "error" => e);
+                        //reject the message
+                        if let Err(e) = self.gossipsub.report_message_validation_result(
+                            &id,
+                            &propagation_source,
+                            MessageAcceptance::Reject,
+                        ) {
+                            warn!(self.log, "Failed to report message validation"; "message_id" => id.to_string(), "peer_id" => propagation_source.to_string(), "error" => format!("{:?}", e));
+                        }
+                    }
                     Ok(msg) => {
                         // Notify the network
                         self.add_event(BehaviourEvent::PubsubMessage {
@@ -409,23 +498,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                 }
             }
             GossipsubEvent::Subscribed { peer_id, topic } => {
-                if let Some(topic_metric) = metrics::get_int_gauge(
-                    &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_COUNT,
-                    &[topic.as_str()],
-                ) {
-                    topic_metric.inc()
-                }
-
                 self.add_event(BehaviourEvent::PeerSubscribed(peer_id, topic));
             }
-            GossipsubEvent::Unsubscribed { peer_id: _, topic } => {
-                if let Some(topic_metric) = metrics::get_int_gauge(
-                    &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_COUNT,
-                    &[topic.as_str()],
-                ) {
-                    topic_metric.dec()
-                }
-            }
+            GossipsubEvent::Unsubscribed { .. } => {}
         }
     }
 
@@ -1039,45 +1114,8 @@ pub enum BehaviourEvent<TSpec: EthSpec> {
     StatusPeer(PeerId),
 }
 
-/// Load metadata from persisted file. Return default metadata if loading fails.
-fn load_or_build_metadata<E: EthSpec>(network_dir: &PathBuf, log: &slog::Logger) -> MetaData<E> {
-    // Default metadata
-    let mut meta_data = MetaData {
-        seq_number: 0,
-        attnets: EnrBitfield::<E>::default(),
-    };
-    // Read metadata from persisted file if available
-    let metadata_path = network_dir.join(METADATA_FILENAME);
-    if let Ok(mut metadata_file) = File::open(metadata_path) {
-        let mut metadata_ssz = Vec::new();
-        if metadata_file.read_to_end(&mut metadata_ssz).is_ok() {
-            match MetaData::<E>::from_ssz_bytes(&metadata_ssz) {
-                Ok(persisted_metadata) => {
-                    meta_data.seq_number = persisted_metadata.seq_number;
-                    // Increment seq number if persisted attnet is not default
-                    if persisted_metadata.attnets != meta_data.attnets {
-                        meta_data.seq_number += 1;
-                    }
-                    debug!(log, "Loaded metadata from disk");
-                }
-                Err(e) => {
-                    debug!(
-                        log,
-                        "Metadata from file could not be decoded";
-                        "error" => format!("{:?}", e),
-                    );
-                }
-            }
-        }
-    };
-
-    debug!(log, "Metadata sequence number"; "seq_num" => meta_data.seq_number);
-    save_metadata_to_disk(network_dir, meta_data.clone(), &log);
-    meta_data
-}
-
 /// Persist metadata to disk
-fn save_metadata_to_disk<E: EthSpec>(dir: &PathBuf, metadata: MetaData<E>, log: &slog::Logger) {
+pub fn save_metadata_to_disk<E: EthSpec>(dir: &PathBuf, metadata: MetaData<E>, log: &slog::Logger) {
     let _ = std::fs::create_dir_all(&dir);
     match File::create(dir.join(METADATA_FILENAME))
         .and_then(|mut f| f.write_all(&metadata.as_ssz_bytes()))

beacon_node/eth2_libp2p/src/config.rs

@@ -1,5 +1,5 @@
 use crate::types::GossipKind;
-use crate::Enr;
+use crate::{Enr, PeerIdSerialized};
 use discv5::{Discv5Config, Discv5ConfigBuilder};
 use libp2p::gossipsub::{
     GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode,
@@ -58,6 +58,9 @@ pub struct Config {
     /// List of libp2p nodes to initially connect to.
     pub libp2p_nodes: Vec<Multiaddr>,
 
+    /// List of trusted libp2p nodes which are not scored.
+    pub trusted_peers: Vec<PeerIdSerialized>,
+
     /// Client version
     pub client_version: String,
 
@@ -85,13 +88,9 @@ impl Default for Config {
         ];
 
         // The function used to generate a gossipsub message id
-        // We use base64(SHA256(data)) for content addressing
-        let gossip_message_id = |message: &GossipsubMessage| {
-            MessageId::from(base64::encode_config(
-                &Sha256::digest(&message.data),
-                base64::URL_SAFE_NO_PAD,
-            ))
-        };
+        // We use the first 8 bytes of SHA256(data) for content addressing
+        let gossip_message_id =
+            |message: &GossipsubMessage| MessageId::from(&Sha256::digest(&message.data)[..8]);
 
         // gossipsub configuration
         // Note: The topics by default are sent as plain strings. Hashes are an optional
@@ -99,19 +98,20 @@ impl Default for Config {
         let gs_config = GossipsubConfigBuilder::new()
             .max_transmit_size(GOSSIP_MAX_SIZE)
             .heartbeat_interval(Duration::from_millis(700))
-            .mesh_n(6)
-            .mesh_n_low(5)
+            .mesh_n(8)
+            .mesh_n_low(6)
             .mesh_n_high(12)
             .gossip_lazy(6)
             .fanout_ttl(Duration::from_secs(60))
             .history_length(6)
             .history_gossip(3)
             .validate_messages() // require validation before propagation
-            .validation_mode(ValidationMode::Permissive)
+            .validation_mode(ValidationMode::Anonymous)
             // prevent duplicates for 550 heartbeats(700millis * 550) = 385 secs
             .duplicate_cache_time(Duration::from_secs(385))
             .message_id_fn(gossip_message_id)
-            .build();
+            .build()
+            .expect("valid gossipsub configuration");
 
         // discv5 configuration
         let discv5_config = Discv5ConfigBuilder::new()
@@ -142,6 +142,7 @@ impl Default for Config {
             boot_nodes_enr: vec![],
             boot_nodes_multiaddr: vec![],
             libp2p_nodes: vec![],
+            trusted_peers: vec![],
             client_version: lighthouse_version::version_with_platform(),
             disable_discovery: false,
             topics,

beacon_node/eth2_libp2p/src/discovery/enr.rs

@@ -6,6 +6,7 @@ use super::enr_ext::CombinedKeyExt;
 use super::ENR_FILENAME;
 use crate::types::{Enr, EnrBitfield};
 use crate::NetworkConfig;
+use discv5::enr::EnrKey;
 use libp2p::core::identity::Keypair;
 use slog::{debug, warn};
 use ssz::{Decode, Encode};
@@ -48,6 +49,56 @@ impl Eth2Enr for Enr {
     }
 }
 
+/// Either use the given ENR or load an ENR from file if it exists and matches the current NodeId
+/// and sequence number.
+/// If an ENR exists, with the same NodeId, this function checks to see if the loaded ENR from
+/// disk is suitable to use, otherwise we increment the given ENR's sequence number.
+pub fn use_or_load_enr(
+    enr_key: &CombinedKey,
+    local_enr: &mut Enr,
+    config: &NetworkConfig,
+    log: &slog::Logger,
+) -> Result<(), String> {
+    let enr_f = config.network_dir.join(ENR_FILENAME);
+    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
+        let mut enr_string = String::new();
+        match enr_file.read_to_string(&mut enr_string) {
+            Err(_) => debug!(log, "Could not read ENR from file"),
+            Ok(_) => {
+                match Enr::from_str(&enr_string) {
+                    Ok(disk_enr) => {
+                        // if the same node id, then we may need to update our sequence number
+                        if local_enr.node_id() == disk_enr.node_id() {
+                            if compare_enr(&local_enr, &disk_enr) {
+                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
+                                // the stored ENR has the same configuration, use it
+                                *local_enr = disk_enr;
+                                return Ok(());
+                            }
+
+                            // same node id, different configuration - update the sequence number
+                            // Note: local_enr is generated with default(0) attnets value,
+                            // so a non default value in persisted enr will also update sequence number.
+                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
+                            local_enr.set_seq(new_seq_no, enr_key).map_err(|e| {
+                                format!("Could not update ENR sequence number: {:?}", e)
+                            })?;
+                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
+                        }
+                    }
+                    Err(e) => {
+                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
+                    }
+                }
+            }
+        }
+    }
+
+    save_enr_to_disk(&config.network_dir, &local_enr, log);
+
+    Ok(())
+}
+
 /// Loads an ENR from file if it exists and matches the current NodeId and sequence number. If none
 /// exists, generates a new one.
 ///
@@ -65,51 +116,11 @@ pub fn build_or_load_enr<T: EthSpec>(
     let enr_key = CombinedKey::from_libp2p(&local_key)?;
     let mut local_enr = build_enr::<T>(&enr_key, config, enr_fork_id)?;
 
-    let enr_f = config.network_dir.join(ENR_FILENAME);
-    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
-        let mut enr_string = String::new();
-        match enr_file.read_to_string(&mut enr_string) {
-            Err(_) => debug!(log, "Could not read ENR from file"),
-            Ok(_) => {
-                match Enr::from_str(&enr_string) {
-                    Ok(disk_enr) => {
-                        // if the same node id, then we may need to update our sequence number
-                        if local_enr.node_id() == disk_enr.node_id() {
-                            if compare_enr(&local_enr, &disk_enr) {
-                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
-                                // the stored ENR has the same configuration, use it
-                                return Ok(disk_enr);
-                            }
-
-                            // same node id, different configuration - update the sequence number
-                            // Note: local_enr is generated with default(0) attnets value,
-                            // so a non default value in persisted enr will also update sequence number.
-                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
-                            local_enr.set_seq(new_seq_no, &enr_key).map_err(|e| {
-                                format!("Could not update ENR sequence number: {:?}", e)
-                            })?;
-                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
-                        }
-                    }
-                    Err(e) => {
-                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
-                    }
-                }
-            }
-        }
-    }
-
-    save_enr_to_disk(&config.network_dir, &local_enr, log);
-
+    use_or_load_enr(&enr_key, &mut local_enr, config, log)?;
     Ok(local_enr)
 }
 
-/// Builds a lighthouse ENR given a `NetworkConfig`.
-pub fn build_enr<T: EthSpec>(
-    enr_key: &CombinedKey,
-    config: &NetworkConfig,
-    enr_fork_id: EnrForkId,
-) -> Result<Enr, String> {
+pub fn create_enr_builder_from_config<T: EnrKey>(config: &NetworkConfig) -> EnrBuilder<T> {
     let mut builder = EnrBuilder::new("v4");
     if let Some(enr_address) = config.enr_address {
         builder.ip(enr_address);
@@ -120,7 +131,17 @@ pub fn build_enr<T: EthSpec>(
     // we always give it our listening tcp port
     // TODO: Add uPnP support to map udp and tcp ports
     let tcp_port = config.enr_tcp_port.unwrap_or_else(|| config.libp2p_port);
-    builder.tcp(tcp_port);
+    builder.tcp(tcp_port).tcp(config.libp2p_port);
+    builder
+}
+
+/// Builds a lighthouse ENR given a `NetworkConfig`.
+pub fn build_enr<T: EthSpec>(
+    enr_key: &CombinedKey,
+    config: &NetworkConfig,
+    enr_fork_id: EnrForkId,
+) -> Result<Enr, String> {
+    let mut builder = create_enr_builder_from_config(config);
 
     // set the `eth2` field on our ENR
     builder.add_value(ETH2_ENR_KEY.into(), enr_fork_id.as_ssz_bytes());
@@ -131,7 +152,6 @@ pub fn build_enr<T: EthSpec>(
     builder.add_value(BITFIELD_ENR_KEY.into(), bitfield.as_ssz_bytes());
 
     builder
-        .tcp(config.libp2p_port)
         .build(enr_key)
         .map_err(|e| format!("Could not build Local ENR: {:?}", e))
 }

beacon_node/eth2_libp2p/src/discovery/mod.rs

@@ -3,12 +3,12 @@ pub(crate) mod enr;
 pub mod enr_ext;
 
 // Allow external use of the lighthouse ENR builder
-pub use enr::{build_enr, CombinedKey, Eth2Enr};
+pub use enr::{build_enr, create_enr_builder_from_config, use_or_load_enr, CombinedKey, Eth2Enr};
 pub use enr_ext::{CombinedKeyExt, EnrExt};
 pub use libp2p::core::identity::Keypair;
 
 use crate::metrics;
-use crate::{error, Enr, NetworkConfig, NetworkGlobals};
+use crate::{error, Enr, NetworkConfig, NetworkGlobals, SubnetDiscovery};
 use discv5::{enr::NodeId, Discv5, Discv5Event};
 use enr::{BITFIELD_ENR_KEY, ETH2_ENR_KEY};
 use futures::prelude::*;
@@ -31,12 +31,12 @@ use tokio::sync::mpsc;
 use types::{EnrForkId, EthSpec, SubnetId};
 
 mod subnet_predicate;
-use subnet_predicate::subnet_predicate;
+pub use subnet_predicate::subnet_predicate;
 
 /// Local ENR storage filename.
 pub const ENR_FILENAME: &str = "enr.dat";
 /// Target number of peers we'd like to have connected to a given long-lived subnet.
-const TARGET_SUBNET_PEERS: usize = 3;
+pub const TARGET_SUBNET_PEERS: usize = 3;
 /// Target number of peers to search for given a grouped subnet query.
 const TARGET_PEERS_FOR_GROUPED_QUERY: usize = 6;
 /// Number of times to attempt a discovery request.
@@ -287,6 +287,11 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         self.discv5.local_enr()
     }
 
+    /// Return the cached enrs.
+    pub fn cached_enrs(&self) -> impl Iterator<Item = (&PeerId, &Enr)> {
+        self.cached_enrs.iter()
+    }
+
     /// This adds a new `FindPeers` query to the queue if one doesn't already exist.
     pub fn discover_peers(&mut self) {
         // If the discv5 service isn't running or we are in the process of a query, don't bother queuing a new one.
@@ -305,12 +310,19 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
     }
 
     /// Processes a request to search for more peers on a subnet.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
+    pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
         // If the discv5 service isn't running, ignore queries
         if !self.started {
             return;
         }
-        self.add_subnet_query(subnet_id, min_ttl, 0);
+        debug!(
+            self.log,
+            "Making discovery query for subnets";
+            "subnets" => format!("{:?}", subnets_to_discover.iter().map(|s| s.subnet_id).collect::<Vec<_>>())
+        );
+        for subnet in subnets_to_discover {
+            self.add_subnet_query(subnet.subnet_id, subnet.min_ttl, 0);
+        }
     }
 
     /// Add an ENR to the routing table of the discovery mechanism.
@@ -514,6 +526,11 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                         // This query is for searching for peers of a particular subnet
                         // Drain subnet_queries so we can re-use it as we continue to process the queue
                         let grouped_queries: Vec<SubnetQuery> = subnet_queries.drain(..).collect();
+                        debug!(
+                            self.log,
+                            "Starting grouped subnet query";
+                            "subnets" => format!("{:?}", grouped_queries.iter().map(|q| q.subnet_id).collect::<Vec<_>>()),
+                        );
                         self.start_subnet_query(grouped_queries);
                     }
                 }
@@ -546,7 +563,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                     .peers_on_subnet(subnet_query.subnet_id)
                     .count();
 
-                if peers_on_subnet > TARGET_SUBNET_PEERS {
+                if peers_on_subnet >= TARGET_SUBNET_PEERS {
                     debug!(self.log, "Discovery ignored";
                         "reason" => "Already connected to desired peers",
                         "connected_peers_on_subnet" => peers_on_subnet,

beacon_node/eth2_libp2p/src/discovery/subnet_predicate.rs

@@ -1,5 +1,6 @@
 ///! The subnet predicate used for searching for a particular subnet.
 use super::*;
+use slog::{debug, trace};
 use std::ops::Deref;
 
 /// Returns the predicate for a given subnet.
@@ -30,7 +31,7 @@ where
                 .collect();
 
             if matches.is_empty() {
-                debug!(
+                trace!(
                     log_clone,
                     "Peer found but not on any of the desired subnets";
                     "peer_id" => format!("{}", enr.peer_id())

beacon_node/eth2_libp2p/src/lib.rs

@@ -14,16 +14,60 @@ pub mod rpc;
 mod service;
 pub mod types;
 
-pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage};
+use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
+use std::str::FromStr;
+
+/// Wrapper over a libp2p `PeerId` which implements `Serialize` and `Deserialize`
+#[derive(Clone, Debug)]
+pub struct PeerIdSerialized(libp2p::PeerId);
+
+impl From<PeerIdSerialized> for PeerId {
+    fn from(peer_id: PeerIdSerialized) -> Self {
+        peer_id.0
+    }
+}
+
+impl FromStr for PeerIdSerialized {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Self(
+            PeerId::from_str(s).map_err(|e| format!("Invalid peer id: {}", e))?,
+        ))
+    }
+}
+
+impl Serialize for PeerIdSerialized {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.0.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for PeerIdSerialized {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s: String = Deserialize::deserialize(deserializer)?;
+        Ok(Self(PeerId::from_str(&s).map_err(|e| {
+            de::Error::custom(format!("Failed to deserialise peer id: {:?}", e))
+        })?))
+    }
+}
+
+pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage, SubnetDiscovery};
 pub use behaviour::{BehaviourEvent, PeerRequestId, Request, Response};
 pub use config::Config as NetworkConfig;
 pub use discovery::{CombinedKeyExt, EnrExt, Eth2Enr};
 pub use discv5;
-pub use libp2p::gossipsub::{MessageId, Topic, TopicHash};
+pub use libp2p::gossipsub::{Gossipsub, MessageAcceptance, MessageId, Topic, TopicHash};
 pub use libp2p::{core::ConnectedPoint, PeerId, Swarm};
 pub use libp2p::{multiaddr, Multiaddr};
 pub use metrics::scrape_discovery_metrics;
 pub use peer_manager::{
     client::Client, score::PeerAction, PeerDB, PeerInfo, PeerSyncStatus, SyncInfo,
 };
-pub use service::{Libp2pEvent, Service, NETWORK_KEY_FILENAME};
+pub use service::{load_private_key, Libp2pEvent, Service, NETWORK_KEY_FILENAME};

beacon_node/eth2_libp2p/src/metrics.rs

@@ -34,9 +34,20 @@ lazy_static! {
         "Unsolicited discovery requests per ip per second",
         &["Addresses"]
     );
-    pub static ref GOSSIPSUB_SUBSCRIBED_PEERS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
-        "gossipsub_peers_per_topic_count",
-        "Peers subscribed per topic",
+    pub static ref PEERS_PER_CLIENT: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "libp2p_peers_per_client",
+        "The connected peers via client implementation",
+        &["Client"]
+    );
+    pub static ref FAILED_ATTESTATION_PUBLISHES_PER_SUBNET: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "gossipsub_failed_attestation_publishes_per_subnet",
+            "Failed attestation publishes per subnet",
+            &["subnet"]
+        );
+    pub static ref FAILED_PUBLISHES_PER_MAIN_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_failed_publishes_per_main_topic",
+        "Failed gossip publishes",
         &["topic_hash"]
     );
 }

beacon_node/eth2_libp2p/src/peer_manager/client.rs

@@ -20,7 +20,7 @@ pub struct Client {
     pub agent_string: Option<String>,
 }
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, PartialEq)]
 pub enum ClientKind {
     /// A lighthouse node (the best kind).
     Lighthouse,
@@ -98,6 +98,12 @@ impl std::fmt::Display for Client {
     }
 }
 
+impl std::fmt::Display for ClientKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
 // helper function to identify clients from their agent_version. Returns the client
 // kind and it's associated version and the OS kind.
 fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String) {

beacon_node/eth2_libp2p/src/peer_manager/mod.rs

@@ -1,10 +1,10 @@
 //! Implementation of a Lighthouse's peer management system.
 
 pub use self::peerdb::*;
-use crate::discovery::{Discovery, DiscoveryEvent};
+use crate::discovery::{subnet_predicate, Discovery, DiscoveryEvent, TARGET_SUBNET_PEERS};
 use crate::rpc::{GoodbyeReason, MetaData, Protocol, RPCError, RPCResponseErrorCode};
 use crate::{error, metrics};
-use crate::{EnrExt, NetworkConfig, NetworkGlobals, PeerId};
+use crate::{EnrExt, NetworkConfig, NetworkGlobals, PeerId, SubnetDiscovery};
 use futures::prelude::*;
 use futures::Stream;
 use hashset_delay::HashSetDelay;
@@ -133,9 +133,9 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     pub fn goodbye_peer(&mut self, peer_id: &PeerId, reason: GoodbyeReason) {
         // get the peer info
         if let Some(info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
-            debug!(self.log, "Sending goodbye to peer"; "peer_id" => peer_id.to_string(), "reason" => reason.to_string(), "score" => info.score.to_string());
+            debug!(self.log, "Sending goodbye to peer"; "peer_id" => peer_id.to_string(), "reason" => reason.to_string(), "score" => info.score().to_string());
             // Goodbye's are fatal
-            info.score.apply_peer_action(PeerAction::Fatal);
+            info.apply_peer_action_to_score(PeerAction::Fatal);
             if info.connection_status.is_connected_or_dialing() {
                 self.events
                     .push(PeerManagerEvent::DisconnectPeer(peer_id.clone(), reason));
@@ -155,12 +155,12 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         let mut unban_peer = None;
 
         if let Some(info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
-            let previous_state = info.score.state();
-            info.score.apply_peer_action(action);
-            if previous_state != info.score.state() {
-                match info.score.state() {
+            let previous_state = info.score_state();
+            info.apply_peer_action_to_score(action);
+            if previous_state != info.score_state() {
+                match info.score_state() {
                     ScoreState::Banned => {
-                        debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string());
+                        debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string());
                         ban_peer = Some(peer_id.clone());
                         if info.connection_status.is_connected_or_dialing() {
                             self.events.push(PeerManagerEvent::DisconnectPeer(
@@ -170,7 +170,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                         }
                     }
                     ScoreState::Disconnected => {
-                        debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
+                        debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
                         // disconnect the peer if it's currently connected or dialing
                         unban_peer = Some(peer_id.clone());
                         if info.connection_status.is_connected_or_dialing() {
@@ -182,13 +182,13 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                         // TODO: Update the peer manager to inform that the peer is disconnecting.
                     }
                     ScoreState::Healthy => {
-                        debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
+                        debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
                         // unban the peer if it was previously banned.
                         unban_peer = Some(peer_id.clone());
                     }
                 }
             } else {
-                debug!(self.log, "Peer score adjusted"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string());
+                debug!(self.log, "Peer score adjusted"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string());
             }
         }
 
@@ -213,17 +213,46 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     }
 
     /// A request to find peers on a given subnet.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
-        // Extend the time to maintain peers if required.
-        if let Some(min_ttl) = min_ttl {
-            self.network_globals
-                .peers
-                .write()
-                .extend_peers_on_subnet(subnet_id, min_ttl);
-        }
+    pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
+        let filtered: Vec<SubnetDiscovery> = subnets_to_discover
+            .into_iter()
+            .filter(|s| {
+                // Extend min_ttl of connected peers on required subnets
+                if let Some(min_ttl) = s.min_ttl {
+                    self.network_globals
+                        .peers
+                        .write()
+                        .extend_peers_on_subnet(s.subnet_id, min_ttl);
+                }
+                // Already have target number of peers, no need for subnet discovery
+                let peers_on_subnet = self
+                    .network_globals
+                    .peers
+                    .read()
+                    .peers_on_subnet(s.subnet_id)
+                    .count();
+                if peers_on_subnet >= TARGET_SUBNET_PEERS {
+                    debug!(
+                        self.log,
+                        "Discovery query ignored";
+                        "subnet_id" => format!("{:?}",s.subnet_id),
+                        "reason" => "Already connected to desired peers",
+                        "connected_peers_on_subnet" => peers_on_subnet,
+                        "target_subnet_peers" => TARGET_SUBNET_PEERS,
+                    );
+                    false
+                // Queue an outgoing connection request to the cached peers that are on `s.subnet_id`.
+                // If we connect to the cached peers before the discovery query starts, then we potentially
+                // save a costly discovery query.
+                } else {
+                    self.dial_cached_enrs_in_subnet(s.subnet_id);
+                    true
+                }
+            })
+            .collect();
 
         // request the subnet query from discovery
-        self.discovery.discover_subnet_peers(subnet_id, min_ttl);
+        self.discovery.discover_subnet_peers(filtered);
     }
 
     /// A STATUS message has been received from a peer. This resets the status timer.
@@ -237,6 +266,27 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// This is also called when dialing a peer fails.
     pub fn notify_disconnect(&mut self, peer_id: &PeerId) {
+        // Decrement the PEERS_PER_CLIENT metric
+        if let Some(kind) = self
+            .network_globals
+            .peers
+            .read()
+            .peer_info(peer_id)
+            .and_then(|peer_info| {
+                if let Connected { .. } = peer_info.connection_status {
+                    Some(peer_info.client.kind.clone())
+                } else {
+                    None
+                }
+            })
+        {
+            if let Some(v) =
+                metrics::get_int_gauge(&metrics::PEERS_PER_CLIENT, &[&kind.to_string()])
+            {
+                v.dec()
+            };
+        }
+
         self.network_globals.peers.write().disconnect(peer_id);
 
         // remove the ping and status timer for the peer
@@ -294,8 +344,25 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// Updates `PeerInfo` with `identify` information.
     pub fn identify(&mut self, peer_id: &PeerId, info: &IdentifyInfo) {
         if let Some(peer_info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
+            let previous_kind = peer_info.client.kind.clone();
             peer_info.client = client::Client::from_identify_info(info);
             peer_info.listening_addresses = info.listen_addrs.clone();
+
+            if previous_kind != peer_info.client.kind {
+                // update the peer client kind metric
+                if let Some(v) = metrics::get_int_gauge(
+                    &metrics::PEERS_PER_CLIENT,
+                    &[&peer_info.client.kind.to_string()],
+                ) {
+                    v.inc()
+                };
+                if let Some(v) = metrics::get_int_gauge(
+                    &metrics::PEERS_PER_CLIENT,
+                    &[&previous_kind.to_string()],
+                ) {
+                    v.dec()
+                };
+            }
         } else {
             crit!(self.log, "Received an Identify response from an unknown peer"; "peer_id" => peer_id.to_string());
         }
@@ -491,6 +558,30 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         self.events.push(PeerManagerEvent::SocketUpdated(multiaddr));
     }
 
+    /// Dial cached enrs in discovery service that are in the given `subnet_id` and aren't
+    /// in Connected, Dialing or Banned state.
+    fn dial_cached_enrs_in_subnet(&mut self, subnet_id: SubnetId) {
+        let predicate = subnet_predicate::<TSpec>(vec![subnet_id], &self.log);
+        let peers_to_dial: Vec<PeerId> = self
+            .discovery()
+            .cached_enrs()
+            .filter_map(|(peer_id, enr)| {
+                let peers = self.network_globals.peers.read();
+                if predicate(enr)
+                    && !peers.is_connected_or_dialing(peer_id)
+                    && !peers.is_banned(peer_id)
+                {
+                    Some(peer_id.clone())
+                } else {
+                    None
+                }
+            })
+            .collect();
+        for peer in &peers_to_dial {
+            self.dial_peer(peer);
+        }
+    }
+
     /// Peers that have been returned by discovery requests are dialed here if they are suitable.
     ///
     /// NOTE: By dialing `PeerId`s and not multiaddrs, libp2p requests the multiaddr associated
@@ -549,7 +640,10 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             }
 
             match connection {
-                ConnectingType::Dialing => peerdb.dialing_peer(peer_id),
+                ConnectingType::Dialing => {
+                    peerdb.dialing_peer(peer_id);
+                    return true;
+                }
                 ConnectingType::IngoingConnected => peerdb.connect_outgoing(peer_id),
                 ConnectingType::OutgoingConnected => peerdb.connect_ingoing(peer_id),
             }
@@ -566,6 +660,21 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             self.network_globals.connected_peers() as i64,
         );
 
+        // Increment the PEERS_PER_CLIENT metric
+        if let Some(kind) = self
+            .network_globals
+            .peers
+            .read()
+            .peer_info(peer_id)
+            .map(|peer_info| peer_info.client.kind.clone())
+        {
+            if let Some(v) =
+                metrics::get_int_gauge(&metrics::PEERS_PER_CLIENT, &[&kind.to_string()])
+            {
+                v.inc()
+            };
+        }
+
         true
     }
 
@@ -580,9 +689,9 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         let mut to_unban_peers = Vec::new();
 
         for (peer_id, info) in pdb.peers_mut() {
-            let previous_state = info.score.state();
+            let previous_state = info.score_state();
             // Update scores
-            info.score.update();
+            info.score_update();
 
             /* TODO: Implement logic about connection lifetimes
             match info.connection_status {
@@ -637,10 +746,10 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             */
 
             // handle score transitions
-            if previous_state != info.score.state() {
-                match info.score.state() {
+            if previous_state != info.score_state() {
+                match info.score_state() {
                     ScoreState::Banned => {
-                        debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string());
+                        debug!(self.log, "Peer has been banned"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string());
                         to_ban_peers.push(peer_id.clone());
                         if info.connection_status.is_connected_or_dialing() {
                             self.events.push(PeerManagerEvent::DisconnectPeer(
@@ -650,7 +759,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                         }
                     }
                     ScoreState::Disconnected => {
-                        debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
+                        debug!(self.log, "Peer transitioned to disconnect state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
                         // disconnect the peer if it's currently connected or dialing
                         to_unban_peers.push(peer_id.clone());
                         if info.connection_status.is_connected_or_dialing() {
@@ -662,7 +771,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                         // TODO: Update peer manager to report that it's disconnecting.
                     }
                     ScoreState::Healthy => {
-                        debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score.to_string(), "past_state" => previous_state.to_string());
+                        debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
                         // unban the peer if it was previously banned.
                         to_unban_peers.push(peer_id.clone());
                     }
@@ -712,7 +821,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                 .take(connected_peer_count - self.target_peers)
                 //we only need to disconnect peers with healthy scores, since the others got already
                 //disconnected in update_peer_scores
-                .filter(|(_, info)| info.score.state() == ScoreState::Healthy)
+                .filter(|(_, info)| info.score_state() == ScoreState::Healthy)
             {
                 self.events.push(PeerManagerEvent::DisconnectPeer(
                     (*peer_id).clone(),

beacon_node/eth2_libp2p/src/peer_manager/peer_info.rs

@@ -1,12 +1,13 @@
 use super::client::Client;
-use super::score::Score;
+use super::score::{PeerAction, Score, ScoreState};
 use super::PeerSyncStatus;
 use crate::rpc::MetaData;
 use crate::Multiaddr;
 use serde::{
-    ser::{SerializeStructVariant, Serializer},
+    ser::{SerializeStruct, Serializer},
     Serialize,
 };
+use std::net::IpAddr;
 use std::time::Instant;
 use types::{EthSpec, SubnetId};
 use PeerConnectionStatus::*;
@@ -18,7 +19,7 @@ pub struct PeerInfo<T: EthSpec> {
     /// The connection status of the peer
     _status: PeerStatus,
     /// The peers reputation
-    pub score: Score,
+    score: Score,
     /// Client managing this peer
     pub client: Client,
     /// Connection status of this peer
@@ -35,6 +36,8 @@ pub struct PeerInfo<T: EthSpec> {
     /// necessary.
     #[serde(skip)]
     pub min_ttl: Option<Instant>,
+    /// Is the peer a trusted peer.
+    pub is_trusted: bool,
 }
 
 impl<TSpec: EthSpec> Default for PeerInfo<TSpec> {
@@ -48,11 +51,21 @@ impl<TSpec: EthSpec> Default for PeerInfo<TSpec> {
             sync_status: PeerSyncStatus::Unknown,
             meta_data: None,
             min_ttl: None,
+            is_trusted: false,
         }
     }
 }
 
 impl<T: EthSpec> PeerInfo<T> {
+    /// Return a PeerInfo struct for a trusted peer.
+    pub fn trusted_peer_info() -> Self {
+        PeerInfo {
+            score: Score::max_score(),
+            is_trusted: true,
+            ..Default::default()
+        }
+    }
+
     /// Returns if the peer is subscribed to a given `SubnetId`
     pub fn on_subnet(&self, subnet_id: SubnetId) -> bool {
         if let Some(meta_data) = &self.meta_data {
@@ -68,6 +81,38 @@ impl<T: EthSpec> PeerInfo<T> {
     pub fn has_future_duty(&self) -> bool {
         self.min_ttl.map_or(false, |i| i >= Instant::now())
     }
+
+    /// Returns score of the peer.
+    pub fn score(&self) -> Score {
+        self.score
+    }
+
+    /// Returns the state of the peer based on the score.
+    pub(crate) fn score_state(&self) -> ScoreState {
+        self.score.state()
+    }
+
+    /// Applies decay rates to a non-trusted peer's score.
+    pub fn score_update(&mut self) {
+        if !self.is_trusted {
+            self.score.update()
+        }
+    }
+
+    /// Apply peer action to a non-trusted peer's score.
+    pub fn apply_peer_action_to_score(&mut self, peer_action: PeerAction) {
+        if !self.is_trusted {
+            self.score.apply_peer_action(peer_action)
+        }
+    }
+
+    #[cfg(test)]
+    /// Add an f64 to a non-trusted peer's score abiding by the limits.
+    pub fn add_to_score(&mut self, score: f64) {
+        if !self.is_trusted {
+            self.score.add(score)
+        }
+    }
 }
 
 #[derive(Clone, Debug, Serialize)]
@@ -104,6 +149,8 @@ pub enum PeerConnectionStatus {
     Banned {
         /// moment when the peer was banned.
         since: Instant,
+        /// ip addresses this peer had a the moment of the ban
+        ip_addresses: Vec<IpAddr>,
     },
     /// We are currently dialing this peer.
     Dialing {
@@ -117,29 +164,51 @@ pub enum PeerConnectionStatus {
 /// Serialization for http requests.
 impl Serialize for PeerConnectionStatus {
     fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let mut s = serializer.serialize_struct("connection_status", 5)?;
         match self {
             Connected { n_in, n_out } => {
-                let mut s = serializer.serialize_struct_variant("", 0, "Connected", 2)?;
-                s.serialize_field("in", n_in)?;
-                s.serialize_field("out", n_out)?;
+                s.serialize_field("status", "connected")?;
+                s.serialize_field("connections_in", n_in)?;
+                s.serialize_field("connections_out", n_out)?;
+                s.serialize_field("last_seen", &0)?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
                 s.end()
             }
             Disconnected { since } => {
-                let mut s = serializer.serialize_struct_variant("", 1, "Disconnected", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "disconnected")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
                 s.end()
             }
-            Banned { since } => {
-                let mut s = serializer.serialize_struct_variant("", 2, "Banned", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+            Banned {
+                since,
+                ip_addresses,
+            } => {
+                s.serialize_field("status", "banned")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.serialize_field("banned_ips", &ip_addresses)?;
                 s.end()
             }
             Dialing { since } => {
-                let mut s = serializer.serialize_struct_variant("", 3, "Dialing", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "dialing")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
+                s.end()
+            }
+            Unknown => {
+                s.serialize_field("status", "unknown")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &0)?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
                 s.end()
             }
-            Unknown => serializer.serialize_unit_variant("", 4, "Unknown"),
         }
     }
 }
@@ -218,15 +287,16 @@ impl PeerConnectionStatus {
     }
 
     /// Modifies the status to Banned
-    pub fn ban(&mut self) {
+    pub fn ban(&mut self, ip_addresses: Vec<IpAddr>) {
         *self = Banned {
             since: Instant::now(),
+            ip_addresses,
         };
     }
 
     /// The score system has unbanned the peer. Update the connection status
     pub fn unban(&mut self) {
-        if let PeerConnectionStatus::Banned { since } = self {
+        if let PeerConnectionStatus::Banned { since, .. } = self {
             *self = PeerConnectionStatus::Disconnected { since: *since }
         }
     }

beacon_node/eth2_libp2p/src/peer_manager/peerdb.rs

@@ -1,11 +1,13 @@
 use super::peer_info::{PeerConnectionStatus, PeerInfo};
 use super::peer_sync_status::PeerSyncStatus;
 use super::score::{Score, ScoreState};
+use crate::multiaddr::Protocol;
 use crate::rpc::methods::MetaData;
 use crate::PeerId;
 use rand::seq::SliceRandom;
 use slog::{crit, debug, trace, warn};
 use std::collections::HashMap;
+use std::net::IpAddr;
 use std::time::Instant;
 use types::{EthSpec, SubnetId};
 
@@ -13,6 +15,9 @@ use types::{EthSpec, SubnetId};
 const MAX_DC_PEERS: usize = 500;
 /// The maximum number of banned nodes to remember.
 const MAX_BANNED_PEERS: usize = 1000;
+/// If there are more than `BANNED_PEERS_PER_IP_THRESHOLD` many banned peers with the same IP we ban
+/// the IP.
+const BANNED_PEERS_PER_IP_THRESHOLD: usize = 5;
 
 /// Storage of known peers, their reputation and information
 pub struct PeerDB<TSpec: EthSpec> {
@@ -20,19 +25,78 @@ pub struct PeerDB<TSpec: EthSpec> {
     peers: HashMap<PeerId, PeerInfo<TSpec>>,
     /// The number of disconnected nodes in the database.
     disconnected_peers: usize,
-    /// The number of banned peers in the database.
-    banned_peers: usize,
+    /// Counts banned peers in total and per ip
+    banned_peers_count: BannedPeersCount,
     /// PeerDB's logger
     log: slog::Logger,
 }
 
+pub struct BannedPeersCount {
+    /// The number of banned peers in the database.
+    banned_peers: usize,
+    /// maps ips to number of banned peers with this ip
+    banned_peers_per_ip: HashMap<IpAddr, usize>,
+}
+
+impl BannedPeersCount {
+    /// Removes the peer from the counts if it is banned. Returns true if the peer was banned and
+    /// false otherwise.
+    pub fn remove_banned_peer(&mut self, connection_status: &PeerConnectionStatus) -> bool {
+        match connection_status {
+            PeerConnectionStatus::Banned { ip_addresses, .. } => {
+                self.banned_peers = self.banned_peers.saturating_sub(1);
+                for address in ip_addresses {
+                    if let Some(count) = self.banned_peers_per_ip.get_mut(address) {
+                        *count = count.saturating_sub(1);
+                    }
+                }
+                true
+            }
+            _ => false, //if not banned do nothing
+        }
+    }
+
+    pub fn add_banned_peer(&mut self, connection_status: &PeerConnectionStatus) {
+        if let PeerConnectionStatus::Banned { ip_addresses, .. } = connection_status {
+            self.banned_peers += 1;
+            for address in ip_addresses {
+                *self.banned_peers_per_ip.entry(*address).or_insert(0) += 1;
+            }
+        }
+    }
+
+    pub fn banned_peers(&self) -> usize {
+        self.banned_peers
+    }
+
+    /// An IP is considered banned if more than BANNED_PEERS_PER_IP_THRESHOLD banned peers
+    /// exist with this IP
+    pub fn ip_is_banned(&self, ip: &IpAddr) -> bool {
+        self.banned_peers_per_ip
+            .get(ip)
+            .map_or(false, |count| *count > BANNED_PEERS_PER_IP_THRESHOLD)
+    }
+
+    pub fn new() -> Self {
+        BannedPeersCount {
+            banned_peers: 0,
+            banned_peers_per_ip: HashMap::new(),
+        }
+    }
+}
+
 impl<TSpec: EthSpec> PeerDB<TSpec> {
-    pub fn new(log: &slog::Logger) -> Self {
+    pub fn new(trusted_peers: Vec<PeerId>, log: &slog::Logger) -> Self {
+        // Initialize the peers hashmap with trusted peers
+        let peers = trusted_peers
+            .into_iter()
+            .map(|peer_id| (peer_id, PeerInfo::trusted_peer_info()))
+            .collect();
         Self {
             log: log.clone(),
             disconnected_peers: 0,
-            banned_peers: 0,
-            peers: HashMap::new(),
+            banned_peers_count: BannedPeersCount::new(),
+            peers,
         }
     }
 
@@ -42,7 +106,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
     pub fn score(&self, peer_id: &PeerId) -> Score {
         self.peers
             .get(peer_id)
-            .map_or(Score::default(), |info| info.score)
+            .map_or(Score::default(), |info| info.score())
     }
 
     /// Returns an iterator over all peers in the db.
@@ -99,17 +163,35 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
 
     /// Returns true if the Peer is banned.
     pub fn is_banned(&self, peer_id: &PeerId) -> bool {
-        match self.peers.get(peer_id).map(|info| info.score.state()) {
-            Some(ScoreState::Banned) => true,
-            _ => false,
+        if let Some(peer) = self.peers.get(peer_id) {
+            match peer.score().state() {
+                ScoreState::Banned => true,
+                _ => self.ip_is_banned(peer),
+            }
+        } else {
+            false
         }
     }
 
+    fn ip_is_banned(&self, peer: &PeerInfo<TSpec>) -> bool {
+        peer.listening_addresses.iter().any(|addr| {
+            addr.iter().any(|p| match p {
+                Protocol::Ip4(ip) => self.banned_peers_count.ip_is_banned(&ip.into()),
+                Protocol::Ip6(ip) => self.banned_peers_count.ip_is_banned(&ip.into()),
+                _ => false,
+            })
+        })
+    }
+
     /// Returns true if the Peer is either banned or in the disconnected state.
     pub fn is_banned_or_disconnected(&self, peer_id: &PeerId) -> bool {
-        match self.peers.get(peer_id).map(|info| info.score.state()) {
-            Some(ScoreState::Banned) | Some(ScoreState::Disconnected) => true,
-            _ => false,
+        if let Some(peer) = self.peers.get(peer_id) {
+            match peer.score().state() {
+                ScoreState::Banned | ScoreState::Disconnected => true,
+                _ => self.ip_is_banned(peer),
+            }
+        } else {
+            false
         }
     }
 
@@ -187,7 +269,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
             .collect::<Vec<_>>();
 
         connected.shuffle(&mut rand::thread_rng());
-        connected.sort_by_key(|(_, info)| info.score);
+        connected.sort_by_key(|(_, info)| info.score());
         connected
     }
 
@@ -202,7 +284,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
             .iter()
             .filter(|(_, info)| is_status(&info.connection_status))
             .collect::<Vec<_>>();
-        by_status.sort_by_key(|(_, info)| info.score);
+        by_status.sort_by_key(|(_, info)| info.score());
         by_status.into_iter().rev().collect()
     }
 
@@ -214,7 +296,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         self.peers
             .iter()
             .filter(|(_, info)| is_status(&info.connection_status))
-            .max_by_key(|(_, info)| info.score)
+            .max_by_key(|(_, info)| info.score())
             .map(|(id, _)| id)
     }
 
@@ -233,9 +315,10 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         if info.connection_status.is_disconnected() {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
-        if info.connection_status.is_banned() {
-            self.banned_peers = self.banned_peers.saturating_sub(1);
-        }
+
+        self.banned_peers_count
+            .remove_banned_peer(&info.connection_status);
+
         info.connection_status = PeerConnectionStatus::Dialing {
             since: Instant::now(),
         };
@@ -284,9 +367,8 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         if info.connection_status.is_disconnected() {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
-        if info.connection_status.is_banned() {
-            self.banned_peers = self.banned_peers.saturating_sub(1);
-        }
+        self.banned_peers_count
+            .remove_banned_peer(&info.connection_status);
         info.connection_status.connect_ingoing();
     }
 
@@ -297,9 +379,8 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         if info.connection_status.is_disconnected() {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
-        if info.connection_status.is_banned() {
-            self.banned_peers = self.banned_peers.saturating_sub(1);
-        }
+        self.banned_peers_count
+            .remove_banned_peer(&info.connection_status);
         info.connection_status.connect_outgoing();
     }
 
@@ -329,8 +410,23 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
         if !info.connection_status.is_banned() {
-            info.connection_status.ban();
-            self.banned_peers += 1;
+            info.connection_status
+                .ban(
+                    info.listening_addresses
+                        .iter()
+                        .fold(Vec::new(), |mut v, a| {
+                            for p in a {
+                                match p {
+                                    Protocol::Ip4(ip) => v.push(ip.into()),
+                                    Protocol::Ip6(ip) => v.push(ip.into()),
+                                    _ => (),
+                                }
+                            }
+                            v
+                        }),
+                );
+            self.banned_peers_count
+                .add_banned_peer(&info.connection_status);
         }
         self.shrink_to_fit();
     }
@@ -345,8 +441,9 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         });
 
         if info.connection_status.is_banned() {
+            self.banned_peers_count
+                .remove_banned_peer(&info.connection_status);
             info.connection_status.unban();
-            self.banned_peers = self.banned_peers.saturating_sub(1);
         }
         self.shrink_to_fit();
     }
@@ -355,26 +452,34 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
     /// Drops the peers with the lowest reputation so that the number of
     /// disconnected peers is less than MAX_DC_PEERS
     pub fn shrink_to_fit(&mut self) {
-        // Remove excess baned peers
-        while self.banned_peers > MAX_BANNED_PEERS {
-            if let Some(to_drop) = self
+        // Remove excess banned peers
+        while self.banned_peers_count.banned_peers() > MAX_BANNED_PEERS {
+            if let Some(to_drop) = if let Some((id, info)) = self
                 .peers
                 .iter()
                 .filter(|(_, info)| info.connection_status.is_banned())
                 .min_by(|(_, info_a), (_, info_b)| {
                     info_a
-                        .score
-                        .partial_cmp(&info_b.score)
+                        .score()
+                        .partial_cmp(&info_b.score())
                         .unwrap_or(std::cmp::Ordering::Equal)
-                })
-                .map(|(id, _)| id.clone())
-            {
+                }) {
+                self.banned_peers_count
+                    .remove_banned_peer(&info.connection_status);
+                Some(id.clone())
+            } else {
+                // If there is no minimum, this is a coding error.
+                crit!(
+                    self.log,
+                    "banned_peers > MAX_BANNED_PEERS despite no banned peers in db!"
+                );
+                // reset banned_peers this will also exit the loop
+                self.banned_peers_count = BannedPeersCount::new();
+                None
+            } {
                 debug!(self.log, "Removing old banned peer"; "peer_id" => to_drop.to_string());
                 self.peers.remove(&to_drop);
             }
-            // If there is no minimum, this is a coding error. For safety we decrease
-            // the count to avoid a potential infinite loop.
-            self.banned_peers = self.banned_peers.saturating_sub(1);
         }
 
         // Remove excess disconnected peers
@@ -385,8 +490,8 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
                 .filter(|(_, info)| info.connection_status.is_disconnected())
                 .min_by(|(_, info_a), (_, info_b)| {
                     info_a
-                        .score
-                        .partial_cmp(&info_b.score)
+                        .score()
+                        .partial_cmp(&info_b.score())
                         .unwrap_or(std::cmp::Ordering::Equal)
                 })
                 .map(|(id, _)| id.clone())
@@ -422,8 +527,11 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use libp2p::core::Multiaddr;
     use slog::{o, Drain};
+    use std::net::{Ipv4Addr, Ipv6Addr};
     use types::MinimalEthSpec;
+
     type M = MinimalEthSpec;
 
     pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger {
@@ -440,13 +548,13 @@ mod tests {
 
     fn add_score<TSpec: EthSpec>(db: &mut PeerDB<TSpec>, peer_id: &PeerId, score: f64) {
         if let Some(info) = db.peer_info_mut(peer_id) {
-            info.score.add(score);
+            info.add_to_score(score);
         }
     }
 
     fn get_db() -> PeerDB<M> {
         let log = build_log(slog::Level::Debug, false);
-        PeerDB::new(&log)
+        PeerDB::new(vec![], &log)
     }
 
     #[test]
@@ -503,13 +611,13 @@ mod tests {
             let p = PeerId::random();
             pdb.connect_ingoing(&p);
         }
-        assert_eq!(pdb.banned_peers, 0);
+        assert_eq!(pdb.banned_peers_count.banned_peers(), 0);
 
         for p in pdb.connected_peer_ids().cloned().collect::<Vec<_>>() {
             pdb.ban(&p);
         }
 
-        assert_eq!(pdb.banned_peers, MAX_BANNED_PEERS);
+        assert_eq!(pdb.banned_peers_count.banned_peers(), MAX_BANNED_PEERS);
     }
 
     #[test]
@@ -608,24 +716,39 @@ mod tests {
         pdb.connect_ingoing(&random_peer2);
         pdb.connect_ingoing(&random_peer3);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.connect_ingoing(&random_peer);
         pdb.disconnect(&random_peer1);
         pdb.ban(&random_peer2);
         pdb.connect_ingoing(&random_peer3);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.ban(&random_peer1);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.connect_outgoing(&random_peer2);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.ban(&random_peer3);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.ban(&random_peer3);
         pdb.connect_ingoing(&random_peer1);
@@ -633,15 +756,215 @@ mod tests {
         pdb.ban(&random_peer3);
         pdb.connect_ingoing(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.disconnect(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.disconnect(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.ban(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
     }
+
+    fn connect_peer_with_ips(pdb: &mut PeerDB<M>, ips: Vec<Vec<IpAddr>>) -> PeerId {
+        let p = PeerId::random();
+        pdb.connect_ingoing(&p);
+        pdb.peers.get_mut(&p).unwrap().listening_addresses = ips
+            .into_iter()
+            .map(|ip_addresses| {
+                let mut addr = Multiaddr::empty();
+                for ip_address in ip_addresses {
+                    addr.push(Protocol::from(ip_address));
+                }
+                addr
+            })
+            .collect();
+        p
+    }
+
+    #[test]
+    fn test_ban_address() {
+        let mut pdb = get_db();
+
+        let ip1: IpAddr = Ipv4Addr::new(1, 2, 3, 4).into();
+        let ip2: IpAddr = Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8).into();
+        let ip3: IpAddr = Ipv4Addr::new(1, 2, 3, 5).into();
+        let ip4: IpAddr = Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 9).into();
+        let ip5: IpAddr = Ipv4Addr::new(2, 2, 3, 4).into();
+
+        let mut peers = Vec::new();
+        for i in 0..BANNED_PEERS_PER_IP_THRESHOLD + 2 {
+            peers.push(connect_peer_with_ips(
+                &mut pdb,
+                if i == 0 {
+                    vec![vec![ip1], vec![ip2]]
+                } else {
+                    vec![vec![ip1, ip2], vec![ip3, ip4]]
+                },
+            ));
+        }
+
+        let p1 = connect_peer_with_ips(&mut pdb, vec![vec![ip1]]);
+        let p2 = connect_peer_with_ips(&mut pdb, vec![vec![ip2, ip5]]);
+        let p3 = connect_peer_with_ips(&mut pdb, vec![vec![ip3], vec![ip5]]);
+        let p4 = connect_peer_with_ips(&mut pdb, vec![vec![ip5, ip4]]);
+        let p5 = connect_peer_with_ips(&mut pdb, vec![vec![ip5]]);
+
+        for p in &peers[..BANNED_PEERS_PER_IP_THRESHOLD + 1] {
+            pdb.ban(p);
+        }
+
+        //check that ip1 and ip2 are banned but ip3-5 not
+        assert!(pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+        assert!(!pdb.is_banned(&p3));
+        assert!(!pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+
+        //ban also the last peer in peers
+        pdb.ban(&peers[BANNED_PEERS_PER_IP_THRESHOLD + 1]);
+
+        //check that ip1-ip4 are banned but ip5 not
+        assert!(pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+        assert!(pdb.is_banned(&p3));
+        assert!(pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+
+        //peers[0] gets unbanned
+        pdb.unban(&peers[0]);
+
+        //nothing changed
+        assert!(pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+        assert!(pdb.is_banned(&p3));
+        assert!(pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+
+        //peers[1] gets unbanned
+        pdb.unban(&peers[1]);
+
+        //all ips are unbanned
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+        assert!(!pdb.is_banned(&p3));
+        assert!(!pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+    }
+
+    #[test]
+    fn test_banned_ip_consistent_after_changing_ips() {
+        let mut pdb = get_db();
+
+        let ip1: IpAddr = Ipv4Addr::new(1, 2, 3, 4).into();
+        let ip2: IpAddr = Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8).into();
+
+        let mut peers = Vec::new();
+        for _ in 0..BANNED_PEERS_PER_IP_THRESHOLD + 1 {
+            peers.push(connect_peer_with_ips(&mut pdb, vec![vec![ip1]]));
+        }
+
+        let p1 = connect_peer_with_ips(&mut pdb, vec![vec![ip1]]);
+        let p2 = connect_peer_with_ips(&mut pdb, vec![vec![ip2]]);
+
+        //ban all peers
+        for p in &peers {
+            pdb.ban(p);
+        }
+
+        //check ip is banned
+        assert!(pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //change addresses of banned peers
+        for p in &peers {
+            pdb.peers.get_mut(p).unwrap().listening_addresses =
+                vec![Multiaddr::empty().with(Protocol::from(ip2))];
+        }
+
+        //check still the same ip is banned
+        assert!(pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //unban a peer
+        pdb.unban(&peers[0]);
+
+        //check not banned anymore
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //check still not banned after new ban
+        pdb.ban(&peers[0]);
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //unban and reban all peers
+        for p in &peers {
+            pdb.unban(p);
+            pdb.ban(p);
+        }
+
+        //ip2 is now banned
+        assert!(!pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+
+        //change ips back again
+        for p in &peers {
+            pdb.peers.get_mut(p).unwrap().listening_addresses =
+                vec![Multiaddr::empty().with(Protocol::from(ip1))];
+        }
+
+        //reban every peer except one
+        for p in &peers[1..] {
+            pdb.unban(p);
+            pdb.ban(p);
+        }
+
+        //nothing is banned
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //reban last peer
+        pdb.unban(&peers[0]);
+        pdb.ban(&peers[0]);
+
+        //ip1 is banned
+        assert!(pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+    }
+
+    #[test]
+    fn test_trusted_peers_score() {
+        let trusted_peer = PeerId::random();
+        let log = build_log(slog::Level::Debug, false);
+        let mut pdb: PeerDB<M> = PeerDB::new(vec![trusted_peer.clone()], &log);
+
+        pdb.connect_ingoing(&trusted_peer);
+
+        // Check trusted status and score
+        assert!(pdb.peer_info(&trusted_peer).unwrap().is_trusted);
+        assert_eq!(
+            pdb.peer_info(&trusted_peer).unwrap().score().score(),
+            Score::max_score().score()
+        );
+
+        // Adding/Subtracting score should have no effect on a trusted peer
+        add_score(&mut pdb, &trusted_peer, -50.0);
+
+        assert_eq!(
+            pdb.peer_info(&trusted_peer).unwrap().score().score(),
+            Score::max_score().score()
+        );
+    }
 }

beacon_node/eth2_libp2p/src/peer_manager/score.rs

@@ -145,6 +145,13 @@ impl std::fmt::Display for ScoreState {
 }
 
 impl Score {
+    /// Return max possible score.
+    pub fn max_score() -> Self {
+        Score {
+            score: MAX_SCORE,
+            last_updated: Instant::now(),
+        }
+    }
     /// Access to the underlying score.
     pub fn score(&self) -> f64 {
         self.score

beacon_node/eth2_libp2p/src/rpc/codec/base.rs

@@ -1,5 +1,6 @@
 //! This handles the various supported encoding mechanism for the Eth 2.0 RPC.
 
+use crate::rpc::methods::ErrorType;
 use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
 use libp2p::bytes::BufMut;
 use libp2p::bytes::BytesMut;
@@ -8,12 +9,12 @@ use tokio_util::codec::{Decoder, Encoder};
 use types::EthSpec;
 
 pub trait OutboundCodec<TItem>: Encoder<TItem> + Decoder {
-    type ErrorType;
+    type CodecErrorType;
 
     fn decode_error(
         &mut self,
         src: &mut BytesMut,
-    ) -> Result<Option<Self::ErrorType>, <Self as Decoder>::Error>;
+    ) -> Result<Option<Self::CodecErrorType>, <Self as Decoder>::Error>;
 }
 
 /* Global Inbound Codec */
@@ -130,8 +131,8 @@ where
 impl<TCodec, TSpec> Decoder for BaseOutboundCodec<TCodec, TSpec>
 where
     TSpec: EthSpec,
-    TCodec:
-        OutboundCodec<RPCRequest<TSpec>, ErrorType = String> + Decoder<Item = RPCResponse<TSpec>>,
+    TCodec: OutboundCodec<RPCRequest<TSpec>, CodecErrorType = ErrorType>
+        + Decoder<Item = RPCResponse<TSpec>>,
 {
     type Item = RPCCodedResponse<TSpec>;
     type Error = <TCodec as Decoder>::Error;

beacon_node/eth2_libp2p/src/rpc/codec/ssz_snappy.rs

@@ -99,6 +99,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
+            // Note: length-prefix of > 10 bytes(uint64) would be a decoding error
             match self.inner.decode(src).map_err(RPCError::from)? {
                 Some(length) => {
                     self.len = Some(length);
@@ -116,7 +117,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
         let mut reader = FrameDecoder::new(Cursor::new(&src));
         let mut decoded_buffer = vec![0; length];
 
-        match reader.read_exact(&mut decoded_buffer) {
+        match read_exact(&mut reader, &mut decoded_buffer, length) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
                 let n = reader.get_ref().position();
@@ -194,8 +195,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
                 }
             }
             Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
+                // Haven't received enough bytes to decode yet, wait for more
                 ErrorKind::UnexpectedEof => Ok(None),
                 _ => Err(e).map_err(RPCError::from),
             },
@@ -277,6 +277,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
+            // Note: length-prefix of > 10 bytes(uint64) would be a decoding error
             match self.inner.decode(src).map_err(RPCError::from)? {
                 Some(length) => {
                     self.len = Some(length as usize);
@@ -293,7 +294,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
         }
         let mut reader = FrameDecoder::new(Cursor::new(&src));
         let mut decoded_buffer = vec![0; length];
-        match reader.read_exact(&mut decoded_buffer) {
+        match read_exact(&mut reader, &mut decoded_buffer, length) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
                 let n = reader.get_ref().position();
@@ -364,8 +365,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
                 }
             }
             Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
+                // Haven't received enough bytes to decode yet, wait for more
                 ErrorKind::UnexpectedEof => Ok(None),
                 _ => Err(e).map_err(RPCError::from),
             },
@@ -374,9 +374,12 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
 }
 
 impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZSnappyOutboundCodec<TSpec> {
-    type ErrorType = String;
+    type CodecErrorType = ErrorType;
 
-    fn decode_error(&mut self, src: &mut BytesMut) -> Result<Option<Self::ErrorType>, RPCError> {
+    fn decode_error(
+        &mut self,
+        src: &mut BytesMut,
+    ) -> Result<Option<Self::CodecErrorType>, RPCError> {
         if self.len.is_none() {
             // Decode the length of the uncompressed bytes from an unsigned varint
             match self.inner.decode(src).map_err(RPCError::from)? {
@@ -395,22 +398,78 @@ impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZSnappyOutboundCodec
         }
         let mut reader = FrameDecoder::new(Cursor::new(&src));
         let mut decoded_buffer = vec![0; length];
-        match reader.read_exact(&mut decoded_buffer) {
+        match read_exact(&mut reader, &mut decoded_buffer, length) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
                 let n = reader.get_ref().position();
                 self.len = None;
                 let _read_bytes = src.split_to(n as usize);
-                Ok(Some(
-                    String::from_utf8_lossy(&<Vec<u8>>::from_ssz_bytes(&decoded_buffer)?).into(),
-                ))
+                Ok(Some(ErrorType(VariableList::from_ssz_bytes(
+                    &decoded_buffer,
+                )?)))
             }
             Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
+                // Haven't received enough bytes to decode yet, wait for more
                 ErrorKind::UnexpectedEof => Ok(None),
                 _ => Err(e).map_err(RPCError::from),
             },
         }
     }
 }
+
+/// Wrapper over `read` implementation of `FrameDecoder`.
+///
+/// Works like the standard `read_exact` implementation, except that it returns an error if length of
+// compressed bytes read from the underlying reader is greater than worst case compression length for snappy.
+fn read_exact<T: std::convert::AsRef<[u8]>>(
+    reader: &mut FrameDecoder<Cursor<T>>,
+    mut buf: &mut [u8],
+    uncompressed_length: usize,
+) -> Result<(), std::io::Error> {
+    // Calculate worst case compression length for given uncompressed length
+    let max_compressed_len = snap::raw::max_compress_len(uncompressed_length) as u64;
+
+    // Initialize the position of the reader
+    let mut pos = reader.get_ref().position();
+    let mut count = 0;
+    while !buf.is_empty() {
+        match reader.read(buf) {
+            Ok(0) => break,
+            Ok(n) => {
+                let tmp = buf;
+                buf = &mut tmp[n..];
+            }
+            Err(ref e) if e.kind() == ErrorKind::Interrupted => {}
+            Err(e) => return Err(e),
+        }
+        // Get current position of reader
+        let curr_pos = reader.get_ref().position();
+        // Note: reader should always advance forward. However, this behaviour
+        // depends on the implementation of `snap::FrameDecoder`, so it is better
+        // to check to avoid underflow panic.
+        if curr_pos > pos {
+            count += reader.get_ref().position() - pos;
+            pos = curr_pos;
+        } else {
+            return Err(std::io::Error::new(
+                ErrorKind::InvalidData,
+                "snappy: reader is not advanced forward while reading",
+            ));
+        }
+
+        if count > max_compressed_len {
+            return Err(std::io::Error::new(
+                ErrorKind::InvalidData,
+                "snappy: compressed data is > max_compressed_len",
+            ));
+        }
+    }
+    if !buf.is_empty() {
+        Err(std::io::Error::new(
+            ErrorKind::UnexpectedEof,
+            "failed to fill whole buffer",
+        ))
+    } else {
+        Ok(())
+    }
+}

beacon_node/eth2_libp2p/src/rpc/handler.rs

@@ -81,7 +81,7 @@ where
     TSpec: EthSpec,
 {
     /// The upgrade for inbound substreams.
-    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>,
+    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
 
     /// Errors occurring on outbound and inbound connections queued for reporting back.
     pending_errors: Vec<HandlerErr>,
@@ -225,7 +225,10 @@ impl<TSpec> RPCHandler<TSpec>
 where
     TSpec: EthSpec,
 {
-    pub fn new(listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>, log: &slog::Logger) -> Self {
+    pub fn new(
+        listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
+        log: &slog::Logger,
+    ) -> Self {
         RPCHandler {
             listen_protocol,
             pending_errors: Vec::new(),
@@ -249,7 +252,7 @@ where
     ///
     /// > **Note**: If you modify the protocol, modifications will only applies to future inbound
     /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_ref(&self) -> &SubstreamProtocol<RPCProtocol<TSpec>> {
+    pub fn listen_protocol_ref(&self) -> &SubstreamProtocol<RPCProtocol<TSpec>, ()> {
         &self.listen_protocol
     }
 
@@ -257,7 +260,7 @@ where
     ///
     /// > **Note**: If you modify the protocol, modifications will only apply to future inbound
     /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_mut(&mut self) -> &mut SubstreamProtocol<RPCProtocol<TSpec>> {
+    pub fn listen_protocol_mut(&mut self) -> &mut SubstreamProtocol<RPCProtocol<TSpec>, ()> {
         &mut self.listen_protocol
     }
 
@@ -344,14 +347,16 @@ where
     type InboundProtocol = RPCProtocol<TSpec>;
     type OutboundProtocol = RPCRequest<TSpec>;
     type OutboundOpenInfo = (RequestId, RPCRequest<TSpec>); // Keep track of the id and the request
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.listen_protocol.clone()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         substream: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         // only accept new peer requests when active
         if !matches!(self.state, HandlerState::Active) {
@@ -863,8 +868,7 @@ where
             let (id, req) = self.dial_queue.remove(0);
             self.dial_queue.shrink_to_fit();
             return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                protocol: SubstreamProtocol::new(req.clone()),
-                info: (id, req),
+                protocol: SubstreamProtocol::new(req.clone(), ()).map_info(|()| (id, req)),
             });
         }
         Poll::Pending

beacon_node/eth2_libp2p/src/rpc/methods.rs

@@ -1,6 +1,7 @@
 //! Available RPC methods types and ids.
 
 use crate::types::EnrBitfield;
+use regex::bytes::Regex;
 use serde::Serialize;
 use ssz_derive::{Decode, Encode};
 use ssz_types::{
@@ -19,7 +20,7 @@ type MaxErrorLen = U256;
 
 /// Wrapper over SSZ List to represent error message in rpc responses.
 #[derive(Debug, Clone)]
-pub struct ErrorType(VariableList<u8, MaxErrorLen>);
+pub struct ErrorType(pub VariableList<u8, MaxErrorLen>);
 
 impl From<String> for ErrorType {
     fn from(s: String) -> Self {
@@ -42,10 +43,9 @@ impl Deref for ErrorType {
 
 impl ToString for ErrorType {
     fn to_string(&self) -> String {
-        match std::str::from_utf8(self.0.deref()) {
-            Ok(s) => s.to_string(),
-            Err(_) => format!("{:?}", self.0.deref()), // Display raw bytes if not a UTF-8 string
-        }
+        #[allow(clippy::invalid_regex)]
+        let re = Regex::new("\\p{C}").expect("Regex is valid");
+        String::from_utf8_lossy(&re.replace_all(self.0.deref(), &b""[..])).to_string()
     }
 }
 
@@ -283,13 +283,13 @@ impl<T: EthSpec> RPCCodedResponse<T> {
     }
 
     /// Builds an RPCCodedResponse from a response code and an ErrorMessage
-    pub fn from_error(response_code: u8, err: String) -> Self {
+    pub fn from_error(response_code: u8, err: ErrorType) -> Self {
         let code = match response_code {
             1 => RPCResponseErrorCode::InvalidRequest,
             2 => RPCResponseErrorCode::ServerError,
             _ => RPCResponseErrorCode::Unknown,
         };
-        RPCCodedResponse::Error(code, err.into())
+        RPCCodedResponse::Error(code, err)
     }
 
     /// Specifies which response allows for multiple chunks for the stream handler.

beacon_node/eth2_libp2p/src/rpc/mod.rs

@@ -169,9 +169,12 @@ where
 
     fn new_handler(&mut self) -> Self::ProtocolsHandler {
         RPCHandler::new(
-            SubstreamProtocol::new(RPCProtocol {
-                phantom: PhantomData,
-            }),
+            SubstreamProtocol::new(
+                RPCProtocol {
+                    phantom: PhantomData,
+                },
+                (),
+            ),
             &self.log,
         )
     }

beacon_node/eth2_libp2p/src/rpc/rate_limiter.rs

@@ -189,7 +189,30 @@ impl RPCRateLimiter {
         request: &RPCRequest<T>,
     ) -> Result<(), RateLimitedErr> {
         let time_since_start = self.init_time.elapsed();
-        let tokens = request.expected_responses().max(1);
+        let mut tokens = request.expected_responses().max(1);
+
+        // Increase the rate limit for blocks by range requests with large step counts.
+        // We count to tokens as a quadratic increase with step size.
+        // Using (step_size/5)^2 + 1 as penalty factor allows step sizes of 1-4 to have no penalty
+        // but step sizes higher than this add a quadratic penalty.
+        // Penalty's go:
+        // Step size | Penalty Factor
+        //     1     |   1
+        //     2     |   1
+        //     3     |   1
+        //     4     |   1
+        //     5     |   2
+        //     6     |   2
+        //     7     |   2
+        //     8     |   3
+        //     9     |   4
+        //     10    |   5
+
+        if let RPCRequest::BlocksByRange(bbr_req) = request {
+            let penalty_factor = (bbr_req.step as f64 / 5.0).powi(2) as u64 + 1;
+            tokens *= penalty_factor;
+        }
+
         let check =
             |limiter: &mut Limiter<PeerId>| limiter.allows(time_since_start, peer_id, tokens);
         let limiter = match request.protocol() {

beacon_node/eth2_libp2p/src/service.rs

@@ -1,8 +1,10 @@
-use crate::behaviour::{Behaviour, BehaviourEvent, PeerRequestId, Request, Response};
+use crate::behaviour::{
+    save_metadata_to_disk, Behaviour, BehaviourEvent, PeerRequestId, Request, Response,
+};
 use crate::discovery::enr;
 use crate::multiaddr::Protocol;
-use crate::rpc::{GoodbyeReason, RPCResponseErrorCode, RequestId};
-use crate::types::{error, GossipKind};
+use crate::rpc::{GoodbyeReason, MetaData, RPCResponseErrorCode, RequestId};
+use crate::types::{error, EnrBitfield, GossipKind};
 use crate::EnrExt;
 use crate::{NetworkConfig, NetworkGlobals, PeerAction};
 use futures::prelude::*;
@@ -15,6 +17,7 @@ use libp2p::{
     PeerId, Swarm, Transport,
 };
 use slog::{crit, debug, info, o, trace, warn};
+use ssz::Decode;
 use std::fs::File;
 use std::io::prelude::*;
 use std::io::{Error, ErrorKind};
@@ -26,6 +29,8 @@ use types::{EnrForkId, EthSpec};
 pub const NETWORK_KEY_FILENAME: &str = "key";
 /// The maximum simultaneous libp2p connections per peer.
 const MAX_CONNECTIONS_PER_PEER: usize = 1;
+/// The filename to store our local metadata.
+pub const METADATA_FILENAME: &str = "metadata";
 
 /// The types of events than can be obtained from polling the libp2p service.
 ///
@@ -36,6 +41,8 @@ pub enum Libp2pEvent<TSpec: EthSpec> {
     Behaviour(BehaviourEvent<TSpec>),
     /// A new listening address has been established.
     NewListenAddr(Multiaddr),
+    /// We reached zero listening addresses.
+    ZeroListeners,
 }
 
 /// The configuration and state of the libp2p components for the beacon node.
@@ -68,11 +75,20 @@ impl<TSpec: EthSpec> Service<TSpec> {
             enr::build_or_load_enr::<TSpec>(local_keypair.clone(), config, enr_fork_id, &log)?;
 
         let local_peer_id = enr.peer_id();
+
+        let meta_data = load_or_build_metadata(&config.network_dir, &log);
+
         // set up a collection of variables accessible outside of the network crate
         let network_globals = Arc::new(NetworkGlobals::new(
             enr.clone(),
             config.libp2p_port,
             config.discovery_port,
+            meta_data,
+            config
+                .trusted_peers
+                .iter()
+                .map(|x| PeerId::from(x.clone()))
+                .collect(),
             &log,
         ));
 
@@ -283,10 +299,17 @@ impl<TSpec: EthSpec> Service<TSpec> {
                     debug!(self.log, "Listen address expired"; "multiaddr" => multiaddr.to_string())
                 }
                 SwarmEvent::ListenerClosed { addresses, reason } => {
-                    crit!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason))
+                    crit!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason));
+                    if Swarm::listeners(&self.swarm).count() == 0 {
+                        return Libp2pEvent::ZeroListeners;
+                    }
                 }
                 SwarmEvent::ListenerError { error } => {
-                    warn!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string()))
+                    // this is non fatal, but we still check
+                    warn!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string()));
+                    if Swarm::listeners(&self.swarm).count() == 0 {
+                        return Libp2pEvent::ZeroListeners;
+                    }
                 }
                 SwarmEvent::Dialing(peer_id) => {
                     debug!(self.log, "Dialing peer"; "peer_id" => peer_id.to_string());
@@ -348,7 +371,7 @@ fn keypair_from_bytes(mut bytes: Vec<u8>) -> error::Result<Keypair> {
 /// generated and is then saved to disk.
 ///
 /// Currently only secp256k1 keys are allowed, as these are the only keys supported by discv5.
-fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
+pub fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
     // check for key from disk
     let network_key_f = config.network_dir.join(NETWORK_KEY_FILENAME);
     if let Ok(mut network_key_file) = File::open(network_key_f.clone()) {
@@ -411,3 +434,43 @@ fn strip_peer_id(addr: &mut Multiaddr) {
         _ => {}
     }
 }
+
+/// Load metadata from persisted file. Return default metadata if loading fails.
+fn load_or_build_metadata<E: EthSpec>(
+    network_dir: &std::path::PathBuf,
+    log: &slog::Logger,
+) -> MetaData<E> {
+    // Default metadata
+    let mut meta_data = MetaData {
+        seq_number: 0,
+        attnets: EnrBitfield::<E>::default(),
+    };
+    // Read metadata from persisted file if available
+    let metadata_path = network_dir.join(METADATA_FILENAME);
+    if let Ok(mut metadata_file) = File::open(metadata_path) {
+        let mut metadata_ssz = Vec::new();
+        if metadata_file.read_to_end(&mut metadata_ssz).is_ok() {
+            match MetaData::<E>::from_ssz_bytes(&metadata_ssz) {
+                Ok(persisted_metadata) => {
+                    meta_data.seq_number = persisted_metadata.seq_number;
+                    // Increment seq number if persisted attnet is not default
+                    if persisted_metadata.attnets != meta_data.attnets {
+                        meta_data.seq_number += 1;
+                    }
+                    debug!(log, "Loaded metadata from disk");
+                }
+                Err(e) => {
+                    debug!(
+                        log,
+                        "Metadata from file could not be decoded";
+                        "error" => format!("{:?}", e),
+                    );
+                }
+            }
+        }
+    };
+
+    debug!(log, "Metadata sequence number"; "seq_num" => meta_data.seq_number);
+    save_metadata_to_disk(network_dir, meta_data.clone(), &log);
+    meta_data
+}

beacon_node/eth2_libp2p/src/types/globals.rs

@@ -1,5 +1,6 @@
 //! A collection of variables that are accessible outside of the network thread itself.
 use crate::peer_manager::PeerDB;
+use crate::rpc::MetaData;
 use crate::types::SyncState;
 use crate::Client;
 use crate::EnrExt;
@@ -22,6 +23,8 @@ pub struct NetworkGlobals<TSpec: EthSpec> {
     pub listen_port_udp: AtomicU16,
     /// The collection of known peers.
     pub peers: RwLock<PeerDB<TSpec>>,
+    // The local meta data of our node.
+    pub local_metadata: RwLock<MetaData<TSpec>>,
     /// The current gossipsub topic subscriptions.
     pub gossipsub_subscriptions: RwLock<HashSet<GossipTopic>>,
     /// The current sync status of the node.
@@ -29,14 +32,22 @@ pub struct NetworkGlobals<TSpec: EthSpec> {
 }
 
 impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
-    pub fn new(enr: Enr, tcp_port: u16, udp_port: u16, log: &slog::Logger) -> Self {
+    pub fn new(
+        enr: Enr,
+        tcp_port: u16,
+        udp_port: u16,
+        local_metadata: MetaData<TSpec>,
+        trusted_peers: Vec<PeerId>,
+        log: &slog::Logger,
+    ) -> Self {
         NetworkGlobals {
             local_enr: RwLock::new(enr.clone()),
             peer_id: RwLock::new(enr.peer_id()),
             listen_multiaddrs: RwLock::new(Vec::new()),
             listen_port_tcp: AtomicU16::new(tcp_port),
             listen_port_udp: AtomicU16::new(udp_port),
-            peers: RwLock::new(PeerDB::new(log)),
+            local_metadata: RwLock::new(local_metadata),
+            peers: RwLock::new(PeerDB::new(trusted_peers, log)),
             gossipsub_subscriptions: RwLock::new(HashSet::new()),
             sync_state: RwLock::new(SyncState::Stalled),
         }

beacon_node/eth2_libp2p/src/types/mod.rs

@@ -1,6 +1,7 @@
 pub mod error;
 mod globals;
 mod pubsub;
+mod subnet;
 mod sync_state;
 mod topics;
 
@@ -13,5 +14,6 @@ pub type Enr = discv5::enr::Enr<discv5::enr::CombinedKey>;
 
 pub use globals::NetworkGlobals;
 pub use pubsub::PubsubMessage;
+pub use subnet::SubnetDiscovery;
 pub use sync_state::SyncState;
 pub use topics::{GossipEncoding, GossipKind, GossipTopic};

beacon_node/eth2_libp2p/src/types/subnet.rs

@@ -0,0 +1,28 @@
+use std::time::{Duration, Instant};
+use types::SubnetId;
+
+const DURATION_DIFFERENCE: Duration = Duration::from_millis(1);
+
+/// A subnet to discover peers on along with the instant after which it's no longer useful.
+#[derive(Debug, Clone)]
+pub struct SubnetDiscovery {
+    pub subnet_id: SubnetId,
+    pub min_ttl: Option<Instant>,
+}
+
+impl PartialEq for SubnetDiscovery {
+    fn eq(&self, other: &SubnetDiscovery) -> bool {
+        self.subnet_id == other.subnet_id
+            && match (self.min_ttl, other.min_ttl) {
+                (Some(min_ttl_instant), Some(other_min_ttl_instant)) => {
+                    min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
+                        < DURATION_DIFFERENCE
+                        && other_min_ttl_instant.saturating_duration_since(min_ttl_instant)
+                            < DURATION_DIFFERENCE
+                }
+                (None, None) => true,
+                (None, Some(_)) => true,
+                (Some(_), None) => true,
+            }
+    }
+}

beacon_node/eth2_libp2p/src/types/topics.rs

@@ -1,4 +1,4 @@
-use libp2p::gossipsub::Topic;
+use libp2p::gossipsub::IdentTopic as Topic;
 use serde_derive::{Deserialize, Serialize};
 use types::SubnetId;
 
@@ -139,7 +139,7 @@ impl GossipTopic {
 
 impl Into<Topic> for GossipTopic {
     fn into(self) -> Topic {
-        Topic::new(self.into())
+        Topic::new(self)
     }
 }
 

beacon_node/eth2_libp2p/tests/common/mod.rs

@@ -10,6 +10,7 @@ use std::time::Duration;
 use types::{EnrForkId, MinimalEthSpec};
 
 type E = MinimalEthSpec;
+use libp2p::gossipsub::GossipsubConfigBuilder;
 use tempdir::TempDir;
 
 pub struct Libp2pInstance(LibP2PService<E>, exit_future::Signal);
@@ -83,8 +84,11 @@ pub fn build_config(port: u16, mut boot_nodes: Vec<Enr>) -> NetworkConfig {
     config.boot_nodes_enr.append(&mut boot_nodes);
     config.network_dir = path.into_path();
     // Reduce gossipsub heartbeat parameters
-    config.gs_config.heartbeat_initial_delay = Duration::from_millis(500);
-    config.gs_config.heartbeat_interval = Duration::from_millis(500);
+    config.gs_config = GossipsubConfigBuilder::from(config.gs_config)
+        .heartbeat_initial_delay(Duration::from_millis(500))
+        .heartbeat_interval(Duration::from_millis(500))
+        .build()
+        .unwrap();
     config
 }
 
@@ -94,8 +98,13 @@ pub async fn build_libp2p_instance(boot_nodes: Vec<Enr>, log: slog::Logger) -> L
     // launch libp2p service
 
     let (signal, exit) = exit_future::signal();
-    let executor =
-        environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone());
+    let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+    let executor = environment::TaskExecutor::new(
+        tokio::runtime::Handle::current(),
+        exit,
+        log.clone(),
+        shutdown_tx,
+    );
     Libp2pInstance(
         LibP2PService::new(executor, &config, EnrForkId::default(), &log)
             .await

beacon_node/network/src/attestation_service/mod.rs

@@ -2,7 +2,7 @@
 //! given time. It schedules subscriptions to shard subnets, requests peer discoveries and
 //! determines whether attestations should be aggregated and/or passed to the beacon node.
 
-use std::collections::VecDeque;
+use std::collections::{HashMap, VecDeque};
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
@@ -13,7 +13,7 @@ use rand::seq::SliceRandom;
 use slog::{crit, debug, error, o, trace, warn};
 
 use beacon_chain::{BeaconChain, BeaconChainTypes};
-use eth2_libp2p::{types::GossipKind, NetworkGlobals};
+use eth2_libp2p::{types::GossipKind, NetworkGlobals, SubnetDiscovery};
 use hashset_delay::HashSetDelay;
 use rest_types::ValidatorSubscription;
 use slot_clock::SlotClock;
@@ -25,11 +25,8 @@ mod tests;
 
 /// The minimum number of slots ahead that we attempt to discover peers for a subscription. If the
 /// slot is less than this number, skip the peer discovery process.
-const MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 1;
-/// The number of slots ahead that we attempt to discover peers for a subscription. If the slot to
-/// attest to is greater than this, we queue a discovery request for this many slots prior to
-/// subscribing.
-const TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 6;
+/// Subnet discovery query takes atmost 30 secs, 2 slots take 24s.
+const MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 2;
 /// The time (in slots) before a last seen validator is considered absent and we unsubscribe from the random
 /// gossip topics that we subscribed to due to the validator connection.
 const LAST_SEEN_VALIDATOR_TIMEOUT: u32 = 150;
@@ -39,12 +36,10 @@ const LAST_SEEN_VALIDATOR_TIMEOUT: u32 = 150;
 /// Note: The time is calculated as `time = milliseconds_per_slot / ADVANCE_SUBSCRIPTION_TIME`.
 const ADVANCE_SUBSCRIBE_TIME: u32 = 3;
 /// The default number of slots before items in hash delay sets used by this class should expire.
+///  36s at 12s slot time
 const DEFAULT_EXPIRATION_TIMEOUT: u32 = 3;
-// 36s at 12s slot time
-/// The duration difference between two instance for them to be considered equal.
-const DURATION_DIFFERENCE: Duration = Duration::from_millis(1);
 
-#[derive(Debug, Eq, Clone)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum AttServiceMessage {
     /// Subscribe to the specified subnet id.
     Subscribe(SubnetId),
@@ -54,44 +49,8 @@ pub enum AttServiceMessage {
     EnrAdd(SubnetId),
     /// Remove the `SubnetId` from the ENR bitfield.
     EnrRemove(SubnetId),
-    /// Discover peers for a particular subnet.
-    /// The includes the `Instant` we need the discovered peer until.
-    DiscoverPeers {
-        subnet_id: SubnetId,
-        min_ttl: Option<Instant>,
-    },
-}
-
-impl PartialEq for AttServiceMessage {
-    fn eq(&self, other: &AttServiceMessage) -> bool {
-        match (self, other) {
-            (&AttServiceMessage::Subscribe(a), &AttServiceMessage::Subscribe(b)) => a == b,
-            (&AttServiceMessage::Unsubscribe(a), &AttServiceMessage::Unsubscribe(b)) => a == b,
-            (&AttServiceMessage::EnrAdd(a), &AttServiceMessage::EnrAdd(b)) => a == b,
-            (&AttServiceMessage::EnrRemove(a), &AttServiceMessage::EnrRemove(b)) => a == b,
-            (
-                &AttServiceMessage::DiscoverPeers { subnet_id, min_ttl },
-                &AttServiceMessage::DiscoverPeers {
-                    subnet_id: other_subnet_id,
-                    min_ttl: other_min_ttl,
-                },
-            ) => {
-                subnet_id == other_subnet_id
-                    && match (min_ttl, other_min_ttl) {
-                        (Some(min_ttl_instant), Some(other_min_ttl_instant)) => {
-                            min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
-                                < DURATION_DIFFERENCE
-                                && other_min_ttl_instant.saturating_duration_since(min_ttl_instant)
-                                    < DURATION_DIFFERENCE
-                        }
-                        (None, None) => true,
-                        (None, Some(_)) => true,
-                        (Some(_), None) => true,
-                    }
-            }
-            _ => false,
-        }
-    }
+    /// Discover peers for a list of `SubnetDiscovery`.
+    DiscoverPeers(Vec<SubnetDiscovery>),
 }
 
 /// A particular subnet at a given slot.
@@ -116,9 +75,6 @@ pub struct AttestationService<T: BeaconChainTypes> {
     /// The collection of currently subscribed random subnets mapped to their expiry deadline.
     random_subnets: HashSetDelay<SubnetId>,
 
-    /// A collection of timeouts for when to start searching for peers for a particular shard.
-    discover_peers: HashSetDelay<ExactSubnet>,
-
     /// A collection of timeouts for when to subscribe to a shard subnet.
     subscriptions: HashSetDelay<ExactSubnet>,
 
@@ -172,7 +128,6 @@ impl<T: BeaconChainTypes> AttestationService<T> {
             network_globals,
             beacon_chain,
             random_subnets: HashSetDelay::new(Duration::from_millis(random_subnet_duration_millis)),
-            discover_peers: HashSetDelay::new(default_timeout),
             subscriptions: HashSetDelay::new(default_timeout),
             unsubscriptions: HashSetDelay::new(default_timeout),
             aggregate_validators_on_subnet: HashSetDelay::new(default_timeout),
@@ -198,6 +153,8 @@ impl<T: BeaconChainTypes> AttestationService<T> {
         &mut self,
         subscriptions: Vec<ValidatorSubscription>,
     ) -> Result<(), String> {
+        // Maps each subnet_id subscription to it's highest slot
+        let mut subnets_to_discover: HashMap<SubnetId, Slot> = HashMap::new();
         for subscription in subscriptions {
             metrics::inc_counter(&metrics::SUBNET_SUBSCRIPTION_REQUESTS);
             //NOTE: We assume all subscriptions have been verified before reaching this service
@@ -226,15 +183,20 @@ impl<T: BeaconChainTypes> AttestationService<T> {
                     continue;
                 }
             };
+            // Ensure each subnet_id inserted into the map has the highest slot as it's value.
+            // Higher slot corresponds to higher min_ttl in the `SubnetDiscovery` entry.
+            if let Some(slot) = subnets_to_discover.get(&subnet_id) {
+                if subscription.slot > *slot {
+                    subnets_to_discover.insert(subnet_id, subscription.slot);
+                }
+            } else {
+                subnets_to_discover.insert(subnet_id, subscription.slot);
+            }
 
             let exact_subnet = ExactSubnet {
                 subnet_id,
                 slot: subscription.slot,
             };
-            // determine if we should run a discovery lookup request and request it if required
-            if let Err(e) = self.discover_peers_request(exact_subnet.clone()) {
-                warn!(self.log, "Discovery lookup request error"; "error" => e);
-            }
 
             // determine if the validator is an aggregator. If so, we subscribe to the subnet and
             // if successful add the validator to a mapping of known aggregators for that exact
@@ -264,6 +226,14 @@ impl<T: BeaconChainTypes> AttestationService<T> {
             }
         }
 
+        if let Err(e) = self.discover_peers_request(
+            subnets_to_discover
+                .into_iter()
+                .map(|(subnet_id, slot)| ExactSubnet { subnet_id, slot }),
+        ) {
+            warn!(self.log, "Discovery lookup request error"; "error" => e);
+        };
+
         // pre-emptively wake the thread to check for new events
         if let Some(waker) = &self.waker {
             waker.wake_by_ref();
@@ -290,114 +260,55 @@ impl<T: BeaconChainTypes> AttestationService<T> {
     /// Checks if there are currently queued discovery requests and the time required to make the
     /// request.
     ///
-    /// If there is sufficient time and no other request exists, queues a peer discovery request
-    /// for the required subnet.
-    fn discover_peers_request(&mut self, exact_subnet: ExactSubnet) -> Result<(), &'static str> {
+    /// If there is sufficient time, queues a peer discovery request for all the required subnets.
+    fn discover_peers_request(
+        &mut self,
+        exact_subnets: impl Iterator<Item = ExactSubnet>,
+    ) -> Result<(), &'static str> {
         let current_slot = self
             .beacon_chain
             .slot_clock
             .now()
             .ok_or_else(|| "Could not get the current slot")?;
-        let slot_duration = self.beacon_chain.slot_clock.slot_duration();
 
-        // if there is enough time to perform a discovery lookup
-        if exact_subnet.slot >= current_slot.saturating_add(MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD) {
-            // check if a discovery request already exists
-            if self.discover_peers.get(&exact_subnet).is_some() {
-                // already a request queued, end
-                return Ok(());
-            }
-
-            // if the slot is more than epoch away, add an event to start looking for peers
-            if exact_subnet.slot
-                < current_slot.saturating_add(TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD)
-            {
-                // add one slot to ensure we keep the peer for the subscription slot
-                let min_ttl = self
-                    .beacon_chain
-                    .slot_clock
-                    .duration_to_slot(exact_subnet.slot + 1)
-                    .map(|duration| std::time::Instant::now() + duration);
-
-                self.send_or_update_discovery_event(exact_subnet.subnet_id, min_ttl);
-            } else {
-                // Queue the discovery event to be executed for
-                // TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD
-
-                let duration_to_discover = {
-                    let duration_to_next_slot = self
+        let discovery_subnets: Vec<SubnetDiscovery> = exact_subnets
+            .filter_map(|exact_subnet| {
+                // check if there is enough time to perform a discovery lookup
+                if exact_subnet.slot
+                    >= current_slot.saturating_add(MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD)
+                {
+                    // if the slot is more than epoch away, add an event to start looking for peers
+                    // add one slot to ensure we keep the peer for the subscription slot
+                    let min_ttl = self
                         .beacon_chain
                         .slot_clock
-                        .duration_to_next_slot()
-                        .ok_or_else(|| "Unable to determine duration to next slot")?;
-                    // The -1 is done here to exclude the current slot duration, as we will use
-                    // `duration_to_next_slot`.
-                    let slots_until_discover = exact_subnet
-                        .slot
-                        .saturating_sub(current_slot)
-                        .saturating_sub(1u64)
-                        .saturating_sub(TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD);
+                        .duration_to_slot(exact_subnet.slot + 1)
+                        .map(|duration| std::time::Instant::now() + duration);
+                    Some(SubnetDiscovery {
+                        subnet_id: exact_subnet.subnet_id,
+                        min_ttl,
+                    })
+                } else {
+                    // TODO: Send the time frame needed to have a peer connected, so that we can
+                    // maintain peers for a least this duration.
+                    // We may want to check the global PeerInfo to see estimated timeouts for each
+                    // peer before they can be removed.
+                    warn!(self.log,
+                        "Not enough time for a discovery search";
+                        "subnet_id" => format!("{:?}", exact_subnet)
+                    );
+                    None
+                }
+            })
+            .collect();
 
-                    duration_to_next_slot + slot_duration * (slots_until_discover.as_u64() as u32)
-                };
-
-                self.discover_peers
-                    .insert_at(exact_subnet, duration_to_discover);
-            }
-        } else {
-            // TODO: Send the time frame needed to have a peer connected, so that we can
-            // maintain peers for a least this duration.
-            // We may want to check the global PeerInfo to see estimated timeouts for each
-            // peer before they can be removed.
-            return Err("Not enough time for a discovery search");
+        if !discovery_subnets.is_empty() {
+            self.events
+                .push_back(AttServiceMessage::DiscoverPeers(discovery_subnets));
         }
         Ok(())
     }
 
-    /// Checks if we have a discover peers event already and sends a new event if necessary
-    ///
-    /// If a message exists for the same subnet, compare the `min_ttl` of the current and
-    /// existing messages and extend the existing message as necessary.
-    fn send_or_update_discovery_event(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
-        // track whether this message already exists in the event queue
-        let mut is_duplicate = false;
-
-        self.events.iter_mut().for_each(|event| {
-            if let AttServiceMessage::DiscoverPeers {
-                subnet_id: other_subnet_id,
-                min_ttl: other_min_ttl,
-            } = event
-            {
-                if subnet_id == *other_subnet_id {
-                    let other_min_ttl_clone = *other_min_ttl;
-                    match (min_ttl, other_min_ttl_clone) {
-                        (Some(min_ttl_instant), Some(other_min_ttl_instant)) =>
-                        // only update the min_ttl if it is greater than the existing min_ttl and a DURATION_DIFFERENCE padding
-                        {
-                            if min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
-                                > DURATION_DIFFERENCE
-                            {
-                                *other_min_ttl = min_ttl;
-                            }
-                        }
-                        (None, Some(_)) => {} // Keep the current one as it has an actual min_ttl
-                        (Some(min_ttl), None) => {
-                            // Update the request to include a min_ttl.
-                            *other_min_ttl = Some(min_ttl);
-                        }
-                        (None, None) => {} // Duplicate message, do nothing.
-                    }
-                    is_duplicate = true;
-                    return;
-                }
-            };
-        });
-        if !is_duplicate {
-            self.events
-                .push_back(AttServiceMessage::DiscoverPeers { subnet_id, min_ttl });
-        }
-    }
-
     /// Checks the current random subnets and subscriptions to determine if a new subscription for this
     /// subnet is required for the given slot.
     ///
@@ -547,7 +458,11 @@ impl<T: BeaconChainTypes> AttestationService<T> {
 
             if !already_subscribed {
                 // send a discovery request and a subscription
-                self.send_or_update_discovery_event(subnet_id, None);
+                self.events
+                    .push_back(AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery {
+                        subnet_id,
+                        min_ttl: None,
+                    }]));
                 self.events
                     .push_back(AttServiceMessage::Subscribe(subnet_id));
             }
@@ -558,20 +473,6 @@ impl<T: BeaconChainTypes> AttestationService<T> {
 
     /* A collection of functions that handle the various timeouts */
 
-    /// Request a discovery query to find peers for a particular subnet.
-    fn handle_discover_peers(&mut self, exact_subnet: ExactSubnet) {
-        debug!(self.log, "Searching for peers for subnet"; "subnet" => *exact_subnet.subnet_id, "target_slot" => exact_subnet.slot);
-
-        // add one slot to ensure we keep the peer for the subscription slot
-        let min_ttl = self
-            .beacon_chain
-            .slot_clock
-            .duration_to_slot(exact_subnet.slot + 1)
-            .map(|duration| std::time::Instant::now() + duration);
-
-        self.send_or_update_discovery_event(exact_subnet.subnet_id, min_ttl)
-    }
-
     /// A queued subscription is ready.
     ///
     /// We add subscriptions events even if we are already subscribed to a random subnet (as these
@@ -731,15 +632,6 @@ impl<T: BeaconChainTypes> Stream for AttestationService<T> {
             self.waker = Some(cx.waker().clone());
         }
 
-        // process any peer discovery events
-        match self.discover_peers.poll_next_unpin(cx) {
-            Poll::Ready(Some(Ok(exact_subnet))) => self.handle_discover_peers(exact_subnet),
-            Poll::Ready(Some(Err(e))) => {
-                error!(self.log, "Failed to check for peer discovery requests"; "error"=> e);
-            }
-            Poll::Ready(None) | Poll::Pending => {}
-        }
-
         // process any subscription events
         match self.subscriptions.poll_next_unpin(cx) {
             Poll::Ready(Some(Ok(exact_subnet))) => self.handle_subscriptions(exact_subnet),

beacon_node/network/src/attestation_service/tests/mod.rs

@@ -8,7 +8,11 @@ mod tests {
         migrate::NullMigrator,
     };
     use eth2_libp2p::discovery::{build_enr, Keypair};
-    use eth2_libp2p::{discovery::CombinedKey, CombinedKeyExt, NetworkConfig, NetworkGlobals};
+    use eth2_libp2p::rpc::methods::MetaData;
+    use eth2_libp2p::types::EnrBitfield;
+    use eth2_libp2p::{
+        discovery::CombinedKey, CombinedKeyExt, NetworkConfig, NetworkGlobals, SubnetDiscovery,
+    };
     use futures::Stream;
     use genesis::{generate_deterministic_keypairs, interop_genesis_state};
     use lazy_static::lazy_static;
@@ -100,7 +104,14 @@ mod tests {
         let enr_key = CombinedKey::from_libp2p(&Keypair::generate_secp256k1()).unwrap();
         let enr = build_enr::<MinimalEthSpec>(&enr_key, &config, EnrForkId::default()).unwrap();
 
-        let network_globals: NetworkGlobals<MinimalEthSpec> = NetworkGlobals::new(enr, 0, 0, &log);
+        // Default metadata
+        let meta_data = MetaData {
+            seq_number: 0,
+            attnets: EnrBitfield::<MinimalEthSpec>::default(),
+        };
+
+        let network_globals: NetworkGlobals<MinimalEthSpec> =
+            NetworkGlobals::new(enr, 0, 0, meta_data, vec![], &log);
         AttestationService::new(beacon_chain, Arc::new(network_globals), &log)
     }
 
@@ -120,23 +131,21 @@ mod tests {
         }
     }
 
-    fn _get_subscriptions(
+    fn get_subscriptions(
         validator_count: u64,
         slot: Slot,
         committee_count_at_slot: u64,
     ) -> Vec<ValidatorSubscription> {
-        let mut subscriptions: Vec<ValidatorSubscription> = Vec::new();
-        for validator_index in 0..validator_count {
-            let is_aggregator = true;
-            subscriptions.push(ValidatorSubscription {
-                validator_index,
-                attestation_committee_index: validator_index,
-                slot,
-                committee_count_at_slot,
-                is_aggregator,
-            });
-        }
-        subscriptions
+        (0..validator_count)
+            .map(|validator_index| {
+                get_subscription(
+                    validator_index,
+                    validator_index,
+                    slot,
+                    committee_count_at_slot,
+                )
+            })
+            .collect()
     }
 
     // gets a number of events from the subscription service, or returns none if it times out after a number
@@ -210,14 +219,7 @@ mod tests {
         let events = get_events(attestation_service, no_events_expected, 1).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any1),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any1), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -270,14 +272,7 @@ mod tests {
         let events = get_events(attestation_service, no_events_expected, 2).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any1),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any1), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -330,19 +325,15 @@ mod tests {
             &attestation_service.beacon_chain.spec,
         )
         .unwrap();
-        let expected = vec![AttServiceMessage::DiscoverPeers { subnet_id, min_ttl }];
+        let expected = vec![AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery {
+            subnet_id,
+            min_ttl,
+        }])];
 
         let events = get_events(attestation_service, no_events_expected, 1).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -396,21 +387,14 @@ mod tests {
         )
         .unwrap();
         let expected = vec![
-            AttServiceMessage::DiscoverPeers { subnet_id, min_ttl },
+            AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery { subnet_id, min_ttl }]),
             AttServiceMessage::Subscribe(subnet_id),
         ];
 
         let events = get_events(attestation_service, no_events_expected, 5).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -454,14 +438,7 @@ mod tests {
 
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -517,20 +494,16 @@ mod tests {
 
         // expect discover peers because we will enter TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD range
         let expected: Vec<AttServiceMessage> =
-            vec![AttServiceMessage::DiscoverPeers { subnet_id, min_ttl }];
+            vec![AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery {
+                subnet_id,
+                min_ttl,
+            }])];
 
         let events = get_events(attestation_service, no_events_expected, 5).await;
 
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -553,7 +526,7 @@ mod tests {
             .now()
             .expect("Could not get current slot");
 
-        let subscriptions = _get_subscriptions(
+        let subscriptions = get_subscriptions(
             subscription_count,
             current_slot + subscription_slot,
             committee_count,
@@ -572,10 +545,9 @@ mod tests {
 
         for event in events {
             match event {
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant,
-                } => discover_peer_count = discover_peer_count + 1,
+                AttServiceMessage::DiscoverPeers(_) => {
+                    discover_peer_count = discover_peer_count + 1
+                }
                 AttServiceMessage::Subscribe(_any_subnet) => subscribe_count = subscribe_count + 1,
                 AttServiceMessage::EnrAdd(_any_subnet) => enr_add_count = enr_add_count + 1,
                 _ => unexpected_msg_count = unexpected_msg_count + 1,
@@ -605,7 +577,7 @@ mod tests {
             .now()
             .expect("Could not get current slot");
 
-        let subscriptions = _get_subscriptions(
+        let subscriptions = get_subscriptions(
             subscription_count,
             current_slot + subscription_slot,
             committee_count,
@@ -624,10 +596,9 @@ mod tests {
 
         for event in events {
             match event {
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant,
-                } => discover_peer_count = discover_peer_count + 1,
+                AttServiceMessage::DiscoverPeers(_) => {
+                    discover_peer_count = discover_peer_count + 1
+                }
                 AttServiceMessage::Subscribe(_any_subnet) => subscribe_count = subscribe_count + 1,
                 AttServiceMessage::EnrAdd(_any_subnet) => enr_add_count = enr_add_count + 1,
                 _ => unexpected_msg_count = unexpected_msg_count + 1,
@@ -639,4 +610,40 @@ mod tests {
         assert_eq!(enr_add_count, 64);
         assert_eq!(unexpected_msg_count, 0);
     }
+
+    #[tokio::test]
+    async fn test_discovery_peers_count() {
+        let subscription_slot = 10;
+        let validator_count = 32;
+        let committee_count = 1;
+        let expected_events = 97;
+
+        // create the attestation service and subscriptions
+        let mut attestation_service = get_attestation_service();
+        let current_slot = attestation_service
+            .beacon_chain
+            .slot_clock
+            .now()
+            .expect("Could not get current slot");
+
+        let subscriptions = get_subscriptions(
+            validator_count,
+            current_slot + subscription_slot,
+            committee_count,
+        );
+
+        // submit sthe subscriptions
+        attestation_service
+            .validator_subscriptions(subscriptions)
+            .unwrap();
+
+        let events = get_events(attestation_service, expected_events, 3).await;
+
+        let event = events.get(96);
+        if let Some(AttServiceMessage::DiscoverPeers(d)) = event {
+            assert_eq!(d.len(), validator_count as usize);
+        } else {
+            panic!("Unexpected event {:?}", event);
+        }
+    }
 }

beacon_node/network/src/beacon_processor/mod.rs

@@ -36,22 +36,22 @@
 //! task.
 
 use crate::{metrics, service::NetworkMessage, sync::SyncMessage};
-use beacon_chain::{
-    attestation_verification::Error as AttnError, BeaconChain, BeaconChainError, BeaconChainTypes,
-    BlockError, ForkChoiceError,
-};
-use chain_segment::handle_chain_segment;
+use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError};
 use environment::TaskExecutor;
 use eth2_libp2p::{MessageId, NetworkGlobals, PeerId};
-use slog::{crit, debug, error, info, trace, warn, Logger};
-use ssz::Encode;
+use slog::{crit, debug, error, trace, warn, Logger};
 use std::collections::VecDeque;
 use std::sync::{Arc, Weak};
 use std::time::{Duration, Instant};
 use tokio::sync::{mpsc, oneshot};
-use types::{Attestation, EthSpec, Hash256, SignedAggregateAndProof, SignedBeaconBlock, SubnetId};
+use types::{
+    Attestation, AttesterSlashing, EthSpec, Hash256, ProposerSlashing, SignedAggregateAndProof,
+    SignedBeaconBlock, SignedVoluntaryExit, SubnetId,
+};
+use worker::Worker;
 
 mod chain_segment;
+mod worker;
 
 pub use chain_segment::ProcessId;
 
@@ -78,6 +78,18 @@ const MAX_AGGREGATED_ATTESTATION_QUEUE_LEN: usize = 1_024;
 /// before we start dropping them.
 const MAX_GOSSIP_BLOCK_QUEUE_LEN: usize = 1_024;
 
+/// The maximum number of queued `SignedVoluntaryExit` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_EXIT_QUEUE_LEN: usize = 4_096;
+
+/// The maximum number of queued `ProposerSlashing` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_PROPOSER_SLASHING_QUEUE_LEN: usize = 4_096;
+
+/// The maximum number of queued `AttesterSlashing` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_ATTESTER_SLASHING_QUEUE_LEN: usize = 4_096;
+
 /// The maximum number of queued `SignedBeaconBlock` objects received from the network RPC that
 /// will be stored before we start dropping them.
 const MAX_RPC_BLOCK_QUEUE_LEN: usize = 1_024;
@@ -242,6 +254,54 @@ impl<E: EthSpec> WorkEvent<E> {
         }
     }
 
+    /// Create a new `Work` event for some exit.
+    pub fn gossip_voluntary_exit(
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: Box<SignedVoluntaryExit>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipVoluntaryExit {
+                message_id,
+                peer_id,
+                voluntary_exit,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some proposer slashing.
+    pub fn gossip_proposer_slashing(
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipProposerSlashing {
+                message_id,
+                peer_id,
+                proposer_slashing,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some attester slashing.
+    pub fn gossip_attester_slashing(
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<E>>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipAttesterSlashing {
+                message_id,
+                peer_id,
+                attester_slashing,
+            },
+        }
+    }
+
     /// Create a new `Work` event for some block, where the result from computation (if any) is
     /// sent to the other side of `result_tx`.
     pub fn rpc_beacon_block(block: Box<SignedBeaconBlock<E>>) -> (Self, BlockResultReceiver<E>) {
@@ -282,6 +342,21 @@ pub enum Work<E: EthSpec> {
         peer_id: PeerId,
         block: Box<SignedBeaconBlock<E>>,
     },
+    GossipVoluntaryExit {
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: Box<SignedVoluntaryExit>,
+    },
+    GossipProposerSlashing {
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
+    },
+    GossipAttesterSlashing {
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<E>>,
+    },
     RpcBlock {
         block: Box<SignedBeaconBlock<E>>,
         result_tx: BlockResultSender<E>,
@@ -299,6 +374,9 @@ impl<E: EthSpec> Work<E> {
             Work::GossipAttestation { .. } => "gossip_attestation",
             Work::GossipAggregate { .. } => "gossip_aggregate",
             Work::GossipBlock { .. } => "gossip_block",
+            Work::GossipVoluntaryExit { .. } => "gossip_voluntary_exit",
+            Work::GossipProposerSlashing { .. } => "gossip_proposer_slashing",
+            Work::GossipAttesterSlashing { .. } => "gossip_attester_slashing",
             Work::RpcBlock { .. } => "rpc_block",
             Work::ChainSegment { .. } => "chain_segment",
         }
@@ -351,21 +429,33 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
     pub fn spawn_manager(mut self, mut event_rx: mpsc::Receiver<WorkEvent<T::EthSpec>>) {
         let (idle_tx, mut idle_rx) = mpsc::channel::<()>(MAX_IDLE_QUEUE_LEN);
 
+        // Using LIFO queues for attestations since validator profits rely upon getting fresh
+        // attestations into blocks. Additionally, later attestations contain more information than
+        // earlier ones, so we consider them more valuable.
         let mut aggregate_queue = LifoQueue::new(MAX_AGGREGATED_ATTESTATION_QUEUE_LEN);
         let mut aggregate_debounce = TimeLatch::default();
-
         let mut attestation_queue = LifoQueue::new(MAX_UNAGGREGATED_ATTESTATION_QUEUE_LEN);
         let mut attestation_debounce = TimeLatch::default();
 
-        let mut gossip_block_queue = FifoQueue::new(MAX_GOSSIP_BLOCK_QUEUE_LEN);
+        // Using a FIFO queue for voluntary exits since it prevents exit censoring. I don't have
+        // a strong feeling about queue type for exits.
+        let mut gossip_voluntary_exit_queue = FifoQueue::new(MAX_GOSSIP_EXIT_QUEUE_LEN);
 
+        // Using a FIFO queue for slashing to prevent people from flushing their slashings from the
+        // queues with lots of junk messages.
+        let mut gossip_proposer_slashing_queue =
+            FifoQueue::new(MAX_GOSSIP_PROPOSER_SLASHING_QUEUE_LEN);
+        let mut gossip_attester_slashing_queue =
+            FifoQueue::new(MAX_GOSSIP_ATTESTER_SLASHING_QUEUE_LEN);
+
+        // Using a FIFO queue since blocks need to be imported sequentially.
         let mut rpc_block_queue = FifoQueue::new(MAX_RPC_BLOCK_QUEUE_LEN);
-
         let mut chain_segment_queue = FifoQueue::new(MAX_CHAIN_SEGMENT_QUEUE_LEN);
+        let mut gossip_block_queue = FifoQueue::new(MAX_GOSSIP_BLOCK_QUEUE_LEN);
 
         let executor = self.executor.clone();
 
-        // The manager future will run on the non-blocking executor and delegate tasks to worker
+        // The manager future will run on the core executor and delegate tasks to worker
         // threads on the blocking executor.
         let manager_future = async move {
             loop {
@@ -452,6 +542,18 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
                             self.spawn_worker(idle_tx.clone(), item);
                         } else if let Some(item) = attestation_queue.pop() {
                             self.spawn_worker(idle_tx.clone(), item);
+                        // Check slashings after all other consensus messages so we prioritize
+                        // following head.
+                        //
+                        // Check attester slashings before proposer slashings since they have the
+                        // potential to slash multiple validators at once.
+                        } else if let Some(item) = gossip_attester_slashing_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        } else if let Some(item) = gossip_proposer_slashing_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check exits last since our validators don't get rewards from them.
+                        } else if let Some(item) = gossip_voluntary_exit_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
                         }
                     }
                     // There is no new work event and we are unable to spawn a new worker.
@@ -491,6 +593,15 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
                             Work::GossipBlock { .. } => {
                                 gossip_block_queue.push(work, work_id, &self.log)
                             }
+                            Work::GossipVoluntaryExit { .. } => {
+                                gossip_voluntary_exit_queue.push(work, work_id, &self.log)
+                            }
+                            Work::GossipProposerSlashing { .. } => {
+                                gossip_proposer_slashing_queue.push(work, work_id, &self.log)
+                            }
+                            Work::GossipAttesterSlashing { .. } => {
+                                gossip_attester_slashing_queue.push(work, work_id, &self.log)
+                            }
                             Work::RpcBlock { .. } => rpc_block_queue.push(work, work_id, &self.log),
                             Work::ChainSegment { .. } => {
                                 chain_segment_queue.push(work, work_id, &self.log)
@@ -523,6 +634,18 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
                     &metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL,
                     chain_segment_queue.len() as i64,
                 );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_EXIT_QUEUE_TOTAL,
+                    gossip_voluntary_exit_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL,
+                    gossip_proposer_slashing_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL,
+                    gossip_attester_slashing_queue.len() as i64,
+                );
 
                 if aggregate_queue.is_full() && aggregate_debounce.elapsed() {
                     error!(
@@ -544,7 +667,7 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
             }
         };
 
-        // Spawn on the non-blocking executor.
+        // Spawn on the core executor.
         executor.spawn(manager_future, MANAGER_TASK_NAME);
     }
 
@@ -574,11 +697,16 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
             return;
         };
 
-        let network_tx = self.network_tx.clone();
-        let sync_tx = self.sync_tx.clone();
         let log = self.log.clone();
         let executor = self.executor.clone();
 
+        let worker = Worker {
+            chain,
+            network_tx: self.network_tx.clone(),
+            sync_tx: self.sync_tx.clone(),
+            log: self.log.clone(),
+        };
+
         trace!(
             self.log,
             "Spawning beacon processor worker";
@@ -589,298 +717,85 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
         executor.spawn_blocking(
             move || {
                 let _worker_timer = worker_timer;
-                let inner_log = log.clone();
 
-                // We use this closure pattern to avoid using a `return` that prevents the
-                // `idle_tx` message from sending.
-                let handler = || {
-                    let log = inner_log.clone();
-                    match work {
-                        /*
-                         * Unaggregated attestation verification.
-                         */
-                        Work::GossipAttestation {
-                            message_id,
-                            peer_id,
-                            attestation,
-                            subnet_id,
-                            should_import,
-                        } => {
-                            let beacon_block_root = attestation.data.beacon_block_root;
-
-                            let attestation = match chain
-                                .verify_unaggregated_attestation_for_gossip(*attestation, subnet_id)
-                            {
-                                Ok(attestation) => attestation,
-                                Err(e) => {
-                                    handle_attestation_verification_failure(
-                                        &log,
-                                        sync_tx,
-                                        peer_id,
-                                        beacon_block_root,
-                                        "unaggregated",
-                                        e,
-                                    );
-                                    return;
-                                }
-                            };
-
-                            // Indicate to the `Network` service that this message is valid and can be
-                            // propagated on the gossip network.
-                            propagate_gossip_message(network_tx, message_id, peer_id.clone(), &log);
-
-                            if !should_import {
-                                return;
-                            }
-
-                            metrics::inc_counter(
-                                &metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL,
-                            );
-
-                            if let Err(e) = chain.apply_attestation_to_fork_choice(&attestation) {
-                                match e {
-                                    BeaconChainError::ForkChoiceError(
-                                        ForkChoiceError::InvalidAttestation(e),
-                                    ) => debug!(
-                                        log,
-                                        "Attestation invalid for fork choice";
-                                        "reason" => format!("{:?}", e),
-                                        "peer" => peer_id.to_string(),
-                                        "beacon_block_root" => format!("{:?}", beacon_block_root)
-                                    ),
-                                    e => error!(
-                                        log,
-                                        "Error applying attestation to fork choice";
-                                        "reason" => format!("{:?}", e),
-                                        "peer" => peer_id.to_string(),
-                                        "beacon_block_root" => format!("{:?}", beacon_block_root)
-                                    ),
-                                }
-                            }
-
-                            if let Err(e) = chain.add_to_naive_aggregation_pool(attestation) {
-                                debug!(
-                                    log,
-                                    "Attestation invalid for agg pool";
-                                    "reason" => format!("{:?}", e),
-                                    "peer" => peer_id.to_string(),
-                                    "beacon_block_root" => format!("{:?}", beacon_block_root)
-                                )
-                            }
-
-                            metrics::inc_counter(
-                                &metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_IMPORTED_TOTAL,
-                            );
-                        }
-                        /*
-                         * Aggregated attestation verification.
-                         */
-                        Work::GossipAggregate {
-                            message_id,
-                            peer_id,
-                            aggregate,
-                        } => {
-                            let beacon_block_root =
-                                aggregate.message.aggregate.data.beacon_block_root;
-
-                            let aggregate =
-                                match chain.verify_aggregated_attestation_for_gossip(*aggregate) {
-                                    Ok(aggregate) => aggregate,
-                                    Err(e) => {
-                                        handle_attestation_verification_failure(
-                                            &log,
-                                            sync_tx,
-                                            peer_id,
-                                            beacon_block_root,
-                                            "aggregated",
-                                            e,
-                                        );
-                                        return;
-                                    }
-                                };
-
-                            // Indicate to the `Network` service that this message is valid and can be
-                            // propagated on the gossip network.
-                            propagate_gossip_message(network_tx, message_id, peer_id.clone(), &log);
-
-                            metrics::inc_counter(
-                                &metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL,
-                            );
-
-                            if let Err(e) = chain.apply_attestation_to_fork_choice(&aggregate) {
-                                match e {
-                                    BeaconChainError::ForkChoiceError(
-                                        ForkChoiceError::InvalidAttestation(e),
-                                    ) => debug!(
-                                        log,
-                                        "Aggregate invalid for fork choice";
-                                        "reason" => format!("{:?}", e),
-                                        "peer" => peer_id.to_string(),
-                                        "beacon_block_root" => format!("{:?}", beacon_block_root)
-                                    ),
-                                    e => error!(
-                                        log,
-                                        "Error applying aggregate to fork choice";
-                                        "reason" => format!("{:?}", e),
-                                        "peer" => peer_id.to_string(),
-                                        "beacon_block_root" => format!("{:?}", beacon_block_root)
-                                    ),
-                                }
-                            }
-
-                            if let Err(e) = chain.add_to_block_inclusion_pool(aggregate) {
-                                debug!(
-                                    log,
-                                    "Attestation invalid for op pool";
-                                    "reason" => format!("{:?}", e),
-                                    "peer" => peer_id.to_string(),
-                                    "beacon_block_root" => format!("{:?}", beacon_block_root)
-                                )
-                            }
-
-                            metrics::inc_counter(
-                                &metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_IMPORTED_TOTAL,
-                            );
-                        }
-                        /*
-                         * Verification for beacon blocks received on gossip.
-                         */
-                        Work::GossipBlock {
-                            message_id,
-                            peer_id,
-                            block,
-                        } => {
-                            let verified_block = match chain.verify_block_for_gossip(*block) {
-                                Ok(verified_block) => {
-                                    info!(
-                                        log,
-                                        "New block received";
-                                        "slot" => verified_block.block.slot(),
-                                        "hash" => verified_block.block_root.to_string()
-                                    );
-                                    propagate_gossip_message(
-                                        network_tx,
-                                        message_id,
-                                        peer_id.clone(),
-                                        &log,
-                                    );
-                                    verified_block
-                                }
-                                Err(BlockError::ParentUnknown(block)) => {
-                                    send_sync_message(
-                                        sync_tx,
-                                        SyncMessage::UnknownBlock(peer_id, block),
-                                        &log,
-                                    );
-                                    return;
-                                }
-                                Err(BlockError::BlockIsAlreadyKnown) => {
-                                    debug!(
-                                        log,
-                                        "Gossip block is already known";
-                                    );
-                                    return;
-                                }
-                                Err(e) => {
-                                    warn!(
-                                        log,
-                                        "Could not verify block for gossip";
-                                        "error" => format!("{:?}", e)
-                                    );
-                                    return;
-                                }
-                            };
-
-                            metrics::inc_counter(
-                                &metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL,
-                            );
-
-                            let block = Box::new(verified_block.block.clone());
-                            match chain.process_block(verified_block) {
-                                Ok(_block_root) => {
-                                    metrics::inc_counter(
-                                        &metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL,
-                                    );
-
-                                    trace!(
-                                        log,
-                                        "Gossipsub block processed";
-                                        "peer_id" => peer_id.to_string()
-                                    );
-
-                                    // TODO: It would be better if we can run this _after_ we publish the block to
-                                    // reduce block propagation latency.
-                                    //
-                                    // The `MessageHandler` would be the place to put this, however it doesn't seem
-                                    // to have a reference to the `BeaconChain`. I will leave this for future
-                                    // works.
-                                    match chain.fork_choice() {
-                                        Ok(()) => trace!(
-                                            log,
-                                            "Fork choice success";
-                                            "location" => "block gossip"
-                                        ),
-                                        Err(e) => error!(
-                                            log,
-                                            "Fork choice failed";
-                                            "error" => format!("{:?}", e),
-                                            "location" => "block gossip"
-                                        ),
-                                    }
-                                }
-                                Err(BlockError::ParentUnknown { .. }) => {
-                                    // Inform the sync manager to find parents for this block
-                                    // This should not occur. It should be checked by `should_forward_block`
-                                    error!(
-                                        log,
-                                        "Block with unknown parent attempted to be processed";
-                                        "peer_id" => peer_id.to_string()
-                                    );
-                                    send_sync_message(
-                                        sync_tx,
-                                        SyncMessage::UnknownBlock(peer_id, block),
-                                        &log,
-                                    );
-                                }
-                                other => {
-                                    debug!(
-                                        log,
-                                        "Invalid gossip beacon block";
-                                        "outcome" => format!("{:?}", other),
-                                        "block root" => format!("{}", block.canonical_root()),
-                                        "block slot" => block.slot()
-                                    );
-                                    trace!(
-                                        log,
-                                        "Invalid gossip beacon block ssz";
-                                        "ssz" => format!("0x{}", hex::encode(block.as_ssz_bytes())),
-                                    );
-                                }
-                            };
-                        }
-                        /*
-                         * Verification for beacon blocks received during syncing via RPC.
-                         */
-                        Work::RpcBlock { block, result_tx } => {
-                            let block_result = chain.process_block(*block);
-
-                            metrics::inc_counter(
-                                &metrics::BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL,
-                            );
-
-                            if result_tx.send(block_result).is_err() {
-                                crit!(log, "Failed return sync block result");
-                            }
-                        }
-                        /*
-                         * Verification for a chain segment (multiple blocks).
-                         */
-                        Work::ChainSegment { process_id, blocks } => {
-                            handle_chain_segment(chain, process_id, blocks, sync_tx, log)
-                        }
-                    };
+                match work {
+                    /*
+                     * Unaggregated attestation verification.
+                     */
+                    Work::GossipAttestation {
+                        message_id,
+                        peer_id,
+                        attestation,
+                        subnet_id,
+                        should_import,
+                    } => worker.process_gossip_attestation(
+                        message_id,
+                        peer_id,
+                        *attestation,
+                        subnet_id,
+                        should_import,
+                    ),
+                    /*
+                     * Aggregated attestation verification.
+                     */
+                    Work::GossipAggregate {
+                        message_id,
+                        peer_id,
+                        aggregate,
+                    } => worker.process_gossip_aggregate(message_id, peer_id, *aggregate),
+                    /*
+                     * Verification for beacon blocks received on gossip.
+                     */
+                    Work::GossipBlock {
+                        message_id,
+                        peer_id,
+                        block,
+                    } => worker.process_gossip_block(message_id, peer_id, *block),
+                    /*
+                     * Voluntary exits received on gossip.
+                     */
+                    Work::GossipVoluntaryExit {
+                        message_id,
+                        peer_id,
+                        voluntary_exit,
+                    } => worker.process_gossip_voluntary_exit(message_id, peer_id, *voluntary_exit),
+                    /*
+                     * Proposer slashings received on gossip.
+                     */
+                    Work::GossipProposerSlashing {
+                        message_id,
+                        peer_id,
+                        proposer_slashing,
+                    } => worker.process_gossip_proposer_slashing(
+                        message_id,
+                        peer_id,
+                        *proposer_slashing,
+                    ),
+                    /*
+                     * Attester slashings received on gossip.
+                     */
+                    Work::GossipAttesterSlashing {
+                        message_id,
+                        peer_id,
+                        attester_slashing,
+                    } => worker.process_gossip_attester_slashing(
+                        message_id,
+                        peer_id,
+                        *attester_slashing,
+                    ),
+                    /*
+                     * Verification for beacon blocks received during syncing via RPC.
+                     */
+                    Work::RpcBlock { block, result_tx } => {
+                        worker.process_rpc_block(*block, result_tx)
+                    }
+                    /*
+                     * Verification for a chain segment (multiple blocks).
+                     */
+                    Work::ChainSegment { process_id, blocks } => {
+                        worker.process_chain_segment(process_id, blocks)
+                    }
                 };
-                handler();
 
                 trace!(
                     log,
@@ -902,300 +817,3 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
         );
     }
 }
-
-/// Send a message on `message_tx` that the `message_id` sent by `peer_id` should be propagated on
-/// the gossip network.
-///
-/// Creates a log if there is an interal error.
-fn propagate_gossip_message<E: EthSpec>(
-    network_tx: mpsc::UnboundedSender<NetworkMessage<E>>,
-    message_id: MessageId,
-    peer_id: PeerId,
-    log: &Logger,
-) {
-    network_tx
-        .send(NetworkMessage::Validate {
-            propagation_source: peer_id,
-            message_id,
-        })
-        .unwrap_or_else(|_| {
-            warn!(
-                log,
-                "Could not send propagation request to the network service"
-            )
-        });
-}
-
-/// Send a message to `sync_tx`.
-///
-/// Creates a log if there is an interal error.
-fn send_sync_message<E: EthSpec>(
-    sync_tx: mpsc::UnboundedSender<SyncMessage<E>>,
-    message: SyncMessage<E>,
-    log: &Logger,
-) {
-    sync_tx
-        .send(message)
-        .unwrap_or_else(|_| error!(log, "Could not send message to the sync service"));
-}
-
-/// Handle an error whilst verifying an `Attestation` or `SignedAggregateAndProof` from the
-/// network.
-pub fn handle_attestation_verification_failure<E: EthSpec>(
-    log: &Logger,
-    sync_tx: mpsc::UnboundedSender<SyncMessage<E>>,
-    peer_id: PeerId,
-    beacon_block_root: Hash256,
-    attestation_type: &str,
-    error: AttnError,
-) {
-    metrics::register_attestation_error(&error);
-    match &error {
-        AttnError::FutureEpoch { .. }
-        | AttnError::PastEpoch { .. }
-        | AttnError::FutureSlot { .. }
-        | AttnError::PastSlot { .. } => {
-            /*
-             * These errors can be triggered by a mismatch between our slot and the peer.
-             *
-             *
-             * The peer has published an invalid consensus message, _only_ if we trust our own clock.
-             */
-        }
-        AttnError::InvalidSelectionProof { .. } | AttnError::InvalidSignature => {
-            /*
-             * These errors are caused by invalid signatures.
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-        AttnError::EmptyAggregationBitfield => {
-            /*
-             * The aggregate had no signatures and is therefore worthless.
-             *
-             * Whilst we don't gossip this attestation, this act is **not** a clear
-             * violation of the spec nor indication of fault.
-             *
-             * This may change soon. Reference:
-             *
-             * https://github.com/ethereum/eth2.0-specs/pull/1732
-             */
-        }
-        AttnError::AggregatorPubkeyUnknown(_) => {
-            /*
-             * The aggregator index was higher than any known validator index. This is
-             * possible in two cases:
-             *
-             * 1. The attestation is malformed
-             * 2. The attestation attests to a beacon_block_root that we do not know.
-             *
-             * It should be impossible to reach (2) without triggering
-             * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
-             * faulty.
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-        AttnError::AggregatorNotInCommittee { .. } => {
-            /*
-             * The aggregator index was higher than any known validator index. This is
-             * possible in two cases:
-             *
-             * 1. The attestation is malformed
-             * 2. The attestation attests to a beacon_block_root that we do not know.
-             *
-             * It should be impossible to reach (2) without triggering
-             * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
-             * faulty.
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-        AttnError::AttestationAlreadyKnown { .. } => {
-            /*
-             * The aggregate attestation has already been observed on the network or in
-             * a block.
-             *
-             * The peer is not necessarily faulty.
-             */
-            trace!(
-                log,
-                "Attestation already known";
-                "peer_id" => peer_id.to_string(),
-                "block" => format!("{}", beacon_block_root),
-                "type" => format!("{:?}", attestation_type),
-            );
-            return;
-        }
-        AttnError::AggregatorAlreadyKnown(_) => {
-            /*
-             * There has already been an aggregate attestation seen from this
-             * aggregator index.
-             *
-             * The peer is not necessarily faulty.
-             */
-            trace!(
-                log,
-                "Aggregator already known";
-                "peer_id" => peer_id.to_string(),
-                "block" => format!("{}", beacon_block_root),
-                "type" => format!("{:?}", attestation_type),
-            );
-            return;
-        }
-        AttnError::PriorAttestationKnown { .. } => {
-            /*
-             * We have already seen an attestation from this validator for this epoch.
-             *
-             * The peer is not necessarily faulty.
-             */
-            trace!(
-                log,
-                "Prior attestation known";
-                "peer_id" => peer_id.to_string(),
-                "block" => format!("{}", beacon_block_root),
-                "type" => format!("{:?}", attestation_type),
-            );
-            return;
-        }
-        AttnError::ValidatorIndexTooHigh(_) => {
-            /*
-             * The aggregator index (or similar field) was higher than the maximum
-             * possible number of validators.
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-        AttnError::UnknownHeadBlock { beacon_block_root } => {
-            // Note: its a little bit unclear as to whether or not this block is unknown or
-            // just old. See:
-            //
-            // https://github.com/sigp/lighthouse/issues/1039
-
-            // TODO: Maintain this attestation and re-process once sync completes
-            debug!(
-                log,
-                "Attestation for unknown block";
-                "peer_id" => peer_id.to_string(),
-                "block" => format!("{}", beacon_block_root)
-            );
-            // we don't know the block, get the sync manager to handle the block lookup
-            sync_tx
-                .send(SyncMessage::UnknownBlockHash(peer_id, *beacon_block_root))
-                .unwrap_or_else(|_| {
-                    warn!(
-                        log,
-                        "Failed to send to sync service";
-                        "msg" => "UnknownBlockHash"
-                    )
-                });
-            return;
-        }
-        AttnError::UnknownTargetRoot(_) => {
-            /*
-             * The block indicated by the target root is not known to us.
-             *
-             * We should always get `AttnError::UnknwonHeadBlock` before we get this
-             * error, so this means we can get this error if:
-             *
-             * 1. The target root does not represent a valid block.
-             * 2. We do not have the target root in our DB.
-             *
-             * For (2), we should only be processing attestations when we should have
-             * all the available information. Note: if we do a weak-subjectivity sync
-             * it's possible that this situation could occur, but I think it's
-             * unlikely. For now, we will declare this to be an invalid message>
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-        AttnError::BadTargetEpoch => {
-            /*
-             * The aggregator index (or similar field) was higher than the maximum
-             * possible number of validators.
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-        AttnError::NoCommitteeForSlotAndIndex { .. } => {
-            /*
-             * It is not possible to attest this the given committee in the given slot.
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-        AttnError::NotExactlyOneAggregationBitSet(_) => {
-            /*
-             * The unaggregated attestation doesn't have only one signature.
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-        AttnError::AttestsToFutureBlock { .. } => {
-            /*
-             * The beacon_block_root is from a higher slot than the attestation.
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-
-        AttnError::InvalidSubnetId { received, expected } => {
-            /*
-             * The attestation was received on an incorrect subnet id.
-             */
-            debug!(
-                log,
-                "Received attestation on incorrect subnet";
-                "expected" => format!("{:?}", expected),
-                "received" => format!("{:?}", received),
-            )
-        }
-        AttnError::Invalid(_) => {
-            /*
-             * The attestation failed the state_processing verification.
-             *
-             * The peer has published an invalid consensus message.
-             */
-        }
-        AttnError::TooManySkippedSlots {
-            head_block_slot,
-            attestation_slot,
-        } => {
-            /*
-             * The attestation references a head block that is too far behind the attestation slot.
-             *
-             * The message is not necessarily invalid, but we choose to ignore it.
-             */
-            debug!(
-                log,
-                "Rejected long skip slot attestation";
-                "head_block_slot" => head_block_slot,
-                "attestation_slot" => attestation_slot,
-            )
-        }
-        AttnError::BeaconChainError(e) => {
-            /*
-             * Lighthouse hit an unexpected error whilst processing the attestation. It
-             * should be impossible to trigger a `BeaconChainError` from the network,
-             * so we have a bug.
-             *
-             * It's not clear if the message is invalid/malicious.
-             */
-            error!(
-                log,
-                "Unable to validate aggregate";
-                "peer_id" => peer_id.to_string(),
-                "error" => format!("{:?}", e),
-            );
-        }
-    }
-
-    debug!(
-        log,
-        "Invalid attestation from network";
-        "reason" => format!("{:?}", error),
-        "block" => format!("{}", beacon_block_root),
-        "peer_id" => peer_id.to_string(),
-        "type" => format!("{:?}", attestation_type),
-    );
-}

beacon_node/network/src/beacon_processor/worker.rs

@@ -0,0 +1,906 @@
+use super::{
+    chain_segment::{handle_chain_segment, ProcessId},
+    BlockResultSender,
+};
+use crate::{metrics, service::NetworkMessage, sync::SyncMessage};
+use beacon_chain::{
+    attestation_verification::Error as AttnError, observed_operations::ObservationOutcome,
+    BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError,
+};
+use eth2_libp2p::{MessageAcceptance, MessageId, PeerAction, PeerId};
+use slog::{crit, debug, error, info, trace, warn, Logger};
+use ssz::Encode;
+use std::sync::Arc;
+use tokio::sync::mpsc;
+use types::{
+    Attestation, AttesterSlashing, Hash256, ProposerSlashing, SignedAggregateAndProof,
+    SignedBeaconBlock, SignedVoluntaryExit, SubnetId,
+};
+
+/// Contains the context necessary to import blocks, attestations, etc to the beacon chain.
+pub struct Worker<T: BeaconChainTypes> {
+    pub chain: Arc<BeaconChain<T>>,
+    pub network_tx: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
+    pub sync_tx: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    pub log: Logger,
+}
+
+impl<T: BeaconChainTypes> Worker<T> {
+    /// Process the unaggregated attestation received from the gossip network and:
+    ///
+    /// - If it passes gossip propagation criteria, tell the network thread to forward it.
+    /// - Attempt to apply it to fork choice.
+    /// - Attempt to add it to the naive aggregation pool.
+    ///
+    /// Raises a log if there are errors.
+    pub fn process_gossip_attestation(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attestation: Attestation<T::EthSpec>,
+        subnet_id: SubnetId,
+        should_import: bool,
+    ) {
+        let beacon_block_root = attestation.data.beacon_block_root;
+
+        let attestation = match self
+            .chain
+            .verify_unaggregated_attestation_for_gossip(attestation, subnet_id)
+        {
+            Ok(attestation) => attestation,
+            Err(e) => {
+                self.handle_attestation_verification_failure(
+                    peer_id,
+                    message_id,
+                    beacon_block_root,
+                    "unaggregated",
+                    e,
+                );
+                return;
+            }
+        };
+
+        // Indicate to the `Network` service that this message is valid and can be
+        // propagated on the gossip network.
+        self.propagate_validation_result(message_id, peer_id.clone(), MessageAcceptance::Accept);
+
+        if !should_import {
+            return;
+        }
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL);
+
+        if let Err(e) = self.chain.apply_attestation_to_fork_choice(&attestation) {
+            match e {
+                BeaconChainError::ForkChoiceError(ForkChoiceError::InvalidAttestation(e)) => {
+                    debug!(
+                        self.log,
+                        "Attestation invalid for fork choice";
+                        "reason" => format!("{:?}", e),
+                        "peer" => peer_id.to_string(),
+                        "beacon_block_root" => format!("{:?}", beacon_block_root)
+                    )
+                }
+                e => error!(
+                    self.log,
+                    "Error applying attestation to fork choice";
+                    "reason" => format!("{:?}", e),
+                    "peer" => peer_id.to_string(),
+                    "beacon_block_root" => format!("{:?}", beacon_block_root)
+                ),
+            }
+        }
+
+        if let Err(e) = self.chain.add_to_naive_aggregation_pool(attestation) {
+            debug!(
+                self.log,
+                "Attestation invalid for agg pool";
+                "reason" => format!("{:?}", e),
+                "peer" => peer_id.to_string(),
+                "beacon_block_root" => format!("{:?}", beacon_block_root)
+            )
+        }
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_IMPORTED_TOTAL);
+    }
+
+    /// Process the aggregated attestation received from the gossip network and:
+    ///
+    /// - If it passes gossip propagation criteria, tell the network thread to forward it.
+    /// - Attempt to apply it to fork choice.
+    /// - Attempt to add it to the block inclusion pool.
+    ///
+    /// Raises a log if there are errors.
+    pub fn process_gossip_aggregate(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        aggregate: SignedAggregateAndProof<T::EthSpec>,
+    ) {
+        let beacon_block_root = aggregate.message.aggregate.data.beacon_block_root;
+
+        let aggregate = match self
+            .chain
+            .verify_aggregated_attestation_for_gossip(aggregate)
+        {
+            Ok(aggregate) => aggregate,
+            Err(e) => {
+                // Report the failure to gossipsub
+                self.handle_attestation_verification_failure(
+                    peer_id,
+                    message_id,
+                    beacon_block_root,
+                    "aggregated",
+                    e,
+                );
+                return;
+            }
+        };
+
+        // Indicate to the `Network` service that this message is valid and can be
+        // propagated on the gossip network.
+        self.propagate_validation_result(message_id, peer_id.clone(), MessageAcceptance::Accept);
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL);
+
+        if let Err(e) = self.chain.apply_attestation_to_fork_choice(&aggregate) {
+            match e {
+                BeaconChainError::ForkChoiceError(ForkChoiceError::InvalidAttestation(e)) => {
+                    debug!(
+                        self.log,
+                        "Aggregate invalid for fork choice";
+                        "reason" => format!("{:?}", e),
+                        "peer" => peer_id.to_string(),
+                        "beacon_block_root" => format!("{:?}", beacon_block_root)
+                    )
+                }
+                e => error!(
+                    self.log,
+                    "Error applying aggregate to fork choice";
+                    "reason" => format!("{:?}", e),
+                    "peer" => peer_id.to_string(),
+                    "beacon_block_root" => format!("{:?}", beacon_block_root)
+                ),
+            }
+        }
+
+        if let Err(e) = self.chain.add_to_block_inclusion_pool(aggregate) {
+            debug!(
+                self.log,
+                "Attestation invalid for op pool";
+                "reason" => format!("{:?}", e),
+                "peer" => peer_id.to_string(),
+                "beacon_block_root" => format!("{:?}", beacon_block_root)
+            )
+        }
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_IMPORTED_TOTAL);
+    }
+
+    /// Process the beacon block received from the gossip network and:
+    ///
+    /// - If it passes gossip propagation criteria, tell the network thread to forward it.
+    /// - Attempt to add it to the beacon chain, informing the sync thread if more blocks need to
+    ///   be downloaded.
+    ///
+    /// Raises a log if there are errors.
+    pub fn process_gossip_block(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        block: SignedBeaconBlock<T::EthSpec>,
+    ) {
+        let verified_block = match self.chain.verify_block_for_gossip(block) {
+            Ok(verified_block) => {
+                info!(
+                    self.log,
+                    "New block received";
+                    "slot" => verified_block.block.slot(),
+                    "hash" => verified_block.block_root.to_string()
+                );
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Accept,
+                );
+                verified_block
+            }
+            Err(BlockError::ParentUnknown(block)) => {
+                self.send_sync_message(SyncMessage::UnknownBlock(peer_id, block));
+                return;
+            }
+            Err(e @ BlockError::FutureSlot { .. })
+            | Err(e @ BlockError::WouldRevertFinalizedSlot { .. })
+            | Err(e @ BlockError::BlockIsAlreadyKnown)
+            | Err(e @ BlockError::RepeatProposal { .. })
+            | Err(e @ BlockError::NotFinalizedDescendant { .. })
+            | Err(e @ BlockError::BeaconChainError(_)) => {
+                warn!(self.log, "Could not verify block for gossip, ignoring the block";
+                            "error" => e.to_string());
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            Err(e @ BlockError::StateRootMismatch { .. })
+            | Err(e @ BlockError::IncorrectBlockProposer { .. })
+            | Err(e @ BlockError::BlockSlotLimitReached)
+            | Err(e @ BlockError::ProposalSignatureInvalid)
+            | Err(e @ BlockError::NonLinearSlots)
+            | Err(e @ BlockError::UnknownValidator(_))
+            | Err(e @ BlockError::PerBlockProcessingError(_))
+            | Err(e @ BlockError::NonLinearParentRoots)
+            | Err(e @ BlockError::BlockIsNotLaterThanParent { .. })
+            | Err(e @ BlockError::InvalidSignature)
+            | Err(e @ BlockError::TooManySkippedSlots { .. })
+            | Err(e @ BlockError::GenesisBlock) => {
+                warn!(self.log, "Could not verify block for gossip, rejecting the block";
+                            "error" => e.to_string());
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id, PeerAction::LowToleranceError);
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL);
+
+        let block = Box::new(verified_block.block.clone());
+        match self.chain.process_block(verified_block) {
+            Ok(_block_root) => {
+                metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL);
+
+                trace!(
+                    self.log,
+                    "Gossipsub block processed";
+                    "peer_id" => peer_id.to_string()
+                );
+
+                // The `MessageHandler` would be the place to put this, however it doesn't seem
+                // to have a reference to the `BeaconChain`. I will leave this for future
+                // works.
+                match self.chain.fork_choice() {
+                    Ok(()) => trace!(
+                        self.log,
+                        "Fork choice success";
+                        "location" => "block gossip"
+                    ),
+                    Err(e) => error!(
+                        self.log,
+                        "Fork choice failed";
+                        "error" => format!("{:?}", e),
+                        "location" => "block gossip"
+                    ),
+                }
+            }
+            Err(BlockError::ParentUnknown { .. }) => {
+                // Inform the sync manager to find parents for this block
+                // This should not occur. It should be checked by `should_forward_block`
+                error!(
+                    self.log,
+                    "Block with unknown parent attempted to be processed";
+                    "peer_id" => peer_id.to_string()
+                );
+                self.send_sync_message(SyncMessage::UnknownBlock(peer_id, block));
+            }
+            other => {
+                debug!(
+                    self.log,
+                    "Invalid gossip beacon block";
+                    "outcome" => format!("{:?}", other),
+                    "block root" => format!("{}", block.canonical_root()),
+                    "block slot" => block.slot()
+                );
+                self.penalize_peer(peer_id, PeerAction::MidToleranceError);
+                trace!(
+                    self.log,
+                    "Invalid gossip beacon block ssz";
+                    "ssz" => format!("0x{}", hex::encode(block.as_ssz_bytes())),
+                );
+            }
+        };
+    }
+
+    pub fn process_gossip_voluntary_exit(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: SignedVoluntaryExit,
+    ) {
+        let validator_index = voluntary_exit.message.validator_index;
+
+        let exit = match self.chain.verify_voluntary_exit_for_gossip(voluntary_exit) {
+            Ok(ObservationOutcome::New(exit)) => exit,
+            Ok(ObservationOutcome::AlreadyKnown) => {
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Ignore,
+                );
+                debug!(
+                    self.log,
+                    "Dropping exit for already exiting validator";
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string()
+                );
+                return;
+            }
+            Err(e) => {
+                debug!(
+                    self.log,
+                    "Dropping invalid exit";
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string(),
+                    "error" => format!("{:?}", e)
+                );
+                // These errors occur due to a fault in the beacon chain. It is not necessarily
+                // the fault on the peer.
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Ignore,
+                );
+                // We still penalize a peer slightly to prevent overuse of invalids.
+                self.penalize_peer(peer_id, PeerAction::HighToleranceError);
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL);
+
+        self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept);
+
+        self.chain.import_voluntary_exit(exit);
+        debug!(self.log, "Successfully imported voluntary exit");
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_EXIT_IMPORTED_TOTAL);
+    }
+
+    pub fn process_gossip_proposer_slashing(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: ProposerSlashing,
+    ) {
+        let validator_index = proposer_slashing.signed_header_1.message.proposer_index;
+
+        let slashing = match self
+            .chain
+            .verify_proposer_slashing_for_gossip(proposer_slashing)
+        {
+            Ok(ObservationOutcome::New(slashing)) => slashing,
+            Ok(ObservationOutcome::AlreadyKnown) => {
+                debug!(
+                    self.log,
+                    "Dropping proposer slashing";
+                    "reason" => "Already seen a proposer slashing for that validator",
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string()
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            Err(e) => {
+                // This is likely a fault with the beacon chain and not necessarily a
+                // malicious message from the peer.
+                debug!(
+                    self.log,
+                    "Dropping invalid proposer slashing";
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string(),
+                    "error" => format!("{:?}", e)
+                );
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Ignore,
+                );
+
+                // Penalize peer slightly for invalids.
+                self.penalize_peer(peer_id, PeerAction::HighToleranceError);
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL);
+
+        self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept);
+
+        self.chain.import_proposer_slashing(slashing);
+        debug!(self.log, "Successfully imported proposer slashing");
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_IMPORTED_TOTAL);
+    }
+
+    pub fn process_gossip_attester_slashing(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: AttesterSlashing<T::EthSpec>,
+    ) {
+        let slashing = match self
+            .chain
+            .verify_attester_slashing_for_gossip(attester_slashing)
+        {
+            Ok(ObservationOutcome::New(slashing)) => slashing,
+            Ok(ObservationOutcome::AlreadyKnown) => {
+                debug!(
+                    self.log,
+                    "Dropping attester slashing";
+                    "reason" => "Slashings already known for all slashed validators",
+                    "peer" => peer_id.to_string()
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            Err(e) => {
+                debug!(
+                    self.log,
+                    "Dropping invalid attester slashing";
+                    "peer" => peer_id.to_string(),
+                    "error" => format!("{:?}", e)
+                );
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Ignore,
+                );
+                // Penalize peer slightly for invalids.
+                self.penalize_peer(peer_id, PeerAction::HighToleranceError);
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL);
+
+        self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept);
+
+        if let Err(e) = self.chain.import_attester_slashing(slashing) {
+            debug!(self.log, "Error importing attester slashing"; "error" => format!("{:?}", e));
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_ERROR_TOTAL);
+        } else {
+            debug!(self.log, "Successfully imported attester slashing");
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_IMPORTED_TOTAL);
+        }
+    }
+
+    /// Attempt to process a block received from a direct RPC request, returning the processing
+    /// result on the `result_tx` channel.
+    ///
+    /// Raises a log if there are errors publishing the result to the channel.
+    pub fn process_rpc_block(
+        self,
+        block: SignedBeaconBlock<T::EthSpec>,
+        result_tx: BlockResultSender<T::EthSpec>,
+    ) {
+        let block_result = self.chain.process_block(block);
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL);
+
+        if result_tx.send(block_result).is_err() {
+            crit!(self.log, "Failed return sync block result");
+        }
+    }
+
+    /// Attempt to import the chain segment (`blocks`) to the beacon chain, informing the sync
+    /// thread if more blocks are needed to process it.
+    pub fn process_chain_segment(
+        self,
+        process_id: ProcessId,
+        blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
+    ) {
+        handle_chain_segment(self.chain, process_id, blocks, self.sync_tx, self.log)
+    }
+
+    /// Send a message on `message_tx` that the `message_id` sent by `peer_id` should be propagated on
+    /// the gossip network.
+    ///
+    /// Creates a log if there is an interal error.
+    /// Propagates the result of the validation fot the given message to the network. If the result
+    /// is valid the message gets forwarded to other peers.
+    fn propagate_validation_result(
+        &self,
+        message_id: MessageId,
+        propagation_source: PeerId,
+        validation_result: MessageAcceptance,
+    ) {
+        self.network_tx
+            .send(NetworkMessage::ValidationResult {
+                propagation_source,
+                message_id,
+                validation_result,
+            })
+            .unwrap_or_else(|_| {
+                warn!(
+                    self.log,
+                    "Could not send propagation request to the network service"
+                )
+            });
+    }
+
+    /// Penalizes a peer for misbehaviour.
+    fn penalize_peer(&self, peer_id: PeerId, action: PeerAction) {
+        self.network_tx
+            .send(NetworkMessage::ReportPeer { peer_id, action })
+            .unwrap_or_else(|_| {
+                warn!(
+                    self.log,
+                    "Could not send peer action to the network service"
+                )
+            });
+    }
+
+    /// Send a message to `sync_tx`.
+    ///
+    /// Creates a log if there is an interal error.
+    fn send_sync_message(&self, message: SyncMessage<T::EthSpec>) {
+        self.sync_tx
+            .send(message)
+            .unwrap_or_else(|_| error!(self.log, "Could not send message to the sync service"));
+    }
+
+    /// Handle an error whilst verifying an `Attestation` or `SignedAggregateAndProof` from the
+    /// network.
+    pub fn handle_attestation_verification_failure(
+        &self,
+        peer_id: PeerId,
+        message_id: MessageId,
+        beacon_block_root: Hash256,
+        attestation_type: &str,
+        error: AttnError,
+    ) {
+        metrics::register_attestation_error(&error);
+        match &error {
+            AttnError::FutureEpoch { .. }
+            | AttnError::PastEpoch { .. }
+            | AttnError::FutureSlot { .. }
+            | AttnError::PastSlot { .. } => {
+                /*
+                 * These errors can be triggered by a mismatch between our slot and the peer.
+                 *
+                 *
+                 * The peer has published an invalid consensus message, _only_ if we trust our own clock.
+                 */
+                trace!(
+                    self.log,
+                    "Attestation is not within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Ignore,
+                );
+            }
+            AttnError::InvalidSelectionProof { .. } | AttnError::InvalidSignature => {
+                /*
+                 * These errors are caused by invalid signatures.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::EmptyAggregationBitfield => {
+                /*
+                 * The aggregate had no signatures and is therefore worthless.
+                 *
+                 * Whilst we don't gossip this attestation, this act is **not** a clear
+                 * violation of the spec nor indication of fault.
+                 *
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::AggregatorPubkeyUnknown(_) => {
+                /*
+                 * The aggregator index was higher than any known validator index. This is
+                 * possible in two cases:
+                 *
+                 * 1. The attestation is malformed
+                 * 2. The attestation attests to a beacon_block_root that we do not know.
+                 *
+                 * It should be impossible to reach (2) without triggering
+                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
+                 * faulty.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::AggregatorNotInCommittee { .. } => {
+                /*
+                 * The aggregator index was higher than any known validator index. This is
+                 * possible in two cases:
+                 *
+                 * 1. The attestation is malformed
+                 * 2. The attestation attests to a beacon_block_root that we do not know.
+                 *
+                 * It should be impossible to reach (2) without triggering
+                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
+                 * faulty.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::AttestationAlreadyKnown { .. } => {
+                /*
+                 * The aggregate attestation has already been observed on the network or in
+                 * a block.
+                 *
+                 * The peer is not necessarily faulty.
+                 */
+                trace!(
+                    self.log,
+                    "Attestation already known";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            AttnError::AggregatorAlreadyKnown(_) => {
+                /*
+                 * There has already been an aggregate attestation seen from this
+                 * aggregator index.
+                 *
+                 * The peer is not necessarily faulty.
+                 */
+                trace!(
+                    self.log,
+                    "Aggregator already known";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            AttnError::PriorAttestationKnown { .. } => {
+                /*
+                 * We have already seen an attestation from this validator for this epoch.
+                 *
+                 * The peer is not necessarily faulty.
+                 */
+                trace!(
+                    self.log,
+                    "Prior attestation known";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            AttnError::ValidatorIndexTooHigh(_) => {
+                /*
+                 * The aggregator index (or similar field) was higher than the maximum
+                 * possible number of validators.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::UnknownHeadBlock { beacon_block_root } => {
+                // Note: its a little bit unclear as to whether or not this block is unknown or
+                // just old. See:
+                //
+                // https://github.com/sigp/lighthouse/issues/1039
+
+                // TODO: Maintain this attestation and re-process once sync completes
+                debug!(
+                    self.log,
+                    "Attestation for unknown block";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root)
+                );
+                // we don't know the block, get the sync manager to handle the block lookup
+                self.sync_tx
+                    .send(SyncMessage::UnknownBlockHash(
+                        peer_id.clone(),
+                        *beacon_block_root,
+                    ))
+                    .unwrap_or_else(|_| {
+                        warn!(
+                            self.log,
+                            "Failed to send to sync service";
+                            "msg" => "UnknownBlockHash"
+                        )
+                    });
+                self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+                return;
+            }
+            AttnError::UnknownTargetRoot(_) => {
+                /*
+                 * The block indicated by the target root is not known to us.
+                 *
+                 * We should always get `AttnError::UnknwonHeadBlock` before we get this
+                 * error, so this means we can get this error if:
+                 *
+                 * 1. The target root does not represent a valid block.
+                 * 2. We do not have the target root in our DB.
+                 *
+                 * For (2), we should only be processing attestations when we should have
+                 * all the available information. Note: if we do a weak-subjectivity sync
+                 * it's possible that this situation could occur, but I think it's
+                 * unlikely. For now, we will declare this to be an invalid message>
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::BadTargetEpoch => {
+                /*
+                 * The aggregator index (or similar field) was higher than the maximum
+                 * possible number of validators.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::NoCommitteeForSlotAndIndex { .. } => {
+                /*
+                 * It is not possible to attest this the given committee in the given slot.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::NotExactlyOneAggregationBitSet(_) => {
+                /*
+                 * The unaggregated attestation doesn't have only one signature.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::AttestsToFutureBlock { .. } => {
+                /*
+                 * The beacon_block_root is from a higher slot than the attestation.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+
+            AttnError::InvalidSubnetId { received, expected } => {
+                /*
+                 * The attestation was received on an incorrect subnet id.
+                 */
+                debug!(
+                    self.log,
+                    "Received attestation on incorrect subnet";
+                    "expected" => format!("{:?}", expected),
+                    "received" => format!("{:?}", received),
+                );
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::Invalid(_) => {
+                /*
+                 * The attestation failed the state_processing verification.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+            }
+            AttnError::TooManySkippedSlots {
+                head_block_slot,
+                attestation_slot,
+            } => {
+                /*
+                 * The attestation references a head block that is too far behind the attestation slot.
+                 *
+                 * The message is not necessarily invalid, but we choose to ignore it.
+                 */
+                debug!(
+                    self.log,
+                    "Rejected long skip slot attestation";
+                    "head_block_slot" => head_block_slot,
+                    "attestation_slot" => attestation_slot,
+                );
+                // In this case we wish to penalize gossipsub peers that do this to avoid future
+                // attestations that have too many skip slots.
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Reject,
+                );
+                self.penalize_peer(peer_id.clone(), PeerAction::MidToleranceError);
+            }
+            AttnError::BeaconChainError(e) => {
+                /*
+                 * Lighthouse hit an unexpected error whilst processing the attestation. It
+                 * should be impossible to trigger a `BeaconChainError` from the network,
+                 * so we have a bug.
+                 *
+                 * It's not clear if the message is invalid/malicious.
+                 */
+                error!(
+                    self.log,
+                    "Unable to validate aggregate";
+                    "peer_id" => peer_id.to_string(),
+                    "error" => format!("{:?}", e),
+                );
+                self.propagate_validation_result(
+                    message_id,
+                    peer_id.clone(),
+                    MessageAcceptance::Ignore,
+                );
+                // Penalize the peer slightly
+                self.penalize_peer(peer_id.clone(), PeerAction::HighToleranceError);
+            }
+        }
+
+        debug!(
+            self.log,
+            "Invalid attestation from network";
+            "reason" => format!("{:?}", error),
+            "block" => format!("{}", beacon_block_root),
+            "peer_id" => peer_id.to_string(),
+            "type" => format!("{:?}", attestation_type),
+        );
+    }
+}

beacon_node/network/src/metrics.rs

@@ -1,36 +1,91 @@
 use beacon_chain::attestation_verification::Error as AttnError;
 pub use lighthouse_metrics::*;
 
+lazy_static! {
+
+    /*
+     * Gossip subnets and scoring
+     */
+    pub static ref PEERS_PER_PROTOCOL: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_peers_per_protocol",
+        "Peers via supported protocol",
+        &["protocol"]
+    );
+
+    pub static ref GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_subscribed_subnets",
+        "Subnets currently subscribed to",
+        &["subnet"]
+    );
+
+    pub static ref GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_peers_per_subnet_topic_count",
+        "Peers subscribed per subnet topic",
+        &["subnet"]
+    );
+
+    pub static ref MESH_PEERS_PER_MAIN_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_mesh_peers_per_main_topic",
+        "Mesh peers per main topic",
+        &["topic_hash"]
+    );
+
+    pub static ref MESH_PEERS_PER_SUBNET_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_mesh_peers_per_subnet_topic",
+        "Mesh peers per subnet topic",
+        &["subnet"]
+    );
+
+    pub static ref AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_avg_peer_score_per_topic",
+        "Average peer's score per topic",
+        &["topic_hash"]
+    );
+
+    pub static ref AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_avg_peer_score_per_subnet_topic",
+        "Average peer's score per subnet topic",
+        &["subnet"]
+    );
+
+    pub static ref ATTESTATIONS_PUBLISHED_PER_SUBNET_PER_SLOT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "gossipsub_attestations_published_per_subnet_per_slot",
+        "Failed attestation publishes per subnet",
+        &["subnet"]
+    );
+}
+
 lazy_static! {
     /*
      * Gossip Rx
      */
     pub static ref GOSSIP_BLOCKS_RX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_blocks_rx_total",
+        "gossipsub_blocks_rx_total",
         "Count of gossip blocks received"
     );
     pub static ref GOSSIP_UNAGGREGATED_ATTESTATIONS_RX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_unaggregated_attestations_rx_total",
+        "gossipsub_unaggregated_attestations_rx_total",
         "Count of gossip unaggregated attestations received"
     );
     pub static ref GOSSIP_AGGREGATED_ATTESTATIONS_RX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_aggregated_attestations_rx_total",
+        "gossipsub_aggregated_attestations_rx_total",
         "Count of gossip aggregated attestations received"
     );
 
+
     /*
      * Gossip Tx
      */
     pub static ref GOSSIP_BLOCKS_TX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_blocks_tx_total",
+        "gossipsub_blocks_tx_total",
         "Count of gossip blocks transmitted"
     );
     pub static ref GOSSIP_UNAGGREGATED_ATTESTATIONS_TX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_unaggregated_attestations_tx_total",
+        "gossipsub_unaggregated_attestations_tx_total",
         "Count of gossip unaggregated attestations transmitted"
     );
     pub static ref GOSSIP_AGGREGATED_ATTESTATIONS_TX: Result<IntCounter> = try_create_int_counter(
-        "network_gossip_aggregated_attestations_tx_total",
+        "gossipsub_aggregated_attestations_tx_total",
         "Count of gossip aggregated attestations transmitted"
     );
 
@@ -38,11 +93,11 @@ lazy_static! {
      * Attestation subnet subscriptions
      */
     pub static ref SUBNET_SUBSCRIPTION_REQUESTS: Result<IntCounter> = try_create_int_counter(
-        "network_subnet_subscriptions_total",
+        "gossipsub_subnet_subscriptions_total",
         "Count of validator subscription requests."
     );
     pub static ref SUBNET_SUBSCRIPTION_AGGREGATOR_REQUESTS: Result<IntCounter> = try_create_int_counter(
-        "network_subnet_subscriptions_aggregator_total",
+        "gossipsub_subnet_subscriptions_aggregator_total",
         "Count of validator subscription requests where the subscriber is an aggregator."
     );
 
@@ -98,6 +153,49 @@ lazy_static! {
         "beacon_processor_gossip_block_imported_total",
         "Total number of gossip blocks imported to fork choice, etc."
     );
+    // Gossip Exits.
+    pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_exit_queue_total",
+        "Count of exits from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_exit_verified_total",
+        "Total number of voluntary exits verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_EXIT_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_exit_imported_total",
+        "Total number of voluntary exits imported to the op pool."
+    );
+    // Gossip proposer slashings.
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_proposer_slashing_queue_total",
+        "Count of proposer slashings from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_proposer_slashing_verified_total",
+        "Total number of proposer slashings verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_proposer_slashing_imported_total",
+        "Total number of proposer slashings imported to the op pool."
+    );
+    // Gossip attester slashings.
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_attester_slashing_queue_total",
+        "Count of attester slashings from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_attester_slashing_verified_total",
+        "Total number of attester slashings verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_attester_slashing_imported_total",
+        "Total number of attester slashings imported to the op pool."
+    );
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_ERROR_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_attester_slashing_error_total",
+        "Total number of attester slashings that raised an error during processing."
+    );
     // Rpc blocks.
     pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
         "beacon_processor_rpc_block_queue_total",
@@ -151,95 +249,95 @@ lazy_static! {
      * Attestation Errors
      */
     pub static ref GOSSIP_ATTESTATION_ERROR_FUTURE_EPOCH: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_future_epoch",
+        "gossipsub_attestation_error_future_epoch",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_PAST_EPOCH: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_past_epoch",
+        "gossipsub_attestation_error_past_epoch",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_FUTURE_SLOT: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_future_slot",
+        "gossipsub_attestation_error_future_slot",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_PAST_SLOT: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_past_slot",
+        "gossipsub_attestation_error_past_slot",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_SELECTION_PROOF: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_invalid_selection_proof",
+        "gossipsub_attestation_error_invalid_selection_proof",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_SIGNATURE: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_invalid_signature",
+        "gossipsub_attestation_error_invalid_signature",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_EMPTY_AGGREGATION_BITFIELD: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_empty_aggregation_bitfield",
+        "gossipsub_attestation_error_empty_aggregation_bitfield",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_AGGREGATOR_PUBKEY_UNKNOWN: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_aggregator_pubkey_unknown",
+        "gossipsub_attestation_error_aggregator_pubkey_unknown",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_AGGREGATOR_NOT_IN_COMMITTEE: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_aggregator_not_in_committee",
+        "gossipsub_attestation_error_aggregator_not_in_committee",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_ATTESTATION_ALREADY_KNOWN: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_attestation_already_known",
+        "gossipsub_attestation_error_attestation_already_known",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_AGGREGATOR_ALREADY_KNOWN: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_aggregator_already_known",
+        "gossipsub_attestation_error_aggregator_already_known",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_PRIOR_ATTESTATION_KNOWN: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_prior_attestation_known",
+        "gossipsub_attestation_error_prior_attestation_known",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_VALIDATOR_INDEX_TOO_HIGH: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_validator_index_too_high",
+        "gossipsub_attestation_error_validator_index_too_high",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_UNKNOWN_HEAD_BLOCK: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_unknown_head_block",
+        "gossipsub_attestation_error_unknown_head_block",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_UNKNOWN_TARGET_ROOT: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_unknown_target_root",
+        "gossipsub_attestation_error_unknown_target_root",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_BAD_TARGET_EPOCH: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_bad_target_epoch",
+        "gossipsub_attestation_error_bad_target_epoch",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_NO_COMMITTEE_FOR_SLOT_AND_INDEX: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_no_committee_for_slot_and_index",
+        "gossipsub_attestation_error_no_committee_for_slot_and_index",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_NOT_EXACTLY_ONE_AGGREGATION_BIT_SET: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_not_exactly_one_aggregation_bit_set",
+        "gossipsub_attestation_error_not_exactly_one_aggregation_bit_set",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_ATTESTS_TO_FUTURE_BLOCK: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_attests_to_future_block",
+        "gossipsub_attestation_error_attests_to_future_block",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_SUBNET_ID: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_invalid_subnet_id",
+        "gossipsub_attestation_error_invalid_subnet_id",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_STATE_PROCESSING: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_invalid_state_processing",
+        "gossipsub_attestation_error_invalid_state_processing",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_TOO_MANY_SKIPPED_SLOTS: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_invalid_too_many_skipped_slots",
+        "gossipsub_attestation_error_invalid_too_many_skipped_slots",
         "Count of a specific error type (see metric name)"
     );
     pub static ref GOSSIP_ATTESTATION_ERROR_BEACON_CHAIN_ERROR: Result<IntCounter> = try_create_int_counter(
-        "gossip_attestation_error_beacon_chain_error",
+        "gossipsub_attestation_error_beacon_chain_error",
         "Count of a specific error type (see metric name)"
     );
 }

beacon_node/network/src/router/mod.rs

@@ -16,7 +16,7 @@ use eth2_libp2p::{
 };
 use futures::prelude::*;
 use processor::Processor;
-use slog::{debug, o, trace, warn};
+use slog::{debug, o, trace};
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use types::EthSpec;
@@ -26,8 +26,6 @@ use types::EthSpec;
 /// passing them to the internal message processor. The message processor spawns a syncing thread
 /// which manages which blocks need to be requested and processed.
 pub struct Router<T: BeaconChainTypes> {
-    /// A channel to the network service to allow for gossip propagation.
-    network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
     /// Access to the peer db for logging.
     network_globals: Arc<NetworkGlobals<T::EthSpec>>,
     /// Processes validated and decoded messages from the network. Has direct access to the
@@ -89,13 +87,12 @@ impl<T: BeaconChainTypes> Router<T> {
             executor.clone(),
             beacon_chain,
             network_globals.clone(),
-            network_send.clone(),
+            network_send,
             &log,
         );
 
         // generate the Message handler
         let mut handler = Router {
-            network_send,
             network_globals,
             processor,
             log: message_handler_log,
@@ -232,13 +229,7 @@ impl<T: BeaconChainTypes> Router<T> {
             }
             PubsubMessage::VoluntaryExit(exit) => {
                 debug!(self.log, "Received a voluntary exit"; "peer_id" => format!("{}", peer_id));
-                if let Some(verified_exit) = self
-                    .processor
-                    .verify_voluntary_exit_for_gossip(&peer_id, *exit)
-                {
-                    self.propagate_message(id, peer_id);
-                    self.processor.import_verified_voluntary_exit(verified_exit);
-                }
+                self.processor.on_voluntary_exit_gossip(id, peer_id, exit);
             }
             PubsubMessage::ProposerSlashing(proposer_slashing) => {
                 debug!(
@@ -246,14 +237,8 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received a proposer slashing";
                     "peer_id" => format!("{}", peer_id)
                 );
-                if let Some(verified_proposer_slashing) = self
-                    .processor
-                    .verify_proposer_slashing_for_gossip(&peer_id, *proposer_slashing)
-                {
-                    self.propagate_message(id, peer_id);
-                    self.processor
-                        .import_verified_proposer_slashing(verified_proposer_slashing);
-                }
+                self.processor
+                    .on_proposer_slashing_gossip(id, peer_id, proposer_slashing);
             }
             PubsubMessage::AttesterSlashing(attester_slashing) => {
                 debug!(
@@ -261,30 +246,9 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received a attester slashing";
                     "peer_id" => format!("{}", peer_id)
                 );
-                if let Some(verified_attester_slashing) = self
-                    .processor
-                    .verify_attester_slashing_for_gossip(&peer_id, *attester_slashing)
-                {
-                    self.propagate_message(id, peer_id);
-                    self.processor
-                        .import_verified_attester_slashing(verified_attester_slashing);
-                }
+                self.processor
+                    .on_attester_slashing_gossip(id, peer_id, attester_slashing);
             }
         }
     }
-
-    /// Informs the network service that the message should be forwarded to other peers (is valid).
-    fn propagate_message(&mut self, message_id: MessageId, propagation_source: PeerId) {
-        self.network_send
-            .send(NetworkMessage::Validate {
-                propagation_source,
-                message_id,
-            })
-            .unwrap_or_else(|_| {
-                warn!(
-                    self.log,
-                    "Could not send propagation request to the network service"
-                )
-            });
-    }
 }

beacon_node/network/src/router/processor.rs

@@ -3,14 +3,13 @@ use crate::beacon_processor::{
 };
 use crate::service::NetworkMessage;
 use crate::sync::{PeerSyncInfo, SyncMessage};
-use beacon_chain::{observed_operations::ObservationOutcome, BeaconChain, BeaconChainTypes};
+use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::rpc::*;
 use eth2_libp2p::{
     MessageId, NetworkGlobals, PeerAction, PeerId, PeerRequestId, Request, Response,
 };
 use itertools::process_results;
 use slog::{debug, error, o, trace, warn};
-use state_processing::SigVerifiedOp;
 use std::cmp;
 use std::sync::Arc;
 use tokio::sync::mpsc;
@@ -19,8 +18,6 @@ use types::{
     SignedAggregateAndProof, SignedBeaconBlock, SignedVoluntaryExit, Slot, SubnetId,
 };
 
-//TODO: Rate limit requests
-
 /// If a block is more than `FUTURE_SLOT_TOLERANCE` slots ahead of our slot clock, we drop it.
 /// Otherwise we queue it.
 pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1;
@@ -377,22 +374,29 @@ impl<T: BeaconChainTypes> Processor<T> {
             }
         };
 
-        // pick out the required blocks, ignoring skip-slots and stepping by the step parameter;
+        // Pick out the required blocks, ignoring skip-slots and stepping by the step parameter.
+        //
+        // NOTE: We don't mind if req.count * req.step overflows as it just ends the iterator early and
+        // the peer will get less blocks.
+        // The step parameter is quadratically weighted in the filter, so large values should be
+        // prevented before reaching this point.
         let mut last_block_root = None;
         let maybe_block_roots = process_results(forwards_block_root_iter, |iter| {
-            iter.take_while(|(_, slot)| slot.as_u64() < req.start_slot + req.count * req.step)
-                // map skip slots to None
-                .map(|(root, _)| {
-                    let result = if Some(root) == last_block_root {
-                        None
-                    } else {
-                        Some(root)
-                    };
-                    last_block_root = Some(root);
-                    result
-                })
-                .step_by(req.step as usize)
-                .collect::<Vec<Option<Hash256>>>()
+            iter.take_while(|(_, slot)| {
+                slot.as_u64() < req.start_slot.saturating_add(req.count * req.step)
+            })
+            // map skip slots to None
+            .map(|(root, _)| {
+                let result = if Some(root) == last_block_root {
+                    None
+                } else {
+                    Some(root)
+                };
+                last_block_root = Some(root);
+                result
+            })
+            .step_by(req.step as usize)
+            .collect::<Vec<Option<Hash256>>>()
         });
 
         let block_roots = match maybe_block_roots {
@@ -585,140 +589,70 @@ impl<T: BeaconChainTypes> Processor<T> {
             })
     }
 
-    /// Verify a voluntary exit before gossiping or processing it.
-    ///
-    /// Errors are logged at debug level.
-    pub fn verify_voluntary_exit_for_gossip(
-        &self,
-        peer_id: &PeerId,
-        voluntary_exit: SignedVoluntaryExit,
-    ) -> Option<SigVerifiedOp<SignedVoluntaryExit>> {
-        let validator_index = voluntary_exit.message.validator_index;
-
-        match self.chain.verify_voluntary_exit_for_gossip(voluntary_exit) {
-            Ok(ObservationOutcome::New(sig_verified_exit)) => Some(sig_verified_exit),
-            Ok(ObservationOutcome::AlreadyKnown) => {
-                debug!(
-                    self.log,
-                    "Dropping exit for already exiting validator";
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string()
-                );
-                None
-            }
-            Err(e) => {
-                debug!(
-                    self.log,
-                    "Dropping invalid exit";
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string(),
-                    "error" => format!("{:?}", e)
-                );
-                None
-            }
-        }
-    }
-
-    /// Import a verified exit into the op pool.
-    pub fn import_verified_voluntary_exit(
-        &self,
-        verified_voluntary_exit: SigVerifiedOp<SignedVoluntaryExit>,
+    pub fn on_voluntary_exit_gossip(
+        &mut self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: Box<SignedVoluntaryExit>,
     ) {
-        self.chain.import_voluntary_exit(verified_voluntary_exit);
-        debug!(self.log, "Successfully imported voluntary exit");
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_voluntary_exit(
+                message_id,
+                peer_id,
+                voluntary_exit,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "voluntary exit gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 
-    /// Verify a proposer slashing before gossiping or processing it.
-    ///
-    /// Errors are logged at debug level.
-    pub fn verify_proposer_slashing_for_gossip(
-        &self,
-        peer_id: &PeerId,
-        proposer_slashing: ProposerSlashing,
-    ) -> Option<SigVerifiedOp<ProposerSlashing>> {
-        let validator_index = proposer_slashing.signed_header_1.message.proposer_index;
-
-        match self
-            .chain
-            .verify_proposer_slashing_for_gossip(proposer_slashing)
-        {
-            Ok(ObservationOutcome::New(verified_slashing)) => Some(verified_slashing),
-            Ok(ObservationOutcome::AlreadyKnown) => {
-                debug!(
-                    self.log,
-                    "Dropping proposer slashing";
-                    "reason" => "Already seen a proposer slashing for that validator",
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string()
-                );
-                None
-            }
-            Err(e) => {
-                debug!(
-                    self.log,
-                    "Dropping invalid proposer slashing";
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string(),
-                    "error" => format!("{:?}", e)
-                );
-                None
-            }
-        }
-    }
-
-    /// Import a verified proposer slashing into the op pool.
-    pub fn import_verified_proposer_slashing(
-        &self,
-        proposer_slashing: SigVerifiedOp<ProposerSlashing>,
+    pub fn on_proposer_slashing_gossip(
+        &mut self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
     ) {
-        self.chain.import_proposer_slashing(proposer_slashing);
-        debug!(self.log, "Successfully imported proposer slashing");
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_proposer_slashing(
+                message_id,
+                peer_id,
+                proposer_slashing,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "proposer slashing gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 
-    /// Verify an attester slashing before gossiping or processing it.
-    ///
-    /// Errors are logged at debug level.
-    pub fn verify_attester_slashing_for_gossip(
-        &self,
-        peer_id: &PeerId,
-        attester_slashing: AttesterSlashing<T::EthSpec>,
-    ) -> Option<SigVerifiedOp<AttesterSlashing<T::EthSpec>>> {
-        match self
-            .chain
-            .verify_attester_slashing_for_gossip(attester_slashing)
-        {
-            Ok(ObservationOutcome::New(verified_slashing)) => Some(verified_slashing),
-            Ok(ObservationOutcome::AlreadyKnown) => {
-                debug!(
-                    self.log,
-                    "Dropping attester slashing";
-                    "reason" => "Slashings already known for all slashed validators",
-                    "peer" => peer_id.to_string()
-                );
-                None
-            }
-            Err(e) => {
-                debug!(
-                    self.log,
-                    "Dropping invalid attester slashing";
-                    "peer" => peer_id.to_string(),
-                    "error" => format!("{:?}", e)
-                );
-                None
-            }
-        }
-    }
-
-    /// Import a verified attester slashing into the op pool.
-    pub fn import_verified_attester_slashing(
-        &self,
-        attester_slashing: SigVerifiedOp<AttesterSlashing<T::EthSpec>>,
+    pub fn on_attester_slashing_gossip(
+        &mut self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<T::EthSpec>>,
     ) {
-        if let Err(e) = self.chain.import_attester_slashing(attester_slashing) {
-            debug!(self.log, "Error importing attester slashing"; "error" => format!("{:?}", e));
-        } else {
-            debug!(self.log, "Successfully imported attester slashing");
-        }
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_attester_slashing(
+                message_id,
+                peer_id,
+                attester_slashing,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "attester slashing gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 }
 

beacon_node/network/src/service.rs

@@ -6,17 +6,18 @@ use crate::{
 };
 use crate::{error, metrics};
 use beacon_chain::{BeaconChain, BeaconChainTypes};
-use eth2_libp2p::Service as LibP2PService;
 use eth2_libp2p::{
     rpc::{GoodbyeReason, RPCResponseErrorCode, RequestId},
     Libp2pEvent, PeerAction, PeerRequestId, PubsubMessage, Request, Response,
 };
-use eth2_libp2p::{BehaviourEvent, MessageId, NetworkGlobals, PeerId};
+use eth2_libp2p::{
+    types::GossipKind, BehaviourEvent, GossipTopic, MessageId, NetworkGlobals, PeerId, TopicHash,
+};
+use eth2_libp2p::{MessageAcceptance, Service as LibP2PService};
 use futures::prelude::*;
 use rest_types::ValidatorSubscription;
 use slog::{debug, error, info, o, trace, warn};
-use std::sync::Arc;
-use std::time::Duration;
+use std::{collections::HashMap, sync::Arc, time::Duration};
 use store::HotColdDB;
 use tokio::sync::mpsc;
 use tokio::time::Delay;
@@ -24,6 +25,9 @@ use types::EthSpec;
 
 mod tests;
 
+/// The interval (in seconds) that various network metrics will update.
+const METRIC_UPDATE_INTERVAL: u64 = 1;
+
 /// Types of messages that the network service can receive.
 #[derive(Debug)]
 pub enum NetworkMessage<T: EthSpec> {
@@ -55,11 +59,13 @@ pub enum NetworkMessage<T: EthSpec> {
     /// Publish a list of messages to the gossipsub protocol.
     Publish { messages: Vec<PubsubMessage<T>> },
     /// Validates a received gossipsub message. This will propagate the message on the network.
-    Validate {
+    ValidationResult {
         /// The peer that sent us the message. We don't send back to this peer.
         propagation_source: PeerId,
         /// The id of the message we are validating and propagating.
         message_id: MessageId,
+        /// The result of the validation
+        validation_result: MessageAcceptance,
     },
     /// Reports a peer to the peer manager for performing an action.
     ReportPeer { peer_id: PeerId, action: PeerAction },
@@ -89,6 +95,8 @@ pub struct NetworkService<T: BeaconChainTypes> {
     network_globals: Arc<NetworkGlobals<T::EthSpec>>,
     /// A delay that expires when a new fork takes place.
     next_fork_update: Option<Delay>,
+    /// A timer for updating various network metrics.
+    metrics_update: tokio::time::Interval,
     /// The logger for the network service.
     log: slog::Logger,
 }
@@ -144,6 +152,9 @@ impl<T: BeaconChainTypes> NetworkService<T> {
         let attestation_service =
             AttestationService::new(beacon_chain.clone(), network_globals.clone(), &network_log);
 
+        // create a timer for updating network metrics
+        let metrics_update = tokio::time::interval(Duration::from_secs(METRIC_UPDATE_INTERVAL));
+
         // create the network service and spawn the task
         let network_log = network_log.new(o!("service" => "network"));
         let network_service = NetworkService {
@@ -155,6 +166,7 @@ impl<T: BeaconChainTypes> NetworkService<T> {
             store,
             network_globals: network_globals.clone(),
             next_fork_update,
+            metrics_update,
             log: network_log,
         };
 
@@ -169,12 +181,12 @@ fn spawn_service<T: BeaconChainTypes>(
     mut service: NetworkService<T>,
 ) -> error::Result<()> {
     let mut exit_rx = executor.exit();
+    let mut shutdown_sender = executor.shutdown_sender();
 
     // spawn on the current executor
     executor.spawn_without_exit(async move {
-        // TODO: there is something with this code that prevents cargo fmt from doing anything at
-        // all. Ok, it is worse, the compiler doesn't show errors over this code beyond ast
-        // checking
+
+        let mut metric_update_counter = 0;
         loop {
             // build the futures to check simultaneously
             tokio::select! {
@@ -203,6 +215,17 @@ fn spawn_service<T: BeaconChainTypes>(
                     info!(service.log, "Network service shutdown");
                     return;
                 }
+                _ = service.metrics_update.next() => {
+                    // update various network metrics
+                    metric_update_counter +=1;
+                    if metric_update_counter* 1000 % T::EthSpec::default_spec().milliseconds_per_slot == 0 {
+                        // if a slot has occurred, reset the metrics
+                        let _ = metrics::ATTESTATIONS_PUBLISHED_PER_SUBNET_PER_SLOT
+                            .as_ref()
+                            .map(|gauge| gauge.reset());
+                    }
+                    update_gossip_metrics::<T::EthSpec>(&service.libp2p.swarm.gs());
+                }
                 // handle a message sent to the network
                 Some(message) = service.network_recv.recv() => {
                     match message {
@@ -215,9 +238,10 @@ fn spawn_service<T: BeaconChainTypes>(
                         NetworkMessage::SendError{ peer_id, error, id, reason } => {
                             service.libp2p.respond_with_error(peer_id, id, error, reason);
                         }
-                        NetworkMessage::Validate {
+                        NetworkMessage::ValidationResult {
                             propagation_source,
                             message_id,
+                            validation_result,
                         } => {
                                 trace!(service.log, "Propagating gossipsub message";
                                     "propagation_peer" => format!("{:?}", propagation_source),
@@ -226,7 +250,9 @@ fn spawn_service<T: BeaconChainTypes>(
                                 service
                                     .libp2p
                                     .swarm
-                                    .validate_message(&propagation_source, message_id);
+                                    .report_message_validation_result(
+                                        &propagation_source, message_id, validation_result
+                                    );
                         }
                         NetworkMessage::Publish { messages } => {
                                 let mut topic_kinds = Vec::new();
@@ -271,8 +297,8 @@ fn spawn_service<T: BeaconChainTypes>(
                         AttServiceMessage::EnrRemove(subnet_id) => {
                             service.libp2p.swarm.update_enr_subnet(subnet_id, false);
                         }
-                        AttServiceMessage::DiscoverPeers{subnet_id, min_ttl} => {
-                            service.libp2p.swarm.discover_subnet_peers(subnet_id, min_ttl);
+                        AttServiceMessage::DiscoverPeers(subnets_to_discover) => {
+                            service.libp2p.swarm.discover_subnet_peers(subnets_to_discover);
                         }
                     }
                 }
@@ -376,6 +402,12 @@ fn spawn_service<T: BeaconChainTypes>(
                         Libp2pEvent::NewListenAddr(multiaddr) => {
                             service.network_globals.listen_multiaddrs.write().push(multiaddr);
                         }
+                        Libp2pEvent::ZeroListeners => {
+                            let _ = shutdown_sender.send("All listeners are closed. Unable to listen").await.map_err(|e| {
+                                warn!(service.log, "failed to send a shutdown signal"; "error" => e.to_string()
+                                )
+                            });
+                        }
                     }
                 }
             }
@@ -412,7 +444,11 @@ fn expose_publish_metrics<T: EthSpec>(messages: &[PubsubMessage<T>]) {
     for message in messages {
         match message {
             PubsubMessage::BeaconBlock(_) => metrics::inc_counter(&metrics::GOSSIP_BLOCKS_TX),
-            PubsubMessage::Attestation(_) => {
+            PubsubMessage::Attestation(subnet_id) => {
+                metrics::inc_counter_vec(
+                    &metrics::ATTESTATIONS_PUBLISHED_PER_SUBNET_PER_SLOT,
+                    &[&subnet_id.0.to_string()],
+                );
                 metrics::inc_counter(&metrics::GOSSIP_UNAGGREGATED_ATTESTATIONS_TX)
             }
             PubsubMessage::AggregateAndProofAttestation(_) => {
@@ -436,3 +472,163 @@ fn expose_receive_metrics<T: EthSpec>(message: &PubsubMessage<T>) {
         _ => {}
     }
 }
+
+fn update_gossip_metrics<T: EthSpec>(gossipsub: &eth2_libp2p::Gossipsub) {
+    // Clear the metrics
+    let _ = metrics::PEERS_PER_PROTOCOL
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = metrics::PEERS_PER_PROTOCOL
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = metrics::MESH_PEERS_PER_MAIN_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+
+    // reset the mesh peers, showing all subnets
+    for subnet_id in 0..T::default_spec().attestation_subnet_count {
+        let _ = metrics::get_int_gauge(
+            &metrics::MESH_PEERS_PER_SUBNET_TOPIC,
+            &[&subnet_id.to_string()],
+        )
+        .map(|v| v.set(0));
+
+        let _ = metrics::get_int_gauge(
+            &metrics::GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
+            &[&subnet_id.to_string()],
+        )
+        .map(|v| v.set(0));
+
+        let _ = metrics::get_int_gauge(
+            &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
+            &[&subnet_id.to_string()],
+        )
+        .map(|v| v.set(0));
+    }
+
+    // Subnet topics subscribed to
+    for topic_hash in gossipsub.topics() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            if let GossipKind::Attestation(subnet_id) = topic.kind() {
+                let _ = metrics::get_int_gauge(
+                    &metrics::GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
+                    &[&subnet_id.to_string()],
+                )
+                .map(|v| v.set(1));
+            }
+        }
+    }
+
+    // Peers per subscribed subnet
+    let mut peers_per_topic: HashMap<TopicHash, usize> = HashMap::new();
+    for (peer_id, topics) in gossipsub.all_peers() {
+        for topic_hash in topics {
+            *peers_per_topic.entry(topic_hash.clone()).or_default() += 1;
+
+            if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+                match topic.kind() {
+                    GossipKind::Attestation(subnet_id) => {
+                        if let Some(v) = metrics::get_int_gauge(
+                            &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
+                            &[&subnet_id.to_string()],
+                        ) {
+                            v.inc()
+                        };
+
+                        // average peer scores
+                        if let Some(score) = gossipsub.peer_score(peer_id) {
+                            if let Some(v) = metrics::get_int_gauge(
+                                &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
+                                &[&subnet_id.to_string()],
+                            ) {
+                                v.add(score as i64)
+                            };
+                        }
+                    }
+                    kind => {
+                        // main topics
+                        if let Some(score) = gossipsub.peer_score(peer_id) {
+                            if let Some(v) = metrics::get_int_gauge(
+                                &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC,
+                                &[&format!("{:?}", kind)],
+                            ) {
+                                v.add(score as i64)
+                            };
+                        }
+                    }
+                }
+            }
+        }
+    }
+    // adjust to average scores by dividing by number of peers
+    for (topic_hash, peers) in peers_per_topic.iter() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::Attestation(subnet_id) => {
+                    // average peer scores
+                    if let Some(v) = metrics::get_int_gauge(
+                        &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
+                        &[&subnet_id.to_string()],
+                    ) {
+                        v.set(v.get() / (*peers as i64))
+                    };
+                }
+                kind => {
+                    // main topics
+                    if let Some(v) = metrics::get_int_gauge(
+                        &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC,
+                        &[&format!("{:?}", kind)],
+                    ) {
+                        v.set(v.get() / (*peers as i64))
+                    };
+                }
+            }
+        }
+    }
+
+    // mesh peers
+    for topic_hash in gossipsub.topics() {
+        let peers = gossipsub.mesh_peers(&topic_hash).count();
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::Attestation(subnet_id) => {
+                    if let Some(v) = metrics::get_int_gauge(
+                        &metrics::MESH_PEERS_PER_SUBNET_TOPIC,
+                        &[&subnet_id.to_string()],
+                    ) {
+                        v.set(peers as i64)
+                    };
+                }
+                kind => {
+                    // main topics
+                    if let Some(v) = metrics::get_int_gauge(
+                        &metrics::MESH_PEERS_PER_MAIN_TOPIC,
+                        &[&format!("{:?}", kind)],
+                    ) {
+                        v.set(peers as i64)
+                    };
+                }
+            }
+        }
+    }
+
+    // protocol peers
+    let mut peers_per_protocol: HashMap<String, i64> = HashMap::new();
+    for (_peer, protocol) in gossipsub.peer_protocol() {
+        *peers_per_protocol.entry(protocol.to_string()).or_default() += 1;
+    }
+
+    for (protocol, peers) in peers_per_protocol.iter() {
+        if let Some(v) =
+            metrics::get_int_gauge(&metrics::PEERS_PER_PROTOCOL, &[&protocol.to_string()])
+        {
+            v.set(*peers)
+        };
+    }
+}

beacon_node/network/src/service/tests.rs

@@ -23,7 +23,7 @@ mod tests {
         let log = get_logger();
 
         let beacon_chain = Arc::new(
-            BeaconChainHarness::new(
+            BeaconChainHarness::new_with_store_config(
                 MinimalEthSpec,
                 generate_deterministic_keypairs(8),
                 StoreConfig::default(),
@@ -40,7 +40,13 @@ mod tests {
         let runtime = Runtime::new().unwrap();
 
         let (signal, exit) = exit_future::signal();
-        let executor = environment::TaskExecutor::new(runtime.handle().clone(), exit, log.clone());
+        let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+        let executor = environment::TaskExecutor::new(
+            runtime.handle().clone(),
+            exit,
+            log.clone(),
+            shutdown_tx,
+        );
 
         let mut config = NetworkConfig::default();
         config.libp2p_port = 21212;

beacon_node/network/src/sync/network_context.rs

@@ -110,7 +110,7 @@ impl<T: EthSpec> SyncNetworkContext<T> {
     }
 
     pub fn report_peer(&mut self, peer_id: PeerId, action: PeerAction) {
-        debug!(self.log, "Sync reporting peer"; "peer_id" => peer_id.to_string(), "action"=> action.to_string());
+        debug!(self.log, "Sync reporting peer"; "peer_id" => peer_id.to_string(), "action" => action.to_string());
         self.network_send
             .send(NetworkMessage::ReportPeer { peer_id, action })
             .unwrap_or_else(|_| {

beacon_node/network/src/sync/range_sync/chain_collection.rs

@@ -248,15 +248,17 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         // Check if any chains become the new syncing chain
         if let Some(index) = self.finalized_syncing_index() {
             // There is a current finalized chain syncing
-            let syncing_chain_peer_count = self.finalized_chains[index].peer_pool.len();
+            let _syncing_chain_peer_count = self.finalized_chains[index].peer_pool.len();
 
             // search for a chain with more peers
             if let Some((new_index, chain)) =
                 self.finalized_chains
                     .iter_mut()
                     .enumerate()
-                    .find(|(iter_index, chain)| {
-                        *iter_index != index && chain.peer_pool.len() > syncing_chain_peer_count
+                    .find(|(_iter_index, _chain)| {
+                        false
+                        //    && *iter_index != index
+                        //    && chain.peer_pool.len() > syncing_chain_peer_count
                     })
             {
                 // A chain has more peers. Swap the syncing chain

beacon_node/network/src/sync/range_sync/range.rs

@@ -172,7 +172,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                 } else {
                     // there is no finalized chain that matches this peer's last finalized target
                     // create a new finalized chain
-                    debug!(self.log, "New finalized chain added to sync"; "peer_id" => format!("{:?}", peer_id), "start_epoch" => local_finalized_slot, "end_slot" => remote_finalized_slot, "finalized_root" => format!("{}", remote_info.finalized_root));
+                    debug!(self.log, "New finalized chain added to sync"; "peer_id" => format!("{:?}", peer_id), "start_slot" => local_finalized_slot, "end_slot" => remote_finalized_slot, "finalized_root" => format!("{}", remote_info.finalized_root));
 
                     self.chains.new_finalized_chain(
                         local_info.finalized_epoch,

beacon_node/rest_api/Cargo.toml

@@ -34,7 +34,6 @@ hex = "0.4.2"
 parking_lot = "0.11.0"
 futures = "0.3.5"
 operation_pool = { path = "../operation_pool" }
-rayon = "1.3.0"
 environment = { path = "../../lighthouse/environment" }
 uhttp_sse = "0.5.1"
 bus = "2.2.3"

beacon_node/rest_api/src/advanced.rs

@@ -1,34 +0,0 @@
-use crate::response_builder::ResponseBuilder;
-use crate::ApiResult;
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use hyper::{Body, Request};
-use operation_pool::PersistedOperationPool;
-use std::sync::Arc;
-
-/// Returns the `proto_array` fork choice struct, encoded as JSON.
-///
-/// Useful for debugging or advanced inspection of the chain.
-pub fn get_fork_choice<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(
-        &*beacon_chain
-            .fork_choice
-            .read()
-            .proto_array()
-            .core_proto_array(),
-    )
-}
-
-/// Returns the `PersistedOperationPool` struct.
-///
-/// Useful for debugging or advanced inspection of the stored operations.
-pub fn get_operation_pool<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&PersistedOperationPool::from_operation_pool(
-        &beacon_chain.op_pool,
-    ))
-}

beacon_node/rest_api/src/beacon.rs

@@ -1,14 +1,13 @@
 use crate::helpers::*;
-use crate::response_builder::ResponseBuilder;
 use crate::validator::get_state_for_epoch;
-use crate::{ApiError, ApiResult, UrlQuery};
+use crate::Context;
+use crate::{ApiError, UrlQuery};
 use beacon_chain::{
     observed_operations::ObservationOutcome, BeaconChain, BeaconChainTypes, StateSkipConfig,
 };
-use bus::BusReader;
 use futures::executor::block_on;
 use hyper::body::Bytes;
-use hyper::{Body, Request, Response};
+use hyper::{Body, Request};
 use rest_types::{
     BlockResponse, CanonicalHeadResponse, Committee, HeadBeaconBlock, StateResponse,
     ValidatorRequest, ValidatorResponse,
@@ -16,20 +15,20 @@ use rest_types::{
 use std::io::Write;
 use std::sync::Arc;
 
-use slog::{error, Logger};
+use slog::error;
 use types::{
     AttesterSlashing, BeaconState, EthSpec, Hash256, ProposerSlashing, PublicKeyBytes,
     RelativeEpoch, SignedBeaconBlockHash, Slot,
 };
 
-/// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`.
+/// Returns a summary of the head of the beacon chain.
 pub fn get_head<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    ctx: Arc<Context<T>>,
+) -> Result<CanonicalHeadResponse, ApiError> {
+    let beacon_chain = &ctx.beacon_chain;
     let chain_head = beacon_chain.head()?;
 
-    let head = CanonicalHeadResponse {
+    Ok(CanonicalHeadResponse {
         slot: chain_head.beacon_state.slot,
         block_root: chain_head.beacon_block_root,
         state_root: chain_head.beacon_state_root,
@@ -51,33 +50,27 @@ pub fn get_head<T: BeaconChainTypes>(
             .epoch
             .start_slot(T::EthSpec::slots_per_epoch()),
         previous_justified_block_root: chain_head.beacon_state.previous_justified_checkpoint.root,
-    };
-
-    ResponseBuilder::new(&req)?.body(&head)
+    })
 }
 
-/// HTTP handler to return a list of head BeaconBlocks.
-pub fn get_heads<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let heads = beacon_chain
+/// Return the list of heads of the beacon chain.
+pub fn get_heads<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Vec<HeadBeaconBlock> {
+    ctx.beacon_chain
         .heads()
         .into_iter()
         .map(|(beacon_block_root, beacon_block_slot)| HeadBeaconBlock {
             beacon_block_root,
             beacon_block_slot,
         })
-        .collect::<Vec<_>>();
-
-    ResponseBuilder::new(&req)?.body(&heads)
+        .collect()
 }
 
 /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`.
 pub fn get_block<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<BlockResponse<T::EthSpec>, ApiError> {
+    let beacon_chain = &ctx.beacon_chain;
     let query_params = ["root", "slot"];
     let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?;
 
@@ -85,7 +78,7 @@ pub fn get_block<T: BeaconChainTypes>(
         ("slot", value) => {
             let target = parse_slot(&value)?;
 
-            block_root_at_slot(&beacon_chain, target)?.ok_or_else(|| {
+            block_root_at_slot(beacon_chain, target)?.ok_or_else(|| {
                 ApiError::NotFound(format!(
                     "Unable to find SignedBeaconBlock for slot {:?}",
                     target
@@ -103,30 +96,26 @@ pub fn get_block<T: BeaconChainTypes>(
         ))
     })?;
 
-    let response = BlockResponse {
+    Ok(BlockResponse {
         root: block_root,
         beacon_block: block,
-    };
-
-    ResponseBuilder::new(&req)?.body(&response)
+    })
 }
 
 /// HTTP handler to return a `SignedBeaconBlock` root at a given `slot`.
 pub fn get_block_root<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Hash256, ApiError> {
     let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?;
     let target = parse_slot(&slot_string)?;
 
-    let root = block_root_at_slot(&beacon_chain, target)?.ok_or_else(|| {
+    block_root_at_slot(&ctx.beacon_chain, target)?.ok_or_else(|| {
         ApiError::NotFound(format!(
             "Unable to find SignedBeaconBlock for slot {:?}",
             target
         ))
-    })?;
-
-    ResponseBuilder::new(&req)?.body(&root)
+    })
 }
 
 fn make_sse_response_chunk(new_head_hash: SignedBeaconBlockHash) -> std::io::Result<Bytes> {
@@ -140,45 +129,27 @@ fn make_sse_response_chunk(new_head_hash: SignedBeaconBlockHash) -> std::io::Res
     Ok(bytes)
 }
 
-pub fn stream_forks<T: BeaconChainTypes>(
-    log: Logger,
-    mut events: BusReader<SignedBeaconBlockHash>,
-) -> ApiResult {
+pub fn stream_forks<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Result<Body, ApiError> {
+    let mut events = ctx.events.lock().add_rx();
     let (mut sender, body) = Body::channel();
     std::thread::spawn(move || {
         while let Ok(new_head_hash) = events.recv() {
             let chunk = match make_sse_response_chunk(new_head_hash) {
                 Ok(chunk) => chunk,
                 Err(e) => {
-                    error!(log, "Failed to make SSE chunk"; "error" => e.to_string());
+                    error!(ctx.log, "Failed to make SSE chunk"; "error" => e.to_string());
                     sender.abort();
                     break;
                 }
             };
             match block_on(sender.send_data(chunk)) {
                 Err(e) if e.is_closed() => break,
-                Err(e) => error!(log, "Couldn't stream piece {:?}", e),
+                Err(e) => error!(ctx.log, "Couldn't stream piece {:?}", e),
                 Ok(_) => (),
             }
         }
     });
-    let response = Response::builder()
-        .status(200)
-        .header("Content-Type", "text/event-stream")
-        .header("Connection", "Keep-Alive")
-        .header("Cache-Control", "no-cache")
-        .header("Access-Control-Allow-Origin", "*")
-        .body(body)
-        .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))?;
-    Ok(response)
-}
-
-/// HTTP handler to return the `Fork` of the current head.
-pub fn get_fork<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&beacon_chain.head()?.beacon_state.fork)
+    Ok(body)
 }
 
 /// HTTP handler to which accepts a query string of a list of validator pubkeys and maps it to a
@@ -187,9 +158,9 @@ pub fn get_fork<T: BeaconChainTypes>(
 /// This method is limited to as many `pubkeys` that can fit in a URL. See `post_validators` for
 /// doing bulk requests.
 pub fn get_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let validator_pubkeys = query
@@ -204,17 +175,14 @@ pub fn get_validators<T: BeaconChainTypes>(
         None
     };
 
-    let validators =
-        validator_responses_by_pubkey(beacon_chain, state_root_opt, validator_pubkeys)?;
-
-    ResponseBuilder::new(&req)?.body(&validators)
+    validator_responses_by_pubkey(&ctx.beacon_chain, state_root_opt, validator_pubkeys)
 }
 
 /// HTTP handler to return all validators, each as a `ValidatorResponse`.
 pub fn get_all_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let state_root_opt = if let Some((_key, value)) = query.first_of_opt(&["state_root"]) {
@@ -223,23 +191,21 @@ pub fn get_all_validators<T: BeaconChainTypes>(
         None
     };
 
-    let mut state = get_state_from_root_opt(&beacon_chain, state_root_opt)?;
+    let mut state = get_state_from_root_opt(&ctx.beacon_chain, state_root_opt)?;
     state.update_pubkey_cache()?;
 
-    let validators = state
+    state
         .validators
         .iter()
         .map(|validator| validator_response_by_pubkey(&state, validator.pubkey.clone()))
-        .collect::<Result<Vec<_>, _>>()?;
-
-    ResponseBuilder::new(&req)?.body(&validators)
+        .collect::<Result<Vec<_>, _>>()
 }
 
 /// HTTP handler to return all active validators, each as a `ValidatorResponse`.
 pub fn get_active_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let state_root_opt = if let Some((_key, value)) = query.first_of_opt(&["state_root"]) {
@@ -248,17 +214,15 @@ pub fn get_active_validators<T: BeaconChainTypes>(
         None
     };
 
-    let mut state = get_state_from_root_opt(&beacon_chain, state_root_opt)?;
+    let mut state = get_state_from_root_opt(&ctx.beacon_chain, state_root_opt)?;
     state.update_pubkey_cache()?;
 
-    let validators = state
+    state
         .validators
         .iter()
         .filter(|validator| validator.is_active_at(state.current_epoch()))
         .map(|validator| validator_response_by_pubkey(&state, validator.pubkey.clone()))
-        .collect::<Result<Vec<_>, _>>()?;
-
-    ResponseBuilder::new(&req)?.body(&validators)
+        .collect::<Result<Vec<_>, _>>()
 }
 
 /// HTTP handler to which accepts a `ValidatorRequest` and returns a `ValidatorResponse` for
@@ -266,17 +230,11 @@ pub fn get_active_validators<T: BeaconChainTypes>(
 ///
 /// This method allows for a basically unbounded list of `pubkeys`, where as the `get_validators`
 /// request is limited by the max number of pubkeys you can fit in a URL.
-pub async fn post_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
-    let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-    serde_json::from_slice::<ValidatorRequest>(&chunks)
+pub fn post_validators<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
+    serde_json::from_slice::<ValidatorRequest>(&req.into_body())
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorRequest: {:?}",
@@ -285,12 +243,11 @@ pub async fn post_validators<T: BeaconChainTypes>(
         })
         .and_then(|bulk_request| {
             validator_responses_by_pubkey(
-                beacon_chain,
+                &ctx.beacon_chain,
                 bulk_request.state_root,
                 bulk_request.pubkeys,
             )
         })
-        .and_then(|validators| response_builder?.body(&validators))
 }
 
 /// Returns either the state given by `state_root_opt`, or the canonical head state if it is
@@ -317,11 +274,11 @@ fn get_state_from_root_opt<T: BeaconChainTypes>(
 /// Maps a vec of `validator_pubkey` to a vec of `ValidatorResponse`, using the state at the given
 /// `state_root`. If `state_root.is_none()`, uses the canonial head state.
 fn validator_responses_by_pubkey<T: BeaconChainTypes>(
-    beacon_chain: Arc<BeaconChain<T>>,
+    beacon_chain: &BeaconChain<T>,
     state_root_opt: Option<Hash256>,
     validator_pubkeys: Vec<PublicKeyBytes>,
 ) -> Result<Vec<ValidatorResponse>, ApiError> {
-    let mut state = get_state_from_root_opt(&beacon_chain, state_root_opt)?;
+    let mut state = get_state_from_root_opt(beacon_chain, state_root_opt)?;
     state.update_pubkey_cache()?;
 
     validator_pubkeys
@@ -372,24 +329,25 @@ fn validator_response_by_pubkey<E: EthSpec>(
 
 /// HTTP handler
 pub fn get_committees<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<Committee>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
 
-    let mut state = get_state_for_epoch(&beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
+    let mut state =
+        get_state_for_epoch(&ctx.beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
 
     let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), epoch).map_err(|e| {
         ApiError::ServerError(format!("Failed to get state suitable for epoch: {:?}", e))
     })?;
 
     state
-        .build_committee_cache(relative_epoch, &beacon_chain.spec)
+        .build_committee_cache(relative_epoch, &ctx.beacon_chain.spec)
         .map_err(|e| ApiError::ServerError(format!("Unable to build committee cache: {:?}", e)))?;
 
-    let committees = state
+    Ok(state
         .get_beacon_committees_at_epoch(relative_epoch)
         .map_err(|e| ApiError::ServerError(format!("Unable to get all committees: {:?}", e)))?
         .into_iter()
@@ -398,9 +356,7 @@ pub fn get_committees<T: BeaconChainTypes>(
             index: c.index,
             committee: c.committee.to_vec(),
         })
-        .collect::<Vec<_>>();
-
-    ResponseBuilder::new(&req)?.body(&committees)
+        .collect::<Vec<_>>())
 }
 
 /// HTTP handler to return a `BeaconState` at a given `root` or `slot`.
@@ -408,10 +364,10 @@ pub fn get_committees<T: BeaconChainTypes>(
 /// Will not return a state if the request slot is in the future. Will return states higher than
 /// the current head by skipping slots.
 pub fn get_state<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let head_state = beacon_chain.head()?.beacon_state;
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<StateResponse<T::EthSpec>, ApiError> {
+    let head_state = ctx.beacon_chain.head()?.beacon_state;
 
     let (key, value) = match UrlQuery::from_request(&req) {
         Ok(query) => {
@@ -429,11 +385,12 @@ pub fn get_state<T: BeaconChainTypes>(
     };
 
     let (root, state): (Hash256, BeaconState<T::EthSpec>) = match (key.as_ref(), value) {
-        ("slot", value) => state_at_slot(&beacon_chain, parse_slot(&value)?)?,
+        ("slot", value) => state_at_slot(&ctx.beacon_chain, parse_slot(&value)?)?,
         ("root", value) => {
             let root = &parse_root(&value)?;
 
-            let state = beacon_chain
+            let state = ctx
+                .beacon_chain
                 .store
                 .get_state(root, None)?
                 .ok_or_else(|| ApiError::NotFound(format!("No state for root: {:?}", root)))?;
@@ -443,12 +400,10 @@ pub fn get_state<T: BeaconChainTypes>(
         _ => return Err(ApiError::ServerError("Unexpected query parameter".into())),
     };
 
-    let response = StateResponse {
+    Ok(StateResponse {
         root,
         beacon_state: state,
-    };
-
-    ResponseBuilder::new(&req)?.body(&response)
+    })
 }
 
 /// HTTP handler to return a `BeaconState` root at a given `slot`.
@@ -456,15 +411,13 @@ pub fn get_state<T: BeaconChainTypes>(
 /// Will not return a state if the request slot is in the future. Will return states higher than
 /// the current head by skipping slots.
 pub fn get_state_root<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Hash256, ApiError> {
     let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?;
     let slot = parse_slot(&slot_string)?;
 
-    let root = state_root_at_slot(&beacon_chain, slot, StateSkipConfig::WithStateRoots)?;
-
-    ResponseBuilder::new(&req)?.body(&root)
+    state_root_at_slot(&ctx.beacon_chain, slot, StateSkipConfig::WithStateRoots)
 }
 
 /// HTTP handler to return a `BeaconState` at the genesis block.
@@ -472,50 +425,28 @@ pub fn get_state_root<T: BeaconChainTypes>(
 /// This is an undocumented convenience method used during testing. For production, simply do a
 /// state request at slot 0.
 pub fn get_genesis_state<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?;
-
-    ResponseBuilder::new(&req)?.body(&state)
+    ctx: Arc<Context<T>>,
+) -> Result<BeaconState<T::EthSpec>, ApiError> {
+    state_at_slot(&ctx.beacon_chain, Slot::new(0)).map(|(_root, state)| state)
 }
 
-/// Read the genesis time from the current beacon chain state.
-pub fn get_genesis_time<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&beacon_chain.head_info()?.genesis_time)
-}
-
-/// Read the `genesis_validators_root` from the current beacon chain state.
-pub fn get_genesis_validators_root<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&beacon_chain.head_info()?.genesis_validators_root)
-}
-
-pub async fn proposer_slashing<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn proposer_slashing<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<bool, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice::<ProposerSlashing>(&chunks)
+    serde_json::from_slice::<ProposerSlashing>(&body)
         .map_err(|e| format!("Unable to parse JSON into ProposerSlashing: {:?}", e))
         .and_then(move |proposer_slashing| {
-            if beacon_chain.eth1_chain.is_some() {
-                let obs_outcome = beacon_chain
+            if ctx.beacon_chain.eth1_chain.is_some() {
+                let obs_outcome = ctx
+                    .beacon_chain
                     .verify_proposer_slashing_for_gossip(proposer_slashing)
                     .map_err(|e| format!("Error while verifying proposer slashing: {:?}", e))?;
                 if let ObservationOutcome::New(verified_proposer_slashing) = obs_outcome {
-                    beacon_chain.import_proposer_slashing(verified_proposer_slashing);
+                    ctx.beacon_chain
+                        .import_proposer_slashing(verified_proposer_slashing);
                     Ok(())
                 } else {
                     Err("Proposer slashing for that validator index already known".into())
@@ -524,22 +455,17 @@ pub async fn proposer_slashing<T: BeaconChainTypes>(
                 Err("Cannot insert proposer slashing on node without Eth1 connection.".to_string())
             }
         })
-        .map_err(ApiError::BadRequest)
-        .and_then(|_| response_builder?.body(&true))
+        .map_err(ApiError::BadRequest)?;
+
+    Ok(true)
 }
 
-pub async fn attester_slashing<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn attester_slashing<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<bool, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-
-    serde_json::from_slice::<AttesterSlashing<T::EthSpec>>(&chunks)
+    serde_json::from_slice::<AttesterSlashing<T::EthSpec>>(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into AttesterSlashing: {:?}",
@@ -547,13 +473,13 @@ pub async fn attester_slashing<T: BeaconChainTypes>(
             ))
         })
         .and_then(move |attester_slashing| {
-            if beacon_chain.eth1_chain.is_some() {
-                beacon_chain
+            if ctx.beacon_chain.eth1_chain.is_some() {
+                ctx.beacon_chain
                     .verify_attester_slashing_for_gossip(attester_slashing)
                     .map_err(|e| format!("Error while verifying attester slashing: {:?}", e))
                     .and_then(|outcome| {
                         if let ObservationOutcome::New(verified_attester_slashing) = outcome {
-                            beacon_chain
+                            ctx.beacon_chain
                                 .import_attester_slashing(verified_attester_slashing)
                                 .map_err(|e| {
                                     format!("Error while importing attester slashing: {:?}", e)
@@ -568,6 +494,7 @@ pub async fn attester_slashing<T: BeaconChainTypes>(
                     "Cannot insert attester slashing on node without Eth1 connection.".to_string(),
                 ))
             }
-        })
-        .and_then(|_| response_builder?.body(&true))
+        })?;
+
+    Ok(true)
 }

beacon_node/rest_api/src/consensus.rs

@@ -1,8 +1,7 @@
 use crate::helpers::*;
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult, UrlQuery};
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use hyper::{Body, Request};
+use crate::{ApiError, Context, UrlQuery};
+use beacon_chain::BeaconChainTypes;
+use hyper::Request;
 use rest_types::{IndividualVotesRequest, IndividualVotesResponse};
 use serde::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -50,38 +49,31 @@ impl Into<VoteCount> for TotalBalances {
 
 /// HTTP handler return a `VoteCount` for some given `Epoch`.
 pub fn get_vote_count<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<VoteCount, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
     // This is the last slot of the given epoch (one prior to the first slot of the next epoch).
     let target_slot = (epoch + 1).start_slot(T::EthSpec::slots_per_epoch()) - 1;
 
-    let (_root, state) = state_at_slot(&beacon_chain, target_slot)?;
-    let spec = &beacon_chain.spec;
+    let (_root, state) = state_at_slot(&ctx.beacon_chain, target_slot)?;
+    let spec = &ctx.beacon_chain.spec;
 
     let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
     validator_statuses.process_attestations(&state, spec)?;
 
-    let report: VoteCount = validator_statuses.total_balances.into();
-
-    ResponseBuilder::new(&req)?.body(&report)
+    Ok(validator_statuses.total_balances.into())
 }
 
-pub async fn post_individual_votes<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn post_individual_votes<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<IndividualVotesResponse>, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice::<IndividualVotesRequest>(&chunks)
+    serde_json::from_slice::<IndividualVotesRequest>(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorDutiesRequest: {:?}",
@@ -94,8 +86,8 @@ pub async fn post_individual_votes<T: BeaconChainTypes>(
             // This is the last slot of the given epoch (one prior to the first slot of the next epoch).
             let target_slot = (epoch + 1).start_slot(T::EthSpec::slots_per_epoch()) - 1;
 
-            let (_root, mut state) = state_at_slot(&beacon_chain, target_slot)?;
-            let spec = &beacon_chain.spec;
+            let (_root, mut state) = state_at_slot(&ctx.beacon_chain, target_slot)?;
+            let spec = &ctx.beacon_chain.spec;
 
             let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
             validator_statuses.process_attestations(&state, spec)?;
@@ -135,5 +127,4 @@ pub async fn post_individual_votes<T: BeaconChainTypes>(
                 })
                 .collect::<Result<Vec<_>, _>>()
         })
-        .and_then(|votes| response_builder?.body_no_ssz(&votes))
 }

beacon_node/rest_api/src/helpers.rs

@@ -1,9 +1,7 @@
-use crate::{ApiError, ApiResult, NetworkChannel};
+use crate::{ApiError, NetworkChannel};
 use beacon_chain::{BeaconChain, BeaconChainTypes, StateSkipConfig};
 use bls::PublicKeyBytes;
 use eth2_libp2p::PubsubMessage;
-use http::header;
-use hyper::{Body, Request};
 use itertools::process_results;
 use network::NetworkMessage;
 use ssz::Decode;
@@ -41,21 +39,6 @@ pub fn parse_committee_index(string: &str) -> Result<CommitteeIndex, ApiError> {
         .map_err(|e| ApiError::BadRequest(format!("Unable to parse committee index: {:?}", e)))
 }
 
-/// Checks the provided request to ensure that the `content-type` header.
-///
-/// The content-type header should either be omitted, in which case JSON is assumed, or it should
-/// explicitly specify `application/json`. If anything else is provided, an error is returned.
-pub fn check_content_type_for_json(req: &Request<Body>) -> Result<(), ApiError> {
-    match req.headers().get(header::CONTENT_TYPE) {
-        Some(h) if h == "application/json" => Ok(()),
-        Some(h) => Err(ApiError::BadRequest(format!(
-            "The provided content-type {:?} is not available, this endpoint only supports json.",
-            h
-        ))),
-        _ => Ok(()),
-    }
-}
-
 /// Parse an SSZ object from some hex-encoded bytes.
 ///
 /// E.g., A signature is `"0x0000000000000000000000000000000000000000000000000000000000000000"`
@@ -228,14 +211,8 @@ pub fn state_root_at_slot<T: BeaconChainTypes>(
     }
 }
 
-pub fn implementation_pending_response(_req: Request<Body>) -> ApiResult {
-    Err(ApiError::NotImplemented(
-        "API endpoint has not yet been implemented, but is planned to be soon.".to_owned(),
-    ))
-}
-
 pub fn publish_beacon_block_to_network<T: BeaconChainTypes + 'static>(
-    chan: NetworkChannel<T::EthSpec>,
+    chan: &NetworkChannel<T::EthSpec>,
     block: SignedBeaconBlock<T::EthSpec>,
 ) -> Result<(), ApiError> {
     // send the block via SSZ encoding

beacon_node/rest_api/src/lib.rs

@@ -1,22 +1,15 @@
 #[macro_use]
-mod macros;
-#[macro_use]
 extern crate lazy_static;
+mod router;
 extern crate network as client_network;
 
-mod advanced;
 mod beacon;
 pub mod config;
 mod consensus;
-mod error;
 mod helpers;
 mod lighthouse;
 mod metrics;
-mod network;
 mod node;
-mod response_builder;
-mod router;
-mod spec;
 mod url_query;
 mod validator;
 
@@ -24,7 +17,6 @@ use beacon_chain::{BeaconChain, BeaconChainTypes};
 use bus::Bus;
 use client_network::NetworkMessage;
 pub use config::ApiEncodingFormat;
-use error::{ApiError, ApiResult};
 use eth2_config::Eth2Config;
 use eth2_libp2p::NetworkGlobals;
 use futures::future::TryFutureExt;
@@ -32,6 +24,7 @@ use hyper::server::conn::AddrStream;
 use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Request, Server};
 use parking_lot::Mutex;
+use rest_types::ApiError;
 use slog::{info, warn};
 use std::net::SocketAddr;
 use std::path::PathBuf;
@@ -42,6 +35,7 @@ use url_query::UrlQuery;
 
 pub use crate::helpers::parse_pubkey_bytes;
 pub use config::Config;
+pub use router::Context;
 
 pub type NetworkChannel<T> = mpsc::UnboundedSender<NetworkMessage<T>>;
 
@@ -63,36 +57,28 @@ pub fn start_server<T: BeaconChainTypes>(
     events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
 ) -> Result<SocketAddr, hyper::Error> {
     let log = executor.log();
-    let inner_log = log.clone();
-    let rest_api_config = Arc::new(config.clone());
     let eth2_config = Arc::new(eth2_config);
 
+    let context = Arc::new(Context {
+        executor: executor.clone(),
+        config: config.clone(),
+        beacon_chain,
+        network_globals: network_info.network_globals.clone(),
+        network_chan: network_info.network_chan,
+        eth2_config,
+        log: log.clone(),
+        db_path,
+        freezer_db_path,
+        events,
+    });
+
     // Define the function that will build the request handler.
     let make_service = make_service_fn(move |_socket: &AddrStream| {
-        let beacon_chain = beacon_chain.clone();
-        let log = inner_log.clone();
-        let rest_api_config = rest_api_config.clone();
-        let eth2_config = eth2_config.clone();
-        let network_globals = network_info.network_globals.clone();
-        let network_channel = network_info.network_chan.clone();
-        let db_path = db_path.clone();
-        let freezer_db_path = freezer_db_path.clone();
-        let events = events.clone();
+        let ctx = context.clone();
 
         async move {
             Ok::<_, hyper::Error>(service_fn(move |req: Request<Body>| {
-                router::route(
-                    req,
-                    beacon_chain.clone(),
-                    network_globals.clone(),
-                    network_channel.clone(),
-                    rest_api_config.clone(),
-                    eth2_config.clone(),
-                    log.clone(),
-                    db_path.clone(),
-                    freezer_db_path.clone(),
-                    events.clone(),
-                )
+                router::on_http_request(req, ctx.clone())
             }))
         }
     });

beacon_node/rest_api/src/lighthouse.rs

@@ -1,24 +1,16 @@
 //! This contains a collection of lighthouse specific HTTP endpoints.
 
-use crate::response_builder::ResponseBuilder;
-use crate::ApiResult;
-use eth2_libp2p::{NetworkGlobals, PeerInfo};
-use hyper::{Body, Request};
+use crate::{ApiError, Context};
+use beacon_chain::BeaconChainTypes;
+use eth2_libp2p::PeerInfo;
 use serde::Serialize;
 use std::sync::Arc;
 use types::EthSpec;
 
-/// The syncing state of the beacon node.
-pub fn syncing<T: EthSpec>(
-    req: Request<Body>,
-    network_globals: Arc<NetworkGlobals<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&network_globals.sync_state())
-}
-
 /// Returns all known peers and corresponding information
-pub fn peers<T: EthSpec>(req: Request<Body>, network_globals: Arc<NetworkGlobals<T>>) -> ApiResult {
-    let peers: Vec<Peer<T>> = network_globals
+pub fn peers<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Result<Vec<Peer<T::EthSpec>>, ApiError> {
+    Ok(ctx
+        .network_globals
         .peers
         .read()
         .peers()
@@ -26,16 +18,15 @@ pub fn peers<T: EthSpec>(req: Request<Body>, network_globals: Arc<NetworkGlobals
             peer_id: peer_id.to_string(),
             peer_info: peer_info.clone(),
         })
-        .collect();
-    ResponseBuilder::new(&req)?.body_no_ssz(&peers)
+        .collect())
 }
 
 /// Returns all known connected peers and their corresponding information
-pub fn connected_peers<T: EthSpec>(
-    req: Request<Body>,
-    network_globals: Arc<NetworkGlobals<T>>,
-) -> ApiResult {
-    let peers: Vec<Peer<T>> = network_globals
+pub fn connected_peers<T: BeaconChainTypes>(
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<Peer<T::EthSpec>>, ApiError> {
+    Ok(ctx
+        .network_globals
         .peers
         .read()
         .connected_peers()
@@ -43,14 +34,13 @@ pub fn connected_peers<T: EthSpec>(
             peer_id: peer_id.to_string(),
             peer_info: peer_info.clone(),
         })
-        .collect();
-    ResponseBuilder::new(&req)?.body_no_ssz(&peers)
+        .collect())
 }
 
 /// Information returned by `peers` and `connected_peers`.
 #[derive(Clone, Debug, Serialize)]
 #[serde(bound = "T: EthSpec")]
-struct Peer<T: EthSpec> {
+pub struct Peer<T: EthSpec> {
     /// The Peer's ID
     peer_id: String,
     /// The PeerInfo associated with the peer.

beacon_node/rest_api/src/macros.rs

@@ -1,11 +0,0 @@
-macro_rules! try_future {
-    ($expr:expr) => {
-        match $expr {
-            core::result::Result::Ok(val) => val,
-            core::result::Result::Err(err) => return Err(std::convert::From::from(err)),
-        }
-    };
-    ($expr:expr,) => {
-        $crate::try_future!($expr)
-    };
-}

beacon_node/rest_api/src/metrics.rs

@@ -1,42 +1,38 @@
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult};
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use hyper::{Body, Request};
+use crate::{ApiError, Context};
+use beacon_chain::BeaconChainTypes;
 use lighthouse_metrics::{Encoder, TextEncoder};
 use rest_types::Health;
-use std::path::PathBuf;
 use std::sync::Arc;
 
 pub use lighthouse_metrics::*;
 
 lazy_static! {
+    pub static ref BEACON_HTTP_API_REQUESTS_TOTAL: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "beacon_http_api_requests_total",
+            "Count of HTTP requests received",
+            &["endpoint"]
+        );
+    pub static ref BEACON_HTTP_API_SUCCESS_TOTAL: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "beacon_http_api_success_total",
+            "Count of HTTP requests that returned 200 OK",
+            &["endpoint"]
+        );
+    pub static ref BEACON_HTTP_API_ERROR_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_http_api_error_total",
+        "Count of HTTP that did not return 200 OK",
+        &["endpoint"]
+    );
+    pub static ref BEACON_HTTP_API_TIMES_TOTAL: Result<HistogramVec> = try_create_histogram_vec(
+        "beacon_http_api_times_total",
+        "Duration to process HTTP requests",
+        &["endpoint"]
+    );
     pub static ref REQUEST_RESPONSE_TIME: Result<Histogram> = try_create_histogram(
         "http_server_request_duration_seconds",
         "Time taken to build a response to a HTTP request"
     );
-    pub static ref REQUEST_COUNT: Result<IntCounter> = try_create_int_counter(
-        "http_server_request_total",
-        "Total count of HTTP requests received"
-    );
-    pub static ref SUCCESS_COUNT: Result<IntCounter> = try_create_int_counter(
-        "http_server_success_total",
-        "Total count of HTTP 200 responses sent"
-    );
-    pub static ref VALIDATOR_GET_BLOCK_REQUEST_RESPONSE_TIME: Result<Histogram> =
-        try_create_histogram(
-            "http_server_validator_block_get_request_duration_seconds",
-            "Time taken to respond to GET /validator/block"
-        );
-    pub static ref VALIDATOR_GET_ATTESTATION_REQUEST_RESPONSE_TIME: Result<Histogram> =
-        try_create_histogram(
-            "http_server_validator_attestation_get_request_duration_seconds",
-            "Time taken to respond to GET /validator/attestation"
-        );
-    pub static ref VALIDATOR_GET_DUTIES_REQUEST_RESPONSE_TIME: Result<Histogram> =
-        try_create_histogram(
-            "http_server_validator_duties_get_request_duration_seconds",
-            "Time taken to respond to GET /validator/duties"
-        );
     pub static ref PROCESS_NUM_THREADS: Result<IntGauge> = try_create_int_gauge(
         "process_num_threads",
         "Number of threads used by the current process"
@@ -77,11 +73,8 @@ lazy_static! {
 ///
 /// This is a HTTP handler method.
 pub fn get_prometheus<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    db_path: PathBuf,
-    freezer_db_path: PathBuf,
-) -> ApiResult {
+    ctx: Arc<Context<T>>,
+) -> std::result::Result<String, ApiError> {
     let mut buffer = vec![];
     let encoder = TextEncoder::new();
 
@@ -101,9 +94,9 @@ pub fn get_prometheus<T: BeaconChainTypes>(
     // using `lighthouse_metrics::gather(..)` to collect the global `DEFAULT_REGISTRY` metrics into
     // a string that can be returned via HTTP.
 
-    slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&beacon_chain.slot_clock);
-    store::scrape_for_metrics(&db_path, &freezer_db_path);
-    beacon_chain::scrape_for_metrics(&beacon_chain);
+    slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&ctx.beacon_chain.slot_clock);
+    store::scrape_for_metrics(&ctx.db_path, &ctx.freezer_db_path);
+    beacon_chain::scrape_for_metrics(&ctx.beacon_chain);
     eth2_libp2p::scrape_discovery_metrics();
 
     // This will silently fail if we are unable to observe the health. This is desired behaviour
@@ -133,6 +126,5 @@ pub fn get_prometheus<T: BeaconChainTypes>(
         .unwrap();
 
     String::from_utf8(buffer)
-        .map(|string| ResponseBuilder::new(&req)?.body_text(string))
-        .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e)))?
+        .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e)))
 }

beacon_node/rest_api/src/network.rs

@@ -1,72 +0,0 @@
-use crate::error::ApiResult;
-use crate::response_builder::ResponseBuilder;
-use crate::NetworkGlobals;
-use beacon_chain::BeaconChainTypes;
-use eth2_libp2p::{Multiaddr, PeerId};
-use hyper::{Body, Request};
-use std::sync::Arc;
-
-/// HTTP handler to return the list of libp2p multiaddr the client is listening on.
-///
-/// Returns a list of `Multiaddr`, serialized according to their `serde` impl.
-pub fn get_listen_addresses<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    let multiaddresses: Vec<Multiaddr> = network.listen_multiaddrs();
-    ResponseBuilder::new(&req)?.body_no_ssz(&multiaddresses)
-}
-
-/// HTTP handler to return the network port the client is listening on.
-///
-/// Returns the TCP port number in its plain form (which is also valid JSON serialization)
-pub fn get_listen_port<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&network.listen_port_tcp())
-}
-
-/// HTTP handler to return the Discv5 ENR from the client's libp2p service.
-///
-/// ENR is encoded as base64 string.
-pub fn get_enr<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&network.local_enr().to_base64())
-}
-
-/// HTTP handler to return the `PeerId` from the client's libp2p service.
-///
-/// PeerId is encoded as base58 string.
-pub fn get_peer_id<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&network.local_peer_id().to_base58())
-}
-
-/// HTTP handler to return the number of peers connected in the client's libp2p service.
-pub fn get_peer_count<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&network.connected_peers())
-}
-
-/// HTTP handler to return the list of peers connected to the client's libp2p service.
-///
-/// Peers are presented as a list of `PeerId::to_string()`.
-pub fn get_peer_list<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    let connected_peers: Vec<String> = network
-        .peers
-        .read()
-        .connected_peer_ids()
-        .map(PeerId::to_string)
-        .collect();
-    ResponseBuilder::new(&req)?.body_no_ssz(&connected_peers)
-}

beacon_node/rest_api/src/node.rs

@@ -1,23 +1,19 @@
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult};
-use eth2_libp2p::{types::SyncState, NetworkGlobals};
-use hyper::{Body, Request};
-use lighthouse_version::version_with_platform;
-use rest_types::{Health, SyncingResponse, SyncingStatus};
+use crate::{ApiError, Context};
+use beacon_chain::BeaconChainTypes;
+use eth2_libp2p::types::SyncState;
+use rest_types::{SyncingResponse, SyncingStatus};
 use std::sync::Arc;
-use types::{EthSpec, Slot};
+use types::Slot;
 
-/// Read the version string from the current Lighthouse build.
-pub fn get_version(req: Request<Body>) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&version_with_platform())
-}
+/// Returns a syncing status.
+pub fn syncing<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Result<SyncingResponse, ApiError> {
+    let current_slot = ctx
+        .beacon_chain
+        .head_info()
+        .map_err(|e| ApiError::ServerError(format!("Unable to read head slot: {:?}", e)))?
+        .slot;
 
-pub fn syncing<T: EthSpec>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T>>,
-    current_slot: Slot,
-) -> ApiResult {
-    let (starting_slot, highest_slot) = match network.sync_state() {
+    let (starting_slot, highest_slot) = match ctx.network_globals.sync_state() {
         SyncState::SyncingFinalized {
             start_slot,
             head_slot,
@@ -36,14 +32,8 @@ pub fn syncing<T: EthSpec>(
         highest_slot,
     };
 
-    ResponseBuilder::new(&req)?.body(&SyncingResponse {
-        is_syncing: network.is_syncing(),
+    Ok(SyncingResponse {
+        is_syncing: ctx.network_globals.is_syncing(),
         sync_status,
     })
 }
-
-pub fn get_health(req: Request<Body>) -> ApiResult {
-    let health = Health::observe().map_err(ApiError::ServerError)?;
-
-    ResponseBuilder::new(&req)?.body_no_ssz(&health)
-}

beacon_node/rest_api/src/response_builder.rs

@@ -1,83 +0,0 @@
-use super::{ApiError, ApiResult};
-use crate::config::ApiEncodingFormat;
-use hyper::header;
-use hyper::{Body, Request, Response, StatusCode};
-use serde::Serialize;
-use ssz::Encode;
-
-pub struct ResponseBuilder {
-    encoding: ApiEncodingFormat,
-}
-
-impl ResponseBuilder {
-    pub fn new(req: &Request<Body>) -> Result<Self, ApiError> {
-        let accept_header: String = req
-            .headers()
-            .get(header::ACCEPT)
-            .map_or(Ok(""), |h| h.to_str())
-            .map_err(|e| {
-                ApiError::BadRequest(format!(
-                    "The Accept header contains invalid characters: {:?}",
-                    e
-                ))
-            })
-            .map(String::from)?;
-
-        // JSON is our default encoding, unless something else is requested.
-        let encoding = ApiEncodingFormat::from(accept_header.as_str());
-        Ok(Self { encoding })
-    }
-
-    pub fn body<T: Serialize + Encode>(self, item: &T) -> ApiResult {
-        match self.encoding {
-            ApiEncodingFormat::SSZ => Response::builder()
-                .status(StatusCode::OK)
-                .header("content-type", "application/ssz")
-                .body(Body::from(item.as_ssz_bytes()))
-                .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))),
-            _ => self.body_no_ssz(item),
-        }
-    }
-
-    pub fn body_no_ssz<T: Serialize>(self, item: &T) -> ApiResult {
-        let (body, content_type) = match self.encoding {
-            ApiEncodingFormat::JSON => (
-                Body::from(serde_json::to_string(&item).map_err(|e| {
-                    ApiError::ServerError(format!(
-                        "Unable to serialize response body as JSON: {:?}",
-                        e
-                    ))
-                })?),
-                "application/json",
-            ),
-            ApiEncodingFormat::SSZ => {
-                return Err(ApiError::UnsupportedType(
-                    "Response cannot be encoded as SSZ.".into(),
-                ));
-            }
-            ApiEncodingFormat::YAML => (
-                Body::from(serde_yaml::to_string(&item).map_err(|e| {
-                    ApiError::ServerError(format!(
-                        "Unable to serialize response body as YAML: {:?}",
-                        e
-                    ))
-                })?),
-                "application/yaml",
-            ),
-        };
-
-        Response::builder()
-            .status(StatusCode::OK)
-            .header("content-type", content_type)
-            .body(body)
-            .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))
-    }
-
-    pub fn body_text(self, text: String) -> ApiResult {
-        Response::builder()
-            .status(StatusCode::OK)
-            .header("content-type", "text/plain; charset=utf-8")
-            .body(Body::from(text))
-            .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))
-    }
-}

beacon_node/rest_api/src/router.rs

@@ -1,241 +1,322 @@
 use crate::{
-    advanced, beacon, config::Config, consensus, error::ApiError, helpers, lighthouse, metrics,
-    network, node, spec, validator, NetworkChannel,
+    beacon, config::Config, consensus, lighthouse, metrics, node, validator, NetworkChannel,
 };
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use bus::Bus;
+use environment::TaskExecutor;
 use eth2_config::Eth2Config;
-use eth2_libp2p::NetworkGlobals;
+use eth2_libp2p::{NetworkGlobals, PeerId};
 use hyper::header::HeaderValue;
 use hyper::{Body, Method, Request, Response};
+use lighthouse_version::version_with_platform;
+use operation_pool::PersistedOperationPool;
 use parking_lot::Mutex;
+use rest_types::{ApiError, Handler, Health};
 use slog::debug;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
-use types::{SignedBeaconBlockHash, Slot};
+use types::{EthSpec, SignedBeaconBlockHash};
 
-// Allowing more than 7 arguments.
-#[allow(clippy::too_many_arguments)]
-pub async fn route<T: BeaconChainTypes>(
+pub struct Context<T: BeaconChainTypes> {
+    pub executor: TaskExecutor,
+    pub config: Config,
+    pub beacon_chain: Arc<BeaconChain<T>>,
+    pub network_globals: Arc<NetworkGlobals<T::EthSpec>>,
+    pub network_chan: NetworkChannel<T::EthSpec>,
+    pub eth2_config: Arc<Eth2Config>,
+    pub log: slog::Logger,
+    pub db_path: PathBuf,
+    pub freezer_db_path: PathBuf,
+    pub events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
+}
+
+pub async fn on_http_request<T: BeaconChainTypes>(
     req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    network_globals: Arc<NetworkGlobals<T::EthSpec>>,
-    network_channel: NetworkChannel<T::EthSpec>,
-    rest_api_config: Arc<Config>,
-    eth2_config: Arc<Eth2Config>,
-    local_log: slog::Logger,
-    db_path: PathBuf,
-    freezer_db_path: PathBuf,
-    events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
+    ctx: Arc<Context<T>>,
 ) -> Result<Response<Body>, ApiError> {
-    metrics::inc_counter(&metrics::REQUEST_COUNT);
-    let received_instant = Instant::now();
-
     let path = req.uri().path().to_string();
 
-    let log = local_log.clone();
-    let result = {
-        let _timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME);
+    let _timer = metrics::start_timer_vec(&metrics::BEACON_HTTP_API_TIMES_TOTAL, &[&path]);
+    metrics::inc_counter_vec(&metrics::BEACON_HTTP_API_REQUESTS_TOTAL, &[&path]);
 
-        match (req.method(), path.as_ref()) {
-            // Methods for Client
-            (&Method::GET, "/node/health") => node::get_health(req),
-            (&Method::GET, "/node/version") => node::get_version(req),
-            (&Method::GET, "/node/syncing") => {
-                // inform the current slot, or set to 0
-                let current_slot = beacon_chain
-                    .head_info()
-                    .map(|info| info.slot)
-                    .unwrap_or_else(|_| Slot::from(0u64));
+    let received_instant = Instant::now();
+    let log = ctx.log.clone();
+    let allow_origin = ctx.config.allow_origin.clone();
 
-                node::syncing::<T::EthSpec>(req, network_globals, current_slot)
-            }
-
-            // Methods for Network
-            (&Method::GET, "/network/enr") => network::get_enr::<T>(req, network_globals),
-            (&Method::GET, "/network/peer_count") => {
-                network::get_peer_count::<T>(req, network_globals)
-            }
-            (&Method::GET, "/network/peer_id") => network::get_peer_id::<T>(req, network_globals),
-            (&Method::GET, "/network/peers") => network::get_peer_list::<T>(req, network_globals),
-            (&Method::GET, "/network/listen_port") => {
-                network::get_listen_port::<T>(req, network_globals)
-            }
-            (&Method::GET, "/network/listen_addresses") => {
-                network::get_listen_addresses::<T>(req, network_globals)
-            }
-
-            // Methods for Beacon Node
-            (&Method::GET, "/beacon/head") => beacon::get_head::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/heads") => beacon::get_heads::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/block") => beacon::get_block::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/block_root") => beacon::get_block_root::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/fork") => beacon::get_fork::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/fork/stream") => {
-                let reader = events.lock().add_rx();
-                beacon::stream_forks::<T>(log, reader)
-            }
-            (&Method::GET, "/beacon/genesis_time") => {
-                beacon::get_genesis_time::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/genesis_validators_root") => {
-                beacon::get_genesis_validators_root::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/validators") => beacon::get_validators::<T>(req, beacon_chain),
-            (&Method::POST, "/beacon/validators") => {
-                beacon::post_validators::<T>(req, beacon_chain).await
-            }
-            (&Method::GET, "/beacon/validators/all") => {
-                beacon::get_all_validators::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/validators/active") => {
-                beacon::get_active_validators::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/state") => beacon::get_state::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/state_root") => beacon::get_state_root::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/state/genesis") => {
-                beacon::get_genesis_state::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/committees") => beacon::get_committees::<T>(req, beacon_chain),
-            (&Method::POST, "/beacon/proposer_slashing") => {
-                beacon::proposer_slashing::<T>(req, beacon_chain).await
-            }
-            (&Method::POST, "/beacon/attester_slashing") => {
-                beacon::attester_slashing::<T>(req, beacon_chain).await
-            }
-
-            // Methods for Validator
-            (&Method::POST, "/validator/duties") => {
-                let timer =
-                    metrics::start_timer(&metrics::VALIDATOR_GET_DUTIES_REQUEST_RESPONSE_TIME);
-                let response = validator::post_validator_duties::<T>(req, beacon_chain);
-                drop(timer);
-                response.await
-            }
-            (&Method::POST, "/validator/subscribe") => {
-                validator::post_validator_subscriptions::<T>(req, network_channel).await
-            }
-            (&Method::GET, "/validator/duties/all") => {
-                validator::get_all_validator_duties::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/validator/duties/active") => {
-                validator::get_active_validator_duties::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/validator/block") => {
-                let timer =
-                    metrics::start_timer(&metrics::VALIDATOR_GET_BLOCK_REQUEST_RESPONSE_TIME);
-                let response = validator::get_new_beacon_block::<T>(req, beacon_chain, log);
-                drop(timer);
-                response
-            }
-            (&Method::POST, "/validator/block") => {
-                validator::publish_beacon_block::<T>(req, beacon_chain, network_channel, log).await
-            }
-            (&Method::GET, "/validator/attestation") => {
-                let timer =
-                    metrics::start_timer(&metrics::VALIDATOR_GET_ATTESTATION_REQUEST_RESPONSE_TIME);
-                let response = validator::get_new_attestation::<T>(req, beacon_chain);
-                drop(timer);
-                response
-            }
-            (&Method::GET, "/validator/aggregate_attestation") => {
-                validator::get_aggregate_attestation::<T>(req, beacon_chain)
-            }
-            (&Method::POST, "/validator/attestations") => {
-                validator::publish_attestations::<T>(req, beacon_chain, network_channel, log).await
-            }
-            (&Method::POST, "/validator/aggregate_and_proofs") => {
-                validator::publish_aggregate_and_proofs::<T>(
-                    req,
-                    beacon_chain,
-                    network_channel,
-                    log,
-                )
-                .await
-            }
-
-            // Methods for consensus
-            (&Method::GET, "/consensus/global_votes") => {
-                consensus::get_vote_count::<T>(req, beacon_chain)
-            }
-            (&Method::POST, "/consensus/individual_votes") => {
-                consensus::post_individual_votes::<T>(req, beacon_chain).await
-            }
-
-            // Methods for bootstrap and checking configuration
-            (&Method::GET, "/spec") => spec::get_spec::<T>(req, beacon_chain),
-            (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::<T>(req),
-            (&Method::GET, "/spec/deposit_contract") => {
-                helpers::implementation_pending_response(req)
-            }
-            (&Method::GET, "/spec/eth2_config") => spec::get_eth2_config::<T>(req, eth2_config),
-
-            // Methods for advanced parameters
-            (&Method::GET, "/advanced/fork_choice") => {
-                advanced::get_fork_choice::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/advanced/operation_pool") => {
-                advanced::get_operation_pool::<T>(req, beacon_chain)
-            }
-
-            (&Method::GET, "/metrics") => {
-                metrics::get_prometheus::<T>(req, beacon_chain, db_path, freezer_db_path)
-            }
-
-            // Lighthouse specific
-            (&Method::GET, "/lighthouse/syncing") => {
-                lighthouse::syncing::<T::EthSpec>(req, network_globals)
-            }
-
-            (&Method::GET, "/lighthouse/peers") => {
-                lighthouse::peers::<T::EthSpec>(req, network_globals)
-            }
-
-            (&Method::GET, "/lighthouse/connected_peers") => {
-                lighthouse::connected_peers::<T::EthSpec>(req, network_globals)
-            }
-            _ => Err(ApiError::NotFound(
-                "Request path and/or method not found.".to_owned(),
-            )),
-        }
-    };
-
-    let request_processing_duration = Instant::now().duration_since(received_instant);
-
-    // Map the Rust-friendly `Result` in to a http-friendly response. In effect, this ensures that
-    // any `Err` returned from our response handlers becomes a valid http response to the client
-    // (e.g., a response with a 404 or 500 status).
-
-    match result {
+    match route(req, ctx).await {
         Ok(mut response) => {
-            if rest_api_config.allow_origin != "" {
+            metrics::inc_counter_vec(&metrics::BEACON_HTTP_API_SUCCESS_TOTAL, &[&path]);
+
+            if allow_origin != "" {
                 let headers = response.headers_mut();
                 headers.insert(
                     hyper::header::ACCESS_CONTROL_ALLOW_ORIGIN,
-                    HeaderValue::from_str(&rest_api_config.allow_origin)?,
+                    HeaderValue::from_str(&allow_origin)?,
                 );
                 headers.insert(hyper::header::VARY, HeaderValue::from_static("Origin"));
             }
 
             debug!(
-                local_log,
+                log,
                 "HTTP API request successful";
                 "path" => path,
-                "duration_ms" => request_processing_duration.as_millis()
+                "duration_ms" => Instant::now().duration_since(received_instant).as_millis()
             );
-            metrics::inc_counter(&metrics::SUCCESS_COUNT);
             Ok(response)
         }
 
         Err(error) => {
+            metrics::inc_counter_vec(&metrics::BEACON_HTTP_API_ERROR_TOTAL, &[&path]);
+
             debug!(
-                local_log,
+                log,
                 "HTTP API request failure";
                 "path" => path,
-                "duration_ms" => request_processing_duration.as_millis()
+                "duration_ms" => Instant::now().duration_since(received_instant).as_millis()
             );
             Ok(error.into())
         }
     }
 }
+
+async fn route<T: BeaconChainTypes>(
+    req: Request<Body>,
+    ctx: Arc<Context<T>>,
+) -> Result<Response<Body>, ApiError> {
+    let path = req.uri().path().to_string();
+    let ctx = ctx.clone();
+    let method = req.method().clone();
+    let executor = ctx.executor.clone();
+    let handler = Handler::new(req, ctx, executor)?;
+
+    match (method, path.as_ref()) {
+        (Method::GET, "/node/version") => handler
+            .static_value(version_with_platform())
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/node/health") => handler
+            .static_value(Health::observe().map_err(ApiError::ServerError)?)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/node/syncing") => handler
+            .allow_body()
+            .in_blocking_task(|_, ctx| node::syncing(ctx))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/enr") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.local_enr().to_base64()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/peer_count") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.connected_peers()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/peer_id") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.local_peer_id().to_base58()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/peers") => handler
+            .in_blocking_task(|_, ctx| {
+                Ok(ctx
+                    .network_globals
+                    .peers
+                    .read()
+                    .connected_peer_ids()
+                    .map(PeerId::to_string)
+                    .collect::<Vec<_>>())
+            })
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/listen_port") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.listen_port_tcp()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/listen_addresses") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.network_globals.listen_multiaddrs()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/beacon/head") => handler
+            .in_blocking_task(|_, ctx| beacon::get_head(ctx))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/heads") => handler
+            .in_blocking_task(|_, ctx| Ok(beacon::get_heads(ctx)))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/block") => handler
+            .in_blocking_task(beacon::get_block)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/block_root") => handler
+            .in_blocking_task(beacon::get_block_root)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/fork") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.head_info()?.fork))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/fork/stream") => {
+            handler.sse_stream(|_, ctx| beacon::stream_forks(ctx)).await
+        }
+        (Method::GET, "/beacon/genesis_time") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.head_info()?.genesis_time))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/genesis_validators_root") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.head_info()?.genesis_validators_root))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/validators") => handler
+            .in_blocking_task(beacon::get_validators)
+            .await?
+            .all_encodings(),
+        (Method::POST, "/beacon/validators") => handler
+            .allow_body()
+            .in_blocking_task(beacon::post_validators)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/validators/all") => handler
+            .in_blocking_task(beacon::get_all_validators)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/validators/active") => handler
+            .in_blocking_task(beacon::get_active_validators)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/state") => handler
+            .in_blocking_task(beacon::get_state)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/state_root") => handler
+            .in_blocking_task(beacon::get_state_root)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/state/genesis") => handler
+            .in_blocking_task(|_, ctx| beacon::get_genesis_state(ctx))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/committees") => handler
+            .in_blocking_task(beacon::get_committees)
+            .await?
+            .all_encodings(),
+        (Method::POST, "/beacon/proposer_slashing") => handler
+            .allow_body()
+            .in_blocking_task(beacon::proposer_slashing)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/beacon/attester_slashing") => handler
+            .allow_body()
+            .in_blocking_task(beacon::attester_slashing)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/duties") => handler
+            .allow_body()
+            .in_blocking_task(validator::post_validator_duties)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/subscribe") => handler
+            .allow_body()
+            .in_blocking_task(validator::post_validator_subscriptions)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/duties/all") => handler
+            .in_blocking_task(validator::get_all_validator_duties)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/duties/active") => handler
+            .in_blocking_task(validator::get_active_validator_duties)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/block") => handler
+            .in_blocking_task(validator::get_new_beacon_block)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/block") => handler
+            .allow_body()
+            .in_blocking_task(validator::publish_beacon_block)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/attestation") => handler
+            .in_blocking_task(validator::get_new_attestation)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/aggregate_attestation") => handler
+            .in_blocking_task(validator::get_aggregate_attestation)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/attestations") => handler
+            .allow_body()
+            .in_blocking_task(validator::publish_attestations)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/aggregate_and_proofs") => handler
+            .allow_body()
+            .in_blocking_task(validator::publish_aggregate_and_proofs)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/consensus/global_votes") => handler
+            .allow_body()
+            .in_blocking_task(consensus::get_vote_count)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/consensus/individual_votes") => handler
+            .allow_body()
+            .in_blocking_task(consensus::post_individual_votes)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/spec") => handler
+            // TODO: this clone is not ideal.
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.spec.clone()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/spec/slots_per_epoch") => handler
+            .static_value(T::EthSpec::slots_per_epoch())
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/spec/eth2_config") => handler
+            // TODO: this clone is not ideal.
+            .in_blocking_task(|_, ctx| Ok(ctx.eth2_config.as_ref().clone()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/advanced/fork_choice") => handler
+            .in_blocking_task(|_, ctx| {
+                Ok(ctx
+                    .beacon_chain
+                    .fork_choice
+                    .read()
+                    .proto_array()
+                    .core_proto_array()
+                    .clone())
+            })
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/advanced/operation_pool") => handler
+            .in_blocking_task(|_, ctx| {
+                Ok(PersistedOperationPool::from_operation_pool(
+                    &ctx.beacon_chain.op_pool,
+                ))
+            })
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/metrics") => handler
+            .in_blocking_task(|_, ctx| metrics::get_prometheus(ctx))
+            .await?
+            .text_encoding(),
+        (Method::GET, "/lighthouse/syncing") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.network_globals.sync_state()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/lighthouse/peers") => handler
+            .in_blocking_task(|_, ctx| lighthouse::peers(ctx))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/lighthouse/connected_peers") => handler
+            .in_blocking_task(|_, ctx| lighthouse::connected_peers(ctx))
+            .await?
+            .serde_encodings(),
+        _ => Err(ApiError::NotFound(
+            "Request path and/or method not found.".to_owned(),
+        )),
+    }
+}

beacon_node/rest_api/src/spec.rs

@@ -1,28 +0,0 @@
-use super::ApiResult;
-use crate::response_builder::ResponseBuilder;
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use eth2_config::Eth2Config;
-use hyper::{Body, Request};
-use std::sync::Arc;
-use types::EthSpec;
-
-/// HTTP handler to return the full spec object.
-pub fn get_spec<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&beacon_chain.spec)
-}
-
-/// HTTP handler to return the full Eth2Config object.
-pub fn get_eth2_config<T: BeaconChainTypes>(
-    req: Request<Body>,
-    eth2_config: Arc<Eth2Config>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(eth2_config.as_ref())
-}
-
-/// HTTP handler to return the full spec object.
-pub fn get_slots_per_epoch<T: BeaconChainTypes>(req: Request<Body>) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&T::EthSpec::slots_per_epoch())
-}

beacon_node/rest_api/src/validator.rs

@@ -1,41 +1,32 @@
-use crate::helpers::{
-    check_content_type_for_json, parse_hex_ssz_bytes, publish_beacon_block_to_network,
-};
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult, NetworkChannel, UrlQuery};
+use crate::helpers::{parse_hex_ssz_bytes, publish_beacon_block_to_network};
+use crate::{ApiError, Context, NetworkChannel, UrlQuery};
 use beacon_chain::{
     attestation_verification::Error as AttnError, BeaconChain, BeaconChainError, BeaconChainTypes,
     BlockError, ForkChoiceError, StateSkipConfig,
 };
 use bls::PublicKeyBytes;
 use eth2_libp2p::PubsubMessage;
-use hyper::{Body, Request};
+use hyper::Request;
 use network::NetworkMessage;
-use rayon::prelude::*;
 use rest_types::{ValidatorDutiesRequest, ValidatorDutyBytes, ValidatorSubscription};
 use slog::{error, info, trace, warn, Logger};
 use std::sync::Arc;
 use types::beacon_state::EthSpec;
 use types::{
-    Attestation, AttestationData, BeaconState, Epoch, RelativeEpoch, SelectionProof,
+    Attestation, AttestationData, BeaconBlock, BeaconState, Epoch, RelativeEpoch, SelectionProof,
     SignedAggregateAndProof, SignedBeaconBlock, SubnetId,
 };
 
 /// HTTP Handler to retrieve the duties for a set of validators during a particular epoch. This
 /// method allows for collecting bulk sets of validator duties without risking exceeding the max
 /// URL length with query pairs.
-pub async fn post_validator_duties<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn post_validator_duties<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice::<ValidatorDutiesRequest>(&chunks)
+    serde_json::from_slice::<ValidatorDutiesRequest>(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorDutiesRequest: {:?}",
@@ -44,29 +35,22 @@ pub async fn post_validator_duties<T: BeaconChainTypes>(
         })
         .and_then(|bulk_request| {
             return_validator_duties(
-                beacon_chain,
+                &ctx.beacon_chain.clone(),
                 bulk_request.epoch,
                 bulk_request.pubkeys.into_iter().map(Into::into).collect(),
             )
         })
-        .and_then(|duties| response_builder?.body_no_ssz(&duties))
 }
 
 /// HTTP Handler to retrieve subscriptions for a set of validators. This allows the node to
 /// organise peer discovery and topic subscription for known validators.
-pub async fn post_validator_subscriptions<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network_chan: NetworkChannel<T::EthSpec>,
-) -> ApiResult {
-    try_future!(check_content_type_for_json(&req));
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn post_validator_subscriptions<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<(), ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice(&chunks)
+    serde_json::from_slice(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorSubscriptions: {:?}",
@@ -74,7 +58,7 @@ pub async fn post_validator_subscriptions<T: BeaconChainTypes>(
             ))
         })
         .and_then(move |subscriptions: Vec<ValidatorSubscription>| {
-            network_chan
+            ctx.network_chan
                 .send(NetworkMessage::Subscribe { subscriptions })
                 .map_err(|e| {
                     ApiError::ServerError(format!(
@@ -84,19 +68,18 @@ pub async fn post_validator_subscriptions<T: BeaconChainTypes>(
                 })?;
             Ok(())
         })
-        .and_then(|_| response_builder?.body_no_ssz(&()))
 }
 
 /// HTTP Handler to retrieve all validator duties for the given epoch.
 pub fn get_all_validator_duties<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
 
-    let state = get_state_for_epoch(&beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
+    let state = get_state_for_epoch(&ctx.beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
 
     let validator_pubkeys = state
         .validators
@@ -104,21 +87,19 @@ pub fn get_all_validator_duties<T: BeaconChainTypes>(
         .map(|validator| validator.pubkey.clone())
         .collect();
 
-    let duties = return_validator_duties(beacon_chain, epoch, validator_pubkeys)?;
-
-    ResponseBuilder::new(&req)?.body_no_ssz(&duties)
+    return_validator_duties(&ctx.beacon_chain, epoch, validator_pubkeys)
 }
 
 /// HTTP Handler to retrieve all active validator duties for the given epoch.
 pub fn get_active_validator_duties<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
 
-    let state = get_state_for_epoch(&beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
+    let state = get_state_for_epoch(&ctx.beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
 
     let validator_pubkeys = state
         .validators
@@ -127,9 +108,7 @@ pub fn get_active_validator_duties<T: BeaconChainTypes>(
         .map(|validator| validator.pubkey.clone())
         .collect();
 
-    let duties = return_validator_duties(beacon_chain, epoch, validator_pubkeys)?;
-
-    ResponseBuilder::new(&req)?.body_no_ssz(&duties)
+    return_validator_duties(&ctx.beacon_chain, epoch, validator_pubkeys)
 }
 
 /// Helper function to return the state that can be used to determine the duties for some `epoch`.
@@ -165,7 +144,7 @@ pub fn get_state_for_epoch<T: BeaconChainTypes>(
 
 /// Helper function to get the duties for some `validator_pubkeys` in some `epoch`.
 fn return_validator_duties<T: BeaconChainTypes>(
-    beacon_chain: Arc<BeaconChain<T>>,
+    beacon_chain: &BeaconChain<T>,
     epoch: Epoch,
     validator_pubkeys: Vec<PublicKeyBytes>,
 ) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
@@ -281,10 +260,9 @@ fn return_validator_duties<T: BeaconChainTypes>(
 
 /// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator.
 pub fn get_new_beacon_block<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    log: Logger,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<BeaconBlock<T::EthSpec>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let slot = query.slot()?;
@@ -296,11 +274,12 @@ pub fn get_new_beacon_block<T: BeaconChainTypes>(
         None
     };
 
-    let (new_block, _state) = beacon_chain
+    let (new_block, _state) = ctx
+        .beacon_chain
         .produce_block(randao_reveal, slot, validator_graffiti)
         .map_err(|e| {
             error!(
-                log,
+                ctx.log,
                 "Error whilst producing block";
                 "error" => format!("{:?}", e)
             );
@@ -311,48 +290,40 @@ pub fn get_new_beacon_block<T: BeaconChainTypes>(
             ))
         })?;
 
-    ResponseBuilder::new(&req)?.body(&new_block)
+    Ok(new_block)
 }
 
 /// HTTP Handler to publish a SignedBeaconBlock, which has been signed by a validator.
-pub async fn publish_beacon_block<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    network_chan: NetworkChannel<T::EthSpec>,
-    log: Logger,
-) -> ApiResult {
-    try_future!(check_content_type_for_json(&req));
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn publish_beacon_block<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<(), ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice(&chunks).map_err(|e| {
+    serde_json::from_slice(&body).map_err(|e| {
                     ApiError::BadRequest(format!("Unable to parse JSON into SignedBeaconBlock: {:?}", e))
                 })
             .and_then(move |block: SignedBeaconBlock<T::EthSpec>| {
                 let slot = block.slot();
-                match beacon_chain.process_block(block.clone()) {
+                match ctx.beacon_chain.process_block(block.clone()) {
                     Ok(block_root) => {
                         // Block was processed, publish via gossipsub
                         info!(
-                            log,
+                            ctx.log,
                             "Block from local validator";
                             "block_root" => format!("{}", block_root),
                             "block_slot" => slot,
                         );
 
-                        publish_beacon_block_to_network::<T>(network_chan, block)?;
+                        publish_beacon_block_to_network::<T>(&ctx.network_chan, block)?;
 
                         // Run the fork choice algorithm and enshrine a new canonical head, if
                         // found.
                         //
                         // The new head may or may not be the block we just received.
-                        if let Err(e) = beacon_chain.fork_choice() {
+                        if let Err(e) = ctx.beacon_chain.fork_choice() {
                             error!(
-                                log,
+                                ctx.log,
                                 "Failed to find beacon chain head";
                                 "error" => format!("{:?}", e)
                             );
@@ -366,9 +337,9 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
                             // - Excessive time between block produce and publish.
                             // - A validator is using another beacon node to produce blocks and
                             // submitting them here.
-                            if beacon_chain.head()?.beacon_block_root != block_root {
+                            if ctx.beacon_chain.head()?.beacon_block_root != block_root {
                                 warn!(
-                                    log,
+                                    ctx.log,
                                     "Block from validator is not head";
                                     "desc" => "potential re-org",
                                 );
@@ -380,7 +351,7 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
                     }
                     Err(BlockError::BeaconChainError(e)) => {
                         error!(
-                            log,
+                            ctx.log,
                             "Error whilst processing block";
                             "error" => format!("{:?}", e)
                         );
@@ -392,7 +363,7 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
                     }
                     Err(other) => {
                         warn!(
-                            log,
+                            ctx.log,
                             "Invalid block from local validator";
                             "outcome" => format!("{:?}", other)
                         );
@@ -404,41 +375,41 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
                     }
                 }
         })
-        .and_then(|_| response_builder?.body_no_ssz(&()))
 }
 
 /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator.
 pub fn get_new_attestation<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Attestation<T::EthSpec>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let slot = query.slot()?;
     let index = query.committee_index()?;
 
-    let attestation = beacon_chain
+    ctx.beacon_chain
         .produce_unaggregated_attestation(slot, index)
-        .map_err(|e| ApiError::BadRequest(format!("Unable to produce attestation: {:?}", e)))?;
-
-    ResponseBuilder::new(&req)?.body(&attestation)
+        .map_err(|e| ApiError::BadRequest(format!("Unable to produce attestation: {:?}", e)))
 }
 
 /// HTTP Handler to retrieve the aggregate attestation for a slot
 pub fn get_aggregate_attestation<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Attestation<T::EthSpec>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let attestation_data = query.attestation_data()?;
 
-    match beacon_chain.get_aggregated_attestation(&attestation_data) {
-        Ok(Some(attestation)) => ResponseBuilder::new(&req)?.body(&attestation),
+    match ctx
+        .beacon_chain
+        .get_aggregated_attestation(&attestation_data)
+    {
+        Ok(Some(attestation)) => Ok(attestation),
         Ok(None) => Err(ApiError::NotFound(format!(
             "No matching aggregate attestation for slot {:?} is known in slot {:?}",
             attestation_data.slot,
-            beacon_chain.slot()
+            ctx.beacon_chain.slot()
         ))),
         Err(e) => Err(ApiError::ServerError(format!(
             "Unable to obtain attestation: {:?}",
@@ -448,22 +419,13 @@ pub fn get_aggregate_attestation<T: BeaconChainTypes>(
 }
 
 /// HTTP Handler to publish a list of Attestations, which have been signed by a number of validators.
-pub async fn publish_attestations<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    network_chan: NetworkChannel<T::EthSpec>,
-    log: Logger,
-) -> ApiResult {
-    try_future!(check_content_type_for_json(&req));
-    let response_builder = ResponseBuilder::new(&req);
+pub fn publish_attestations<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<(), ApiError> {
+    let bytes = req.into_body();
 
-    let body = req.into_body();
-    let chunk = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-
-    let chunks = chunk.iter().cloned().collect::<Vec<u8>>();
-    serde_json::from_slice(&chunks.as_slice())
+    serde_json::from_slice(&bytes)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to deserialize JSON into a list of attestations: {:?}",
@@ -474,16 +436,16 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
         .map(
             move |attestations: Vec<(Attestation<T::EthSpec>, SubnetId)>| {
                 attestations
-                    .into_par_iter()
+                    .into_iter()
                     .enumerate()
                     .map(|(i, (attestation, subnet_id))| {
                         process_unaggregated_attestation(
-                            &beacon_chain,
-                            network_chan.clone(),
+                            &ctx.beacon_chain,
+                            ctx.network_chan.clone(),
                             attestation,
                             subnet_id,
                             i,
-                            &log,
+                            &ctx.log,
                         )
                     })
                     .collect::<Vec<Result<_, _>>>()
@@ -493,7 +455,7 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
         //
         // Note: this will only provide info about the _first_ failure, not all failures.
         .and_then(|processing_results| processing_results.into_iter().try_for_each(|result| result))
-        .and_then(|_| response_builder?.body_no_ssz(&()))
+        .map(|_| ())
 }
 
 /// Processes an unaggregrated attestation that was included in a list of attestations with the
@@ -566,21 +528,13 @@ fn process_unaggregated_attestation<T: BeaconChainTypes>(
 }
 
 /// HTTP Handler to publish an Attestation, which has been signed by a validator.
-#[allow(clippy::redundant_clone)] // false positives in this function.
-pub async fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    network_chan: NetworkChannel<T::EthSpec>,
-    log: Logger,
-) -> ApiResult {
-    try_future!(check_content_type_for_json(&req));
-    let response_builder = ResponseBuilder::new(&req);
+pub fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<(), ApiError> {
     let body = req.into_body();
-    let chunk = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-    let chunks = chunk.iter().cloned().collect::<Vec<u8>>();
-    serde_json::from_slice(&chunks.as_slice())
+
+    serde_json::from_slice(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to deserialize JSON into a list of SignedAggregateAndProof: {:?}",
@@ -591,15 +545,15 @@ pub async fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
         .map(
             move |signed_aggregates: Vec<SignedAggregateAndProof<T::EthSpec>>| {
                 signed_aggregates
-                    .into_par_iter()
+                    .into_iter()
                     .enumerate()
                     .map(|(i, signed_aggregate)| {
                         process_aggregated_attestation(
-                            &beacon_chain,
-                            network_chan.clone(),
+                            &ctx.beacon_chain,
+                            ctx.network_chan.clone(),
                             signed_aggregate,
                             i,
-                            &log,
+                            &ctx.log,
                         )
                     })
                     .collect::<Vec<Result<_, _>>>()
@@ -609,7 +563,6 @@ pub async fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
         //
         // Note: this will only provide info about the _first_ failure, not all failures.
         .and_then(|processing_results| processing_results.into_iter().try_for_each(|result| result))
-        .and_then(|_| response_builder?.body_no_ssz(&()))
 }
 
 /// Processes an aggregrated attestation that was included in a list of attestations with the index

beacon_node/rest_api/tests/test.rs

@@ -804,7 +804,7 @@ fn get_version() {
     let version = env
         .runtime()
         .block_on(remote_node.http.node().get_version())
-        .expect("should fetch eth2 config from http api");
+        .expect("should fetch version from http api");
 
     assert_eq!(
         lighthouse_version::version_with_platform(),

beacon_node/src/cli.rs

@@ -131,7 +131,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .help("Disables the discv5 discovery protocol. The node will not search for new peers or participate in the discovery protocol.")
                 .takes_value(false),
         )
-
+        .arg(
+            Arg::with_name("trusted-peers")
+                .long("trusted-peers")
+                .value_name("TRUSTED_PEERS")
+                .help("One or more comma-delimited trusted peer ids which always have the highest score according to the peer scoring system.")
+                .takes_value(true),
+        )
         /* REST API related arguments */
         .arg(
             Arg::with_name("http")

beacon_node/src/config.rs

@@ -2,7 +2,7 @@ use beacon_chain::builder::PUBKEY_CACHE_FILENAME;
 use clap::ArgMatches;
 use clap_utils::BAD_TESTNET_DIR_MESSAGE;
 use client::{config::DEFAULT_DATADIR, ClientConfig, ClientGenesis};
-use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr};
+use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized};
 use eth2_testnet_config::Eth2TestnetConfig;
 use slog::{crit, info, Logger};
 use ssz::Encode;
@@ -75,148 +75,13 @@ pub fn get_config<E: EthSpec>(
     /*
      * Networking
      */
-    // If a network dir has been specified, override the `datadir` definition.
-    if let Some(dir) = cli_args.value_of("network-dir") {
-        client_config.network.network_dir = PathBuf::from(dir);
-    } else {
-        client_config.network.network_dir = client_config.data_dir.join(NETWORK_DIR);
-    };
-
-    if let Some(listen_address_str) = cli_args.value_of("listen-address") {
-        let listen_address = listen_address_str
-            .parse()
-            .map_err(|_| format!("Invalid listen address: {:?}", listen_address_str))?;
-        client_config.network.listen_address = listen_address;
-    }
-
-    if let Some(target_peers_str) = cli_args.value_of("target-peers") {
-        client_config.network.target_peers = target_peers_str
-            .parse::<usize>()
-            .map_err(|_| format!("Invalid number of target peers: {}", target_peers_str))?;
-    }
-
-    if let Some(port_str) = cli_args.value_of("port") {
-        let port = port_str
-            .parse::<u16>()
-            .map_err(|_| format!("Invalid port: {}", port_str))?;
-        client_config.network.libp2p_port = port;
-        client_config.network.discovery_port = port;
-    }
-
-    if let Some(port_str) = cli_args.value_of("discovery-port") {
-        let port = port_str
-            .parse::<u16>()
-            .map_err(|_| format!("Invalid port: {}", port_str))?;
-        client_config.network.discovery_port = port;
-    }
-
-    if let Some(boot_enr_str) = cli_args.value_of("boot-nodes") {
-        let mut enrs: Vec<Enr> = vec![];
-        let mut multiaddrs: Vec<Multiaddr> = vec![];
-        for addr in boot_enr_str.split(',') {
-            match addr.parse() {
-                Ok(enr) => enrs.push(enr),
-                Err(_) => {
-                    // parsing as ENR failed, try as Multiaddr
-                    let multi: Multiaddr = addr
-                        .parse()
-                        .map_err(|_| format!("Not valid as ENR nor Multiaddr: {}", addr))?;
-                    if !multi.iter().any(|proto| matches!(proto, Protocol::Udp(_))) {
-                        slog::error!(log, "Missing UDP in Multiaddr {}", multi.to_string());
-                    }
-                    if !multi.iter().any(|proto| matches!(proto, Protocol::P2p(_))) {
-                        slog::error!(log, "Missing P2P in Multiaddr {}", multi.to_string());
-                    }
-                    multiaddrs.push(multi);
-                }
-            }
-        }
-        client_config.network.boot_nodes_enr = enrs;
-        client_config.network.boot_nodes_multiaddr = multiaddrs;
-    }
-
-    if let Some(libp2p_addresses_str) = cli_args.value_of("libp2p-addresses") {
-        client_config.network.libp2p_nodes = libp2p_addresses_str
-            .split(',')
-            .map(|multiaddr| {
-                multiaddr
-                    .parse()
-                    .map_err(|_| format!("Invalid Multiaddr: {}", multiaddr))
-            })
-            .collect::<Result<Vec<Multiaddr>, _>>()?;
-    }
-
-    if let Some(enr_udp_port_str) = cli_args.value_of("enr-udp-port") {
-        client_config.network.enr_udp_port = Some(
-            enr_udp_port_str
-                .parse::<u16>()
-                .map_err(|_| format!("Invalid discovery port: {}", enr_udp_port_str))?,
-        );
-    }
-
-    if let Some(enr_tcp_port_str) = cli_args.value_of("enr-tcp-port") {
-        client_config.network.enr_tcp_port = Some(
-            enr_tcp_port_str
-                .parse::<u16>()
-                .map_err(|_| format!("Invalid ENR TCP port: {}", enr_tcp_port_str))?,
-        );
-    }
-
-    if cli_args.is_present("enr-match") {
-        // set the enr address to localhost if the address is 0.0.0.0
-        if client_config.network.listen_address
-            == "0.0.0.0".parse::<IpAddr>().expect("valid ip addr")
-        {
-            client_config.network.enr_address =
-                Some("127.0.0.1".parse::<IpAddr>().expect("valid ip addr"));
-        } else {
-            client_config.network.enr_address = Some(client_config.network.listen_address);
-        }
-        client_config.network.enr_udp_port = Some(client_config.network.discovery_port);
-    }
-
-    if let Some(enr_address) = cli_args.value_of("enr-address") {
-        let resolved_addr = match enr_address.parse::<IpAddr>() {
-            Ok(addr) => addr, // // Input is an IpAddr
-            Err(_) => {
-                let mut addr = enr_address.to_string();
-                // Appending enr-port to the dns hostname to appease `to_socket_addrs()` parsing.
-                // Since enr-update is disabled with a dns address, not setting the enr-udp-port
-                // will make the node undiscoverable.
-                if let Some(enr_udp_port) = client_config.network.enr_udp_port {
-                    addr.push_str(&format!(":{}", enr_udp_port.to_string()));
-                } else {
-                    return Err(
-                        "enr-udp-port must be set for node to be discoverable with dns address"
-                            .into(),
-                    );
-                }
-                // `to_socket_addr()` does the dns resolution
-                // Note: `to_socket_addrs()` is a blocking call
-                let resolved_addr = if let Ok(mut resolved_addrs) = addr.to_socket_addrs() {
-                    // Pick the first ip from the list of resolved addresses
-                    resolved_addrs
-                        .next()
-                        .map(|a| a.ip())
-                        .ok_or_else(|| "Resolved dns addr contains no entries".to_string())?
-                } else {
-                    return Err(format!("Failed to parse enr-address: {}", enr_address));
-                };
-                client_config.network.discv5_config.enr_update = false;
-                resolved_addr
-            }
-        };
-        client_config.network.enr_address = Some(resolved_addr);
-    }
-
-    if cli_args.is_present("disable_enr_auto_update") {
-        client_config.network.discv5_config.enr_update = false;
-    }
-
-    if cli_args.is_present("disable-discovery") {
-        client_config.network.disable_discovery = true;
-        slog::warn!(log, "Discovery is disabled. New peers will not be found");
-    }
+    set_network_config(
+        &mut client_config.network,
+        cli_args,
+        &client_config.data_dir,
+        &log,
+        false,
+    )?;
 
     /*
      * Http server
@@ -399,6 +264,174 @@ pub fn get_config<E: EthSpec>(
     Ok(client_config)
 }
 
+/// Sets the network config from the command line arguments
+pub fn set_network_config(
+    config: &mut NetworkConfig,
+    cli_args: &ArgMatches,
+    data_dir: &PathBuf,
+    log: &Logger,
+    use_listening_port_as_enr_port_by_default: bool,
+) -> Result<(), String> {
+    // If a network dir has been specified, override the `datadir` definition.
+    if let Some(dir) = cli_args.value_of("network-dir") {
+        config.network_dir = PathBuf::from(dir);
+    } else {
+        config.network_dir = data_dir.join(NETWORK_DIR);
+    };
+
+    if let Some(listen_address_str) = cli_args.value_of("listen-address") {
+        let listen_address = listen_address_str
+            .parse()
+            .map_err(|_| format!("Invalid listen address: {:?}", listen_address_str))?;
+        config.listen_address = listen_address;
+    }
+
+    if let Some(target_peers_str) = cli_args.value_of("target-peers") {
+        config.target_peers = target_peers_str
+            .parse::<usize>()
+            .map_err(|_| format!("Invalid number of target peers: {}", target_peers_str))?;
+    }
+
+    if let Some(port_str) = cli_args.value_of("port") {
+        let port = port_str
+            .parse::<u16>()
+            .map_err(|_| format!("Invalid port: {}", port_str))?;
+        config.libp2p_port = port;
+        config.discovery_port = port;
+    }
+
+    if let Some(port_str) = cli_args.value_of("discovery-port") {
+        let port = port_str
+            .parse::<u16>()
+            .map_err(|_| format!("Invalid port: {}", port_str))?;
+        config.discovery_port = port;
+    }
+
+    if let Some(boot_enr_str) = cli_args.value_of("boot-nodes") {
+        let mut enrs: Vec<Enr> = vec![];
+        let mut multiaddrs: Vec<Multiaddr> = vec![];
+        for addr in boot_enr_str.split(',') {
+            match addr.parse() {
+                Ok(enr) => enrs.push(enr),
+                Err(_) => {
+                    // parsing as ENR failed, try as Multiaddr
+                    let multi: Multiaddr = addr
+                        .parse()
+                        .map_err(|_| format!("Not valid as ENR nor Multiaddr: {}", addr))?;
+                    if !multi.iter().any(|proto| matches!(proto, Protocol::Udp(_))) {
+                        slog::error!(log, "Missing UDP in Multiaddr {}", multi.to_string());
+                    }
+                    if !multi.iter().any(|proto| matches!(proto, Protocol::P2p(_))) {
+                        slog::error!(log, "Missing P2P in Multiaddr {}", multi.to_string());
+                    }
+                    multiaddrs.push(multi);
+                }
+            }
+        }
+        config.boot_nodes_enr = enrs;
+        config.boot_nodes_multiaddr = multiaddrs;
+    }
+
+    if let Some(libp2p_addresses_str) = cli_args.value_of("libp2p-addresses") {
+        config.libp2p_nodes = libp2p_addresses_str
+            .split(',')
+            .map(|multiaddr| {
+                multiaddr
+                    .parse()
+                    .map_err(|_| format!("Invalid Multiaddr: {}", multiaddr))
+            })
+            .collect::<Result<Vec<Multiaddr>, _>>()?;
+    }
+
+    if let Some(trusted_peers_str) = cli_args.value_of("trusted-peers") {
+        config.trusted_peers = trusted_peers_str
+            .split(',')
+            .map(|peer_id| {
+                peer_id
+                    .parse()
+                    .map_err(|_| format!("Invalid trusted peer id: {}", peer_id))
+            })
+            .collect::<Result<Vec<PeerIdSerialized>, _>>()?;
+    }
+
+    if let Some(enr_udp_port_str) = cli_args.value_of("enr-udp-port") {
+        config.enr_udp_port = Some(
+            enr_udp_port_str
+                .parse::<u16>()
+                .map_err(|_| format!("Invalid discovery port: {}", enr_udp_port_str))?,
+        );
+    }
+
+    if let Some(enr_tcp_port_str) = cli_args.value_of("enr-tcp-port") {
+        config.enr_tcp_port = Some(
+            enr_tcp_port_str
+                .parse::<u16>()
+                .map_err(|_| format!("Invalid ENR TCP port: {}", enr_tcp_port_str))?,
+        );
+    }
+
+    if cli_args.is_present("enr-match") {
+        // set the enr address to localhost if the address is 0.0.0.0
+        if config.listen_address == "0.0.0.0".parse::<IpAddr>().expect("valid ip addr") {
+            config.enr_address = Some("127.0.0.1".parse::<IpAddr>().expect("valid ip addr"));
+        } else {
+            config.enr_address = Some(config.listen_address);
+        }
+        config.enr_udp_port = Some(config.discovery_port);
+    }
+
+    if let Some(enr_address) = cli_args.value_of("enr-address") {
+        let resolved_addr = match enr_address.parse::<IpAddr>() {
+            Ok(addr) => addr, // // Input is an IpAddr
+            Err(_) => {
+                let mut addr = enr_address.to_string();
+                // Appending enr-port to the dns hostname to appease `to_socket_addrs()` parsing.
+                // Since enr-update is disabled with a dns address, not setting the enr-udp-port
+                // will make the node undiscoverable.
+                if let Some(enr_udp_port) = config.enr_udp_port.or_else(|| {
+                    if use_listening_port_as_enr_port_by_default {
+                        Some(config.discovery_port)
+                    } else {
+                        None
+                    }
+                }) {
+                    addr.push_str(&format!(":{}", enr_udp_port.to_string()));
+                } else {
+                    return Err(
+                        "enr-udp-port must be set for node to be discoverable with dns address"
+                            .into(),
+                    );
+                }
+                // `to_socket_addr()` does the dns resolution
+                // Note: `to_socket_addrs()` is a blocking call
+                let resolved_addr = if let Ok(mut resolved_addrs) = addr.to_socket_addrs() {
+                    // Pick the first ip from the list of resolved addresses
+                    resolved_addrs
+                        .next()
+                        .map(|a| a.ip())
+                        .ok_or_else(|| "Resolved dns addr contains no entries".to_string())?
+                } else {
+                    return Err(format!("Failed to parse enr-address: {}", enr_address));
+                };
+                config.discv5_config.enr_update = false;
+                resolved_addr
+            }
+        };
+        config.enr_address = Some(resolved_addr);
+    }
+
+    if cli_args.is_present("disable_enr_auto_update") {
+        config.discv5_config.enr_update = false;
+    }
+
+    if cli_args.is_present("disable-discovery") {
+        config.disable_discovery = true;
+        slog::warn!(log, "Discovery is disabled. New peers will not be found");
+    }
+
+    Ok(())
+}
+
 /// Gets the datadir which should be used.
 pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf {
     // Read the `--datadir` flag.

beacon_node/src/lib.rs

@@ -7,7 +7,7 @@ mod config;
 pub use beacon_chain;
 pub use cli::cli_app;
 pub use client::{Client, ClientBuilder, ClientConfig, ClientGenesis};
-pub use config::{get_data_dir, get_eth2_testnet_config};
+pub use config::{get_data_dir, get_eth2_testnet_config, set_network_config};
 pub use eth2_config::Eth2Config;
 
 use beacon_chain::events::TeeEventHandler;
@@ -100,7 +100,9 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
                 "endpoint" => &client_config.eth1.endpoint,
                 "method" => "json rpc via http"
             );
-            builder.caching_eth1_backend(client_config.eth1.clone())?
+            builder
+                .caching_eth1_backend(client_config.eth1.clone())
+                .await?
         } else if client_config.dummy_eth1_backend {
             warn!(
                 log,

beacon_node/store/Cargo.toml

@@ -15,7 +15,7 @@ rayon = "1.3.0"
 
 [dependencies]
 db-key = "0.0.5"
-leveldb = "0.8.5"
+leveldb = "0.8.6"
 parking_lot = "0.11.0"
 itertools = "0.9.0"
 eth2_ssz = "0.1.2"

beacon_node/store/src/hot_cold_store.rs

@@ -14,7 +14,7 @@ use crate::{
 };
 use lru::LruCache;
 use parking_lot::{Mutex, RwLock};
-use slog::{debug, error, trace, warn, Logger};
+use slog::{debug, error, info, trace, warn, Logger};
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
 use state_processing::{
@@ -147,6 +147,12 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
         // Load the previous split slot from the database (if any). This ensures we can
         // stop and restart correctly.
         if let Some(split) = db.load_split()? {
+            info!(
+                db.log,
+                "Hot-Cold DB initialized";
+                "split_slot" => split.slot,
+                "split_state" => format!("{:?}", split.state_root)
+            );
             *db.split.write() = split;
         }
         Ok(db)
@@ -819,7 +825,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
 }
 
 /// Advance the split point of the store, moving new finalized states to the freezer.
-pub fn process_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
+pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
     store: Arc<HotColdDB<E, Hot, Cold>>,
     frozen_head_root: Hash256,
     frozen_head: &BeaconState<E>,

beacon_node/store/src/lib.rs

@@ -27,7 +27,7 @@ pub mod iter;
 use std::borrow::Cow;
 
 pub use self::config::StoreConfig;
-pub use self::hot_cold_store::{HotColdDB, HotStateSummary, Split};
+pub use self::hot_cold_store::{BlockReplay, HotColdDB, HotStateSummary, Split};
 pub use self::leveldb_store::LevelDB;
 pub use self::memory_store::MemoryStore;
 pub use self::partial_beacon_state::PartialBeaconState;

book/src/SUMMARY.md

@@ -11,6 +11,7 @@
 * [Key Management](./key-management.md)
     * [Create a wallet](./wallet-create.md)
     * [Create a validator](./validator-create.md)
+    * [Key recovery](./key-recovery.md)
 * [Validator Management](./validator-management.md)
 	* [Importing from the Eth2 Launchpad](./validator-import-launchpad.md)
 * [Local Testnets](./local-testnets.md)

book/src/become-a-validator-source.md

@@ -88,7 +88,7 @@ validator](./validator-create.md). A two-step example follows:
 Create a wallet with:
 
 ```bash
-lighthouse --testnet medalla account wallet create --name my-validators --passphrase-file my-validators.pass
+lighthouse --testnet medalla account wallet create --name my-validators --password-file my-validators.pass
 ```
 
 The output will look like this:
@@ -124,7 +124,7 @@ used to restore your validator if there is a data loss.
 Create a validator from the wallet with:
 
 ```bash
-lighthouse --testnet medalla account validator create --wallet-name my-validators --wallet-passphrase my-validators.pass --count 1
+lighthouse --testnet medalla account validator create --wallet-name my-validators --wallet-password my-validators.pass --count 1
 ```
 
 The output will look like this:

book/src/cross-compiling.md

@@ -18,17 +18,15 @@ project.
 
 The `Makefile` in the project contains four targets for cross-compiling:
 
-- `build-x86_64`: builds an optimized version for x86_64 processors (suitable
-	for most users).
-- `build-x86_64-portable`: builds a version x86_64 processors which avoids
-	using some modern CPU instructions that might cause an "illegal
-	instruction" error on older CPUs.
-- `build-aarch64`: builds an optimized version for 64bit ARM processors
+- `build-x86_64`: builds an optimized version for x86_64 processors (suitable for most users).
+  Supports Intel Broadwell (2014) and newer, and AMD Ryzen (2017) and newer.
+- `build-x86_64-portable`: builds a version for x86_64 processors which avoids using some modern CPU
+  instructions that are incompatible with older CPUs. Suitable for pre-Broadwell/Ryzen CPUs.
+- `build-aarch64`: builds an optimized version for 64-bit ARM processors
 	(suitable for Raspberry Pi 4).
-- `build-aarch64-portable`: builds a version 64 bit ARM processors which avoids
-	using some modern CPU instructions that might cause an "illegal
-	instruction" error on older CPUs.
-
+- `build-aarch64-portable`: builds a version for 64-bit ARM processors which avoids using some
+  modern CPU instructions. In practice, very few ARM processors lack the instructions necessary to
+  run the faster non-portable build.
 
 ### Example
 

book/src/intro.md

@@ -2,12 +2,10 @@
 
 _Documentation for Lighthouse users and developers._
 
-[![Doc Status]][Doc Link] [![Chat Badge]][Chat Link]
+[![Chat Badge]][Chat Link]
 
 [Chat Badge]: https://img.shields.io/badge/chat-discord-%237289da
 [Chat Link]: https://discord.gg/cyAszAh
-[Doc Status]:https://img.shields.io/badge/rust--docs-master-orange
-[Doc Link]: http://lighthouse-docs.sigmaprime.io/
 
 Lighthouse is an **Ethereum 2.0 client** that connects to other Ethereum 2.0
 clients to form a resilient and decentralized proof-of-stake blockchain.
@@ -24,7 +22,6 @@ You may read this book from start to finish, or jump to some of these topics:
 - Utilize the whole stack by starting a [local testnet](./local-testnets.md).
 - Query the [RESTful HTTP API](./http.md) using `curl`.
 - Listen to events with the [JSON WebSocket API](./websockets.md).
-- View the [Rust code docs](http://lighthouse-docs.sigmaprime.io/).
 
 
 Prospective contributors can read the [Contributing](./contributing.md) section

book/src/key-management.md

@@ -35,16 +35,16 @@ items, starting at one easy-to-backup mnemonic and ending with multiple
 keypairs. Creating a single validator looks like this:
 
 1. Create a **wallet** and record the **mnemonic**:
-    - `lighthouse account wallet create --name wally --passphrase-file wally.pass`
+    - `lighthouse account wallet create --name wally --password-file wally.pass`
 1. Create the voting and withdrawal **keystores** for one validator:
-	- `lighthouse account validator create --wallet-name wally --wallet-passphrase wally.pass --count 1`
+	- `lighthouse account validator create --wallet-name wally --wallet-password wally.pass --count 1`
 
 
 In step (1), we created a wallet in `~/.lighthouse/wallets` with the name
 `wally`. We encrypted this using a pre-defined password in the
 `wally.pass` file. Then, in step (2), we created one new validator in the
 `~/.lighthouse/validators` directory using `wally` (unlocking it with
-`mywallet.pass`) and storing the passwords to the validators voting key in
+`wally.pass`) and storing the passwords to the validators voting key in
 `~/.lighthouse/secrets`.
 
 Thanks to the hierarchical key derivation scheme, we can delete all of the

book/src/key-recovery.md

@@ -0,0 +1,65 @@
+# Key recovery
+
+
+Generally, validator keystore files are generated alongside a *mnemonic*. If
+the keystore and/or the keystore password are lost this mnemonic can
+regenerate a new, equivalent keystore with a new password.
+
+There are two ways to recover keys using the `lighthouse` CLI:
+
+- `lighthouse account validator recover`: recover one or more EIP-2335 keystores from a mnemonic.
+	These keys can be used directly in a validator client.
+- `lighthouse account wallet recover`: recover an EIP-2386 wallet from a
+	mnemonic.
+
+## ⚠️ Warning
+
+**Recovering validator keys from a mnemonic should only be used as a last
+resort.** Key recovery entails significant risks:
+
+- Exposing your mnemonic to a computer at any time puts it at risk of being
+	compromised. Your mnemonic is **not encrypted** and is a target for theft.
+- It's completely possible to regenerate a validator keypairs that is already active
+	on some other validator client. Running the same keypairs on two different
+	validator clients is very likely to result in slashing.
+
+## Recover EIP-2335 validator keystores
+
+A single mnemonic can generate a practically unlimited number of validator
+keystores using an *index*. Generally, the first time you generate a keystore
+you'll use index 0, the next time you'll use index 1, and so on. Using the same
+index on the same mnemonic always results in the same validator keypair being
+generated (see [EIP-2334](https://eips.ethereum.org/EIPS/eip-2334) for more
+detail).
+
+
+Using the `lighthouse account validator recover` command you can generate the
+keystores that correspond to one or more indices in the mnemonic:
+
+- `lighthouse account validator recover`: recover only index `0`.
+- `lighthouse account validator recover --count 2`: recover indices `0, 1`.
+- `lighthouse account validator recover --first-index 1`: recover only index `1`.
+- `lighthouse account validator recover --first-index 1 --count 2`: recover indices `1, 2`.
+
+
+For each of the indices recovered in the above commands, a directory will be
+created in the `--validator-dir` location (default `~/.lighthouse/validator`)
+which contains all the information necessary to run a validator using the
+`lighthouse vc` command. The password to this new keystore will be placed in
+the `--secrets-dir` (default `~/.lighthouse/secrets`).
+
+## Recover a EIP-2386 wallet
+
+Instead of creating EIP-2335 keystores directly, an EIP-2386 wallet can be
+generated from the mnemonic. This wallet can then be used to generate validator
+keystores, if desired. For example, the following command will create an
+encrypted wallet named `wally-recovered` from a mnemonic:
+
+```
+lighthouse account wallet recover --name wally-recovered
+```
+
+**⚠️ Warning:** the wallet will be created with a `nextaccount` value of `0`.
+This means that if you have already generated `n` validators, then the next `n`
+validators generated by this wallet will be duplicates. As mentioned
+previously, running duplicate validators is likely to result in slashing.