Bump to v0.2.7 (#1561)

## Issue Addressed

NA

## Proposed Changes

- Update to v0.2.7
- Add script to make update easy.

## Additional Info

NA

.dockerignore

@@ -2,4 +2,3 @@ tests/ef_tests/eth2.0-spec-tests
 target/
 *.data
 *.tar.gz
-.git

.gitignore

@@ -6,3 +6,4 @@ target/
 flamegraph.svg
 perf.data*
 *.tar.gz
+bin/

Cargo.toml

@@ -28,6 +28,7 @@ members = [
     "common/lighthouse_metrics",
     "common/lighthouse_version",
     "common/logging",
+    "common/lru_cache",
     "common/remote_beacon_node",
     "common/rest_types",
     "common/slot_clock",

Cross.toml

@@ -0,0 +1,4 @@
+[build.env]
+passthrough = [
+    "RUSTFLAGS",
+]

Makefile

@@ -2,6 +2,13 @@
 
 EF_TESTS = "testing/ef_tests"
 STATE_TRANSITION_VECTORS = "testing/state_transition_vectors"
+GIT_TAG := $(shell git describe --tags --candidates 1)
+BIN_DIR = "bin"
+
+X86_64_TAG = "x86_64-unknown-linux-gnu"
+BUILD_PATH_X86_64 = "target/$(X86_64_TAG)/release"
+AARCH64_TAG = "aarch64-unknown-linux-gnu"
+BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
 
 # Builds the Lighthouse binary in release (optimized).
 #
@@ -21,6 +28,53 @@ else
 	cargo install --path lcli --force --locked
 endif
 
+# The following commands use `cross` to build a cross-compile.
+#
+# These commands require that:
+#
+# - `cross` is installed (`cargo install cross`).
+# - Docker is running.
+# - The current user is in the `docker` group.
+#
+# The resulting binaries will be created in the `target/` directory.
+#
+# The *-portable options compile the blst library *without* the use of some
+# optimized CPU functions that may not be available on some systems. This
+# results in a more portable binary with ~20% slower BLS verification.
+build-x86_64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu
+build-x86_64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features portable
+build-aarch64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu
+build-aarch64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu --features portable
+
+# Create a `.tar.gz` containing a binary for a specific target.
+define tarball_release_binary
+	cp $(1)/lighthouse $(BIN_DIR)/lighthouse
+	cd $(BIN_DIR) && \
+		tar -czf lighthouse-$(GIT_TAG)-$(2)$(3).tar.gz lighthouse && \
+		rm lighthouse
+endef
+
+# Create a series of `.tar.gz` files in the BIN_DIR directory, each containing
+# a `lighthouse` binary for a different target.
+#
+# The current git tag will be used as the version in the output file names. You
+# will likely need to use `git tag` and create a semver tag (e.g., `v0.2.3`).
+build-release-tarballs:
+	[ -d $(BIN_DIR) ] || mkdir -p $(BIN_DIR)
+	$(MAKE) build-x86_64
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"")
+	$(MAKE) build-x86_64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"-portable")
+	$(MAKE) build-aarch64
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"")
+	$(MAKE) build-aarch64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"-portable")
+
+
 # Runs the full workspace tests in **release**, without downloading any additional
 # test vectors.
 test-release:

README.md

@@ -22,8 +22,7 @@ An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prim
 Lighthouse is:
 
 - Fully open-source, licensed under Apache 2.0.
-- Security-focused. Fuzzing has begun and security reviews are planned
-	for late-2019.
+- Security-focused. Fuzzing has begun and security reviews are underway.
 - Built in [Rust](https://www.rust-lang.org/), a modern language providing unique safety guarantees and
 	excellent performance (comparable to C++).
 - Funded by various organisations, including Sigma Prime, the

account_manager/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "account_manager"
-version = "0.0.1"
+version = "0.2.7"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
 edition = "2018"
 

beacon_node/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "beacon_node"
-version = "0.1.2"
-authors = ["Paul Hauner <paul@paulhauner.com>", "Sigma Prime <contact@sigmaprime.io>"]
+version = "0.2.7"
+authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
 edition = "2018"
 
 [lib]

beacon_node/beacon_chain/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "beacon_chain"
-version = "0.1.2"
-authors = ["Paul Hauner <paul@paulhauner.com>"]
+version = "0.2.0"
+authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com>"]
 edition = "2018"
 
 [features]

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -220,6 +220,12 @@ pub enum Error {
     ///
     /// The peer has sent an invalid message.
     Invalid(AttestationValidationError),
+    /// The attestation head block is too far behind the attestation slot, causing many skip slots.
+    /// This is deemed a DoS risk.
+    TooManySkippedSlots {
+        head_block_slot: Slot,
+        attestation_slot: Slot,
+    },
     /// There was an error whilst processing the attestation. It is not known if it is valid or invalid.
     ///
     /// ## Peer scoring
@@ -319,6 +325,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         }?;
 
         // Ensure the block being voted for (attestation.data.beacon_block_root) passes validation.
+        // Don't enforce the skip slot restriction for aggregates.
         //
         // This indirectly checks to see if the `attestation.data.beacon_block_root` is in our fork
         // choice. Any known, non-finalized, processed block should be in fork choice, so this
@@ -327,7 +334,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         //
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
-        verify_head_block_is_known(chain, &attestation)?;
+        verify_head_block_is_known(chain, &attestation, None)?;
 
         // Ensure that the attestation has participants.
         if attestation.aggregation_bits.is_zero() {
@@ -433,7 +440,9 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
-        verify_head_block_is_known(chain, &attestation)?;
+        //
+        // Enforce a maximum skip distance for unaggregated attestations.
+        verify_head_block_is_known(chain, &attestation, chain.config.import_max_skip_slots)?;
 
         let (indexed_attestation, committees_per_slot) =
             obtain_indexed_attestation_and_committees_per_slot(chain, &attestation)?;
@@ -531,12 +540,22 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 fn verify_head_block_is_known<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
+    max_skip_slots: Option<u64>,
 ) -> Result<(), Error> {
-    if chain
+    if let Some(block) = chain
         .fork_choice
         .read()
-        .contains_block(&attestation.data.beacon_block_root)
+        .get_block(&attestation.data.beacon_block_root)
     {
+        // Reject any block that exceeds our limit on skipped slots.
+        if let Some(max_skip_slots) = max_skip_slots {
+            if attestation.data.slot > block.slot + max_skip_slots {
+                return Err(Error::TooManySkippedSlots {
+                    head_block_slot: block.slot,
+                    attestation_slot: attestation.data.slot,
+                });
+            }
+        }
         Ok(())
     } else {
         Err(Error::UnknownHeadBlock {
@@ -775,7 +794,12 @@ where
             metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_READ_TIMES);
 
         let mut state = chain
-            .get_state(&target_block.state_root, Some(target_block.slot))?
+            .store
+            .get_inconsistent_state_for_attestation_verification_only(
+                &target_block.state_root,
+                Some(target_block.slot),
+            )
+            .map_err(BeaconChainError::from)?
             .ok_or_else(|| BeaconChainError::MissingBeaconState(target_block.state_root))?;
 
         metrics::stop_timer(state_read_timer);

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -3,9 +3,11 @@ use crate::attestation_verification::{
     VerifiedUnaggregatedAttestation,
 };
 use crate::block_verification::{
-    check_block_relevancy, get_block_root, signature_verify_chain_segment, BlockError,
-    FullyVerifiedBlock, GossipVerifiedBlock, IntoFullyVerifiedBlock,
+    check_block_is_finalized_descendant, check_block_relevancy, get_block_root,
+    signature_verify_chain_segment, BlockError, FullyVerifiedBlock, GossipVerifiedBlock,
+    IntoFullyVerifiedBlock,
 };
+use crate::chain_config::ChainConfig;
 use crate::errors::{BeaconChainError as Error, BlockProductionError};
 use crate::eth1_chain::{Eth1Chain, Eth1ChainBackend};
 use crate::events::{EventHandler, EventKind};
@@ -70,15 +72,14 @@ pub const ETH1_CACHE_DB_KEY: [u8; 32] = [0; 32];
 pub const FORK_CHOICE_DB_KEY: [u8; 32] = [0; 32];
 
 /// The result of a chain segment processing.
-#[derive(Debug)]
-pub enum ChainSegmentResult {
+pub enum ChainSegmentResult<T: EthSpec> {
     /// Processing this chain segment finished successfully.
     Successful { imported_blocks: usize },
     /// There was an error processing this chain segment. Before the error, some blocks could
     /// have been imported.
     Failed {
         imported_blocks: usize,
-        error: BlockError,
+        error: BlockError<T>,
     },
 }
 
@@ -161,6 +162,8 @@ pub trait BeaconChainTypes: Send + Sync + 'static {
 /// operations and chooses a canonical head.
 pub struct BeaconChain<T: BeaconChainTypes> {
     pub spec: ChainSpec,
+    /// Configuration for `BeaconChain` runtime behaviour.
+    pub config: ChainConfig,
     /// Persistent storage for blocks, states, etc. Typically an on-disk store, such as LevelDB.
     pub store: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
     /// Database migrator for running background maintenance on the store.
@@ -175,7 +178,7 @@ pub struct BeaconChain<T: BeaconChainTypes> {
     ///
     /// This pool accepts `Attestation` objects that only have one aggregation bit set and provides
     /// a method to get an aggregated `Attestation` for some `AttestationData`.
-    pub naive_aggregation_pool: NaiveAggregationPool<T::EthSpec>,
+    pub naive_aggregation_pool: RwLock<NaiveAggregationPool<T::EthSpec>>,
     /// Contains a store of attestations which have been observed by the beacon chain.
     pub observed_attestations: ObservedAttestations<T::EthSpec>,
     /// Maintains a record of which validators have been seen to attest in recent epochs.
@@ -747,7 +750,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         &self,
         data: &AttestationData,
     ) -> Result<Option<Attestation<T::EthSpec>>, Error> {
-        self.naive_aggregation_pool.get(data).map_err(Into::into)
+        self.naive_aggregation_pool
+            .read()
+            .get(data)
+            .map_err(Into::into)
     }
 
     /// Produce an unaggregated `Attestation` that is valid for the given `slot` and `index`.
@@ -937,7 +943,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         let attestation = unaggregated_attestation.attestation();
 
-        match self.naive_aggregation_pool.insert(attestation) {
+        match self.naive_aggregation_pool.write().insert(attestation) {
             Ok(outcome) => trace!(
                 self.log,
                 "Stored unaggregated attestation";
@@ -1153,7 +1159,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     pub fn process_chain_segment(
         &self,
         chain_segment: Vec<SignedBeaconBlock<T::EthSpec>>,
-    ) -> ChainSegmentResult {
+    ) -> ChainSegmentResult<T::EthSpec> {
         let mut filtered_chain_segment = Vec::with_capacity(chain_segment.len());
         let mut imported_blocks = 0;
 
@@ -1212,6 +1218,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 // However, we will potentially get a `ParentUnknown` on a later block. The sync
                 // protocol will need to ensure this is handled gracefully.
                 Err(BlockError::WouldRevertFinalizedSlot { .. }) => continue,
+                // The block has a known parent that does not descend from the finalized block.
+                // There is no need to process this block or any children.
+                Err(BlockError::NotFinalizedDescendant { block_parent_root }) => {
+                    return ChainSegmentResult::Failed {
+                        imported_blocks,
+                        error: BlockError::NotFinalizedDescendant { block_parent_root },
+                    }
+                }
                 // If there was an error whilst determining if the block was invalid, return that
                 // error.
                 Err(BlockError::BeaconChainError(e)) => {
@@ -1286,7 +1300,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     pub fn verify_block_for_gossip(
         &self,
         block: SignedBeaconBlock<T::EthSpec>,
-    ) -> Result<GossipVerifiedBlock<T>, BlockError> {
+    ) -> Result<GossipVerifiedBlock<T>, BlockError<T::EthSpec>> {
         let slot = block.message.slot;
         let graffiti_string = String::from_utf8(block.message.body.graffiti[..].to_vec())
             .unwrap_or_else(|_| format!("{:?}", &block.message.body.graffiti[..]));
@@ -1307,7 +1321,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 debug!(
                     self.log,
                     "Rejected gossip block";
-                    "error" => format!("{:?}", e),
+                    "error" => e.to_string(),
                     "graffiti" => graffiti_string,
                     "slot" => slot,
                 );
@@ -1332,7 +1346,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     pub fn process_block<B: IntoFullyVerifiedBlock<T>>(
         &self,
         unverified_block: B,
-    ) -> Result<Hash256, BlockError> {
+    ) -> Result<Hash256, BlockError<T::EthSpec>> {
         // Start the Prometheus timer.
         let _full_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_TIMES);
 
@@ -1343,7 +1357,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let block = unverified_block.block().clone();
 
         // A small closure to group the verification and import errors.
-        let import_block = |unverified_block: B| -> Result<Hash256, BlockError> {
+        let import_block = |unverified_block: B| -> Result<Hash256, BlockError<T::EthSpec>> {
             let fully_verified = unverified_block.into_fully_verified_block(self)?;
             self.import_block(fully_verified)
         };
@@ -1390,11 +1404,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 trace!(
                     self.log,
                     "Beacon block rejected";
-                    "reason" => format!("{:?}", other),
+                    "reason" => other.to_string(),
                 );
 
                 let _ = self.event_handler.register(EventKind::BeaconBlockRejected {
-                    reason: format!("Invalid block: {:?}", other),
+                    reason: format!("Invalid block: {}", other),
                     block: Box::new(block),
                 });
 
@@ -1411,9 +1425,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     fn import_block(
         &self,
         fully_verified_block: FullyVerifiedBlock<T>,
-    ) -> Result<Hash256, BlockError> {
+    ) -> Result<Hash256, BlockError<T::EthSpec>> {
         let signed_block = fully_verified_block.block;
-        let block = &signed_block.message;
         let block_root = fully_verified_block.block_root;
         let state = fully_verified_block.state;
         let parent_block = fully_verified_block.parent_block;
@@ -1425,7 +1438,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         // Iterate through the attestations in the block and register them as an "observed
         // attestation". This will stop us from propagating them on the gossip network.
-        for a in &block.body.attestations {
+        for a in &signed_block.message.body.attestations {
             match self.observed_attestations.observe_attestation(a, None) {
                 // If the observation was successful or if the slot for the attestation was too
                 // low, continue.
@@ -1475,6 +1488,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         let mut fork_choice = self.fork_choice.write();
 
+        // Do not import a block that doesn't descend from the finalized root.
+        let signed_block =
+            check_block_is_finalized_descendant::<T, _>(signed_block, &fork_choice, &self.store)?;
+        let block = &signed_block.message;
+
         // Register the new block with the fork choice service.
         {
             let _fork_choice_block_timer =
@@ -1562,12 +1580,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         &self,
         randao_reveal: Signature,
         slot: Slot,
+        validator_graffiti: Option<Graffiti>,
     ) -> Result<BeaconBlockAndState<T::EthSpec>, BlockProductionError> {
         let state = self
             .state_at_slot(slot - 1, StateSkipConfig::WithStateRoots)
             .map_err(|_| BlockProductionError::UnableToProduceAtSlot(slot))?;
 
-        self.produce_block_on_state(state, slot, randao_reveal)
+        self.produce_block_on_state(state, slot, randao_reveal, validator_graffiti)
     }
 
     /// Produce a block for some `slot` upon the given `state`.
@@ -1583,6 +1602,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         mut state: BeaconState<T::EthSpec>,
         produce_at_slot: Slot,
         randao_reveal: Signature,
+        validator_graffiti: Option<Graffiti>,
     ) -> Result<BeaconBlockAndState<T::EthSpec>, BlockProductionError> {
         metrics::inc_counter(&metrics::BLOCK_PRODUCTION_REQUESTS);
         let timer = metrics::start_timer(&metrics::BLOCK_PRODUCTION_TIMES);
@@ -1632,6 +1652,30 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 })
         };
 
+        // Iterate through the naive aggregation pool and ensure all the attestations from there
+        // are included in the operation pool.
+        for attestation in self.naive_aggregation_pool.read().iter() {
+            if let Err(e) = self.op_pool.insert_attestation(
+                attestation.clone(),
+                &state.fork,
+                state.genesis_validators_root,
+                &self.spec,
+            ) {
+                // Don't stop block production if there's an error, just create a log.
+                error!(
+                    self.log,
+                    "Attestation did not transfer to op pool";
+                    "reason" => format!("{:?}", e)
+                );
+            }
+        }
+
+        // Override the beacon node's graffiti with graffiti from the validator, if present.
+        let graffiti = match validator_graffiti {
+            Some(graffiti) => graffiti,
+            None => self.graffiti,
+        };
+
         let mut block = SignedBeaconBlock {
             message: BeaconBlock {
                 slot: state.slot,
@@ -1641,7 +1685,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 body: BeaconBlockBody {
                     randao_reveal,
                     eth1_data,
-                    graffiti: self.graffiti,
+                    graffiti,
                     proposer_slashings: proposer_slashings.into(),
                     attester_slashings: attester_slashings.into(),
                     attestations: self
@@ -1852,7 +1896,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     pub fn per_slot_task(&self) {
         trace!(self.log, "Running beacon chain per slot tasks");
         if let Some(slot) = self.slot_clock.now() {
-            self.naive_aggregation_pool.prune(slot);
+            self.naive_aggregation_pool.write().prune(slot);
         }
     }
 
@@ -2133,8 +2177,8 @@ impl From<BeaconStateError> for Error {
     }
 }
 
-impl ChainSegmentResult {
-    pub fn into_block_error(self) -> Result<(), BlockError> {
+impl<T: EthSpec> ChainSegmentResult<T> {
+    pub fn into_block_error(self) -> Result<(), BlockError<T>> {
         match self {
             ChainSegmentResult::Failed { error, .. } => Err(error),
             ChainSegmentResult::Successful { .. } => Ok(()),

beacon_node/beacon_chain/src/block_verification.rs

@@ -48,6 +48,7 @@ use crate::{
     },
     metrics, BeaconChain, BeaconChainError, BeaconChainTypes, BeaconSnapshot,
 };
+use fork_choice::{ForkChoice, ForkChoiceStore};
 use parking_lot::RwLockReadGuard;
 use slog::{error, Logger};
 use slot_clock::SlotClock;
@@ -62,7 +63,7 @@ use std::borrow::Cow;
 use std::convert::TryFrom;
 use std::fs;
 use std::io::Write;
-use store::{Error as DBError, HotStateSummary, StoreOp};
+use store::{Error as DBError, HotColdDB, HotStateSummary, StoreOp};
 use tree_hash::TreeHash;
 use types::{
     BeaconBlock, BeaconState, BeaconStateError, ChainSpec, CloneConfig, EthSpec, Hash256,
@@ -83,14 +84,16 @@ const WRITE_BLOCK_PROCESSING_SSZ: bool = cfg!(feature = "write_ssz_files");
 /// - The block is malformed/invalid (indicated by all results other than `BeaconChainError`.
 /// - We encountered an error whilst trying to verify the block (a `BeaconChainError`).
 #[derive(Debug)]
-pub enum BlockError {
+pub enum BlockError<T: EthSpec> {
     /// The parent block was unknown.
     ///
     /// ## Peer scoring
     ///
     /// It's unclear if this block is valid, but it cannot be processed without already knowing
     /// its parent.
-    ParentUnknown(Hash256),
+    ParentUnknown(Box<SignedBeaconBlock<T>>),
+    /// The block skips too many slots and is a DoS risk.
+    TooManySkippedSlots { parent_slot: Slot, block_slot: Slot },
     /// The block slot is greater than the present slot.
     ///
     /// ## Peer scoring
@@ -118,6 +121,13 @@ pub enum BlockError {
         block_slot: Slot,
         finalized_slot: Slot,
     },
+    /// The block conflicts with finalization, no need to propagate.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this block is valid, but it conflicts with finality and shouldn't be
+    /// imported.
+    NotFinalizedDescendant { block_parent_root: Hash256 },
     /// Block is already known, no need to re-import.
     ///
     /// ## Peer scoring
@@ -199,7 +209,18 @@ pub enum BlockError {
     BeaconChainError(BeaconChainError),
 }
 
-impl From<BlockSignatureVerifierError> for BlockError {
+impl<T: EthSpec> std::fmt::Display for BlockError<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BlockError::ParentUnknown(block) => {
+                write!(f, "ParentUnknown(parent_root:{})", block.parent_root())
+            }
+            other => write!(f, "{:?}", other),
+        }
+    }
+}
+
+impl<T: EthSpec> From<BlockSignatureVerifierError> for BlockError<T> {
     fn from(e: BlockSignatureVerifierError) -> Self {
         match e {
             // Make a special distinction for `IncorrectBlockProposer` since it indicates an
@@ -216,25 +237,25 @@ impl From<BlockSignatureVerifierError> for BlockError {
     }
 }
 
-impl From<BeaconChainError> for BlockError {
+impl<T: EthSpec> From<BeaconChainError> for BlockError<T> {
     fn from(e: BeaconChainError) -> Self {
         BlockError::BeaconChainError(e)
     }
 }
 
-impl From<BeaconStateError> for BlockError {
+impl<T: EthSpec> From<BeaconStateError> for BlockError<T> {
     fn from(e: BeaconStateError) -> Self {
         BlockError::BeaconChainError(BeaconChainError::BeaconStateError(e))
     }
 }
 
-impl From<SlotProcessingError> for BlockError {
+impl<T: EthSpec> From<SlotProcessingError> for BlockError<T> {
     fn from(e: SlotProcessingError) -> Self {
         BlockError::BeaconChainError(BeaconChainError::SlotProcessingError(e))
     }
 }
 
-impl From<DBError> for BlockError {
+impl<T: EthSpec> From<DBError> for BlockError<T> {
     fn from(e: DBError) -> Self {
         BlockError::BeaconChainError(BeaconChainError::DBError(e))
     }
@@ -251,15 +272,17 @@ impl From<DBError> for BlockError {
 /// The given `chain_segment` must span no more than two epochs, otherwise an error will be
 /// returned.
 pub fn signature_verify_chain_segment<T: BeaconChainTypes>(
-    chain_segment: Vec<(Hash256, SignedBeaconBlock<T::EthSpec>)>,
+    mut chain_segment: Vec<(Hash256, SignedBeaconBlock<T::EthSpec>)>,
     chain: &BeaconChain<T>,
-) -> Result<Vec<SignatureVerifiedBlock<T>>, BlockError> {
-    let (mut parent, slot) = if let Some(block) = chain_segment.first().map(|(_, block)| block) {
-        let parent = load_parent(&block.message, chain)?;
-        (parent, block.slot())
-    } else {
+) -> Result<Vec<SignatureVerifiedBlock<T>>, BlockError<T::EthSpec>> {
+    if chain_segment.is_empty() {
         return Ok(vec![]);
-    };
+    }
+
+    let (first_root, first_block) = chain_segment.remove(0);
+    let (mut parent, first_block) = load_parent(first_block, chain)?;
+    let slot = first_block.slot();
+    chain_segment.insert(0, (first_root, first_block));
 
     let highest_slot = chain_segment
         .last()
@@ -343,7 +366,7 @@ pub trait IntoFullyVerifiedBlock<T: BeaconChainTypes> {
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError>;
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>>;
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec>;
 }
@@ -356,7 +379,7 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
     pub fn new(
         block: SignedBeaconBlock<T::EthSpec>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, BlockError> {
+    ) -> Result<Self, BlockError<T::EthSpec>> {
         // Do not gossip or process blocks from future slots.
         let present_slot_with_tolerance = chain
             .slot_clock
@@ -369,9 +392,22 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
             });
         }
 
+        let block_root = get_block_root(&block);
+
         // Do not gossip a block from a finalized slot.
         check_block_against_finalized_slot(&block.message, chain)?;
 
+        // Check if the block is already known. We know it is post-finalization, so it is
+        // sufficient to check the fork choice.
+        //
+        // In normal operation this isn't necessary, however it is useful immediately after a
+        // reboot if the `observed_block_producers` cache is empty. In that case, without this
+        // check, we will load the parent and state from disk only to find out later that we
+        // already know this block.
+        if chain.fork_choice.read().contains_block(&block_root) {
+            return Err(BlockError::BlockIsAlreadyKnown);
+        }
+
         // Check that we have not already received a block with a valid signature for this slot.
         if chain
             .observed_block_producers
@@ -384,8 +420,19 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
             });
         }
 
-        let mut parent = load_parent(&block.message, chain)?;
-        let block_root = get_block_root(&block);
+        // Do not process a block that doesn't descend from the finalized root.
+        //
+        // We check this *before* we load the parent so that we can return a more detailed error.
+        let block = check_block_is_finalized_descendant::<T, _>(
+            block,
+            &chain.fork_choice.read(),
+            &chain.store,
+        )?;
+
+        let (mut parent, block) = load_parent(block, chain)?;
+
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
 
         let state = cheap_state_advance_to_obtain_committees(
             &mut parent.beacon_state,
@@ -453,7 +500,7 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for GossipVerifiedBlock<T> {
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError> {
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
         let fully_verified = SignatureVerifiedBlock::from_gossip_verified_block(self, chain)?;
         fully_verified.into_fully_verified_block(chain)
     }
@@ -471,8 +518,12 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
     pub fn new(
         block: SignedBeaconBlock<T::EthSpec>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, BlockError> {
-        let mut parent = load_parent(&block.message, chain)?;
+    ) -> Result<Self, BlockError<T::EthSpec>> {
+        let (mut parent, block) = load_parent(block, chain)?;
+
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
+
         let block_root = get_block_root(&block);
 
         let state = cheap_state_advance_to_obtain_committees(
@@ -503,7 +554,7 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
     pub fn from_gossip_verified_block(
         from: GossipVerifiedBlock<T>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, BlockError> {
+    ) -> Result<Self, BlockError<T::EthSpec>> {
         let mut parent = from.parent;
         let block = from.block;
 
@@ -536,12 +587,12 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignatureVerifiedBlock<T
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError> {
-        let block = self.block;
-        let parent = self
-            .parent
-            .map(Result::Ok)
-            .unwrap_or_else(|| load_parent(&block.message, chain))?;
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
+        let (parent, block) = if let Some(parent) = self.parent {
+            (parent, self.block)
+        } else {
+            load_parent(self.block, chain)?
+        };
 
         FullyVerifiedBlock::from_signature_verified_components(
             block,
@@ -562,7 +613,7 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignedBeaconBlock<T::Eth
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError> {
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
         SignatureVerifiedBlock::new(self, chain)?.into_fully_verified_block(chain)
     }
 
@@ -584,7 +635,7 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
         block_root: Hash256,
         parent: BeaconSnapshot<T::EthSpec>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, BlockError> {
+    ) -> Result<Self, BlockError<T::EthSpec>> {
         // Reject any block if its parent is not known to fork choice.
         //
         // A block that is not in fork choice is either:
@@ -600,9 +651,12 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
             .read()
             .contains_block(&block.parent_root())
         {
-            return Err(BlockError::ParentUnknown(block.parent_root()));
+            return Err(BlockError::ParentUnknown(Box::new(block)));
         }
 
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
+
         /*
          *  Perform cursory checks to see if the block is even worth processing.
          */
@@ -642,7 +696,10 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
                 let state_root = state.update_tree_hash_cache()?;
 
                 let op = if state.slot % T::EthSpec::slots_per_epoch() == 0 {
-                    StoreOp::PutState(state_root.into(), Cow::Owned(state.clone()))
+                    StoreOp::PutState(
+                        state_root.into(),
+                        Cow::Owned(state.clone_with(CloneConfig::committee_caches_only())),
+                    )
                 } else {
                     StoreOp::PutStateSummary(
                         state_root.into(),
@@ -742,6 +799,30 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
     }
 }
 
+/// Check that the count of skip slots between the block and its parent does not exceed our maximum
+/// value.
+///
+/// Whilst this is not part of the specification, we include this to help prevent us from DoS
+/// attacks. In times of dire network circumstance, the user can configure the
+/// `import_max_skip_slots` value.
+fn check_block_skip_slots<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    parent: &BeaconBlock<T::EthSpec>,
+    block: &BeaconBlock<T::EthSpec>,
+) -> Result<(), BlockError<T::EthSpec>> {
+    // Reject any block that exceeds our limit on skipped slots.
+    if let Some(max_skip_slots) = chain.config.import_max_skip_slots {
+        if block.slot > parent.slot + max_skip_slots {
+            return Err(BlockError::TooManySkippedSlots {
+                parent_slot: parent.slot,
+                block_slot: block.slot,
+            });
+        }
+    }
+
+    Ok(())
+}
+
 /// Returns `Ok(())` if the block is later than the finalized slot on `chain`.
 ///
 /// Returns an error if the block is earlier or equal to the finalized slot, or there was an error
@@ -749,7 +830,7 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
 fn check_block_against_finalized_slot<T: BeaconChainTypes>(
     block: &BeaconBlock<T::EthSpec>,
     chain: &BeaconChain<T>,
-) -> Result<(), BlockError> {
+) -> Result<(), BlockError<T::EthSpec>> {
     let finalized_slot = chain
         .head_info()?
         .finalized_checkpoint
@@ -766,6 +847,36 @@ fn check_block_against_finalized_slot<T: BeaconChainTypes>(
     }
 }
 
+/// Returns `Ok(block)` if the block descends from the finalized root.
+pub fn check_block_is_finalized_descendant<T: BeaconChainTypes, F: ForkChoiceStore<T::EthSpec>>(
+    block: SignedBeaconBlock<T::EthSpec>,
+    fork_choice: &ForkChoice<F, T::EthSpec>,
+    store: &HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>,
+) -> Result<SignedBeaconBlock<T::EthSpec>, BlockError<T::EthSpec>> {
+    if fork_choice.is_descendant_of_finalized(block.parent_root()) {
+        Ok(block)
+    } else {
+        // If fork choice does *not* consider the parent to be a descendant of the finalized block,
+        // then there are two more cases:
+        //
+        // 1. We have the parent stored in our database. Because fork-choice has confirmed the
+        //    parent is *not* in our post-finalization DAG, all other blocks must be either
+        //    pre-finalization or conflicting with finalization.
+        // 2. The parent is unknown to us, we probably want to download it since it might actually
+        //    descend from the finalized root.
+        if store
+            .item_exists::<SignedBeaconBlock<T::EthSpec>>(&block.parent_root())
+            .map_err(|e| BlockError::BeaconChainError(e.into()))?
+        {
+            Err(BlockError::NotFinalizedDescendant {
+                block_parent_root: block.parent_root(),
+            })
+        } else {
+            Err(BlockError::ParentUnknown(Box::new(block)))
+        }
+    }
+}
+
 /// Performs simple, cheap checks to ensure that the block is relevant to be imported.
 ///
 /// `Ok(block_root)` is returned if the block passes these checks and should progress with
@@ -777,7 +888,7 @@ pub fn check_block_relevancy<T: BeaconChainTypes>(
     signed_block: &SignedBeaconBlock<T::EthSpec>,
     block_root: Option<Hash256>,
     chain: &BeaconChain<T>,
-) -> Result<Hash256, BlockError> {
+) -> Result<Hash256, BlockError<T::EthSpec>> {
     let block = &signed_block.message;
 
     // Do not process blocks from the future.
@@ -830,12 +941,11 @@ pub fn get_block_root<E: EthSpec>(block: &SignedBeaconBlock<E>) -> Hash256 {
 ///
 /// Returns `Err(BlockError::ParentUnknown)` if the parent is not found, or if an error occurs
 /// whilst attempting the operation.
+#[allow(clippy::type_complexity)]
 fn load_parent<T: BeaconChainTypes>(
-    block: &BeaconBlock<T::EthSpec>,
+    block: SignedBeaconBlock<T::EthSpec>,
     chain: &BeaconChain<T>,
-) -> Result<BeaconSnapshot<T::EthSpec>, BlockError> {
-    let db_read_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_READ);
-
+) -> Result<(BeaconSnapshot<T::EthSpec>, SignedBeaconBlock<T::EthSpec>), BlockError<T::EthSpec>> {
     // Reject any block if its parent is not known to fork choice.
     //
     // A block that is not in fork choice is either:
@@ -846,50 +956,58 @@ fn load_parent<T: BeaconChainTypes>(
     //  because it will revert finalization. Note that the finalized block is stored in fork
     //  choice, so we will not reject any child of the finalized block (this is relevant during
     //  genesis).
-    if !chain.fork_choice.read().contains_block(&block.parent_root) {
-        return Err(BlockError::ParentUnknown(block.parent_root));
+    if !chain
+        .fork_choice
+        .read()
+        .contains_block(&block.parent_root())
+    {
+        return Err(BlockError::ParentUnknown(Box::new(block)));
     }
 
-    // Load the parent block and state from disk, returning early if it's not available.
-    let result = chain
+    let db_read_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_READ);
+
+    let result = if let Some(snapshot) = chain
         .snapshot_cache
         .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
-        .and_then(|mut snapshot_cache| snapshot_cache.try_remove(block.parent_root))
-        .map(|snapshot| Ok(Some(snapshot)))
-        .unwrap_or_else(|| {
-            // Load the blocks parent block from the database, returning invalid if that block is not
-            // found.
-            //
-            // We don't return a DBInconsistent error here since it's possible for a block to
-            // exist in fork choice but not in the database yet. In such a case we simply
-            // indicate that we don't yet know the parent.
-            let parent_block = if let Some(block) = chain.get_block(&block.parent_root)? {
-                block
-            } else {
-                return Ok(None);
-            };
+        .and_then(|mut snapshot_cache| snapshot_cache.try_remove(block.parent_root()))
+    {
+        Ok((snapshot, block))
+    } else {
+        // Load the blocks parent block from the database, returning invalid if that block is not
+        // found.
+        //
+        // We don't return a DBInconsistent error here since it's possible for a block to
+        // exist in fork choice but not in the database yet. In such a case we simply
+        // indicate that we don't yet know the parent.
+        let root = block.parent_root();
+        let parent_block = if let Some(block) = chain
+            .get_block(&block.parent_root())
+            .map_err(BlockError::BeaconChainError)?
+        {
+            block
+        } else {
+            return Err(BlockError::ParentUnknown(Box::new(block)));
+        };
 
-            // Load the parent blocks state from the database, returning an error if it is not found.
-            // It is an error because if we know the parent block we should also know the parent state.
-            let parent_state_root = parent_block.state_root();
-            let parent_state = chain
-                .get_state(&parent_state_root, Some(parent_block.slot()))?
-                .ok_or_else(|| {
-                    BeaconChainError::DBInconsistent(format!(
-                        "Missing state {:?}",
-                        parent_state_root
-                    ))
-                })?;
+        // Load the parent blocks state from the database, returning an error if it is not found.
+        // It is an error because if we know the parent block we should also know the parent state.
+        let parent_state_root = parent_block.state_root();
+        let parent_state = chain
+            .get_state(&parent_state_root, Some(parent_block.slot()))?
+            .ok_or_else(|| {
+                BeaconChainError::DBInconsistent(format!("Missing state {:?}", parent_state_root))
+            })?;
 
-            Ok(Some(BeaconSnapshot {
+        Ok((
+            BeaconSnapshot {
                 beacon_block: parent_block,
-                beacon_block_root: block.parent_root,
+                beacon_block_root: root,
                 beacon_state: parent_state,
                 beacon_state_root: parent_state_root,
-            }))
-        })
-        .map_err(BlockError::BeaconChainError)?
-        .ok_or_else(|| BlockError::ParentUnknown(block.parent_root));
+            },
+            block,
+        ))
+    };
 
     metrics::stop_timer(db_read_timer);
 
@@ -911,7 +1029,7 @@ fn cheap_state_advance_to_obtain_committees<'a, E: EthSpec>(
     state: &'a mut BeaconState<E>,
     block_slot: Slot,
     spec: &ChainSpec,
-) -> Result<Cow<'a, BeaconState<E>>, BlockError> {
+) -> Result<Cow<'a, BeaconState<E>>, BlockError<E>> {
     let block_epoch = block_slot.epoch(E::slots_per_epoch());
 
     if state.current_epoch() == block_epoch {
@@ -943,7 +1061,7 @@ fn cheap_state_advance_to_obtain_committees<'a, E: EthSpec>(
 /// Obtains a read-locked `ValidatorPubkeyCache` from the `chain`.
 fn get_validator_pubkey_cache<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
-) -> Result<RwLockReadGuard<ValidatorPubkeyCache>, BlockError> {
+) -> Result<RwLockReadGuard<ValidatorPubkeyCache>, BlockError<T::EthSpec>> {
     chain
         .validator_pubkey_cache
         .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)

beacon_node/beacon_chain/src/builder.rs

@@ -11,6 +11,7 @@ use crate::shuffling_cache::ShufflingCache;
 use crate::snapshot_cache::{SnapshotCache, DEFAULT_SNAPSHOT_CACHE_SIZE};
 use crate::timeout_rw_lock::TimeoutRwLock;
 use crate::validator_pubkey_cache::ValidatorPubkeyCache;
+use crate::ChainConfig;
 use crate::{
     BeaconChain, BeaconChainTypes, BeaconForkChoiceStore, BeaconSnapshot, Eth1Chain,
     Eth1ChainBackend, EventHandler,
@@ -110,6 +111,7 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     pubkey_cache_path: Option<PathBuf>,
     validator_pubkey_cache: Option<ValidatorPubkeyCache>,
     spec: ChainSpec,
+    chain_config: ChainConfig,
     disabled_forks: Vec<String>,
     log: Option<Logger>,
     graffiti: Graffiti,
@@ -157,6 +159,7 @@ where
             disabled_forks: Vec::new(),
             validator_pubkey_cache: None,
             spec: TEthSpec::default_spec(),
+            chain_config: ChainConfig::default(),
             log: None,
             graffiti: Graffiti::default(),
         }
@@ -171,6 +174,15 @@ where
         self
     }
 
+    /// Sets the maximum number of blocks that will be skipped when processing
+    /// some consensus messages.
+    ///
+    /// Set to `None` for no limit.
+    pub fn import_max_skip_slots(mut self, n: Option<u64>) -> Self {
+        self.chain_config.import_max_skip_slots = n;
+        self
+    }
+
     /// Sets the store (database).
     ///
     /// Should generally be called early in the build chain.
@@ -406,6 +418,12 @@ where
         self
     }
 
+    /// Sets the `ChainConfig` that determines `BeaconChain` runtime behaviour.
+    pub fn chain_config(mut self, config: ChainConfig) -> Self {
+        self.chain_config = config;
+        self
+    }
+
     /// Consumes `self`, returning a `BeaconChain` if all required parameters have been supplied.
     ///
     /// An error will be returned at runtime if all required parameters have not been configured.
@@ -489,6 +507,7 @@ where
 
         let beacon_chain = BeaconChain {
             spec: self.spec,
+            config: self.chain_config,
             store,
             store_migrator: self
                 .store_migrator

beacon_node/beacon_chain/src/chain_config.rs

@@ -0,0 +1,21 @@
+use serde_derive::{Deserialize, Serialize};
+
+/// There is a 693 block skip in the current canonical Medalla chain, we use 700 to be safe.
+pub const DEFAULT_IMPORT_BLOCK_MAX_SKIP_SLOTS: u64 = 700;
+
+#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
+pub struct ChainConfig {
+    /// Maximum number of slots to skip when importing a consensus message (e.g., block,
+    /// attestation, etc).
+    ///
+    /// If `None`, there is no limit.
+    pub import_max_skip_slots: Option<u64>,
+}
+
+impl Default for ChainConfig {
+    fn default() -> Self {
+        Self {
+            import_max_skip_slots: Some(DEFAULT_IMPORT_BLOCK_MAX_SKIP_SLOTS),
+        }
+    }
+}

beacon_node/beacon_chain/src/lib.rs

@@ -8,6 +8,7 @@ mod beacon_fork_choice_store;
 mod beacon_snapshot;
 mod block_verification;
 pub mod builder;
+pub mod chain_config;
 mod errors;
 pub mod eth1_chain;
 pub mod events;
@@ -32,6 +33,7 @@ pub use self::beacon_chain::{
     ForkChoiceError, StateSkipConfig,
 };
 pub use self::beacon_snapshot::BeaconSnapshot;
+pub use self::chain_config::ChainConfig;
 pub use self::errors::{BeaconChainError, BlockProductionError};
 pub use attestation_verification::Error as AttestationError;
 pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError};

beacon_node/beacon_chain/src/migrate.rs

@@ -202,11 +202,6 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
         old_finalized_block_hash: SignedBeaconBlockHash,
         new_finalized_block_hash: SignedBeaconBlockHash,
     ) {
-        if let Err(e) = process_finalization(self.db.clone(), state_root, &new_finalized_state) {
-            // This migrator is only used for testing, so we just log to stderr without a logger.
-            eprintln!("Migration error: {:?}", e);
-        }
-
         if let Err(e) = Self::prune_abandoned_forks(
             self.db.clone(),
             head_tracker,
@@ -216,6 +211,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
         ) {
             eprintln!("Pruning error: {:?}", e);
         }
+
+        if let Err(e) = process_finalization(self.db.clone(), state_root, &new_finalized_state) {
+            // This migrator is only used for testing, so we just log to stderr without a logger.
+            eprintln!("Migration error: {:?}", e);
+        }
     }
 }
 
@@ -325,6 +325,17 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                 new_finalized_slot,
             )) = rx.recv()
             {
+                match Self::prune_abandoned_forks(
+                    db.clone(),
+                    head_tracker,
+                    old_finalized_block_hash,
+                    new_finalized_block_hash,
+                    new_finalized_slot,
+                ) {
+                    Ok(()) => {}
+                    Err(e) => warn!(log, "Block pruning failed: {:?}", e),
+                }
+
                 match process_finalization(db.clone(), state_root, &state) {
                     Ok(()) => {}
                     Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
@@ -342,17 +353,6 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                         );
                     }
                 };
-
-                match Self::prune_abandoned_forks(
-                    db.clone(),
-                    head_tracker,
-                    old_finalized_block_hash,
-                    new_finalized_block_hash,
-                    new_finalized_slot,
-                ) {
-                    Ok(()) => {}
-                    Err(e) => warn!(log, "Block pruning failed: {:?}", e),
-                }
             }
         });
 

beacon_node/beacon_chain/src/naive_aggregation_pool.rs

@@ -1,5 +1,4 @@
 use crate::metrics;
-use parking_lot::RwLock;
 use std::collections::HashMap;
 use types::{Attestation, AttestationData, EthSpec, Slot};
 
@@ -120,6 +119,11 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
         Ok(self.map.get(data).cloned())
     }
 
+    /// Iterate all attestations in `self`.
+    pub fn iter(&self) -> impl Iterator<Item = &Attestation<E>> {
+        self.map.iter().map(|(_key, attestation)| attestation)
+    }
+
     pub fn len(&self) -> usize {
         self.map.len()
     }
@@ -147,15 +151,15 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
 /// than that will also be refused. Pruning is done automatically based upon the attestations it
 /// receives and it can be triggered manually.
 pub struct NaiveAggregationPool<E: EthSpec> {
-    lowest_permissible_slot: RwLock<Slot>,
-    maps: RwLock<HashMap<Slot, AggregatedAttestationMap<E>>>,
+    lowest_permissible_slot: Slot,
+    maps: HashMap<Slot, AggregatedAttestationMap<E>>,
 }
 
 impl<E: EthSpec> Default for NaiveAggregationPool<E> {
     fn default() -> Self {
         Self {
-            lowest_permissible_slot: RwLock::new(Slot::new(0)),
-            maps: RwLock::new(HashMap::new()),
+            lowest_permissible_slot: Slot::new(0),
+            maps: HashMap::new(),
         }
     }
 }
@@ -168,10 +172,10 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
     ///
     /// The pool may be pruned if the given `attestation.data` has a slot higher than any
     /// previously seen.
-    pub fn insert(&self, attestation: &Attestation<E>) -> Result<InsertOutcome, Error> {
+    pub fn insert(&mut self, attestation: &Attestation<E>) -> Result<InsertOutcome, Error> {
         let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_INSERT);
         let slot = attestation.data.slot;
-        let lowest_permissible_slot: Slot = *self.lowest_permissible_slot.read();
+        let lowest_permissible_slot = self.lowest_permissible_slot;
 
         // Reject any attestations that are too old.
         if slot < lowest_permissible_slot {
@@ -183,16 +187,16 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
 
         let lock_timer =
             metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_MAPS_WRITE_LOCK);
-        let mut maps = self.maps.write();
         drop(lock_timer);
 
-        let outcome = if let Some(map) = maps.get_mut(&slot) {
+        let outcome = if let Some(map) = self.maps.get_mut(&slot) {
             map.insert(attestation)
         } else {
             let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CREATE_MAP);
             // To avoid re-allocations, try and determine a rough initial capacity for the new item
             // by obtaining the mean size of all items in earlier epoch.
-            let (count, sum) = maps
+            let (count, sum) = self
+                .maps
                 .iter()
                 // Only include epochs that are less than the given slot in the average. This should
                 // generally avoid including recent epochs that are still "filling up".
@@ -205,12 +209,11 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
 
             let mut item = AggregatedAttestationMap::new(initial_capacity);
             let outcome = item.insert(attestation);
-            maps.insert(slot, item);
+            self.maps.insert(slot, item);
 
             outcome
         };
 
-        drop(maps);
         self.prune(slot);
 
         outcome
@@ -219,16 +222,20 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
     /// Returns an aggregated `Attestation` with the given `data`, if any.
     pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
         self.maps
-            .read()
             .iter()
             .find(|(slot, _map)| **slot == data.slot)
             .map(|(_slot, map)| map.get(data))
             .unwrap_or_else(|| Ok(None))
     }
 
+    /// Iterate all attestations in all slots of `self`.
+    pub fn iter(&self) -> impl Iterator<Item = &Attestation<E>> {
+        self.maps.iter().map(|(_slot, map)| map.iter()).flatten()
+    }
+
     /// Removes any attestations with a slot lower than `current_slot` and bars any future
     /// attestations with a slot lower than `current_slot - SLOTS_RETAINED`.
-    pub fn prune(&self, current_slot: Slot) {
+    pub fn prune(&mut self, current_slot: Slot) {
         let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_PRUNE);
 
         // Taking advantage of saturating subtraction on `Slot`.
@@ -236,30 +243,34 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
 
         // No need to prune if the lowest permissible slot has not changed and the queue length is
         // less than the maximum
-        if *self.lowest_permissible_slot.read() == lowest_permissible_slot
-            && self.maps.read().len() <= SLOTS_RETAINED
+        if self.lowest_permissible_slot == lowest_permissible_slot
+            && self.maps.len() <= SLOTS_RETAINED
         {
             return;
         }
 
-        *self.lowest_permissible_slot.write() = lowest_permissible_slot;
-        let mut maps = self.maps.write();
+        self.lowest_permissible_slot = lowest_permissible_slot;
 
         // Remove any maps that are definitely expired.
-        maps.retain(|slot, _map| *slot >= lowest_permissible_slot);
+        self.maps
+            .retain(|slot, _map| *slot >= lowest_permissible_slot);
 
         // If we have too many maps, remove the lowest amount to ensure we only have
         // `SLOTS_RETAINED` left.
-        if maps.len() > SLOTS_RETAINED {
-            let mut slots = maps.iter().map(|(slot, _map)| *slot).collect::<Vec<_>>();
+        if self.maps.len() > SLOTS_RETAINED {
+            let mut slots = self
+                .maps
+                .iter()
+                .map(|(slot, _map)| *slot)
+                .collect::<Vec<_>>();
             // Sort is generally pretty slow, however `SLOTS_RETAINED` is quite low so it should be
             // negligible.
             slots.sort_unstable();
             slots
                 .into_iter()
-                .take(maps.len().saturating_sub(SLOTS_RETAINED))
+                .take(self.maps.len().saturating_sub(SLOTS_RETAINED))
                 .for_each(|slot| {
-                    maps.remove(&slot);
+                    self.maps.remove(&slot);
                 })
         }
     }
@@ -304,7 +315,7 @@ mod tests {
     fn single_attestation() {
         let mut a = get_attestation(Slot::new(0));
 
-        let pool = NaiveAggregationPool::default();
+        let mut pool = NaiveAggregationPool::default();
 
         assert_eq!(
             pool.insert(&a),
@@ -352,7 +363,7 @@ mod tests {
         sign(&mut a_0, 0, genesis_validators_root);
         sign(&mut a_1, 1, genesis_validators_root);
 
-        let pool = NaiveAggregationPool::default();
+        let mut pool = NaiveAggregationPool::default();
 
         assert_eq!(
             pool.insert(&a_0),
@@ -409,7 +420,7 @@ mod tests {
         let mut base = get_attestation(Slot::new(0));
         sign(&mut base, 0, Hash256::random());
 
-        let pool = NaiveAggregationPool::default();
+        let mut pool = NaiveAggregationPool::default();
 
         for i in 0..SLOTS_RETAINED * 2 {
             let slot = Slot::from(i);
@@ -424,22 +435,16 @@ mod tests {
 
             if i < SLOTS_RETAINED {
                 let len = i + 1;
-                assert_eq!(
-                    pool.maps.read().len(),
-                    len,
-                    "the pool should have length {}",
-                    len
-                );
+                assert_eq!(pool.maps.len(), len, "the pool should have length {}", len);
             } else {
                 assert_eq!(
-                    pool.maps.read().len(),
+                    pool.maps.len(),
                     SLOTS_RETAINED,
                     "the pool should have length SLOTS_RETAINED"
                 );
 
                 let mut pool_slots = pool
                     .maps
-                    .read()
                     .iter()
                     .map(|(slot, _map)| *slot)
                     .collect::<Vec<_>>();
@@ -463,7 +468,7 @@ mod tests {
         let mut base = get_attestation(Slot::new(0));
         sign(&mut base, 0, Hash256::random());
 
-        let pool = NaiveAggregationPool::default();
+        let mut pool = NaiveAggregationPool::default();
 
         for i in 0..=MAX_ATTESTATIONS_PER_SLOT {
             let mut a = base.clone();

beacon_node/beacon_chain/src/test_utils.rs

@@ -152,6 +152,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
         let chain = BeaconChainBuilder::new(eth_spec_instance)
             .logger(log.clone())
             .custom_spec(spec.clone())
+            .import_max_skip_slots(None)
             .store(store.clone())
             .store_migrator(BlockingMigrator::new(store, log.clone()))
             .data_dir(data_dir.path().to_path_buf())
@@ -190,6 +191,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
         let chain = BeaconChainBuilder::new(eth_spec_instance)
             .logger(log.clone())
             .custom_spec(spec)
+            .import_max_skip_slots(None)
             .store(store.clone())
             .store_migrator(<BlockingMigrator<_, _, _> as Migrate<E, _, _>>::new(
                 store,
@@ -520,7 +522,7 @@ where
 
         let (block, state) = self
             .chain
-            .produce_block_on_state(state, slot, randao_reveal)
+            .produce_block_on_state(state, slot, randao_reveal, None)
             .expect("should produce block");
 
         let signed_block = block.sign(sk, &state.fork, state.genesis_validators_root, &self.spec);

beacon_node/beacon_chain/tests/block_verification.rs

@@ -18,8 +18,9 @@ use types::{
 type E = MainnetEthSpec;
 
 // Should ideally be divisible by 3.
-pub const VALIDATOR_COUNT: usize = 24;
-pub const CHAIN_SEGMENT_LENGTH: usize = 64 * 5;
+const VALIDATOR_COUNT: usize = 24;
+const CHAIN_SEGMENT_LENGTH: usize = 64 * 5;
+const BLOCK_INDICES: &[usize] = &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT_LENGTH - 1];
 
 lazy_static! {
     /// A cached set of keys.
@@ -272,17 +273,73 @@ fn chain_segment_non_linear_slots() {
     );
 }
 
+fn assert_invalid_signature(
+    harness: &BeaconChainHarness<HarnessType<E>>,
+    block_index: usize,
+    snapshots: &[BeaconSnapshot<E>],
+    item: &str,
+) {
+    let blocks = snapshots
+        .iter()
+        .map(|snapshot| snapshot.beacon_block.clone())
+        .collect();
+
+    // Ensure the block will be rejected if imported in a chain segment.
+    assert!(
+        matches!(
+            harness
+                .chain
+                .process_chain_segment(blocks)
+                .into_block_error(),
+            Err(BlockError::InvalidSignature)
+        ),
+        "should not import chain segment with an invalid {} signature",
+        item
+    );
+
+    // Ensure the block will be rejected if imported on its own (without gossip checking).
+    let ancestor_blocks = CHAIN_SEGMENT
+        .iter()
+        .take(block_index)
+        .map(|snapshot| snapshot.beacon_block.clone())
+        .collect();
+    // We don't care if this fails, we just call this to ensure that all prior blocks have been
+    // imported prior to this test.
+    let _ = harness.chain.process_chain_segment(ancestor_blocks);
+    assert!(
+        matches!(
+            harness
+                .chain
+                .process_block(snapshots[block_index].beacon_block.clone()),
+            Err(BlockError::InvalidSignature)
+        ),
+        "should not import individual block with an invalid {} signature",
+        item
+    );
+
+    // NOTE: we choose not to check gossip verification here. It only checks one signature
+    // (proposal) and that is already tested elsewhere in this file.
+    //
+    // It's not trivial to just check gossip verification since it will start refusing
+    // blocks as soon as it has seen one valid proposal signature for a given (validator,
+    // slot) tuple.
+}
+
+fn get_invalid_sigs_harness() -> BeaconChainHarness<HarnessType<E>> {
+    let harness = get_harness(VALIDATOR_COUNT);
+    harness
+        .chain
+        .slot_clock
+        .set_slot(CHAIN_SEGMENT.last().unwrap().beacon_block.slot().as_u64());
+    harness
+}
 #[test]
-fn invalid_signatures() {
-    let mut checked_attestation = false;
-
-    for &block_index in &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT.len() - 1] {
-        let harness = get_harness(VALIDATOR_COUNT);
-        harness
-            .chain
-            .slot_clock
-            .set_slot(CHAIN_SEGMENT.last().unwrap().beacon_block.slot().as_u64());
-
+fn invalid_signature_gossip_block() {
+    for &block_index in BLOCK_INDICES {
+        // Ensure the block will be rejected if imported on its own (without gossip checking).
+        let harness = get_invalid_sigs_harness();
+        let mut snapshots = CHAIN_SEGMENT.clone();
+        snapshots[block_index].beacon_block.signature = junk_signature();
         // Import all the ancestors before the `block_index` block.
         let ancestor_blocks = CHAIN_SEGMENT
             .iter()
@@ -294,75 +351,6 @@ fn invalid_signatures() {
             .process_chain_segment(ancestor_blocks)
             .into_block_error()
             .expect("should import all blocks prior to the one being tested");
-
-        // For the given snapshots, test the following:
-        //
-        // - The `process_chain_segment` function returns `InvalidSignature`.
-        // - The `process_block` function returns `InvalidSignature` when importing the
-        //    `SignedBeaconBlock` directly.
-        // - The `verify_block_for_gossip` function does _not_ return an error.
-        // - The `process_block` function returns `InvalidSignature` when verifying the
-        //    `GossipVerifiedBlock`.
-        let assert_invalid_signature = |snapshots: &[BeaconSnapshot<E>], item: &str| {
-            let blocks = snapshots
-                .iter()
-                .map(|snapshot| snapshot.beacon_block.clone())
-                .collect();
-
-            // Ensure the block will be rejected if imported in a chain segment.
-            assert!(
-                matches!(
-                    harness
-                        .chain
-                        .process_chain_segment(blocks)
-                        .into_block_error(),
-                    Err(BlockError::InvalidSignature)
-                ),
-                "should not import chain segment with an invalid {} signature",
-                item
-            );
-
-            // Ensure the block will be rejected if imported on its own (without gossip checking).
-            assert!(
-                matches!(
-                    harness
-                        .chain
-                        .process_block(snapshots[block_index].beacon_block.clone()),
-                    Err(BlockError::InvalidSignature)
-                ),
-                "should not import individual block with an invalid {} signature",
-                item
-            );
-
-            // NOTE: we choose not to check gossip verification here. It only checks one signature
-            // (proposal) and that is already tested elsewhere in this file.
-            //
-            // It's not trivial to just check gossip verification since it will start refusing
-            // blocks as soon as it has seen one valid proposal signature for a given (validator,
-            // slot) tuple.
-        };
-
-        /*
-         * Block proposal
-         */
-        let mut snapshots = CHAIN_SEGMENT.clone();
-        snapshots[block_index].beacon_block.signature = junk_signature();
-        let blocks = snapshots
-            .iter()
-            .map(|snapshot| snapshot.beacon_block.clone())
-            .collect();
-        // Ensure the block will be rejected if imported in a chain segment.
-        assert!(
-            matches!(
-                harness
-                    .chain
-                    .process_chain_segment(blocks)
-                    .into_block_error(),
-                Err(BlockError::InvalidSignature)
-            ),
-            "should not import chain segment with an invalid gossip signature",
-        );
-        // Ensure the block will be rejected if imported on its own (without gossip checking).
         assert!(
             matches!(
                 harness
@@ -372,10 +360,37 @@ fn invalid_signatures() {
             ),
             "should not import individual block with an invalid gossip signature",
         );
+    }
+}
 
-        /*
-         * Randao reveal
-         */
+#[test]
+fn invalid_signature_block_proposal() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
+        let mut snapshots = CHAIN_SEGMENT.clone();
+        snapshots[block_index].beacon_block.signature = junk_signature();
+        let blocks = snapshots
+            .iter()
+            .map(|snapshot| snapshot.beacon_block.clone())
+            .collect::<Vec<_>>();
+        // Ensure the block will be rejected if imported in a chain segment.
+        assert!(
+            matches!(
+                harness
+                    .chain
+                    .process_chain_segment(blocks)
+                    .into_block_error(),
+                Err(BlockError::InvalidSignature)
+            ),
+            "should not import chain segment with an invalid block signature",
+        );
+    }
+}
+
+#[test]
+fn invalid_signature_randao_reveal() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         snapshots[block_index]
             .beacon_block
@@ -384,11 +399,14 @@ fn invalid_signatures() {
             .randao_reveal = junk_signature();
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "randao");
+        assert_invalid_signature(&harness, block_index, &snapshots, "randao");
+    }
+}
 
-        /*
-         * Proposer slashing
-         */
+#[test]
+fn invalid_signature_proposer_slashing() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let proposer_slashing = ProposerSlashing {
             signed_header_1: SignedBeaconBlockHeader {
@@ -409,11 +427,14 @@ fn invalid_signatures() {
             .expect("should update proposer slashing");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "proposer slashing");
+        assert_invalid_signature(&harness, block_index, &snapshots, "proposer slashing");
+    }
+}
 
-        /*
-         * Attester slashing
-         */
+#[test]
+fn invalid_signature_attester_slashing() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let indexed_attestation = IndexedAttestation {
             attesting_indices: vec![0].into(),
@@ -445,11 +466,16 @@ fn invalid_signatures() {
             .expect("should update attester slashing");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "attester slashing");
+        assert_invalid_signature(&harness, block_index, &snapshots, "attester slashing");
+    }
+}
 
-        /*
-         * Attestation
-         */
+#[test]
+fn invalid_signature_attestation() {
+    let mut checked_attestation = false;
+
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         if let Some(attestation) = snapshots[block_index]
             .beacon_block
@@ -461,15 +487,22 @@ fn invalid_signatures() {
             attestation.signature = junk_aggregate_signature();
             update_parent_roots(&mut snapshots);
             update_proposal_signatures(&mut snapshots, &harness);
-            assert_invalid_signature(&snapshots, "attestation");
+            assert_invalid_signature(&harness, block_index, &snapshots, "attestation");
             checked_attestation = true;
         }
+    }
 
-        /*
-         * Deposit
-         *
-         * Note: an invalid deposit signature is permitted!
-         */
+    assert!(
+        checked_attestation,
+        "the test should check an attestation signature"
+    )
+}
+
+#[test]
+fn invalid_signature_deposit() {
+    for &block_index in BLOCK_INDICES {
+        // Note: an invalid deposit signature is permitted!
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let deposit = Deposit {
             proof: vec![Hash256::zero(); DEPOSIT_TREE_DEPTH + 1].into(),
@@ -503,10 +536,13 @@ fn invalid_signatures() {
             ),
             "should not throw an invalid signature error for a bad deposit signature"
         );
+    }
+}
 
-        /*
-         * Voluntary exit
-         */
+#[test]
+fn invalid_signature_exit() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let epoch = snapshots[block_index].beacon_state.current_epoch();
         snapshots[block_index]
@@ -524,13 +560,8 @@ fn invalid_signatures() {
             .expect("should update deposit");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "voluntary exit");
+        assert_invalid_signature(&harness, block_index, &snapshots, "voluntary exit");
     }
-
-    assert!(
-        checked_attestation,
-        "the test should check an attestation signature"
-    )
 }
 
 fn unwrap_err<T, E>(result: Result<T, E>) -> E {
@@ -641,6 +672,48 @@ fn block_gossip_verification() {
         "should not import a block with an invalid proposal signature"
     );
 
+    /*
+     * This test ensures that:
+     *
+     * Spec v0.12.2
+     *
+     * The block's parent (defined by block.parent_root) passes validation.
+     */
+
+    let mut block = CHAIN_SEGMENT[block_index].beacon_block.clone();
+    let parent_root = Hash256::from_low_u64_be(42);
+    block.message.parent_root = parent_root;
+    assert!(
+        matches!(
+            unwrap_err(harness.chain.verify_block_for_gossip(block)),
+            BlockError::ParentUnknown(block)
+            if block.parent_root() == parent_root
+        ),
+        "should not import a block for an unknown parent"
+    );
+
+    /*
+     * This test ensures that:
+     *
+     * Spec v0.12.2
+     *
+     * The current finalized_checkpoint is an ancestor of block -- i.e. get_ancestor(store,
+     * block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) ==
+     * store.finalized_checkpoint.root
+     */
+
+    let mut block = CHAIN_SEGMENT[block_index].beacon_block.clone();
+    let parent_root = CHAIN_SEGMENT[0].beacon_block_root;
+    block.message.parent_root = parent_root;
+    assert!(
+        matches!(
+            unwrap_err(harness.chain.verify_block_for_gossip(block)),
+            BlockError::NotFinalizedDescendant { block_parent_root }
+            if block_parent_root == parent_root
+        ),
+        "should not import a block that conflicts with finality"
+    );
+
     /*
      * This test ensures that:
      *

beacon_node/beacon_chain/tests/store_tests.rs

@@ -743,7 +743,7 @@ fn prunes_abandoned_fork_between_two_finalized_checkpoints() {
     let (stray_blocks, stray_states, _, stray_head, _) = harness.add_stray_blocks(
         harness.get_head_state(),
         slot,
-        slots_per_epoch - 1,
+        slots_per_epoch - 3,
         &faulty_validators,
     );
 

beacon_node/client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "client"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
@@ -31,7 +31,7 @@ slog-async = "2.5.0"
 tokio = "0.2.21"
 dirs = "2.0.2"
 futures = "0.3.5"
-reqwest = "0.10.4"
+reqwest = { version = "0.10.4", features = ["native-tls-vendored"] }
 url = "2.1.1"
 eth1 = { path = "../eth1" }
 genesis = { path = "../genesis" }

beacon_node/client/src/builder.rs

@@ -135,6 +135,7 @@ where
         let eth_spec_instance = self.eth_spec_instance.clone();
         let data_dir = config.data_dir.clone();
         let disabled_forks = config.disabled_forks.clone();
+        let chain_config = config.chain.clone();
         let graffiti = config.graffiti;
 
         let store =
@@ -153,6 +154,7 @@ where
             .store_migrator(store_migrator)
             .data_dir(data_dir)
             .custom_spec(spec.clone())
+            .chain_config(chain_config)
             .disabled_forks(disabled_forks)
             .graffiti(graffiti);
 
@@ -232,8 +234,8 @@ where
         Ok(self)
     }
 
-    /// Immediately starts the networking stack.
-    pub fn network(mut self, config: &NetworkConfig) -> Result<Self, String> {
+    /// Starts the networking stack.
+    pub async fn network(mut self, config: &NetworkConfig) -> Result<Self, String> {
         let beacon_chain = self
             .beacon_chain
             .clone()
@@ -246,6 +248,7 @@ where
 
         let (network_globals, network_send) =
             NetworkService::start(beacon_chain, config, context.executor)
+                .await
                 .map_err(|e| format!("Failed to start network: {:?}", e))?;
 
         self.network_globals = Some(network_globals);

beacon_node/client/src/config.rs

@@ -64,6 +64,7 @@ pub struct Config {
     pub store: store::StoreConfig,
     pub network: network::NetworkConfig,
     pub rest_api: rest_api::Config,
+    pub chain: beacon_chain::ChainConfig,
     pub websocket_server: websocket_server::Config,
     pub eth1: eth1::Config,
 }
@@ -78,6 +79,7 @@ impl Default for Config {
             genesis: <_>::default(),
             store: <_>::default(),
             network: NetworkConfig::default(),
+            chain: <_>::default(),
             rest_api: <_>::default(),
             websocket_server: <_>::default(),
             spec_constants: TESTNET_SPEC_CONSTANTS.into(),

beacon_node/client/src/notifier.rs

@@ -122,14 +122,28 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                     head_distance.as_u64(),
                     slot_distance_pretty(head_distance, slot_duration)
                 );
-                info!(
-                    log,
-                    "Syncing";
-                    "peers" => peer_count_pretty(connected_peer_count),
-                    "distance" => distance,
-                    "speed" => sync_speed_pretty(speedo.slots_per_second()),
-                    "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
-                );
+
+                let speed = speedo.slots_per_second();
+                let display_speed = speed.map_or(false, |speed| speed != 0.0);
+
+                if display_speed {
+                    info!(
+                        log,
+                        "Syncing";
+                        "peers" => peer_count_pretty(connected_peer_count),
+                        "distance" => distance,
+                        "speed" => sync_speed_pretty(speed),
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                    );
+                } else {
+                    info!(
+                        log,
+                        "Syncing";
+                        "peers" => peer_count_pretty(connected_peer_count),
+                        "distance" => distance,
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                    );
+                }
             } else if sync_state.is_synced() {
                 let block_info = if current_slot > head_slot {
                     "   …  empty".to_string()
@@ -220,14 +234,37 @@ fn seconds_pretty(secs: f64) -> String {
     let hours = d.whole_hours();
     let minutes = d.whole_minutes();
 
+    let week_string = if weeks == 1 { "week" } else { "weeks" };
+    let day_string = if days == 1 { "day" } else { "days" };
+    let hour_string = if hours == 1 { "hr" } else { "hrs" };
+    let min_string = if minutes == 1 { "min" } else { "mins" };
+
     if weeks > 0 {
-        format!("{:.0} weeks {:.0} days", weeks, days % DAYS_PER_WEEK)
+        format!(
+            "{:.0} {} {:.0} {}",
+            weeks,
+            week_string,
+            days % DAYS_PER_WEEK,
+            day_string
+        )
     } else if days > 0 {
-        format!("{:.0} days {:.0} hrs", days, hours % HOURS_PER_DAY)
+        format!(
+            "{:.0} {} {:.0} {}",
+            days,
+            day_string,
+            hours % HOURS_PER_DAY,
+            hour_string
+        )
     } else if hours > 0 {
-        format!("{:.0} hrs {:.0} mins", hours, minutes % MINUTES_PER_HOUR)
+        format!(
+            "{:.0} {} {:.0} {}",
+            hours,
+            hour_string,
+            minutes % MINUTES_PER_HOUR,
+            min_string
+        )
     } else {
-        format!("{:.0} mins", minutes)
+        format!("{:.0} {}", minutes, min_string)
     }
 }
 

beacon_node/eth1/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "eth1"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
@@ -11,7 +11,7 @@ web3 = "0.11.0"
 sloggers = "1.0.0"
 
 [dependencies]
-reqwest = "0.10.4"
+reqwest = { version = "0.10.4", features = ["native-tls-vendored"] }
 futures = { version = "0.3.5", features = ["compat"] }
 serde_json = "1.0.52"
 serde = { version = "1.0.110", features = ["derive"] }

beacon_node/eth2_libp2p/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "eth2_libp2p"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
@@ -32,18 +32,18 @@ snap = "1.0.0"
 void = "1.0.2"
 tokio-io-timeout = "0.4.0"
 tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
-discv5 = { version = "0.1.0-alpha.7", features = ["libp2p"] }
+discv5 = { version = "0.1.0-alpha.8", features = ["libp2p", "openssl-vendored"] }
 tiny-keccak = "2.0.2"
 environment = { path = "../../lighthouse/environment" }
 # TODO: Remove rand crate for mainnet
 rand = "0.7.3"
 
 [dependencies.libp2p]
-#version = "0.19.1"
+#version = "0.23.0"
 git = "https://github.com/sigp/rust-libp2p"
-rev = "f1b660a1a96c1b6198cd62062e75d357893faf16"
+rev = "bbf0cfbaff2f733b3ae7bfed3caba8b7ee542803"
 default-features = false
-features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "secio", "tcp-tokio"]
+features = ["websocket", "identify", "mplex", "noise", "gossipsub", "dns", "tcp-tokio"]
 
 [dev-dependencies]
 tokio = { version = "0.2.21", features = ["full"] }

beacon_node/eth2_libp2p/src/behaviour/handler/delegate.rs

@@ -49,7 +49,7 @@ impl<TSpec: EthSpec> DelegatingHandler<TSpec> {
     }
 
     /// Gives access to identify's handler.
-    pub fn identify(&self) -> &IdentifyHandler {
+    pub fn _identify(&self) -> &IdentifyHandler {
         &self.identify_handler
     }
 }

beacon_node/eth2_libp2p/src/behaviour/handler/mod.rs

@@ -77,7 +77,7 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
     fn inject_event(&mut self, event: Self::InEvent) {
         match event {
             BehaviourHandlerIn::Delegate(delegated_ev) => self.delegate.inject_event(delegated_ev),
-            /* Events comming from the behaviour */
+            /* Events coming from the behaviour */
             BehaviourHandlerIn::Shutdown(last_message) => {
                 self.shutting_down = true;
                 self.delegate.rpc_mut().shutdown(last_message);
@@ -113,12 +113,9 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
         >,
     > {
         // Disconnect if the sub-handlers are ready.
-        if self.shutting_down {
-            let rpc_keep_alive = self.delegate.rpc().connection_keep_alive();
-            let identify_keep_alive = self.delegate.identify().connection_keep_alive();
-            if KeepAlive::No == rpc_keep_alive.max(identify_keep_alive) {
-                return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::Disconnected));
-            }
+        // Currently we only respect the RPC handler.
+        if self.shutting_down && KeepAlive::No == self.delegate.rpc().connection_keep_alive() {
+            return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::Disconnected));
         }
 
         match self.delegate.poll(cx) {

beacon_node/eth2_libp2p/src/behaviour/mod.rs

@@ -1,6 +1,6 @@
 use crate::peer_manager::{score::PeerAction, PeerManager, PeerManagerEvent};
 use crate::rpc::*;
-use crate::types::{GossipEncoding, GossipKind, GossipTopic};
+use crate::types::{EnrBitfield, GossipEncoding, GossipKind, GossipTopic, SubnetDiscovery};
 use crate::Eth2Enr;
 use crate::{error, metrics, Enr, NetworkConfig, NetworkGlobals, PubsubMessage, TopicHash};
 use futures::prelude::*;
@@ -19,19 +19,23 @@ use libp2p::{
     },
     PeerId,
 };
-use slog::{crit, debug, o, trace};
+use slog::{crit, debug, o, trace, warn};
+use ssz::{Decode, Encode};
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::PathBuf;
 use std::{
     collections::VecDeque,
     marker::PhantomData,
     sync::Arc,
     task::{Context, Poll},
-    time::Instant,
 };
 use types::{EnrForkId, EthSpec, SignedBeaconBlock, SubnetId};
 
 mod handler;
 
 const MAX_IDENTIFY_ADDRESSES: usize = 10;
+const METADATA_FILENAME: &str = "metadata";
 
 /// Builds the network behaviour that manages the core protocols of eth2.
 /// This core behaviour is managed by `Behaviour` which adds peer management to all core
@@ -61,13 +65,15 @@ pub struct Behaviour<TSpec: EthSpec> {
     enr_fork_id: EnrForkId,
     /// The waker for the current thread.
     waker: Option<std::task::Waker>,
+    /// Directory where metadata is stored
+    network_dir: PathBuf,
     /// Logger for behaviour actions.
     log: slog::Logger,
 }
 
 /// Implements the combined behaviour for the libp2p service.
 impl<TSpec: EthSpec> Behaviour<TSpec> {
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         net_conf: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
@@ -86,15 +92,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .eth2()
             .expect("Local ENR must have a fork id");
 
-        let attnets = network_globals
-            .local_enr()
-            .bitfield::<TSpec>()
-            .expect("Local ENR must have subnet bitfield");
-
-        let meta_data = MetaData {
-            seq_number: 1,
-            attnets,
-        };
+        let meta_data = load_or_build_metadata(&net_conf.network_dir, &log);
 
         // TODO: Until other clients support no author, we will use a 0 peer_id as our author.
         let message_author = PeerId::from_bytes(vec![0, 1, 0]).expect("Valid peer id");
@@ -106,13 +104,15 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                 net_conf.gs_config.clone(),
             ),
             identify,
-            peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)?,
+            peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)
+                .await?,
             events: VecDeque::new(),
             peers_to_dc: VecDeque::new(),
             meta_data,
             network_globals,
             enr_fork_id,
             waker: None,
+            network_dir: net_conf.network_dir.clone(),
             log: behaviour_log,
         })
     }
@@ -123,7 +123,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
     ///
     /// All external dials, dial a multiaddr. This is currently unused but kept here in case any
     /// part of lighthouse needs to connect to a peer_id in the future.
-    pub fn _dial(&mut self, peer_id: &PeerId) {
+    pub fn dial(&mut self, peer_id: &PeerId) {
         self.peer_manager.dial_peer(peer_id);
     }
 
@@ -300,8 +300,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Attempts to discover new peers for a given subnet. The `min_ttl` gives the time at which we
     /// would like to retain the peers for.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
-        self.peer_manager.discover_subnet_peers(subnet_id, min_ttl)
+    pub fn discover_subnet_peers(&mut self, subnet_subscriptions: Vec<SubnetDiscovery>) {
+        self.peer_manager
+            .discover_subnet_peers(subnet_subscriptions)
     }
 
     /// Updates the local ENR's "eth2" field with the latest EnrForkId.
@@ -345,6 +346,8 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             .local_enr()
             .bitfield::<TSpec>()
             .expect("Local discovery must have bitfield");
+        // Save the updated metadata to disk
+        save_metadata_to_disk(&self.network_dir, self.meta_data.clone(), &self.log);
     }
 
     /// Sends a Ping request to the peer.
@@ -694,15 +697,28 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
         conn_id: &ConnectionId,
         endpoint: &ConnectedPoint,
     ) {
+        // If the peer manager (and therefore the behaviour's) believe this peer connected, inform
+        // about the disconnection.
+        if self.network_globals.peers.read().is_connected(&peer_id) {
+            return;
+        }
         delegate_to_behaviours!(self, inject_connection_closed, peer_id, conn_id, endpoint);
     }
 
     // This gets called once there are no more active connections.
     fn inject_disconnected(&mut self, peer_id: &PeerId) {
+        // If the application/behaviour layers thinks this peer has connected inform it of the disconnect.
+        if self.network_globals.peers.read().is_connected(&peer_id) {
+            // Inform the application.
+            self.add_event(BehaviourEvent::PeerDisconnected(peer_id.clone()));
+            // Inform the behaviour.
+            delegate_to_behaviours!(self, inject_disconnected, peer_id);
+        }
         // Inform the peer manager.
+        // NOTE: It may be the case that a rejected node, due to too many peers is disconnected
+        // here and the peer manager has no knowledge of its connection. We insert it here for
+        // reference so that peer manager can track this peer.
         self.peer_manager.notify_disconnect(&peer_id);
-        // Inform the application.
-        self.add_event(BehaviourEvent::PeerDisconnected(peer_id.clone()));
 
         // Update the prometheus metrics
         metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT);
@@ -710,9 +726,6 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
             &metrics::PEERS_CONNECTED,
             self.network_globals.connected_peers() as i64,
         );
-
-        // Inform the behaviour.
-        delegate_to_behaviours!(self, inject_disconnected, peer_id);
     }
 
     // This gets called every time a connection is established.
@@ -733,14 +746,15 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
                 .peer_info(peer_id)
                 .map_or(true, |i| !i.has_future_duty())
         {
-            //If we are at our peer limit and we don't need the peer for a future validator
-            //duty, send goodbye with reason TooManyPeers
+            // If we are at our peer limit and we don't need the peer for a future validator
+            // duty, send goodbye with reason TooManyPeers
             Some(GoodbyeReason::TooManyPeers)
         } else {
             None
         };
 
         if goodbye_reason.is_some() {
+            debug!(self.log, "Disconnecting newly connected peer"; "peer_id" => peer_id.to_string(), "reason" => goodbye_reason.as_ref().expect("Is some").to_string());
             self.peers_to_dc
                 .push_back((peer_id.clone(), goodbye_reason));
             return;
@@ -771,18 +785,8 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
 
     // This gets called on the initial connection establishment.
     fn inject_connected(&mut self, peer_id: &PeerId) {
-        // Drop any connection from a banned peer. The goodbye and disconnects are handled in
-        // `inject_connection_established()`, which gets called first.
-        // The same holds if we reached the peer limit and the connected peer has no future duty.
-        if self.peer_manager.is_banned(peer_id)
-            || (self.peer_manager.peer_limit_reached()
-                && self
-                    .network_globals
-                    .peers
-                    .read()
-                    .peer_info(peer_id)
-                    .map_or(true, |i| !i.has_future_duty()))
-        {
+        // If the PeerManager has connected this peer, inform the behaviours
+        if !self.network_globals.peers.read().is_connected(&peer_id) {
             return;
         }
 
@@ -1034,3 +1038,60 @@ pub enum BehaviourEvent<TSpec: EthSpec> {
     /// Inform the network to send a Status to this peer.
     StatusPeer(PeerId),
 }
+
+/// Load metadata from persisted file. Return default metadata if loading fails.
+fn load_or_build_metadata<E: EthSpec>(network_dir: &PathBuf, log: &slog::Logger) -> MetaData<E> {
+    // Default metadata
+    let mut meta_data = MetaData {
+        seq_number: 0,
+        attnets: EnrBitfield::<E>::default(),
+    };
+    // Read metadata from persisted file if available
+    let metadata_path = network_dir.join(METADATA_FILENAME);
+    if let Ok(mut metadata_file) = File::open(metadata_path) {
+        let mut metadata_ssz = Vec::new();
+        if metadata_file.read_to_end(&mut metadata_ssz).is_ok() {
+            match MetaData::<E>::from_ssz_bytes(&metadata_ssz) {
+                Ok(persisted_metadata) => {
+                    meta_data.seq_number = persisted_metadata.seq_number;
+                    // Increment seq number if persisted attnet is not default
+                    if persisted_metadata.attnets != meta_data.attnets {
+                        meta_data.seq_number += 1;
+                    }
+                    debug!(log, "Loaded metadata from disk");
+                }
+                Err(e) => {
+                    debug!(
+                        log,
+                        "Metadata from file could not be decoded";
+                        "error" => format!("{:?}", e),
+                    );
+                }
+            }
+        }
+    };
+
+    debug!(log, "Metadata sequence number"; "seq_num" => meta_data.seq_number);
+    save_metadata_to_disk(network_dir, meta_data.clone(), &log);
+    meta_data
+}
+
+/// Persist metadata to disk
+fn save_metadata_to_disk<E: EthSpec>(dir: &PathBuf, metadata: MetaData<E>, log: &slog::Logger) {
+    let _ = std::fs::create_dir_all(&dir);
+    match File::create(dir.join(METADATA_FILENAME))
+        .and_then(|mut f| f.write_all(&metadata.as_ssz_bytes()))
+    {
+        Ok(_) => {
+            debug!(log, "Metadata written to disk");
+        }
+        Err(e) => {
+            warn!(
+                log,
+                "Could not write metadata to disk";
+                "file" => format!("{:?}{:?}",dir, METADATA_FILENAME),
+                "error" => format!("{}", e)
+            );
+        }
+    }
+}

beacon_node/eth2_libp2p/src/config.rs

@@ -50,7 +50,10 @@ pub struct Config {
     pub discv5_config: Discv5Config,
 
     /// List of nodes to initially connect to.
-    pub boot_nodes: Vec<Enr>,
+    pub boot_nodes_enr: Vec<Enr>,
+
+    /// List of nodes to initially connect to, on Multiaddr format.
+    pub boot_nodes_multiaddr: Vec<Multiaddr>,
 
     /// List of libp2p nodes to initially connect to.
     pub libp2p_nodes: Vec<Multiaddr>,
@@ -95,11 +98,18 @@ impl Default for Config {
         // parameter.
         let gs_config = GossipsubConfigBuilder::new()
             .max_transmit_size(GOSSIP_MAX_SIZE)
-            .heartbeat_interval(Duration::from_secs(1))
+            .heartbeat_interval(Duration::from_millis(700))
+            .mesh_n(6)
+            .mesh_n_low(5)
+            .mesh_n_high(12)
+            .gossip_lazy(6)
+            .fanout_ttl(Duration::from_secs(60))
+            .history_length(6)
+            .history_gossip(3)
             .validate_messages() // require validation before propagation
             .validation_mode(ValidationMode::Permissive)
-            // Prevent duplicates by caching messages from an epoch + 1 slot amount of time (33*12)
-            .duplicate_cache_time(Duration::from_secs(396))
+            // prevent duplicates for 550 heartbeats(700millis * 550) = 385 secs
+            .duplicate_cache_time(Duration::from_secs(385))
             .message_id_fn(gossip_message_id)
             .build();
 
@@ -129,7 +139,8 @@ impl Default for Config {
             target_peers: 50,
             gs_config,
             discv5_config,
-            boot_nodes: vec![],
+            boot_nodes_enr: vec![],
+            boot_nodes_multiaddr: vec![],
             libp2p_nodes: vec![],
             client_version: lighthouse_version::version_with_platform(),
             disable_discovery: false,

beacon_node/eth2_libp2p/src/discovery/enr.rs

@@ -6,6 +6,7 @@ use super::enr_ext::CombinedKeyExt;
 use super::ENR_FILENAME;
 use crate::types::{Enr, EnrBitfield};
 use crate::NetworkConfig;
+use discv5::enr::EnrKey;
 use libp2p::core::identity::Keypair;
 use slog::{debug, warn};
 use ssz::{Decode, Encode};
@@ -48,6 +49,56 @@ impl Eth2Enr for Enr {
     }
 }
 
+/// Either use the given ENR or load an ENR from file if it exists and matches the current NodeId
+/// and sequence number.
+/// If an ENR exists, with the same NodeId, this function checks to see if the loaded ENR from
+/// disk is suitable to use, otherwise we increment the given ENR's sequence number.
+pub fn use_or_load_enr(
+    enr_key: &CombinedKey,
+    local_enr: &mut Enr,
+    config: &NetworkConfig,
+    log: &slog::Logger,
+) -> Result<(), String> {
+    let enr_f = config.network_dir.join(ENR_FILENAME);
+    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
+        let mut enr_string = String::new();
+        match enr_file.read_to_string(&mut enr_string) {
+            Err(_) => debug!(log, "Could not read ENR from file"),
+            Ok(_) => {
+                match Enr::from_str(&enr_string) {
+                    Ok(disk_enr) => {
+                        // if the same node id, then we may need to update our sequence number
+                        if local_enr.node_id() == disk_enr.node_id() {
+                            if compare_enr(&local_enr, &disk_enr) {
+                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
+                                // the stored ENR has the same configuration, use it
+                                *local_enr = disk_enr;
+                                return Ok(());
+                            }
+
+                            // same node id, different configuration - update the sequence number
+                            // Note: local_enr is generated with default(0) attnets value,
+                            // so a non default value in persisted enr will also update sequence number.
+                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
+                            local_enr.set_seq(new_seq_no, enr_key).map_err(|e| {
+                                format!("Could not update ENR sequence number: {:?}", e)
+                            })?;
+                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
+                        }
+                    }
+                    Err(e) => {
+                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
+                    }
+                }
+            }
+        }
+    }
+
+    save_enr_to_disk(&config.network_dir, &local_enr, log);
+
+    Ok(())
+}
+
 /// Loads an ENR from file if it exists and matches the current NodeId and sequence number. If none
 /// exists, generates a new one.
 ///
@@ -65,49 +116,11 @@ pub fn build_or_load_enr<T: EthSpec>(
     let enr_key = CombinedKey::from_libp2p(&local_key)?;
     let mut local_enr = build_enr::<T>(&enr_key, config, enr_fork_id)?;
 
-    let enr_f = config.network_dir.join(ENR_FILENAME);
-    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
-        let mut enr_string = String::new();
-        match enr_file.read_to_string(&mut enr_string) {
-            Err(_) => debug!(log, "Could not read ENR from file"),
-            Ok(_) => {
-                match Enr::from_str(&enr_string) {
-                    Ok(disk_enr) => {
-                        // if the same node id, then we may need to update our sequence number
-                        if local_enr.node_id() == disk_enr.node_id() {
-                            if compare_enr(&local_enr, &disk_enr) {
-                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
-                                // the stored ENR has the same configuration, use it
-                                return Ok(disk_enr);
-                            }
-
-                            // same node id, different configuration - update the sequence number
-                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
-                            local_enr.set_seq(new_seq_no, &enr_key).map_err(|e| {
-                                format!("Could not update ENR sequence number: {:?}", e)
-                            })?;
-                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
-                        }
-                    }
-                    Err(e) => {
-                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
-                    }
-                }
-            }
-        }
-    }
-
-    save_enr_to_disk(&config.network_dir, &local_enr, log);
-
+    use_or_load_enr(&enr_key, &mut local_enr, config, log)?;
     Ok(local_enr)
 }
 
-/// Builds a lighthouse ENR given a `NetworkConfig`.
-pub fn build_enr<T: EthSpec>(
-    enr_key: &CombinedKey,
-    config: &NetworkConfig,
-    enr_fork_id: EnrForkId,
-) -> Result<Enr, String> {
+pub fn create_enr_builder_from_config<T: EnrKey>(config: &NetworkConfig) -> EnrBuilder<T> {
     let mut builder = EnrBuilder::new("v4");
     if let Some(enr_address) = config.enr_address {
         builder.ip(enr_address);
@@ -118,7 +131,17 @@ pub fn build_enr<T: EthSpec>(
     // we always give it our listening tcp port
     // TODO: Add uPnP support to map udp and tcp ports
     let tcp_port = config.enr_tcp_port.unwrap_or_else(|| config.libp2p_port);
-    builder.tcp(tcp_port);
+    builder.tcp(tcp_port).tcp(config.libp2p_port);
+    builder
+}
+
+/// Builds a lighthouse ENR given a `NetworkConfig`.
+pub fn build_enr<T: EthSpec>(
+    enr_key: &CombinedKey,
+    config: &NetworkConfig,
+    enr_fork_id: EnrForkId,
+) -> Result<Enr, String> {
+    let mut builder = create_enr_builder_from_config(config);
 
     // set the `eth2` field on our ENR
     builder.add_value(ETH2_ENR_KEY.into(), enr_fork_id.as_ssz_bytes());
@@ -129,7 +152,6 @@ pub fn build_enr<T: EthSpec>(
     builder.add_value(BITFIELD_ENR_KEY.into(), bitfield.as_ssz_bytes());
 
     builder
-        .tcp(config.libp2p_port)
         .build(enr_key)
         .map_err(|e| format!("Could not build Local ENR: {:?}", e))
 }

beacon_node/eth2_libp2p/src/discovery/mod.rs

@@ -3,19 +3,19 @@ pub(crate) mod enr;
 pub mod enr_ext;
 
 // Allow external use of the lighthouse ENR builder
-pub use enr::{build_enr, CombinedKey, Eth2Enr};
+pub use enr::{build_enr, create_enr_builder_from_config, use_or_load_enr, CombinedKey, Eth2Enr};
 pub use enr_ext::{CombinedKeyExt, EnrExt};
 pub use libp2p::core::identity::Keypair;
 
 use crate::metrics;
-use crate::{error, Enr, NetworkConfig, NetworkGlobals};
+use crate::{error, Enr, NetworkConfig, NetworkGlobals, SubnetDiscovery};
 use discv5::{enr::NodeId, Discv5, Discv5Event};
 use enr::{BITFIELD_ENR_KEY, ETH2_ENR_KEY};
 use futures::prelude::*;
 use futures::stream::FuturesUnordered;
 use libp2p::core::PeerId;
 use lru::LruCache;
-use slog::{crit, debug, info, warn};
+use slog::{crit, debug, error, info, warn};
 use ssz::{Decode, Encode};
 use ssz_types::BitVector;
 use std::{
@@ -163,7 +163,7 @@ pub struct Discovery<TSpec: EthSpec> {
 
 impl<TSpec: EthSpec> Discovery<TSpec> {
     /// NOTE: Creating discovery requires running within a tokio execution environment.
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         config: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
@@ -189,35 +189,84 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             .map_err(|e| format!("Discv5 service failed. Error: {:?}", e))?;
 
         // Add bootnodes to routing table
-        for bootnode_enr in config.boot_nodes.clone() {
+        for bootnode_enr in config.boot_nodes_enr.clone() {
             debug!(
                 log,
                 "Adding node to routing table";
-                "node_id" => format!("{}", bootnode_enr.node_id()),
-                "peer_id" => format!("{}", bootnode_enr.peer_id()),
+                "node_id" => bootnode_enr.node_id().to_string(),
+                "peer_id" => bootnode_enr.peer_id().to_string(),
                 "ip" => format!("{:?}", bootnode_enr.ip()),
                 "udp" => format!("{:?}", bootnode_enr.udp()),
                 "tcp" => format!("{:?}", bootnode_enr.tcp())
             );
+            let repr = bootnode_enr.to_string();
             let _ = discv5.add_enr(bootnode_enr).map_err(|e| {
-                debug!(
+                error!(
                     log,
                     "Could not add peer to the local routing table";
-                    "error" => e.to_string()
+                    "addr" => repr,
+                    "error" => e.to_string(),
                 )
             });
         }
 
         // Start the discv5 service and obtain an event stream
         let event_stream = if !config.disable_discovery {
-            discv5.start(listen_socket);
+            discv5.start(listen_socket).map_err(|e| e.to_string())?;
             debug!(log, "Discovery service started");
             EventStream::Awaiting(Box::pin(discv5.event_stream()))
         } else {
             EventStream::InActive
         };
 
-        // Obtain the event stream
+        if !config.boot_nodes_multiaddr.is_empty() {
+            info!(log, "Contacting Multiaddr boot-nodes for their ENR");
+        }
+
+        // get futures for requesting the Enrs associated to these multiaddr and wait for their
+        // completion
+        let mut fut_coll = config
+            .boot_nodes_multiaddr
+            .iter()
+            .map(|addr| addr.to_string())
+            // request the ENR for this multiaddr and keep the original for logging
+            .map(|addr| {
+                futures::future::join(
+                    discv5.request_enr(addr.clone()),
+                    futures::future::ready(addr),
+                )
+            })
+            .collect::<FuturesUnordered<_>>();
+
+        while let Some((result, original_addr)) = fut_coll.next().await {
+            match result {
+                Ok(Some(enr)) => {
+                    debug!(
+                        log,
+                        "Adding node to routing table";
+                        "node_id" => enr.node_id().to_string(),
+                        "peer_id" => enr.peer_id().to_string(),
+                        "ip" => format!("{:?}", enr.ip()),
+                        "udp" => format!("{:?}", enr.udp()),
+                        "tcp" => format!("{:?}", enr.tcp())
+                    );
+                    let _ = discv5.add_enr(enr).map_err(|e| {
+                        error!(
+                            log,
+                            "Could not add peer to the local routing table";
+                            "addr" => original_addr.to_string(),
+                            "error" => e.to_string(),
+                        )
+                    });
+                }
+                Ok(None) => {
+                    error!(log, "No ENR found for MultiAddr"; "addr" => original_addr.to_string())
+                }
+                Err(e) => {
+                    error!(log, "Error getting mapping to ENR"; "multiaddr" => original_addr.to_string(), "error" => e.to_string())
+                }
+            }
+        }
 
         Ok(Self {
             cached_enrs: LruCache::new(50),
@@ -256,12 +305,19 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
     }
 
     /// Processes a request to search for more peers on a subnet.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
+    pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
         // If the discv5 service isn't running, ignore queries
         if !self.started {
             return;
         }
-        self.add_subnet_query(subnet_id, min_ttl, 0);
+        debug!(
+            self.log,
+            "Making discovery query for subnets";
+            "subnets" => format!("{:?}", subnets_to_discover.iter().map(|s| s.subnet_id).collect::<Vec<_>>())
+        );
+        for subnet in subnets_to_discover {
+            self.add_subnet_query(subnet.subnet_id, subnet.min_ttl, 0);
+        }
     }
 
     /// Add an ENR to the routing table of the discovery mechanism.
@@ -335,6 +391,9 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         // replace the global version
         *self.network_globals.local_enr.write() = self.discv5.local_enr();
+
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
         Ok(())
     }
 
@@ -367,6 +426,9 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         // replace the global version with discovery version
         *self.network_globals.local_enr.write() = self.discv5.local_enr();
+
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
     }
 
     /* Internal Functions */
@@ -459,6 +521,11 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                         // This query is for searching for peers of a particular subnet
                         // Drain subnet_queries so we can re-use it as we continue to process the queue
                         let grouped_queries: Vec<SubnetQuery> = subnet_queries.drain(..).collect();
+                        debug!(
+                            self.log,
+                            "Starting grouped subnet query";
+                            "subnets" => format!("{:?}", grouped_queries.iter().map(|q| q.subnet_id).collect::<Vec<_>>()),
+                        );
                         self.start_subnet_query(grouped_queries);
                     }
                 }
@@ -733,8 +800,8 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                             if enr.eth2() == self.local_enr().eth2() {
                                 trace!(self.log, "Peer found in process of query"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
                             } else {
-                                // this is temporary warning for debugging the DHT
-                                warn!(self.log, "Found peer during discovery not on correct fork"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
+                            // this is temporary warning for debugging the DHT
+                            warn!(self.log, "Found peer during discovery not on correct fork"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
                             }
                             */
                         }

beacon_node/eth2_libp2p/src/discovery/subnet_predicate.rs

@@ -1,5 +1,6 @@
 ///! The subnet predicate used for searching for a particular subnet.
 use super::*;
+use slog::{debug, trace};
 use std::ops::Deref;
 
 /// Returns the predicate for a given subnet.
@@ -30,7 +31,7 @@ where
                 .collect();
 
             if matches.is_empty() {
-                debug!(
+                trace!(
                     log_clone,
                     "Peer found but not on any of the desired subnets";
                     "peer_id" => format!("{}", enr.peer_id())

beacon_node/eth2_libp2p/src/lib.rs

@@ -14,7 +14,7 @@ pub mod rpc;
 mod service;
 pub mod types;
 
-pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage};
+pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage, SubnetDiscovery};
 pub use behaviour::{BehaviourEvent, PeerRequestId, Request, Response};
 pub use config::Config as NetworkConfig;
 pub use discovery::{CombinedKeyExt, EnrExt, Eth2Enr};
@@ -26,4 +26,4 @@ pub use metrics::scrape_discovery_metrics;
 pub use peer_manager::{
     client::Client, score::PeerAction, PeerDB, PeerInfo, PeerSyncStatus, SyncInfo,
 };
-pub use service::{Libp2pEvent, Service, NETWORK_KEY_FILENAME};
+pub use service::{load_private_key, Libp2pEvent, Service, NETWORK_KEY_FILENAME};

beacon_node/eth2_libp2p/src/peer_manager/client.rs

@@ -30,6 +30,8 @@ pub enum ClientKind {
     Teku,
     /// A Prysm node.
     Prysm,
+    /// A lodestar node.
+    Lodestar,
     /// An unknown client.
     Unknown,
 }
@@ -84,6 +86,7 @@ impl std::fmt::Display for Client {
                 "Prysm: version: {}, os_version: {}",
                 self.version, self.os_version
             ),
+            ClientKind::Lodestar => write!(f, "Lodestar: version: {}", self.version),
             ClientKind::Unknown => {
                 if let Some(agent_string) = &self.agent_string {
                     write!(f, "Unknown: {}", agent_string)
@@ -157,6 +160,18 @@ fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String
             }
             (kind, version, os_version)
         }
+        Some("js-libp2p") => {
+            let kind = ClientKind::Lodestar;
+            let mut version = String::from("unknown");
+            let mut os_version = version.clone();
+            if let Some(agent_version) = agent_split.next() {
+                version = agent_version.into();
+                if let Some(agent_os_version) = agent_split.next() {
+                    os_version = agent_os_version.into();
+                }
+            }
+            (kind, version, os_version)
+        }
         _ => {
             let unknown = String::from("unknown");
             (ClientKind::Unknown, unknown.clone(), unknown)

beacon_node/eth2_libp2p/src/peer_manager/mod.rs

@@ -4,7 +4,7 @@ pub use self::peerdb::*;
 use crate::discovery::{Discovery, DiscoveryEvent};
 use crate::rpc::{GoodbyeReason, MetaData, Protocol, RPCError, RPCResponseErrorCode};
 use crate::{error, metrics};
-use crate::{EnrExt, NetworkConfig, NetworkGlobals, PeerId};
+use crate::{EnrExt, NetworkConfig, NetworkGlobals, PeerId, SubnetDiscovery};
 use futures::prelude::*;
 use futures::Stream;
 use hashset_delay::HashSetDelay;
@@ -19,7 +19,7 @@ use std::{
     task::{Context, Poll},
     time::{Duration, Instant},
 };
-use types::{EthSpec, SubnetId};
+use types::EthSpec;
 
 pub use libp2p::core::{identity::Keypair, Multiaddr};
 
@@ -88,14 +88,14 @@ pub enum PeerManagerEvent {
 
 impl<TSpec: EthSpec> PeerManager<TSpec> {
     // NOTE: Must be run inside a tokio executor.
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         config: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
         log: &slog::Logger,
     ) -> error::Result<Self> {
         // start the discovery service
-        let mut discovery = Discovery::new(local_key, config, network_globals.clone(), log)?;
+        let mut discovery = Discovery::new(local_key, config, network_globals.clone(), log).await?;
 
         // start searching for peers
         discovery.discover_peers();
@@ -213,17 +213,19 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     }
 
     /// A request to find peers on a given subnet.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
+    pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
         // Extend the time to maintain peers if required.
-        if let Some(min_ttl) = min_ttl {
-            self.network_globals
-                .peers
-                .write()
-                .extend_peers_on_subnet(subnet_id, min_ttl);
+        for s in subnets_to_discover.iter() {
+            if let Some(min_ttl) = s.min_ttl {
+                self.network_globals
+                    .peers
+                    .write()
+                    .extend_peers_on_subnet(s.subnet_id, min_ttl);
+            }
         }
 
         // request the subnet query from discovery
-        self.discovery.discover_subnet_peers(subnet_id, min_ttl);
+        self.discovery.discover_subnet_peers(subnets_to_discover);
     }
 
     /// A STATUS message has been received from a peer. This resets the status timer.
@@ -539,11 +541,8 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     ///
     /// This is called by `connect_ingoing` and `connect_outgoing`.
     ///
-    /// This informs if the peer was accepted in to the db or not.
+    /// Informs if the peer was accepted in to the db or not.
     fn connect_peer(&mut self, peer_id: &PeerId, connection: ConnectingType) -> bool {
-        // TODO: remove after timed updates
-        //self.update_reputations();
-
         {
             let mut peerdb = self.network_globals.peers.write();
             if peerdb.connection_status(peer_id).map(|c| c.is_banned()) == Some(true) {
@@ -690,7 +689,8 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// NOTE: Discovery will only add a new query if one isn't already queued.
     fn heartbeat(&mut self) {
         // TODO: Provide a back-off time for discovery queries. I.e Queue many initially, then only
-        // perform discoveries over a larger fixed interval. Perhaps one every 6 heartbeats
+        // perform discoveries over a larger fixed interval. Perhaps one every 6 heartbeats. This
+        // is achievable with a leaky bucket
         let peer_count = self.network_globals.connected_or_dialing_peers();
         if peer_count < self.target_peers {
             // If we need more peers, queue a discovery lookup.

beacon_node/eth2_libp2p/src/peer_manager/peer_info.rs

@@ -7,6 +7,7 @@ use serde::{
     ser::{SerializeStructVariant, Serializer},
     Serialize,
 };
+use std::net::IpAddr;
 use std::time::Instant;
 use types::{EthSpec, SubnetId};
 use PeerConnectionStatus::*;
@@ -104,6 +105,8 @@ pub enum PeerConnectionStatus {
     Banned {
         /// moment when the peer was banned.
         since: Instant,
+        /// ip addresses this peer had a the moment of the ban
+        ip_addresses: Vec<IpAddr>,
     },
     /// We are currently dialing this peer.
     Dialing {
@@ -129,7 +132,7 @@ impl Serialize for PeerConnectionStatus {
                 s.serialize_field("since", &since.elapsed().as_secs())?;
                 s.end()
             }
-            Banned { since } => {
+            Banned { since, .. } => {
                 let mut s = serializer.serialize_struct_variant("", 2, "Banned", 1)?;
                 s.serialize_field("since", &since.elapsed().as_secs())?;
                 s.end()
@@ -218,15 +221,16 @@ impl PeerConnectionStatus {
     }
 
     /// Modifies the status to Banned
-    pub fn ban(&mut self) {
+    pub fn ban(&mut self, ip_addresses: Vec<IpAddr>) {
         *self = Banned {
             since: Instant::now(),
+            ip_addresses,
         };
     }
 
     /// The score system has unbanned the peer. Update the connection status
     pub fn unban(&mut self) {
-        if let PeerConnectionStatus::Banned { since } = self {
+        if let PeerConnectionStatus::Banned { since, .. } = self {
             *self = PeerConnectionStatus::Disconnected { since: *since }
         }
     }

beacon_node/eth2_libp2p/src/peer_manager/peerdb.rs

@@ -1,11 +1,13 @@
 use super::peer_info::{PeerConnectionStatus, PeerInfo};
 use super::peer_sync_status::PeerSyncStatus;
 use super::score::{Score, ScoreState};
+use crate::multiaddr::Protocol;
 use crate::rpc::methods::MetaData;
 use crate::PeerId;
 use rand::seq::SliceRandom;
 use slog::{crit, debug, trace, warn};
 use std::collections::HashMap;
+use std::net::IpAddr;
 use std::time::Instant;
 use types::{EthSpec, SubnetId};
 
@@ -13,6 +15,9 @@ use types::{EthSpec, SubnetId};
 const MAX_DC_PEERS: usize = 500;
 /// The maximum number of banned nodes to remember.
 const MAX_BANNED_PEERS: usize = 1000;
+/// If there are more than `BANNED_PEERS_PER_IP_THRESHOLD` many banned peers with the same IP we ban
+/// the IP.
+const BANNED_PEERS_PER_IP_THRESHOLD: usize = 5;
 
 /// Storage of known peers, their reputation and information
 pub struct PeerDB<TSpec: EthSpec> {
@@ -20,18 +25,72 @@ pub struct PeerDB<TSpec: EthSpec> {
     peers: HashMap<PeerId, PeerInfo<TSpec>>,
     /// The number of disconnected nodes in the database.
     disconnected_peers: usize,
-    /// The number of banned peers in the database.
-    banned_peers: usize,
+    /// Counts banned peers in total and per ip
+    banned_peers_count: BannedPeersCount,
     /// PeerDB's logger
     log: slog::Logger,
 }
 
+pub struct BannedPeersCount {
+    /// The number of banned peers in the database.
+    banned_peers: usize,
+    /// maps ips to number of banned peers with this ip
+    banned_peers_per_ip: HashMap<IpAddr, usize>,
+}
+
+impl BannedPeersCount {
+    /// Removes the peer from the counts if it is banned. Returns true if the peer was banned and
+    /// false otherwise.
+    pub fn remove_banned_peer(&mut self, connection_status: &PeerConnectionStatus) -> bool {
+        match connection_status {
+            PeerConnectionStatus::Banned { ip_addresses, .. } => {
+                self.banned_peers = self.banned_peers.saturating_sub(1);
+                for address in ip_addresses {
+                    if let Some(count) = self.banned_peers_per_ip.get_mut(address) {
+                        *count = count.saturating_sub(1);
+                    }
+                }
+                true
+            }
+            _ => false, //if not banned do nothing
+        }
+    }
+
+    pub fn add_banned_peer(&mut self, connection_status: &PeerConnectionStatus) {
+        if let PeerConnectionStatus::Banned { ip_addresses, .. } = connection_status {
+            self.banned_peers += 1;
+            for address in ip_addresses {
+                *self.banned_peers_per_ip.entry(*address).or_insert(0) += 1;
+            }
+        }
+    }
+
+    pub fn banned_peers(&self) -> usize {
+        self.banned_peers
+    }
+
+    /// An IP is considered banned if more than BANNED_PEERS_PER_IP_THRESHOLD banned peers
+    /// exist with this IP
+    pub fn ip_is_banned(&self, ip: &IpAddr) -> bool {
+        self.banned_peers_per_ip
+            .get(ip)
+            .map_or(false, |count| *count > BANNED_PEERS_PER_IP_THRESHOLD)
+    }
+
+    pub fn new() -> Self {
+        BannedPeersCount {
+            banned_peers: 0,
+            banned_peers_per_ip: HashMap::new(),
+        }
+    }
+}
+
 impl<TSpec: EthSpec> PeerDB<TSpec> {
     pub fn new(log: &slog::Logger) -> Self {
         Self {
             log: log.clone(),
             disconnected_peers: 0,
-            banned_peers: 0,
+            banned_peers_count: BannedPeersCount::new(),
             peers: HashMap::new(),
         }
     }
@@ -99,17 +158,35 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
 
     /// Returns true if the Peer is banned.
     pub fn is_banned(&self, peer_id: &PeerId) -> bool {
-        match self.peers.get(peer_id).map(|info| info.score.state()) {
-            Some(ScoreState::Banned) => true,
-            _ => false,
+        if let Some(peer) = self.peers.get(peer_id) {
+            match peer.score.state() {
+                ScoreState::Banned => true,
+                _ => self.ip_is_banned(peer),
+            }
+        } else {
+            false
         }
     }
 
+    fn ip_is_banned(&self, peer: &PeerInfo<TSpec>) -> bool {
+        peer.listening_addresses.iter().any(|addr| {
+            addr.iter().any(|p| match p {
+                Protocol::Ip4(ip) => self.banned_peers_count.ip_is_banned(&ip.into()),
+                Protocol::Ip6(ip) => self.banned_peers_count.ip_is_banned(&ip.into()),
+                _ => false,
+            })
+        })
+    }
+
     /// Returns true if the Peer is either banned or in the disconnected state.
     pub fn is_banned_or_disconnected(&self, peer_id: &PeerId) -> bool {
-        match self.peers.get(peer_id).map(|info| info.score.state()) {
-            Some(ScoreState::Banned) | Some(ScoreState::Disconnected) => true,
-            _ => false,
+        if let Some(peer) = self.peers.get(peer_id) {
+            match peer.score.state() {
+                ScoreState::Banned | ScoreState::Disconnected => true,
+                _ => self.ip_is_banned(peer),
+            }
+        } else {
+            false
         }
     }
 
@@ -233,9 +310,10 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         if info.connection_status.is_disconnected() {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
-        if info.connection_status.is_banned() {
-            self.banned_peers = self.banned_peers.saturating_sub(1);
-        }
+
+        self.banned_peers_count
+            .remove_banned_peer(&info.connection_status);
+
         info.connection_status = PeerConnectionStatus::Dialing {
             since: Instant::now(),
         };
@@ -284,9 +362,8 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         if info.connection_status.is_disconnected() {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
-        if info.connection_status.is_banned() {
-            self.banned_peers = self.banned_peers.saturating_sub(1);
-        }
+        self.banned_peers_count
+            .remove_banned_peer(&info.connection_status);
         info.connection_status.connect_ingoing();
     }
 
@@ -297,9 +374,8 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         if info.connection_status.is_disconnected() {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
-        if info.connection_status.is_banned() {
-            self.banned_peers = self.banned_peers.saturating_sub(1);
-        }
+        self.banned_peers_count
+            .remove_banned_peer(&info.connection_status);
         info.connection_status.connect_outgoing();
     }
 
@@ -329,8 +405,23 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
             self.disconnected_peers = self.disconnected_peers.saturating_sub(1);
         }
         if !info.connection_status.is_banned() {
-            info.connection_status.ban();
-            self.banned_peers += 1;
+            info.connection_status
+                .ban(
+                    info.listening_addresses
+                        .iter()
+                        .fold(Vec::new(), |mut v, a| {
+                            for p in a {
+                                match p {
+                                    Protocol::Ip4(ip) => v.push(ip.into()),
+                                    Protocol::Ip6(ip) => v.push(ip.into()),
+                                    _ => (),
+                                }
+                            }
+                            v
+                        }),
+                );
+            self.banned_peers_count
+                .add_banned_peer(&info.connection_status);
         }
         self.shrink_to_fit();
     }
@@ -345,8 +436,9 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
         });
 
         if info.connection_status.is_banned() {
+            self.banned_peers_count
+                .remove_banned_peer(&info.connection_status);
             info.connection_status.unban();
-            self.banned_peers = self.banned_peers.saturating_sub(1);
         }
         self.shrink_to_fit();
     }
@@ -355,9 +447,9 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
     /// Drops the peers with the lowest reputation so that the number of
     /// disconnected peers is less than MAX_DC_PEERS
     pub fn shrink_to_fit(&mut self) {
-        // Remove excess baned peers
-        while self.banned_peers > MAX_BANNED_PEERS {
-            if let Some(to_drop) = self
+        // Remove excess banned peers
+        while self.banned_peers_count.banned_peers() > MAX_BANNED_PEERS {
+            if let Some(to_drop) = if let Some((id, info)) = self
                 .peers
                 .iter()
                 .filter(|(_, info)| info.connection_status.is_banned())
@@ -366,15 +458,23 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
                         .score
                         .partial_cmp(&info_b.score)
                         .unwrap_or(std::cmp::Ordering::Equal)
-                })
-                .map(|(id, _)| id.clone())
-            {
+                }) {
+                self.banned_peers_count
+                    .remove_banned_peer(&info.connection_status);
+                Some(id.clone())
+            } else {
+                // If there is no minimum, this is a coding error.
+                crit!(
+                    self.log,
+                    "banned_peers > MAX_BANNED_PEERS despite no banned peers in db!"
+                );
+                // reset banned_peers this will also exit the loop
+                self.banned_peers_count = BannedPeersCount::new();
+                None
+            } {
                 debug!(self.log, "Removing old banned peer"; "peer_id" => to_drop.to_string());
                 self.peers.remove(&to_drop);
             }
-            // If there is no minimum, this is a coding error. For safety we decrease
-            // the count to avoid a potential infinite loop.
-            self.banned_peers = self.banned_peers.saturating_sub(1);
         }
 
         // Remove excess disconnected peers
@@ -422,8 +522,11 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use libp2p::core::Multiaddr;
     use slog::{o, Drain};
+    use std::net::{Ipv4Addr, Ipv6Addr};
     use types::MinimalEthSpec;
+
     type M = MinimalEthSpec;
 
     pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger {
@@ -503,13 +606,13 @@ mod tests {
             let p = PeerId::random();
             pdb.connect_ingoing(&p);
         }
-        assert_eq!(pdb.banned_peers, 0);
+        assert_eq!(pdb.banned_peers_count.banned_peers(), 0);
 
         for p in pdb.connected_peer_ids().cloned().collect::<Vec<_>>() {
             pdb.ban(&p);
         }
 
-        assert_eq!(pdb.banned_peers, MAX_BANNED_PEERS);
+        assert_eq!(pdb.banned_peers_count.banned_peers(), MAX_BANNED_PEERS);
     }
 
     #[test]
@@ -608,24 +711,39 @@ mod tests {
         pdb.connect_ingoing(&random_peer2);
         pdb.connect_ingoing(&random_peer3);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.connect_ingoing(&random_peer);
         pdb.disconnect(&random_peer1);
         pdb.ban(&random_peer2);
         pdb.connect_ingoing(&random_peer3);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.ban(&random_peer1);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.connect_outgoing(&random_peer2);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.ban(&random_peer3);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.ban(&random_peer3);
         pdb.connect_ingoing(&random_peer1);
@@ -633,15 +751,191 @@ mod tests {
         pdb.ban(&random_peer3);
         pdb.connect_ingoing(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.disconnect(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
 
         pdb.disconnect(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
-        assert_eq!(pdb.banned_peers, pdb.banned_peers().count());
+        assert_eq!(
+            pdb.banned_peers_count.banned_peers(),
+            pdb.banned_peers().count()
+        );
         pdb.ban(&random_peer);
         assert_eq!(pdb.disconnected_peers, pdb.disconnected_peers().count());
     }
+
+    fn connect_peer_with_ips(pdb: &mut PeerDB<M>, ips: Vec<Vec<IpAddr>>) -> PeerId {
+        let p = PeerId::random();
+        pdb.connect_ingoing(&p);
+        pdb.peers.get_mut(&p).unwrap().listening_addresses = ips
+            .into_iter()
+            .map(|ip_addresses| {
+                let mut addr = Multiaddr::empty();
+                for ip_address in ip_addresses {
+                    addr.push(Protocol::from(ip_address));
+                }
+                addr
+            })
+            .collect();
+        p
+    }
+
+    #[test]
+    fn test_ban_address() {
+        let mut pdb = get_db();
+
+        let ip1: IpAddr = Ipv4Addr::new(1, 2, 3, 4).into();
+        let ip2: IpAddr = Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8).into();
+        let ip3: IpAddr = Ipv4Addr::new(1, 2, 3, 5).into();
+        let ip4: IpAddr = Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 9).into();
+        let ip5: IpAddr = Ipv4Addr::new(2, 2, 3, 4).into();
+
+        let mut peers = Vec::new();
+        for i in 0..BANNED_PEERS_PER_IP_THRESHOLD + 2 {
+            peers.push(connect_peer_with_ips(
+                &mut pdb,
+                if i == 0 {
+                    vec![vec![ip1], vec![ip2]]
+                } else {
+                    vec![vec![ip1, ip2], vec![ip3, ip4]]
+                },
+            ));
+        }
+
+        let p1 = connect_peer_with_ips(&mut pdb, vec![vec![ip1]]);
+        let p2 = connect_peer_with_ips(&mut pdb, vec![vec![ip2, ip5]]);
+        let p3 = connect_peer_with_ips(&mut pdb, vec![vec![ip3], vec![ip5]]);
+        let p4 = connect_peer_with_ips(&mut pdb, vec![vec![ip5, ip4]]);
+        let p5 = connect_peer_with_ips(&mut pdb, vec![vec![ip5]]);
+
+        for p in &peers[..BANNED_PEERS_PER_IP_THRESHOLD + 1] {
+            pdb.ban(p);
+        }
+
+        //check that ip1 and ip2 are banned but ip3-5 not
+        assert!(pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+        assert!(!pdb.is_banned(&p3));
+        assert!(!pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+
+        //ban also the last peer in peers
+        pdb.ban(&peers[BANNED_PEERS_PER_IP_THRESHOLD + 1]);
+
+        //check that ip1-ip4 are banned but ip5 not
+        assert!(pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+        assert!(pdb.is_banned(&p3));
+        assert!(pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+
+        //peers[0] gets unbanned
+        pdb.unban(&peers[0]);
+
+        //nothing changed
+        assert!(pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+        assert!(pdb.is_banned(&p3));
+        assert!(pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+
+        //peers[1] gets unbanned
+        pdb.unban(&peers[1]);
+
+        //all ips are unbanned
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+        assert!(!pdb.is_banned(&p3));
+        assert!(!pdb.is_banned(&p4));
+        assert!(!pdb.is_banned(&p5));
+    }
+
+    #[test]
+    fn test_banned_ip_consistent_after_changing_ips() {
+        let mut pdb = get_db();
+
+        let ip1: IpAddr = Ipv4Addr::new(1, 2, 3, 4).into();
+        let ip2: IpAddr = Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8).into();
+
+        let mut peers = Vec::new();
+        for _ in 0..BANNED_PEERS_PER_IP_THRESHOLD + 1 {
+            peers.push(connect_peer_with_ips(&mut pdb, vec![vec![ip1]]));
+        }
+
+        let p1 = connect_peer_with_ips(&mut pdb, vec![vec![ip1]]);
+        let p2 = connect_peer_with_ips(&mut pdb, vec![vec![ip2]]);
+
+        //ban all peers
+        for p in &peers {
+            pdb.ban(p);
+        }
+
+        //check ip is banned
+        assert!(pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //change addresses of banned peers
+        for p in &peers {
+            pdb.peers.get_mut(p).unwrap().listening_addresses =
+                vec![Multiaddr::empty().with(Protocol::from(ip2))];
+        }
+
+        //check still the same ip is banned
+        assert!(pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //unban a peer
+        pdb.unban(&peers[0]);
+
+        //check not banned anymore
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //check still not banned after new ban
+        pdb.ban(&peers[0]);
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //unban and reban all peers
+        for p in &peers {
+            pdb.unban(p);
+            pdb.ban(p);
+        }
+
+        //ip2 is now banned
+        assert!(!pdb.is_banned(&p1));
+        assert!(pdb.is_banned(&p2));
+
+        //change ips back again
+        for p in &peers {
+            pdb.peers.get_mut(p).unwrap().listening_addresses =
+                vec![Multiaddr::empty().with(Protocol::from(ip1))];
+        }
+
+        //reban every peer except one
+        for p in &peers[1..] {
+            pdb.unban(p);
+            pdb.ban(p);
+        }
+
+        //nothing is banned
+        assert!(!pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+
+        //reban last peer
+        pdb.unban(&peers[0]);
+        pdb.ban(&peers[0]);
+
+        //ip1 is banned
+        assert!(pdb.is_banned(&p1));
+        assert!(!pdb.is_banned(&p2));
+    }
 }

beacon_node/eth2_libp2p/src/rpc/codec/base.rs

@@ -174,7 +174,6 @@ where
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::*;
     use super::super::ssz_snappy::*;
     use super::*;
     use crate::rpc::protocol::*;
@@ -189,29 +188,22 @@ mod tests {
 
         let snappy_protocol_id =
             ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy);
-        let ssz_protocol_id = ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZ);
 
         let mut snappy_outbound_codec =
             SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, 1_048_576);
-        let mut ssz_outbound_codec = SSZOutboundCodec::<Spec>::new(ssz_protocol_id, 1_048_576);
 
         // decode message just as snappy message
         let snappy_decoded_message = snappy_outbound_codec.decode(&mut buf.clone());
         // decode message just a ssz message
-        let ssz_decoded_message = ssz_outbound_codec.decode(&mut buf.clone());
 
         // build codecs for entire chunk
         let mut snappy_base_outbound_codec = BaseOutboundCodec::new(snappy_outbound_codec);
-        let mut ssz_base_outbound_codec = BaseOutboundCodec::new(ssz_outbound_codec);
 
         // decode message as ssz snappy chunk
         let snappy_decoded_chunk = snappy_base_outbound_codec.decode(&mut buf.clone());
         // decode message just a ssz chunk
-        let ssz_decoded_chunk = ssz_base_outbound_codec.decode(&mut buf.clone());
 
         let _ = dbg!(snappy_decoded_message);
-        let _ = dbg!(ssz_decoded_message);
         let _ = dbg!(snappy_decoded_chunk);
-        let _ = dbg!(ssz_decoded_chunk);
     }
 }

beacon_node/eth2_libp2p/src/rpc/codec/mod.rs

@@ -1,9 +1,7 @@
 pub(crate) mod base;
-pub(crate) mod ssz;
 pub(crate) mod ssz_snappy;
 
 use self::base::{BaseInboundCodec, BaseOutboundCodec};
-use self::ssz::{SSZInboundCodec, SSZOutboundCodec};
 use self::ssz_snappy::{SSZSnappyInboundCodec, SSZSnappyOutboundCodec};
 use crate::rpc::protocol::RPCError;
 use crate::rpc::{RPCCodedResponse, RPCRequest};
@@ -14,12 +12,10 @@ use types::EthSpec;
 // Known types of codecs
 pub enum InboundCodec<TSpec: EthSpec> {
     SSZSnappy(BaseInboundCodec<SSZSnappyInboundCodec<TSpec>, TSpec>),
-    SSZ(BaseInboundCodec<SSZInboundCodec<TSpec>, TSpec>),
 }
 
 pub enum OutboundCodec<TSpec: EthSpec> {
     SSZSnappy(BaseOutboundCodec<SSZSnappyOutboundCodec<TSpec>, TSpec>),
-    SSZ(BaseOutboundCodec<SSZOutboundCodec<TSpec>, TSpec>),
 }
 
 impl<T: EthSpec> Encoder<RPCCodedResponse<T>> for InboundCodec<T> {
@@ -27,7 +23,6 @@ impl<T: EthSpec> Encoder<RPCCodedResponse<T>> for InboundCodec<T> {
 
     fn encode(&mut self, item: RPCCodedResponse<T>, dst: &mut BytesMut) -> Result<(), Self::Error> {
         match self {
-            InboundCodec::SSZ(codec) => codec.encode(item, dst),
             InboundCodec::SSZSnappy(codec) => codec.encode(item, dst),
         }
     }
@@ -39,7 +34,6 @@ impl<TSpec: EthSpec> Decoder for InboundCodec<TSpec> {
 
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         match self {
-            InboundCodec::SSZ(codec) => codec.decode(src),
             InboundCodec::SSZSnappy(codec) => codec.decode(src),
         }
     }
@@ -50,7 +44,6 @@ impl<TSpec: EthSpec> Encoder<RPCRequest<TSpec>> for OutboundCodec<TSpec> {
 
     fn encode(&mut self, item: RPCRequest<TSpec>, dst: &mut BytesMut) -> Result<(), Self::Error> {
         match self {
-            OutboundCodec::SSZ(codec) => codec.encode(item, dst),
             OutboundCodec::SSZSnappy(codec) => codec.encode(item, dst),
         }
     }
@@ -62,7 +55,6 @@ impl<T: EthSpec> Decoder for OutboundCodec<T> {
 
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         match self {
-            OutboundCodec::SSZ(codec) => codec.decode(src),
             OutboundCodec::SSZSnappy(codec) => codec.decode(src),
         }
     }

beacon_node/eth2_libp2p/src/rpc/codec/ssz.rs

@@ -1,324 +0,0 @@
-use crate::rpc::methods::*;
-use crate::rpc::{
-    codec::base::OutboundCodec,
-    protocol::{
-        Encoding, Protocol, ProtocolId, RPCError, Version, BLOCKS_BY_ROOT_REQUEST_MAX,
-        BLOCKS_BY_ROOT_REQUEST_MIN, SIGNED_BEACON_BLOCK_MAX, SIGNED_BEACON_BLOCK_MIN,
-    },
-};
-use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
-use libp2p::bytes::{BufMut, Bytes, BytesMut};
-use ssz::{Decode, Encode};
-use ssz_types::VariableList;
-use std::marker::PhantomData;
-use tokio_util::codec::{Decoder, Encoder};
-use types::{EthSpec, SignedBeaconBlock};
-use unsigned_varint::codec::UviBytes;
-
-/* Inbound Codec */
-
-pub struct SSZInboundCodec<TSpec: EthSpec> {
-    inner: UviBytes,
-    protocol: ProtocolId,
-    phantom: PhantomData<TSpec>,
-}
-
-impl<TSpec: EthSpec> SSZInboundCodec<TSpec> {
-    pub fn new(protocol: ProtocolId, max_packet_size: usize) -> Self {
-        let mut uvi_codec = UviBytes::default();
-        uvi_codec.set_max_len(max_packet_size);
-
-        // this encoding only applies to ssz.
-        debug_assert_eq!(protocol.encoding, Encoding::SSZ);
-
-        SSZInboundCodec {
-            inner: uvi_codec,
-            protocol,
-            phantom: PhantomData,
-        }
-    }
-}
-
-// Encoder for inbound streams: Encodes RPC Responses sent to peers.
-impl<TSpec: EthSpec> Encoder<RPCCodedResponse<TSpec>> for SSZInboundCodec<TSpec> {
-    type Error = RPCError;
-
-    fn encode(
-        &mut self,
-        item: RPCCodedResponse<TSpec>,
-        dst: &mut BytesMut,
-    ) -> Result<(), Self::Error> {
-        let bytes = match item {
-            RPCCodedResponse::Success(resp) => match resp {
-                RPCResponse::Status(res) => res.as_ssz_bytes(),
-                RPCResponse::BlocksByRange(res) => res.as_ssz_bytes(),
-                RPCResponse::BlocksByRoot(res) => res.as_ssz_bytes(),
-                RPCResponse::Pong(res) => res.data.as_ssz_bytes(),
-                RPCResponse::MetaData(res) => res.as_ssz_bytes(),
-            },
-            RPCCodedResponse::Error(_, err) => err.as_ssz_bytes(),
-            RPCCodedResponse::StreamTermination(_) => {
-                unreachable!("Code error - attempting to encode a stream termination")
-            }
-        };
-        if !bytes.is_empty() {
-            // length-prefix and return
-            return self
-                .inner
-                .encode(Bytes::from(bytes), dst)
-                .map_err(RPCError::from);
-        } else {
-            // payload is empty, add a 0-byte length prefix
-            dst.reserve(1);
-            dst.put_u8(0);
-        }
-        Ok(())
-    }
-}
-
-// Decoder for inbound streams: Decodes RPC requests from peers
-impl<TSpec: EthSpec> Decoder for SSZInboundCodec<TSpec> {
-    type Item = RPCRequest<TSpec>;
-    type Error = RPCError;
-
-    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        match self.inner.decode(src).map_err(RPCError::from) {
-            Ok(Some(packet)) => match self.protocol.message_name {
-                Protocol::Status => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                            Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
-                                &packet,
-                            )?)))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::Goodbye => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() == <GoodbyeReason as Encode>::ssz_fixed_len() {
-                            Ok(Some(RPCRequest::Goodbye(GoodbyeReason::from_ssz_bytes(
-                                &packet,
-                            )?)))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::BlocksByRange => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() == <BlocksByRangeRequest as Encode>::ssz_fixed_len() {
-                            Ok(Some(RPCRequest::BlocksByRange(
-                                BlocksByRangeRequest::from_ssz_bytes(&packet)?,
-                            )))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::BlocksByRoot => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() >= *BLOCKS_BY_ROOT_REQUEST_MIN
-                            && packet.len() <= *BLOCKS_BY_ROOT_REQUEST_MAX
-                        {
-                            Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
-                                block_roots: VariableList::from_ssz_bytes(&packet)?,
-                            })))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::Ping => match self.protocol.version {
-                    Version::V1 => {
-                        if packet.len() == <Ping as Encode>::ssz_fixed_len() {
-                            Ok(Some(RPCRequest::Ping(Ping {
-                                data: u64::from_ssz_bytes(&packet)?,
-                            })))
-                        } else {
-                            Err(RPCError::InvalidData)
-                        }
-                    }
-                },
-                Protocol::MetaData => match self.protocol.version {
-                    Version::V1 => {
-                        if !packet.is_empty() {
-                            Err(RPCError::InvalidData)
-                        } else {
-                            Ok(Some(RPCRequest::MetaData(PhantomData)))
-                        }
-                    }
-                },
-            },
-            Ok(None) => Ok(None),
-            Err(e) => Err(e),
-        }
-    }
-}
-
-/* Outbound Codec: Codec for initiating RPC requests */
-
-pub struct SSZOutboundCodec<TSpec: EthSpec> {
-    inner: UviBytes,
-    protocol: ProtocolId,
-    phantom: PhantomData<TSpec>,
-}
-
-impl<TSpec: EthSpec> SSZOutboundCodec<TSpec> {
-    pub fn new(protocol: ProtocolId, max_packet_size: usize) -> Self {
-        let mut uvi_codec = UviBytes::default();
-        uvi_codec.set_max_len(max_packet_size);
-
-        // this encoding only applies to ssz.
-        debug_assert_eq!(protocol.encoding, Encoding::SSZ);
-
-        SSZOutboundCodec {
-            inner: uvi_codec,
-            protocol,
-            phantom: PhantomData,
-        }
-    }
-}
-
-// Encoder for outbound streams: Encodes RPC Requests to peers
-impl<TSpec: EthSpec> Encoder<RPCRequest<TSpec>> for SSZOutboundCodec<TSpec> {
-    type Error = RPCError;
-
-    fn encode(&mut self, item: RPCRequest<TSpec>, dst: &mut BytesMut) -> Result<(), Self::Error> {
-        let bytes = match item {
-            RPCRequest::Status(req) => req.as_ssz_bytes(),
-            RPCRequest::Goodbye(req) => req.as_ssz_bytes(),
-            RPCRequest::BlocksByRange(req) => req.as_ssz_bytes(),
-            RPCRequest::BlocksByRoot(req) => req.block_roots.as_ssz_bytes(),
-            RPCRequest::Ping(req) => req.as_ssz_bytes(),
-            RPCRequest::MetaData(_) => return Ok(()), // no metadata to encode
-        };
-        // length-prefix
-        self.inner
-            .encode(libp2p::bytes::Bytes::from(bytes), dst)
-            .map_err(RPCError::from)
-    }
-}
-
-// Decoder for outbound streams: Decodes RPC responses from peers.
-//
-// The majority of the decoding has now been pushed upstream due to the changing specification.
-// We prefer to decode blocks and attestations with extra knowledge about the chain to perform
-// faster verification checks before decoding entire blocks/attestations.
-impl<TSpec: EthSpec> Decoder for SSZOutboundCodec<TSpec> {
-    type Item = RPCResponse<TSpec>;
-    type Error = RPCError;
-
-    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        if src.len() == 1 && src[0] == 0_u8 {
-            // the object is empty. We return the empty object if this is the case
-            // clear the buffer and return an empty object
-            src.clear();
-            match self.protocol.message_name {
-                Protocol::Status => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty HELLO message. The stream has terminated unexpectedly
-                },
-                Protocol::Goodbye => Err(RPCError::InvalidData),
-                Protocol::BlocksByRange => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty block message.
-                },
-                Protocol::BlocksByRoot => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty block message.
-                },
-                Protocol::Ping => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty block message.
-                },
-                Protocol::MetaData => match self.protocol.version {
-                    Version::V1 => Err(RPCError::IncompleteStream), // cannot have an empty block message.
-                },
-            }
-        } else {
-            match self.inner.decode(src).map_err(RPCError::from) {
-                Ok(Some(mut packet)) => {
-                    // take the bytes from the buffer
-                    let raw_bytes = packet.split();
-
-                    match self.protocol.message_name {
-                        Protocol::Status => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                                    Ok(Some(RPCResponse::Status(StatusMessage::from_ssz_bytes(
-                                        &raw_bytes,
-                                    )?)))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                        Protocol::Goodbye => Err(RPCError::InvalidData),
-                        Protocol::BlocksByRange => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                    && raw_bytes.len() <= *SIGNED_BEACON_BLOCK_MAX
-                                {
-                                    Ok(Some(RPCResponse::BlocksByRange(Box::new(
-                                        SignedBeaconBlock::from_ssz_bytes(&raw_bytes)?,
-                                    ))))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                        Protocol::BlocksByRoot => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                    && raw_bytes.len() <= *SIGNED_BEACON_BLOCK_MAX
-                                {
-                                    Ok(Some(RPCResponse::BlocksByRoot(Box::new(
-                                        SignedBeaconBlock::from_ssz_bytes(&raw_bytes)?,
-                                    ))))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                        Protocol::Ping => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() == <Ping as Encode>::ssz_fixed_len() {
-                                    Ok(Some(RPCResponse::Pong(Ping {
-                                        data: u64::from_ssz_bytes(&raw_bytes)?,
-                                    })))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                        Protocol::MetaData => match self.protocol.version {
-                            Version::V1 => {
-                                if raw_bytes.len() == <MetaData<TSpec> as Encode>::ssz_fixed_len() {
-                                    Ok(Some(RPCResponse::MetaData(MetaData::from_ssz_bytes(
-                                        &raw_bytes,
-                                    )?)))
-                                } else {
-                                    Err(RPCError::InvalidData)
-                                }
-                            }
-                        },
-                    }
-                }
-                Ok(None) => Ok(None), // waiting for more bytes
-                Err(e) => Err(e),
-            }
-        }
-    }
-}
-
-impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZOutboundCodec<TSpec> {
-    type ErrorType = String;
-
-    fn decode_error(&mut self, src: &mut BytesMut) -> Result<Option<Self::ErrorType>, RPCError> {
-        match self.inner.decode(src).map_err(RPCError::from) {
-            Ok(Some(packet)) => Ok(Some(
-                String::from_utf8_lossy(&<Vec<u8>>::from_ssz_bytes(&packet)?).into(),
-            )),
-            Ok(None) => Ok(None),
-            Err(e) => Err(e),
-        }
-    }
-}

beacon_node/eth2_libp2p/src/rpc/handler.rs

@@ -116,9 +116,6 @@ where
     /// Maximum number of concurrent outbound substreams being opened. Value is never modified.
     max_dial_negotiated: u32,
 
-    /// Value to return from `connection_keep_alive`.
-    keep_alive: KeepAlive,
-
     /// State of the handler.
     state: HandlerState,
 
@@ -243,7 +240,6 @@ where
             current_outbound_substream_id: SubstreamId(0),
             state: HandlerState::Active,
             max_dial_negotiated: 8,
-            keep_alive: KeepAlive::Yes,
             outbound_io_error_retries: 0,
             log: log.clone(),
         }
@@ -287,7 +283,6 @@ where
                 TInstant::now() + Duration::from_secs(SHUTDOWN_TIMEOUT_SECS as u64),
             ));
         }
-        self.update_keep_alive();
     }
 
     /// Opens an outbound substream with a request.
@@ -295,7 +290,6 @@ where
         match self.state {
             HandlerState::Active => {
                 self.dial_queue.push((id, req));
-                self.update_keep_alive();
             }
             _ => {
                 self.pending_errors.push(HandlerErr::Outbound {
@@ -338,38 +332,6 @@ where
         }
         inbound_info.pending_items.push(response);
     }
-
-    /// Updates the `KeepAlive` returned by `connection_keep_alive`.
-    ///
-    /// The handler stays alive as long as there are inbound/outbound substreams established and no
-    /// items dialing/to be dialed. Otherwise it is given a grace period of inactivity of
-    /// `self.inactive_timeout`.
-    fn update_keep_alive(&mut self) {
-        // Check that we don't have outbound items pending for dialing, nor dialing, nor
-        // established. Also check that there are no established inbound substreams.
-        // Errors and events need to be reported back, so check those too.
-        let should_shutdown = if let HandlerState::ShuttingDown(_) = self.state {
-            self.dial_queue.is_empty()
-                && self.outbound_substreams.is_empty()
-                && self.inbound_substreams.is_empty()
-                && self.pending_errors.is_empty()
-                && self.events_out.is_empty()
-                && self.dial_negotiated == 0
-        } else {
-            false
-        };
-
-        match self.keep_alive {
-            KeepAlive::Yes if should_shutdown => self.keep_alive = KeepAlive::No,
-            KeepAlive::Yes => {} // We continue being active
-            KeepAlive::Until(_) if should_shutdown => self.keep_alive = KeepAlive::No, // Already deemed inactive
-            KeepAlive::Until(_) => {
-                // No longer idle
-                self.keep_alive = KeepAlive::Yes;
-            }
-            KeepAlive::No => {} // currently not used
-        }
-    }
 }
 
 impl<TSpec> ProtocolsHandler for RPCHandler<TSpec>
@@ -422,8 +384,6 @@ where
         self.events_out
             .push(RPCReceived::Request(self.current_inbound_substream_id, req));
         self.current_inbound_substream_id.0 += 1;
-
-        self.update_keep_alive();
     }
 
     fn inject_fully_negotiated_outbound(
@@ -481,8 +441,6 @@ where
             }
             self.current_outbound_substream_id.0 += 1;
         }
-
-        self.update_keep_alive();
     }
 
     fn inject_event(&mut self, rpc_event: Self::InEvent) {
@@ -510,7 +468,6 @@ where
 
         // This dialing is now considered failed
         self.dial_negotiated -= 1;
-        self.update_keep_alive();
 
         self.outbound_io_error_retries = 0;
         // map the error
@@ -543,7 +500,29 @@ where
     }
 
     fn connection_keep_alive(&self) -> KeepAlive {
-        self.keep_alive
+        // Check that we don't have outbound items pending for dialing, nor dialing, nor
+        // established. Also check that there are no established inbound substreams.
+        // Errors and events need to be reported back, so check those too.
+        let should_shutdown = match self.state {
+            HandlerState::ShuttingDown(_) => {
+                self.dial_queue.is_empty()
+                    && self.outbound_substreams.is_empty()
+                    && self.inbound_substreams.is_empty()
+                    && self.pending_errors.is_empty()
+                    && self.events_out.is_empty()
+                    && self.dial_negotiated == 0
+            }
+            HandlerState::Deactivated => {
+                // Regardless of events, the timeout has expired. Force the disconnect.
+                true
+            }
+            _ => false,
+        };
+        if should_shutdown {
+            KeepAlive::No
+        } else {
+            KeepAlive::Yes
+        }
     }
 
     fn poll(
@@ -619,8 +598,6 @@ where
                     if let Some(OutboundInfo { proto, req_id, .. }) =
                         self.outbound_substreams.remove(outbound_id.get_ref())
                     {
-                        self.update_keep_alive();
-
                         let outbound_err = HandlerErr::Outbound {
                             id: req_id,
                             proto,
@@ -719,7 +696,6 @@ where
             self.inbound_substreams.remove(&inbound_id);
         }
 
-        self.update_keep_alive();
         // drive outbound streams that need to be processed
         for outbound_id in self.outbound_substreams.keys().copied().collect::<Vec<_>>() {
             // get the state and mark it as poisoned
@@ -808,7 +784,6 @@ where
                         let request_id = entry.get().req_id;
                         self.outbound_substreams_delay.remove(delay_key);
                         entry.remove_entry();
-                        self.update_keep_alive();
                         // notify the application error
                         if request.expected_responses() > 1 {
                             // return an end of stream result
@@ -839,7 +814,6 @@ where
                             error: e,
                         };
                         entry.remove_entry();
-                        self.update_keep_alive();
                         return Poll::Ready(ProtocolsHandlerEvent::Custom(Err(outbound_err)));
                     }
                 },
@@ -852,7 +826,6 @@ where
                             let request_id = entry.get().req_id;
                             self.outbound_substreams_delay.remove(delay_key);
                             entry.remove_entry();
-                            self.update_keep_alive();
 
                             // report the stream termination to the user
                             //
@@ -889,7 +862,6 @@ where
             self.dial_negotiated += 1;
             let (id, req) = self.dial_queue.remove(0);
             self.dial_queue.shrink_to_fit();
-            self.update_keep_alive();
             return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
                 protocol: SubstreamProtocol::new(req.clone()),
                 info: (id, req),

beacon_node/eth2_libp2p/src/rpc/methods.rs

@@ -366,7 +366,7 @@ impl<T: EthSpec> std::fmt::Display for RPCCodedResponse<T> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             RPCCodedResponse::Success(res) => write!(f, "{}", res),
-            RPCCodedResponse::Error(code, err) => write!(f, "{}: {:?}", code, err),
+            RPCCodedResponse::Error(code, err) => write!(f, "{}: {}", code, err.to_string()),
             RPCCodedResponse::StreamTermination(_) => write!(f, "Stream Termination"),
         }
     }

beacon_node/eth2_libp2p/src/rpc/protocol.rs

@@ -2,7 +2,6 @@ use super::methods::*;
 use crate::rpc::{
     codec::{
         base::{BaseInboundCodec, BaseOutboundCodec},
-        ssz::{SSZInboundCodec, SSZOutboundCodec},
         ssz_snappy::{SSZSnappyInboundCodec, SSZSnappyOutboundCodec},
         InboundCodec, OutboundCodec,
     },
@@ -91,7 +90,6 @@ pub enum Version {
 /// RPC Encondings supported.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Encoding {
-    SSZ,
     SSZSnappy,
 }
 
@@ -112,7 +110,6 @@ impl std::fmt::Display for Protocol {
 impl std::fmt::Display for Encoding {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let repr = match self {
-            Encoding::SSZ => "ssz",
             Encoding::SSZSnappy => "ssz_snappy",
         };
         f.write_str(repr)
@@ -141,17 +138,11 @@ impl<TSpec: EthSpec> UpgradeInfo for RPCProtocol<TSpec> {
     fn protocol_info(&self) -> Self::InfoIter {
         vec![
             ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::Goodbye, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::Goodbye, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::Ping, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::Ping, Version::V1, Encoding::SSZ),
             ProtocolId::new(Protocol::MetaData, Version::V1, Encoding::SSZSnappy),
-            ProtocolId::new(Protocol::MetaData, Version::V1, Encoding::SSZ),
         ]
     }
 }
@@ -224,11 +215,6 @@ where
                         BaseInboundCodec::new(SSZSnappyInboundCodec::new(protocol, MAX_RPC_SIZE));
                     InboundCodec::SSZSnappy(ssz_snappy_codec)
                 }
-                Encoding::SSZ => {
-                    let ssz_codec =
-                        BaseInboundCodec::new(SSZInboundCodec::new(protocol, MAX_RPC_SIZE));
-                    InboundCodec::SSZ(ssz_codec)
-                }
             };
             let mut timed_socket = TimeoutStream::new(socket);
             timed_socket.set_read_timeout(Some(Duration::from_secs(TTFB_TIMEOUT)));
@@ -286,30 +272,36 @@ impl<TSpec: EthSpec> RPCRequest<TSpec> {
     pub fn supported_protocols(&self) -> Vec<ProtocolId> {
         match self {
             // add more protocols when versions/encodings are supported
-            RPCRequest::Status(_) => vec![
-                ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::Goodbye(_) => vec![
-                ProtocolId::new(Protocol::Goodbye, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::Goodbye, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::BlocksByRange(_) => vec![
-                ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::BlocksByRoot(_) => vec![
-                ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::Ping(_) => vec![
-                ProtocolId::new(Protocol::Ping, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::Ping, Version::V1, Encoding::SSZ),
-            ],
-            RPCRequest::MetaData(_) => vec![
-                ProtocolId::new(Protocol::MetaData, Version::V1, Encoding::SSZSnappy),
-                ProtocolId::new(Protocol::MetaData, Version::V1, Encoding::SSZ),
-            ],
+            RPCRequest::Status(_) => vec![ProtocolId::new(
+                Protocol::Status,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::Goodbye(_) => vec![ProtocolId::new(
+                Protocol::Goodbye,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::BlocksByRange(_) => vec![ProtocolId::new(
+                Protocol::BlocksByRange,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::BlocksByRoot(_) => vec![ProtocolId::new(
+                Protocol::BlocksByRoot,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::Ping(_) => vec![ProtocolId::new(
+                Protocol::Ping,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
+            RPCRequest::MetaData(_) => vec![ProtocolId::new(
+                Protocol::MetaData,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
         }
     }
 
@@ -379,11 +371,6 @@ where
                     BaseOutboundCodec::new(SSZSnappyOutboundCodec::new(protocol, MAX_RPC_SIZE));
                 OutboundCodec::SSZSnappy(ssz_snappy_codec)
             }
-            Encoding::SSZ => {
-                let ssz_codec =
-                    BaseOutboundCodec::new(SSZOutboundCodec::new(protocol, MAX_RPC_SIZE));
-                OutboundCodec::SSZ(ssz_codec)
-            }
         };
 
         let mut socket = Framed::new(socket, codec);

beacon_node/eth2_libp2p/src/service.rs

@@ -7,14 +7,10 @@ use crate::EnrExt;
 use crate::{NetworkConfig, NetworkGlobals, PeerAction};
 use futures::prelude::*;
 use libp2p::core::{
-    identity::Keypair,
-    multiaddr::Multiaddr,
-    muxing::StreamMuxerBox,
-    transport::boxed::Boxed,
-    upgrade::{InboundUpgradeExt, OutboundUpgradeExt},
+    identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::boxed::Boxed,
 };
 use libp2p::{
-    core, noise, secio,
+    core, noise,
     swarm::{SwarmBuilder, SwarmEvent},
     PeerId, Swarm, Transport,
 };
@@ -40,6 +36,8 @@ pub enum Libp2pEvent<TSpec: EthSpec> {
     Behaviour(BehaviourEvent<TSpec>),
     /// A new listening address has been established.
     NewListenAddr(Multiaddr),
+    /// We reached zero listening addresses.
+    ZeroListeners,
 }
 
 /// The configuration and state of the libp2p components for the beacon node.
@@ -55,7 +53,7 @@ pub struct Service<TSpec: EthSpec> {
 }
 
 impl<TSpec: EthSpec> Service<TSpec> {
-    pub fn new(
+    pub async fn new(
         executor: environment::TaskExecutor,
         config: &NetworkConfig,
         enr_fork_id: EnrForkId,
@@ -80,7 +78,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
             &log,
         ));
 
-        info!(log, "Libp2p Service"; "peer_id" => format!("{:?}", enr.peer_id()));
+        info!(log, "Libp2p Service"; "peer_id" => enr.peer_id().to_string());
         let discovery_string = if config.disable_discovery {
             "None".into()
         } else {
@@ -89,11 +87,12 @@ impl<TSpec: EthSpec> Service<TSpec> {
         debug!(log, "Attempting to open listening ports"; "address" => format!("{}", config.listen_address), "tcp_port" => config.libp2p_port, "udp_port" => discovery_string);
 
         let mut swarm = {
-            // Set up the transport - tcp/ws with noise and yamux/mplex
+            // Set up the transport - tcp/ws with noise and mplex
             let transport = build_transport(local_keypair.clone())
                 .map_err(|e| format!("Failed to build transport: {:?}", e))?;
             // Lighthouse network behaviour
-            let behaviour = Behaviour::new(&local_keypair, config, network_globals.clone(), &log)?;
+            let behaviour =
+                Behaviour::new(&local_keypair, config, network_globals.clone(), &log).await?;
 
             // use the executor for libp2p
             struct Executor(environment::TaskExecutor);
@@ -106,6 +105,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
                 .notify_handler_buffer_size(std::num::NonZeroUsize::new(32).expect("Not zero"))
                 .connection_event_buffer_size(64)
                 .incoming_connection_limit(10)
+                .outgoing_connection_limit(config.target_peers * 2)
                 .peer_connection_limit(MAX_CONNECTIONS_PER_PEER)
                 .executor(Box::new(Executor(executor)))
                 .build()
@@ -154,7 +154,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
         }
 
         // attempt to connect to any specified boot-nodes
-        let mut boot_nodes = config.boot_nodes.clone();
+        let mut boot_nodes = config.boot_nodes_enr.clone();
         boot_nodes.dedup();
 
         for bootnode_enr in boot_nodes {
@@ -175,6 +175,16 @@ impl<TSpec: EthSpec> Service<TSpec> {
             }
         }
 
+        for multiaddr in &config.boot_nodes_multiaddr {
+            // check TCP support for dialing
+            if multiaddr
+                .iter()
+                .any(|proto| matches!(proto, Protocol::Tcp(_)))
+            {
+                dial_addr(multiaddr.clone());
+            }
+        }
+
         let mut subscribed_topics: Vec<GossipKind> = vec![];
         for topic_kind in &config.topics {
             if swarm.subscribe_kind(topic_kind.clone()) {
@@ -239,7 +249,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
                     endpoint: _,
                     num_established,
                 } => {
-                    debug!(self.log, "Connection closed"; "peer_id"=> peer_id.to_string(), "cause" => cause.to_string(), "connections" => num_established);
+                    debug!(self.log, "Connection closed"; "peer_id"=> peer_id.to_string(), "cause" => format!("{:?}", cause), "connections" => num_established);
                 }
                 SwarmEvent::NewListenAddr(multiaddr) => {
                     return Libp2pEvent::NewListenAddr(multiaddr)
@@ -275,10 +285,17 @@ impl<TSpec: EthSpec> Service<TSpec> {
                     debug!(self.log, "Listen address expired"; "multiaddr" => multiaddr.to_string())
                 }
                 SwarmEvent::ListenerClosed { addresses, reason } => {
-                    debug!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason))
+                    crit!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason));
+                    if Swarm::listeners(&self.swarm).count() == 0 {
+                        return Libp2pEvent::ZeroListeners;
+                    }
                 }
                 SwarmEvent::ListenerError { error } => {
-                    debug!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string()))
+                    // this is non fatal, but we still check
+                    warn!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string()));
+                    if Swarm::listeners(&self.swarm).count() == 0 {
+                        return Libp2pEvent::ZeroListeners;
+                    }
                 }
                 SwarmEvent::Dialing(peer_id) => {
                     debug!(self.log, "Dialing peer"; "peer_id" => peer_id.to_string());
@@ -289,8 +306,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
 }
 
 /// The implementation supports TCP/IP, WebSockets over TCP/IP, noise as the encryption layer, and
-/// yamux or mplex as the multiplexing layer.
-
+/// mplex as the multiplexing layer.
 fn build_transport(
     local_private_key: Keypair,
 ) -> Result<Boxed<(PeerId, StreamMuxerBox), Error>, Error> {
@@ -302,47 +318,15 @@ fn build_transport(
         transport.or_transport(libp2p::websocket::WsConfig::new(trans_clone))
     };
     // Authentication
-    let transport = transport
-        .and_then(move |stream, endpoint| {
-            let upgrade = core::upgrade::SelectUpgrade::new(
-                secio::SecioConfig::new(local_private_key.clone()),
-                generate_noise_config(&local_private_key),
-            );
-            core::upgrade::apply(stream, upgrade, endpoint, core::upgrade::Version::V1).and_then(
-                |out| async move {
-                    match out {
-                        // Secio was negotiated
-                        core::either::EitherOutput::First((remote_id, out)) => {
-                            Ok((core::either::EitherOutput::First(out), remote_id))
-                        }
-                        // Noise was negotiated
-                        core::either::EitherOutput::Second((remote_id, out)) => {
-                            Ok((core::either::EitherOutput::Second(out), remote_id))
-                        }
-                    }
-                },
-            )
-        })
-        .timeout(Duration::from_secs(20));
-
-    // Multiplexing
-    let transport = transport
-        .and_then(move |(stream, peer_id), endpoint| {
-            let peer_id2 = peer_id.clone();
-            let upgrade = core::upgrade::SelectUpgrade::new(
-                libp2p::mplex::MplexConfig::new(),
-                libp2p::yamux::Config::default(),
-            )
-            .map_inbound(move |muxer| (peer_id, muxer))
-            .map_outbound(move |muxer| (peer_id2, muxer));
-
-            core::upgrade::apply(stream, upgrade, endpoint, core::upgrade::Version::V1)
-                .map_ok(|(id, muxer)| (id, core::muxing::StreamMuxerBox::new(muxer)))
-        })
-        .timeout(Duration::from_secs(20))
+    Ok(transport
+        .upgrade(core::upgrade::Version::V1)
+        .authenticate(generate_noise_config(&local_private_key))
+        .multiplex(libp2p::mplex::MplexConfig::new())
+        .map(|(peer, muxer), _| (peer, core::muxing::StreamMuxerBox::new(muxer)))
+        .timeout(Duration::from_secs(10))
+        .timeout(Duration::from_secs(10))
         .map_err(|err| Error::new(ErrorKind::Other, err))
-        .boxed();
-    Ok(transport)
+        .boxed())
 }
 
 // Useful helper functions for debugging. Currently not used in the client.
@@ -373,7 +357,7 @@ fn keypair_from_bytes(mut bytes: Vec<u8>) -> error::Result<Keypair> {
 /// generated and is then saved to disk.
 ///
 /// Currently only secp256k1 keys are allowed, as these are the only keys supported by discv5.
-fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
+pub fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair {
     // check for key from disk
     let network_key_f = config.network_dir.join(NETWORK_KEY_FILENAME);
     if let Ok(mut network_key_file) = File::open(network_key_f.clone()) {

beacon_node/eth2_libp2p/src/types/globals.rs

@@ -1,10 +1,9 @@
 //! A collection of variables that are accessible outside of the network thread itself.
 use crate::peer_manager::PeerDB;
-use crate::rpc::methods::MetaData;
 use crate::types::SyncState;
 use crate::Client;
 use crate::EnrExt;
-use crate::{Enr, Eth2Enr, GossipTopic, Multiaddr, PeerId};
+use crate::{Enr, GossipTopic, Multiaddr, PeerId};
 use parking_lot::RwLock;
 use std::collections::HashSet;
 use std::sync::atomic::{AtomicU16, Ordering};
@@ -13,8 +12,6 @@ use types::EthSpec;
 pub struct NetworkGlobals<TSpec: EthSpec> {
     /// The current local ENR.
     pub local_enr: RwLock<Enr>,
-    /// The current node's meta-data.
-    pub meta_data: RwLock<MetaData<TSpec>>,
     /// The local peer_id.
     pub peer_id: RwLock<PeerId>,
     /// Listening multiaddrs.
@@ -33,17 +30,8 @@ pub struct NetworkGlobals<TSpec: EthSpec> {
 
 impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
     pub fn new(enr: Enr, tcp_port: u16, udp_port: u16, log: &slog::Logger) -> Self {
-        // set up the local meta data of the node
-        let meta_data = RwLock::new(MetaData {
-            seq_number: 0,
-            attnets: enr
-                .bitfield::<TSpec>()
-                .expect("Local ENR must have a bitfield specified"),
-        });
-
         NetworkGlobals {
             local_enr: RwLock::new(enr.clone()),
-            meta_data,
             peer_id: RwLock::new(enr.peer_id()),
             listen_multiaddrs: RwLock::new(Vec::new()),
             listen_port_tcp: AtomicU16::new(tcp_port),

beacon_node/eth2_libp2p/src/types/mod.rs

@@ -1,6 +1,7 @@
 pub mod error;
 mod globals;
 mod pubsub;
+mod subnet;
 mod sync_state;
 mod topics;
 
@@ -13,5 +14,6 @@ pub type Enr = discv5::enr::Enr<discv5::enr::CombinedKey>;
 
 pub use globals::NetworkGlobals;
 pub use pubsub::PubsubMessage;
+pub use subnet::SubnetDiscovery;
 pub use sync_state::SyncState;
 pub use topics::{GossipEncoding, GossipKind, GossipTopic};

beacon_node/eth2_libp2p/src/types/subnet.rs

@@ -0,0 +1,28 @@
+use std::time::{Duration, Instant};
+use types::SubnetId;
+
+const DURATION_DIFFERENCE: Duration = Duration::from_millis(1);
+
+/// A subnet to discover peers on along with the instant after which it's no longer useful.
+#[derive(Debug, Clone)]
+pub struct SubnetDiscovery {
+    pub subnet_id: SubnetId,
+    pub min_ttl: Option<Instant>,
+}
+
+impl PartialEq for SubnetDiscovery {
+    fn eq(&self, other: &SubnetDiscovery) -> bool {
+        self.subnet_id == other.subnet_id
+            && match (self.min_ttl, other.min_ttl) {
+                (Some(min_ttl_instant), Some(other_min_ttl_instant)) => {
+                    min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
+                        < DURATION_DIFFERENCE
+                        && other_min_ttl_instant.saturating_duration_since(min_ttl_instant)
+                            < DURATION_DIFFERENCE
+                }
+                (None, None) => true,
+                (None, Some(_)) => true,
+                (Some(_), None) => true,
+            }
+    }
+}

beacon_node/eth2_libp2p/tests/common/mod.rs

@@ -80,7 +80,7 @@ pub fn build_config(port: u16, mut boot_nodes: Vec<Enr>) -> NetworkConfig {
     config.enr_tcp_port = Some(port);
     config.enr_udp_port = Some(port);
     config.enr_address = Some("127.0.0.1".parse().unwrap());
-    config.boot_nodes.append(&mut boot_nodes);
+    config.boot_nodes_enr.append(&mut boot_nodes);
     config.network_dir = path.into_path();
     // Reduce gossipsub heartbeat parameters
     config.gs_config.heartbeat_initial_delay = Duration::from_millis(500);
@@ -88,16 +88,22 @@ pub fn build_config(port: u16, mut boot_nodes: Vec<Enr>) -> NetworkConfig {
     config
 }
 
-pub fn build_libp2p_instance(boot_nodes: Vec<Enr>, log: slog::Logger) -> Libp2pInstance {
+pub async fn build_libp2p_instance(boot_nodes: Vec<Enr>, log: slog::Logger) -> Libp2pInstance {
     let port = unused_port("tcp").unwrap();
     let config = build_config(port, boot_nodes);
     // launch libp2p service
 
     let (signal, exit) = exit_future::signal();
-    let executor =
-        environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone());
+    let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+    let executor = environment::TaskExecutor::new(
+        tokio::runtime::Handle::current(),
+        exit,
+        log.clone(),
+        shutdown_tx,
+    );
     Libp2pInstance(
         LibP2PService::new(executor, &config, EnrForkId::default(), &log)
+            .await
             .expect("should build libp2p instance")
             .1,
         signal,
@@ -112,10 +118,11 @@ pub fn get_enr(node: &LibP2PService<E>) -> Enr {
 
 // Returns `n` libp2p peers in fully connected topology.
 #[allow(dead_code)]
-pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
-    let mut nodes: Vec<_> = (0..n)
-        .map(|_| build_libp2p_instance(vec![], log.clone()))
-        .collect();
+pub async fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+    let mut nodes = Vec::with_capacity(n);
+    for _ in 0..n {
+        nodes.push(build_libp2p_instance(vec![], log.clone()).await);
+    }
     let multiaddrs: Vec<Multiaddr> = nodes
         .iter()
         .map(|x| get_enr(&x).multiaddr()[1].clone())
@@ -141,8 +148,8 @@ pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInsta
     let sender_log = log.new(o!("who" => "sender"));
     let receiver_log = log.new(o!("who" => "receiver"));
 
-    let mut sender = build_libp2p_instance(vec![], sender_log);
-    let mut receiver = build_libp2p_instance(vec![], receiver_log);
+    let mut sender = build_libp2p_instance(vec![], sender_log).await;
+    let mut receiver = build_libp2p_instance(vec![], receiver_log).await;
 
     let receiver_multiaddr = receiver.swarm.local_enr().multiaddr()[1].clone();
 
@@ -181,10 +188,12 @@ pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInsta
 
 // Returns `n` peers in a linear topology
 #[allow(dead_code)]
-pub fn build_linear(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
-    let mut nodes: Vec<_> = (0..n)
-        .map(|_| build_libp2p_instance(vec![], log.clone()))
-        .collect();
+pub async fn build_linear(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+    let mut nodes = Vec::with_capacity(n);
+    for _ in 0..n {
+        nodes.push(build_libp2p_instance(vec![], log.clone()).await);
+    }
+
     let multiaddrs: Vec<Multiaddr> = nodes
         .iter()
         .map(|x| get_enr(&x).multiaddr()[1].clone())

beacon_node/genesis/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "genesis"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 

beacon_node/network/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "network"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
@@ -39,3 +39,5 @@ lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
 environment = { path = "../../lighthouse/environment" }
 itertools = "0.9.0"
+num_cpus = "1.13.0"
+lru_cache = { path = "../../common/lru_cache" }

beacon_node/network/src/attestation_service/mod.rs

@@ -2,7 +2,7 @@
 //! given time. It schedules subscriptions to shard subnets, requests peer discoveries and
 //! determines whether attestations should be aggregated and/or passed to the beacon node.
 
-use std::collections::VecDeque;
+use std::collections::{HashMap, VecDeque};
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
@@ -13,7 +13,7 @@ use rand::seq::SliceRandom;
 use slog::{crit, debug, error, o, trace, warn};
 
 use beacon_chain::{BeaconChain, BeaconChainTypes};
-use eth2_libp2p::{types::GossipKind, NetworkGlobals};
+use eth2_libp2p::{types::GossipKind, NetworkGlobals, SubnetDiscovery};
 use hashset_delay::HashSetDelay;
 use rest_types::ValidatorSubscription;
 use slot_clock::SlotClock;
@@ -25,11 +25,8 @@ mod tests;
 
 /// The minimum number of slots ahead that we attempt to discover peers for a subscription. If the
 /// slot is less than this number, skip the peer discovery process.
-const MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 1;
-/// The number of slots ahead that we attempt to discover peers for a subscription. If the slot to
-/// attest to is greater than this, we queue a discovery request for this many slots prior to
-/// subscribing.
-const TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 6;
+/// Subnet discovery query takes atmost 30 secs, 2 slots take 24s.
+const MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 2;
 /// The time (in slots) before a last seen validator is considered absent and we unsubscribe from the random
 /// gossip topics that we subscribed to due to the validator connection.
 const LAST_SEEN_VALIDATOR_TIMEOUT: u32 = 150;
@@ -39,12 +36,10 @@ const LAST_SEEN_VALIDATOR_TIMEOUT: u32 = 150;
 /// Note: The time is calculated as `time = milliseconds_per_slot / ADVANCE_SUBSCRIPTION_TIME`.
 const ADVANCE_SUBSCRIBE_TIME: u32 = 3;
 /// The default number of slots before items in hash delay sets used by this class should expire.
+///  36s at 12s slot time
 const DEFAULT_EXPIRATION_TIMEOUT: u32 = 3;
-// 36s at 12s slot time
-/// The duration difference between two instance for them to be considered equal.
-const DURATION_DIFFERENCE: Duration = Duration::from_millis(1);
 
-#[derive(Debug, Eq, Clone)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum AttServiceMessage {
     /// Subscribe to the specified subnet id.
     Subscribe(SubnetId),
@@ -54,44 +49,8 @@ pub enum AttServiceMessage {
     EnrAdd(SubnetId),
     /// Remove the `SubnetId` from the ENR bitfield.
     EnrRemove(SubnetId),
-    /// Discover peers for a particular subnet.
-    /// The includes the `Instant` we need the discovered peer until.
-    DiscoverPeers {
-        subnet_id: SubnetId,
-        min_ttl: Option<Instant>,
-    },
-}
-
-impl PartialEq for AttServiceMessage {
-    fn eq(&self, other: &AttServiceMessage) -> bool {
-        match (self, other) {
-            (&AttServiceMessage::Subscribe(a), &AttServiceMessage::Subscribe(b)) => a == b,
-            (&AttServiceMessage::Unsubscribe(a), &AttServiceMessage::Unsubscribe(b)) => a == b,
-            (&AttServiceMessage::EnrAdd(a), &AttServiceMessage::EnrAdd(b)) => a == b,
-            (&AttServiceMessage::EnrRemove(a), &AttServiceMessage::EnrRemove(b)) => a == b,
-            (
-                &AttServiceMessage::DiscoverPeers { subnet_id, min_ttl },
-                &AttServiceMessage::DiscoverPeers {
-                    subnet_id: other_subnet_id,
-                    min_ttl: other_min_ttl,
-                },
-            ) => {
-                subnet_id == other_subnet_id
-                    && match (min_ttl, other_min_ttl) {
-                        (Some(min_ttl_instant), Some(other_min_ttl_instant)) => {
-                            min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
-                                < DURATION_DIFFERENCE
-                                && other_min_ttl_instant.saturating_duration_since(min_ttl_instant)
-                                    < DURATION_DIFFERENCE
-                        }
-                        (None, None) => true,
-                        (None, Some(_)) => true,
-                        (Some(_), None) => true,
-                    }
-            }
-            _ => false,
-        }
-    }
+    /// Discover peers for a list of `SubnetDiscovery`.
+    DiscoverPeers(Vec<SubnetDiscovery>),
 }
 
 /// A particular subnet at a given slot.
@@ -116,9 +75,6 @@ pub struct AttestationService<T: BeaconChainTypes> {
     /// The collection of currently subscribed random subnets mapped to their expiry deadline.
     random_subnets: HashSetDelay<SubnetId>,
 
-    /// A collection of timeouts for when to start searching for peers for a particular shard.
-    discover_peers: HashSetDelay<ExactSubnet>,
-
     /// A collection of timeouts for when to subscribe to a shard subnet.
     subscriptions: HashSetDelay<ExactSubnet>,
 
@@ -172,7 +128,6 @@ impl<T: BeaconChainTypes> AttestationService<T> {
             network_globals,
             beacon_chain,
             random_subnets: HashSetDelay::new(Duration::from_millis(random_subnet_duration_millis)),
-            discover_peers: HashSetDelay::new(default_timeout),
             subscriptions: HashSetDelay::new(default_timeout),
             unsubscriptions: HashSetDelay::new(default_timeout),
             aggregate_validators_on_subnet: HashSetDelay::new(default_timeout),
@@ -198,6 +153,8 @@ impl<T: BeaconChainTypes> AttestationService<T> {
         &mut self,
         subscriptions: Vec<ValidatorSubscription>,
     ) -> Result<(), String> {
+        // Maps each subnet_id subscription to it's highest slot
+        let mut subnets_to_discover: HashMap<SubnetId, Slot> = HashMap::new();
         for subscription in subscriptions {
             metrics::inc_counter(&metrics::SUBNET_SUBSCRIPTION_REQUESTS);
             //NOTE: We assume all subscriptions have been verified before reaching this service
@@ -226,15 +183,20 @@ impl<T: BeaconChainTypes> AttestationService<T> {
                     continue;
                 }
             };
+            // Ensure each subnet_id inserted into the map has the highest slot as it's value.
+            // Higher slot corresponds to higher min_ttl in the `SubnetDiscovery` entry.
+            if let Some(slot) = subnets_to_discover.get(&subnet_id) {
+                if subscription.slot > *slot {
+                    subnets_to_discover.insert(subnet_id, subscription.slot);
+                }
+            } else {
+                subnets_to_discover.insert(subnet_id, subscription.slot);
+            }
 
             let exact_subnet = ExactSubnet {
                 subnet_id,
                 slot: subscription.slot,
             };
-            // determine if we should run a discovery lookup request and request it if required
-            if let Err(e) = self.discover_peers_request(exact_subnet.clone()) {
-                warn!(self.log, "Discovery lookup request error"; "error" => e);
-            }
 
             // determine if the validator is an aggregator. If so, we subscribe to the subnet and
             // if successful add the validator to a mapping of known aggregators for that exact
@@ -264,6 +226,14 @@ impl<T: BeaconChainTypes> AttestationService<T> {
             }
         }
 
+        if let Err(e) = self.discover_peers_request(
+            subnets_to_discover
+                .into_iter()
+                .map(|(subnet_id, slot)| ExactSubnet { subnet_id, slot }),
+        ) {
+            warn!(self.log, "Discovery lookup request error"; "error" => e);
+        };
+
         // pre-emptively wake the thread to check for new events
         if let Some(waker) = &self.waker {
             waker.wake_by_ref();
@@ -290,114 +260,55 @@ impl<T: BeaconChainTypes> AttestationService<T> {
     /// Checks if there are currently queued discovery requests and the time required to make the
     /// request.
     ///
-    /// If there is sufficient time and no other request exists, queues a peer discovery request
-    /// for the required subnet.
-    fn discover_peers_request(&mut self, exact_subnet: ExactSubnet) -> Result<(), &'static str> {
+    /// If there is sufficient time, queues a peer discovery request for all the required subnets.
+    fn discover_peers_request(
+        &mut self,
+        exact_subnets: impl Iterator<Item = ExactSubnet>,
+    ) -> Result<(), &'static str> {
         let current_slot = self
             .beacon_chain
             .slot_clock
             .now()
             .ok_or_else(|| "Could not get the current slot")?;
-        let slot_duration = self.beacon_chain.slot_clock.slot_duration();
 
-        // if there is enough time to perform a discovery lookup
-        if exact_subnet.slot >= current_slot.saturating_add(MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD) {
-            // check if a discovery request already exists
-            if self.discover_peers.get(&exact_subnet).is_some() {
-                // already a request queued, end
-                return Ok(());
-            }
-
-            // if the slot is more than epoch away, add an event to start looking for peers
-            if exact_subnet.slot
-                < current_slot.saturating_add(TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD)
-            {
-                // add one slot to ensure we keep the peer for the subscription slot
-                let min_ttl = self
-                    .beacon_chain
-                    .slot_clock
-                    .duration_to_slot(exact_subnet.slot + 1)
-                    .map(|duration| std::time::Instant::now() + duration);
-
-                self.send_or_update_discovery_event(exact_subnet.subnet_id, min_ttl);
-            } else {
-                // Queue the discovery event to be executed for
-                // TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD
-
-                let duration_to_discover = {
-                    let duration_to_next_slot = self
+        let discovery_subnets: Vec<SubnetDiscovery> = exact_subnets
+            .filter_map(|exact_subnet| {
+                // check if there is enough time to perform a discovery lookup
+                if exact_subnet.slot
+                    >= current_slot.saturating_add(MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD)
+                {
+                    // if the slot is more than epoch away, add an event to start looking for peers
+                    // add one slot to ensure we keep the peer for the subscription slot
+                    let min_ttl = self
                         .beacon_chain
                         .slot_clock
-                        .duration_to_next_slot()
-                        .ok_or_else(|| "Unable to determine duration to next slot")?;
-                    // The -1 is done here to exclude the current slot duration, as we will use
-                    // `duration_to_next_slot`.
-                    let slots_until_discover = exact_subnet
-                        .slot
-                        .saturating_sub(current_slot)
-                        .saturating_sub(1u64)
-                        .saturating_sub(TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD);
+                        .duration_to_slot(exact_subnet.slot + 1)
+                        .map(|duration| std::time::Instant::now() + duration);
+                    Some(SubnetDiscovery {
+                        subnet_id: exact_subnet.subnet_id,
+                        min_ttl,
+                    })
+                } else {
+                    // TODO: Send the time frame needed to have a peer connected, so that we can
+                    // maintain peers for a least this duration.
+                    // We may want to check the global PeerInfo to see estimated timeouts for each
+                    // peer before they can be removed.
+                    warn!(self.log,
+                        "Not enough time for a discovery search";
+                        "subnet_id" => format!("{:?}", exact_subnet)
+                    );
+                    None
+                }
+            })
+            .collect();
 
-                    duration_to_next_slot + slot_duration * (slots_until_discover.as_u64() as u32)
-                };
-
-                self.discover_peers
-                    .insert_at(exact_subnet, duration_to_discover);
-            }
-        } else {
-            // TODO: Send the time frame needed to have a peer connected, so that we can
-            // maintain peers for a least this duration.
-            // We may want to check the global PeerInfo to see estimated timeouts for each
-            // peer before they can be removed.
-            return Err("Not enough time for a discovery search");
+        if !discovery_subnets.is_empty() {
+            self.events
+                .push_back(AttServiceMessage::DiscoverPeers(discovery_subnets));
         }
         Ok(())
     }
 
-    /// Checks if we have a discover peers event already and sends a new event if necessary
-    ///
-    /// If a message exists for the same subnet, compare the `min_ttl` of the current and
-    /// existing messages and extend the existing message as necessary.
-    fn send_or_update_discovery_event(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
-        // track whether this message already exists in the event queue
-        let mut is_duplicate = false;
-
-        self.events.iter_mut().for_each(|event| {
-            if let AttServiceMessage::DiscoverPeers {
-                subnet_id: other_subnet_id,
-                min_ttl: other_min_ttl,
-            } = event
-            {
-                if subnet_id == *other_subnet_id {
-                    let other_min_ttl_clone = *other_min_ttl;
-                    match (min_ttl, other_min_ttl_clone) {
-                        (Some(min_ttl_instant), Some(other_min_ttl_instant)) =>
-                        // only update the min_ttl if it is greater than the existing min_ttl and a DURATION_DIFFERENCE padding
-                        {
-                            if min_ttl_instant.saturating_duration_since(other_min_ttl_instant)
-                                > DURATION_DIFFERENCE
-                            {
-                                *other_min_ttl = min_ttl;
-                            }
-                        }
-                        (None, Some(_)) => {} // Keep the current one as it has an actual min_ttl
-                        (Some(min_ttl), None) => {
-                            // Update the request to include a min_ttl.
-                            *other_min_ttl = Some(min_ttl);
-                        }
-                        (None, None) => {} // Duplicate message, do nothing.
-                    }
-                    is_duplicate = true;
-                    return;
-                }
-            };
-        });
-        if !is_duplicate {
-            self.events
-                .push_back(AttServiceMessage::DiscoverPeers { subnet_id, min_ttl });
-        }
-    }
-
     /// Checks the current random subnets and subscriptions to determine if a new subscription for this
     /// subnet is required for the given slot.
     ///
@@ -547,7 +458,11 @@ impl<T: BeaconChainTypes> AttestationService<T> {
 
             if !already_subscribed {
                 // send a discovery request and a subscription
-                self.send_or_update_discovery_event(subnet_id, None);
+                self.events
+                    .push_back(AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery {
+                        subnet_id,
+                        min_ttl: None,
+                    }]));
                 self.events
                     .push_back(AttServiceMessage::Subscribe(subnet_id));
             }
@@ -558,20 +473,6 @@ impl<T: BeaconChainTypes> AttestationService<T> {
 
     /* A collection of functions that handle the various timeouts */
 
-    /// Request a discovery query to find peers for a particular subnet.
-    fn handle_discover_peers(&mut self, exact_subnet: ExactSubnet) {
-        debug!(self.log, "Searching for peers for subnet"; "subnet" => *exact_subnet.subnet_id, "target_slot" => exact_subnet.slot);
-
-        // add one slot to ensure we keep the peer for the subscription slot
-        let min_ttl = self
-            .beacon_chain
-            .slot_clock
-            .duration_to_slot(exact_subnet.slot + 1)
-            .map(|duration| std::time::Instant::now() + duration);
-
-        self.send_or_update_discovery_event(exact_subnet.subnet_id, min_ttl)
-    }
-
     /// A queued subscription is ready.
     ///
     /// We add subscriptions events even if we are already subscribed to a random subnet (as these
@@ -731,15 +632,6 @@ impl<T: BeaconChainTypes> Stream for AttestationService<T> {
             self.waker = Some(cx.waker().clone());
         }
 
-        // process any peer discovery events
-        match self.discover_peers.poll_next_unpin(cx) {
-            Poll::Ready(Some(Ok(exact_subnet))) => self.handle_discover_peers(exact_subnet),
-            Poll::Ready(Some(Err(e))) => {
-                error!(self.log, "Failed to check for peer discovery requests"; "error"=> e);
-            }
-            Poll::Ready(None) | Poll::Pending => {}
-        }
-
         // process any subscription events
         match self.subscriptions.poll_next_unpin(cx) {
             Poll::Ready(Some(Ok(exact_subnet))) => self.handle_subscriptions(exact_subnet),

beacon_node/network/src/attestation_service/tests/mod.rs

@@ -8,7 +8,9 @@ mod tests {
         migrate::NullMigrator,
     };
     use eth2_libp2p::discovery::{build_enr, Keypair};
-    use eth2_libp2p::{discovery::CombinedKey, CombinedKeyExt, NetworkConfig, NetworkGlobals};
+    use eth2_libp2p::{
+        discovery::CombinedKey, CombinedKeyExt, NetworkConfig, NetworkGlobals, SubnetDiscovery,
+    };
     use futures::Stream;
     use genesis::{generate_deterministic_keypairs, interop_genesis_state};
     use lazy_static::lazy_static;
@@ -120,23 +122,21 @@ mod tests {
         }
     }
 
-    fn _get_subscriptions(
+    fn get_subscriptions(
         validator_count: u64,
         slot: Slot,
         committee_count_at_slot: u64,
     ) -> Vec<ValidatorSubscription> {
-        let mut subscriptions: Vec<ValidatorSubscription> = Vec::new();
-        for validator_index in 0..validator_count {
-            let is_aggregator = true;
-            subscriptions.push(ValidatorSubscription {
-                validator_index,
-                attestation_committee_index: validator_index,
-                slot,
-                committee_count_at_slot,
-                is_aggregator,
-            });
-        }
-        subscriptions
+        (0..validator_count)
+            .map(|validator_index| {
+                get_subscription(
+                    validator_index,
+                    validator_index,
+                    slot,
+                    committee_count_at_slot,
+                )
+            })
+            .collect()
     }
 
     // gets a number of events from the subscription service, or returns none if it times out after a number
@@ -210,14 +210,7 @@ mod tests {
         let events = get_events(attestation_service, no_events_expected, 1).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any1),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any1), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -270,14 +263,7 @@ mod tests {
         let events = get_events(attestation_service, no_events_expected, 2).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any1),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any1), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -330,19 +316,15 @@ mod tests {
             &attestation_service.beacon_chain.spec,
         )
         .unwrap();
-        let expected = vec![AttServiceMessage::DiscoverPeers { subnet_id, min_ttl }];
+        let expected = vec![AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery {
+            subnet_id,
+            min_ttl,
+        }])];
 
         let events = get_events(attestation_service, no_events_expected, 1).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -396,21 +378,14 @@ mod tests {
         )
         .unwrap();
         let expected = vec![
-            AttServiceMessage::DiscoverPeers { subnet_id, min_ttl },
+            AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery { subnet_id, min_ttl }]),
             AttServiceMessage::Subscribe(subnet_id),
         ];
 
         let events = get_events(attestation_service, no_events_expected, 5).await;
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -454,14 +429,7 @@ mod tests {
 
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -517,20 +485,16 @@ mod tests {
 
         // expect discover peers because we will enter TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD range
         let expected: Vec<AttServiceMessage> =
-            vec![AttServiceMessage::DiscoverPeers { subnet_id, min_ttl }];
+            vec![AttServiceMessage::DiscoverPeers(vec![SubnetDiscovery {
+                subnet_id,
+                min_ttl,
+            }])];
 
         let events = get_events(attestation_service, no_events_expected, 5).await;
 
         assert_matches!(
             events[..3],
-            [
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant
-                },
-                AttServiceMessage::Subscribe(_any2),
-                AttServiceMessage::EnrAdd(_any3)
-            ]
+            [AttServiceMessage::DiscoverPeers(_), AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3)]
         );
         // if there are fewer events than expected, there's been a collision
         if events.len() == no_events_expected {
@@ -553,7 +517,7 @@ mod tests {
             .now()
             .expect("Could not get current slot");
 
-        let subscriptions = _get_subscriptions(
+        let subscriptions = get_subscriptions(
             subscription_count,
             current_slot + subscription_slot,
             committee_count,
@@ -572,10 +536,9 @@ mod tests {
 
         for event in events {
             match event {
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant,
-                } => discover_peer_count = discover_peer_count + 1,
+                AttServiceMessage::DiscoverPeers(_) => {
+                    discover_peer_count = discover_peer_count + 1
+                }
                 AttServiceMessage::Subscribe(_any_subnet) => subscribe_count = subscribe_count + 1,
                 AttServiceMessage::EnrAdd(_any_subnet) => enr_add_count = enr_add_count + 1,
                 _ => unexpected_msg_count = unexpected_msg_count + 1,
@@ -605,7 +568,7 @@ mod tests {
             .now()
             .expect("Could not get current slot");
 
-        let subscriptions = _get_subscriptions(
+        let subscriptions = get_subscriptions(
             subscription_count,
             current_slot + subscription_slot,
             committee_count,
@@ -624,10 +587,9 @@ mod tests {
 
         for event in events {
             match event {
-                AttServiceMessage::DiscoverPeers {
-                    subnet_id: _any_subnet,
-                    min_ttl: _any_instant,
-                } => discover_peer_count = discover_peer_count + 1,
+                AttServiceMessage::DiscoverPeers(_) => {
+                    discover_peer_count = discover_peer_count + 1
+                }
                 AttServiceMessage::Subscribe(_any_subnet) => subscribe_count = subscribe_count + 1,
                 AttServiceMessage::EnrAdd(_any_subnet) => enr_add_count = enr_add_count + 1,
                 _ => unexpected_msg_count = unexpected_msg_count + 1,
@@ -639,4 +601,40 @@ mod tests {
         assert_eq!(enr_add_count, 64);
         assert_eq!(unexpected_msg_count, 0);
     }
+
+    #[tokio::test]
+    async fn test_discovery_peers_count() {
+        let subscription_slot = 10;
+        let validator_count = 32;
+        let committee_count = 1;
+        let expected_events = 97;
+
+        // create the attestation service and subscriptions
+        let mut attestation_service = get_attestation_service();
+        let current_slot = attestation_service
+            .beacon_chain
+            .slot_clock
+            .now()
+            .expect("Could not get current slot");
+
+        let subscriptions = get_subscriptions(
+            validator_count,
+            current_slot + subscription_slot,
+            committee_count,
+        );
+
+        // submit sthe subscriptions
+        attestation_service
+            .validator_subscriptions(subscriptions)
+            .unwrap();
+
+        let events = get_events(attestation_service, expected_events, 3).await;
+
+        let event = events.get(96);
+        if let Some(AttServiceMessage::DiscoverPeers(d)) = event {
+            assert_eq!(d.len(), validator_count as usize);
+        } else {
+            panic!("Unexpected event {:?}", event);
+        }
+    }
 }

beacon_node/network/src/beacon_processor/chain_segment.rs

@@ -0,0 +1,237 @@
+use crate::metrics;
+use crate::router::processor::FUTURE_SLOT_TOLERANCE;
+use crate::sync::manager::SyncMessage;
+use crate::sync::{BatchProcessResult, ChainId};
+use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, ChainSegmentResult};
+use eth2_libp2p::PeerId;
+use slog::{debug, error, trace, warn};
+use std::sync::Arc;
+use tokio::sync::mpsc;
+use types::{Epoch, EthSpec, Hash256, SignedBeaconBlock};
+
+/// Id associated to a block processing request, either a batch or a single block.
+#[derive(Clone, Debug, PartialEq)]
+pub enum ProcessId {
+    /// Processing Id of a range syncing batch.
+    RangeBatchId(ChainId, Epoch),
+    /// Processing Id of the parent lookup of a block.
+    ParentLookup(PeerId, Hash256),
+}
+
+pub fn handle_chain_segment<T: BeaconChainTypes>(
+    chain: Arc<BeaconChain<T>>,
+    process_id: ProcessId,
+    downloaded_blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
+    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    log: slog::Logger,
+) {
+    match process_id {
+        // this a request from the range sync
+        ProcessId::RangeBatchId(chain_id, epoch) => {
+            let len = downloaded_blocks.len();
+            let start_slot = if len > 0 {
+                downloaded_blocks[0].message.slot.as_u64()
+            } else {
+                0
+            };
+            let end_slot = if len > 0 {
+                downloaded_blocks[len - 1].message.slot.as_u64()
+            } else {
+                0
+            };
+
+            debug!(log, "Processing batch"; "batch_epoch" => epoch, "blocks" => downloaded_blocks.len(),  "first_block_slot" => start_slot, "last_block_slot" => end_slot, "service" => "sync");
+            let result = match process_blocks(chain, downloaded_blocks.iter(), &log) {
+                (_, Ok(_)) => {
+                    debug!(log, "Batch processed"; "batch_epoch" => epoch , "first_block_slot" => start_slot, "last_block_slot" => end_slot, "service"=> "sync");
+                    BatchProcessResult::Success
+                }
+                (imported_blocks, Err(e)) if imported_blocks > 0 => {
+                    debug!(log, "Batch processing failed but imported some blocks";
+                        "batch_epoch" => epoch, "error" => e, "imported_blocks"=> imported_blocks, "service" => "sync");
+                    BatchProcessResult::Partial
+                }
+                (_, Err(e)) => {
+                    debug!(log, "Batch processing failed"; "batch_epoch" => epoch, "error" => e, "service" => "sync");
+                    BatchProcessResult::Failed
+                }
+            };
+
+            let msg = SyncMessage::BatchProcessed {
+                chain_id,
+                epoch,
+                downloaded_blocks,
+                result,
+            };
+            sync_send.send(msg).unwrap_or_else(|_| {
+                debug!(
+                    log,
+                    "Block processor could not inform range sync result. Likely shutting down."
+                );
+            });
+        }
+        // this a parent lookup request from the sync manager
+        ProcessId::ParentLookup(peer_id, chain_head) => {
+            debug!(
+                log, "Processing parent lookup";
+                "last_peer_id" => format!("{}", peer_id),
+                "blocks" => downloaded_blocks.len()
+            );
+            // parent blocks are ordered from highest slot to lowest, so we need to process in
+            // reverse
+            match process_blocks(chain, downloaded_blocks.iter().rev(), &log) {
+                (_, Err(e)) => {
+                    debug!(log, "Parent lookup failed"; "last_peer_id" => format!("{}", peer_id), "error" => e);
+                    sync_send
+                        .send(SyncMessage::ParentLookupFailed{peer_id, chain_head})
+                        .unwrap_or_else(|_| {
+                            // on failure, inform to downvote the peer
+                            debug!(
+                                log,
+                                "Block processor could not inform parent lookup result. Likely shutting down."
+                            );
+                        });
+                }
+                (_, Ok(_)) => {
+                    debug!(log, "Parent lookup processed successfully");
+                }
+            }
+        }
+    }
+}
+
+/// Helper function to process blocks batches which only consumes the chain and blocks to process.
+fn process_blocks<
+    'a,
+    T: BeaconChainTypes,
+    I: Iterator<Item = &'a SignedBeaconBlock<T::EthSpec>>,
+>(
+    chain: Arc<BeaconChain<T>>,
+    downloaded_blocks: I,
+    log: &slog::Logger,
+) -> (usize, Result<(), String>) {
+    let blocks = downloaded_blocks.cloned().collect::<Vec<_>>();
+    match chain.process_chain_segment(blocks) {
+        ChainSegmentResult::Successful { imported_blocks } => {
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_SUCCESS_TOTAL);
+            if imported_blocks == 0 {
+                debug!(log, "All blocks already known");
+            } else {
+                debug!(
+                    log, "Imported blocks from network";
+                    "count" => imported_blocks,
+                );
+                // Batch completed successfully with at least one block, run fork choice.
+                run_fork_choice(chain, log);
+            }
+
+            (imported_blocks, Ok(()))
+        }
+        ChainSegmentResult::Failed {
+            imported_blocks,
+            error,
+        } => {
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_FAILED_TOTAL);
+            let r = handle_failed_chain_segment(error, log);
+            if imported_blocks > 0 {
+                run_fork_choice(chain, log);
+            }
+            (imported_blocks, r)
+        }
+    }
+}
+
+/// Runs fork-choice on a given chain. This is used during block processing after one successful
+/// block import.
+fn run_fork_choice<T: BeaconChainTypes>(chain: Arc<BeaconChain<T>>, log: &slog::Logger) {
+    match chain.fork_choice() {
+        Ok(()) => trace!(
+            log,
+            "Fork choice success";
+            "location" => "batch processing"
+        ),
+        Err(e) => error!(
+            log,
+            "Fork choice failed";
+            "error" => format!("{:?}", e),
+            "location" => "batch import error"
+        ),
+    }
+}
+
+/// Helper function to handle a `BlockError` from `process_chain_segment`
+fn handle_failed_chain_segment<T: EthSpec>(
+    error: BlockError<T>,
+    log: &slog::Logger,
+) -> Result<(), String> {
+    match error {
+        BlockError::ParentUnknown(block) => {
+            // blocks should be sequential and all parents should exist
+
+            Err(format!(
+                "Block has an unknown parent: {}",
+                block.parent_root()
+            ))
+        }
+        BlockError::BlockIsAlreadyKnown => {
+            // This can happen for many reasons. Head sync's can download multiples and parent
+            // lookups can download blocks before range sync
+            Ok(())
+        }
+        BlockError::FutureSlot {
+            present_slot,
+            block_slot,
+        } => {
+            if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot {
+                // The block is too far in the future, drop it.
+                warn!(
+                    log, "Block is ahead of our slot clock";
+                    "msg" => "block for future slot rejected, check your time",
+                    "present_slot" => present_slot,
+                    "block_slot" => block_slot,
+                    "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE,
+                );
+            } else {
+                // The block is in the future, but not too far.
+                debug!(
+                    log, "Block is slightly ahead of our slot clock, ignoring.";
+                    "present_slot" => present_slot,
+                    "block_slot" => block_slot,
+                    "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE,
+                );
+            }
+
+            Err(format!(
+                "Block with slot {} is higher than the current slot {}",
+                block_slot, present_slot
+            ))
+        }
+        BlockError::WouldRevertFinalizedSlot { .. } => {
+            debug!( log, "Finalized or earlier block processed";);
+
+            Ok(())
+        }
+        BlockError::GenesisBlock => {
+            debug!(log, "Genesis block was processed");
+            Ok(())
+        }
+        BlockError::BeaconChainError(e) => {
+            warn!(
+                log, "BlockProcessingFailure";
+                "msg" => "unexpected condition in processing block.",
+                "outcome" => format!("{:?}", e)
+            );
+
+            Err(format!("Internal error whilst processing block: {:?}", e))
+        }
+        other => {
+            debug!(
+                log, "Invalid block received";
+                "msg" => "peer sent invalid block",
+                "outcome" => format!("{:?}", other),
+            );
+
+            Err(format!("Peer sent invalid block. Reason: {:?}", other))
+        }
+    }
+}

beacon_node/network/src/beacon_processor/mod.rs

@@ -0,0 +1,819 @@
+//! Provides the `BeaconProcessor`, a mutli-threaded processor for messages received on the network
+//! that need to be processed by the `BeaconChain`.
+//!
+//! Uses `tokio` tasks (instead of raw threads) to provide the following tasks:
+//!
+//! - A "manager" task, which either spawns worker tasks or enqueues work.
+//! - One or more "worker" tasks which perform time-intensive work on the `BeaconChain`.
+//!
+//! ## Purpose
+//!
+//! The purpose of the `BeaconProcessor` is to provide two things:
+//!
+//! 1. Moving long-running, blocking tasks off the main `tokio` executor.
+//! 2. A fixed-length buffer for consensus messages.
+//!
+//! (1) ensures that we don't clog up the networking stack with long-running tasks, potentially
+//! causing timeouts. (2) means that we can easily and explicitly reject messages when we're
+//! overloaded and also distribute load across time.
+//!
+//! ## Detail
+//!
+//! There is a single "manager" thread who listens to two event channels. These events are either:
+//!
+//! - A new parcel of work (work event).
+//! - Indication that a worker has finished a parcel of work (worker idle).
+//!
+//! Then, there is a maximum of `n` "worker" blocking threads, where `n` is the CPU count.
+//!
+//! Whenever the manager receives a new parcel of work, it either:
+//!
+//! - Provided to a newly-spawned worker tasks (if we are not already at `n` workers).
+//! - Added to a queue.
+//!
+//! Whenever the manager receives a notification that a worker has finished a parcel of work, it
+//! checks the queues to see if there are more parcels of work that can be spawned in a new worker
+//! task.
+
+use crate::{metrics, service::NetworkMessage, sync::SyncMessage};
+use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError};
+use environment::TaskExecutor;
+use eth2_libp2p::{MessageId, NetworkGlobals, PeerId};
+use slog::{crit, debug, error, trace, warn, Logger};
+use std::collections::VecDeque;
+use std::sync::{Arc, Weak};
+use std::time::{Duration, Instant};
+use tokio::sync::{mpsc, oneshot};
+use types::{
+    Attestation, AttesterSlashing, EthSpec, Hash256, ProposerSlashing, SignedAggregateAndProof,
+    SignedBeaconBlock, SignedVoluntaryExit, SubnetId,
+};
+use worker::Worker;
+
+mod chain_segment;
+mod worker;
+
+pub use chain_segment::ProcessId;
+
+/// The maximum size of the channel for work events to the `BeaconProcessor`.
+///
+/// Setting this too low will cause consensus messages to be dropped.
+pub const MAX_WORK_EVENT_QUEUE_LEN: usize = 16_384;
+
+/// The maximum size of the channel for idle events to the `BeaconProcessor`.
+///
+/// Setting this too low will prevent new workers from being spawned. It *should* only need to be
+/// set to the CPU count, but we set it high to be safe.
+const MAX_IDLE_QUEUE_LEN: usize = 16_384;
+
+/// The maximum number of queued `Attestation` objects that will be stored before we start dropping
+/// them.
+const MAX_UNAGGREGATED_ATTESTATION_QUEUE_LEN: usize = 16_384;
+
+/// The maximum number of queued `SignedAggregateAndProof` objects that will be stored before we
+/// start dropping them.
+const MAX_AGGREGATED_ATTESTATION_QUEUE_LEN: usize = 1_024;
+
+/// The maximum number of queued `SignedBeaconBlock` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_BLOCK_QUEUE_LEN: usize = 1_024;
+
+/// The maximum number of queued `SignedVoluntaryExit` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_EXIT_QUEUE_LEN: usize = 4_096;
+
+/// The maximum number of queued `ProposerSlashing` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_PROPOSER_SLASHING_QUEUE_LEN: usize = 4_096;
+
+/// The maximum number of queued `AttesterSlashing` objects received on gossip that will be stored
+/// before we start dropping them.
+const MAX_GOSSIP_ATTESTER_SLASHING_QUEUE_LEN: usize = 4_096;
+
+/// The maximum number of queued `SignedBeaconBlock` objects received from the network RPC that
+/// will be stored before we start dropping them.
+const MAX_RPC_BLOCK_QUEUE_LEN: usize = 1_024;
+
+/// The maximum number of queued `Vec<SignedBeaconBlock>` objects received during syncing that will
+/// be stored before we start dropping them.
+const MAX_CHAIN_SEGMENT_QUEUE_LEN: usize = 64;
+
+/// The name of the manager tokio task.
+const MANAGER_TASK_NAME: &str = "beacon_gossip_processor_manager";
+/// The name of the worker tokio tasks.
+const WORKER_TASK_NAME: &str = "beacon_gossip_processor_worker";
+
+/// The minimum interval between log messages indicating that a queue is full.
+const LOG_DEBOUNCE_INTERVAL: Duration = Duration::from_secs(30);
+
+/// Used to send/receive results from a rpc block import in a blocking task.
+pub type BlockResultSender<E> = oneshot::Sender<Result<Hash256, BlockError<E>>>;
+pub type BlockResultReceiver<E> = oneshot::Receiver<Result<Hash256, BlockError<E>>>;
+
+/// A simple first-in-first-out queue with a maximum length.
+struct FifoQueue<T> {
+    queue: VecDeque<T>,
+    max_length: usize,
+}
+
+impl<T> FifoQueue<T> {
+    /// Create a new, empty queue with the given length.
+    pub fn new(max_length: usize) -> Self {
+        Self {
+            queue: VecDeque::default(),
+            max_length,
+        }
+    }
+
+    /// Add a new item to the queue.
+    ///
+    /// Drops `item` if the queue is full.
+    pub fn push(&mut self, item: T, item_desc: &str, log: &Logger) {
+        if self.queue.len() == self.max_length {
+            error!(
+                log,
+                "Block queue full";
+                "msg" => "the system has insufficient resources for load",
+                "queue_len" => self.max_length,
+                "queue" => item_desc,
+            )
+        } else {
+            self.queue.push_back(item);
+        }
+    }
+
+    /// Remove the next item from the queue.
+    pub fn pop(&mut self) -> Option<T> {
+        self.queue.pop_front()
+    }
+
+    /// Returns the current length of the queue.
+    pub fn len(&self) -> usize {
+        self.queue.len()
+    }
+}
+
+/// A simple last-in-first-out queue with a maximum length.
+struct LifoQueue<T> {
+    queue: VecDeque<T>,
+    max_length: usize,
+}
+
+impl<T> LifoQueue<T> {
+    /// Create a new, empty queue with the given length.
+    pub fn new(max_length: usize) -> Self {
+        Self {
+            queue: VecDeque::default(),
+            max_length,
+        }
+    }
+
+    /// Add a new item to the front of the queue.
+    ///
+    /// If the queue is full, the item at the back of the queue is dropped.
+    pub fn push(&mut self, item: T) {
+        if self.queue.len() == self.max_length {
+            self.queue.pop_back();
+        }
+        self.queue.push_front(item);
+    }
+
+    /// Remove the next item from the queue.
+    pub fn pop(&mut self) -> Option<T> {
+        self.queue.pop_front()
+    }
+
+    /// Returns `true` if the queue is full.
+    pub fn is_full(&self) -> bool {
+        self.queue.len() >= self.max_length
+    }
+
+    /// Returns the current length of the queue.
+    pub fn len(&self) -> usize {
+        self.queue.len()
+    }
+}
+
+/// An event to be processed by the manager task.
+#[derive(Debug)]
+pub struct WorkEvent<E: EthSpec> {
+    drop_during_sync: bool,
+    work: Work<E>,
+}
+
+impl<E: EthSpec> WorkEvent<E> {
+    /// Create a new `Work` event for some unaggregated attestation.
+    pub fn unaggregated_attestation(
+        message_id: MessageId,
+        peer_id: PeerId,
+        attestation: Attestation<E>,
+        subnet_id: SubnetId,
+        should_import: bool,
+    ) -> Self {
+        Self {
+            drop_during_sync: true,
+            work: Work::GossipAttestation {
+                message_id,
+                peer_id,
+                attestation: Box::new(attestation),
+                subnet_id,
+                should_import,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some aggregated attestation.
+    pub fn aggregated_attestation(
+        message_id: MessageId,
+        peer_id: PeerId,
+        aggregate: SignedAggregateAndProof<E>,
+    ) -> Self {
+        Self {
+            drop_during_sync: true,
+            work: Work::GossipAggregate {
+                message_id,
+                peer_id,
+                aggregate: Box::new(aggregate),
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some block.
+    pub fn gossip_beacon_block(
+        message_id: MessageId,
+        peer_id: PeerId,
+        block: Box<SignedBeaconBlock<E>>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipBlock {
+                message_id,
+                peer_id,
+                block,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some exit.
+    pub fn gossip_voluntary_exit(
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: Box<SignedVoluntaryExit>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipVoluntaryExit {
+                message_id,
+                peer_id,
+                voluntary_exit,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some proposer slashing.
+    pub fn gossip_proposer_slashing(
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipProposerSlashing {
+                message_id,
+                peer_id,
+                proposer_slashing,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some attester slashing.
+    pub fn gossip_attester_slashing(
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<E>>,
+    ) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::GossipAttesterSlashing {
+                message_id,
+                peer_id,
+                attester_slashing,
+            },
+        }
+    }
+
+    /// Create a new `Work` event for some block, where the result from computation (if any) is
+    /// sent to the other side of `result_tx`.
+    pub fn rpc_beacon_block(block: Box<SignedBeaconBlock<E>>) -> (Self, BlockResultReceiver<E>) {
+        let (result_tx, result_rx) = oneshot::channel();
+        let event = Self {
+            drop_during_sync: false,
+            work: Work::RpcBlock { block, result_tx },
+        };
+        (event, result_rx)
+    }
+
+    /// Create a new work event to import `blocks` as a beacon chain segment.
+    pub fn chain_segment(process_id: ProcessId, blocks: Vec<SignedBeaconBlock<E>>) -> Self {
+        Self {
+            drop_during_sync: false,
+            work: Work::ChainSegment { process_id, blocks },
+        }
+    }
+}
+
+/// A consensus message (or multiple) from the network that requires processing.
+#[derive(Debug)]
+pub enum Work<E: EthSpec> {
+    GossipAttestation {
+        message_id: MessageId,
+        peer_id: PeerId,
+        attestation: Box<Attestation<E>>,
+        subnet_id: SubnetId,
+        should_import: bool,
+    },
+    GossipAggregate {
+        message_id: MessageId,
+        peer_id: PeerId,
+        aggregate: Box<SignedAggregateAndProof<E>>,
+    },
+    GossipBlock {
+        message_id: MessageId,
+        peer_id: PeerId,
+        block: Box<SignedBeaconBlock<E>>,
+    },
+    GossipVoluntaryExit {
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: Box<SignedVoluntaryExit>,
+    },
+    GossipProposerSlashing {
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
+    },
+    GossipAttesterSlashing {
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<E>>,
+    },
+    RpcBlock {
+        block: Box<SignedBeaconBlock<E>>,
+        result_tx: BlockResultSender<E>,
+    },
+    ChainSegment {
+        process_id: ProcessId,
+        blocks: Vec<SignedBeaconBlock<E>>,
+    },
+}
+
+impl<E: EthSpec> Work<E> {
+    /// Provides a `&str` that uniquely identifies each enum variant.
+    fn str_id(&self) -> &'static str {
+        match self {
+            Work::GossipAttestation { .. } => "gossip_attestation",
+            Work::GossipAggregate { .. } => "gossip_aggregate",
+            Work::GossipBlock { .. } => "gossip_block",
+            Work::GossipVoluntaryExit { .. } => "gossip_voluntary_exit",
+            Work::GossipProposerSlashing { .. } => "gossip_proposer_slashing",
+            Work::GossipAttesterSlashing { .. } => "gossip_attester_slashing",
+            Work::RpcBlock { .. } => "rpc_block",
+            Work::ChainSegment { .. } => "chain_segment",
+        }
+    }
+}
+
+/// Provides de-bounce functionality for logging.
+#[derive(Default)]
+struct TimeLatch(Option<Instant>);
+
+impl TimeLatch {
+    /// Only returns true once every `LOG_DEBOUNCE_INTERVAL`.
+    fn elapsed(&mut self) -> bool {
+        let now = Instant::now();
+
+        let is_elapsed = self.0.map_or(false, |elapse_time| now > elapse_time);
+
+        if is_elapsed || self.0.is_none() {
+            self.0 = Some(now + LOG_DEBOUNCE_INTERVAL);
+        }
+
+        is_elapsed
+    }
+}
+
+/// A mutli-threaded processor for messages received on the network
+/// that need to be processed by the `BeaconChain`
+///
+/// See module level documentation for more information.
+pub struct BeaconProcessor<T: BeaconChainTypes> {
+    pub beacon_chain: Weak<BeaconChain<T>>,
+    pub network_tx: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
+    pub sync_tx: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    pub network_globals: Arc<NetworkGlobals<T::EthSpec>>,
+    pub executor: TaskExecutor,
+    pub max_workers: usize,
+    pub current_workers: usize,
+    pub log: Logger,
+}
+
+impl<T: BeaconChainTypes> BeaconProcessor<T> {
+    /// Spawns the "manager" task which checks the receiver end of the returned `Sender` for
+    /// messages which contain some new work which will be:
+    ///
+    /// - Performed immediately, if a worker is available.
+    /// - Queued for later processing, if no worker is currently available.
+    ///
+    /// Only `self.max_workers` will ever be spawned at one time. Each worker is a `tokio` task
+    /// started with `spawn_blocking`.
+    pub fn spawn_manager(mut self, mut event_rx: mpsc::Receiver<WorkEvent<T::EthSpec>>) {
+        let (idle_tx, mut idle_rx) = mpsc::channel::<()>(MAX_IDLE_QUEUE_LEN);
+
+        // Using LIFO queues for attestations since validator profits rely upon getting fresh
+        // attestations into blocks. Additionally, later attestations contain more information than
+        // earlier ones, so we consider them more valuable.
+        let mut aggregate_queue = LifoQueue::new(MAX_AGGREGATED_ATTESTATION_QUEUE_LEN);
+        let mut aggregate_debounce = TimeLatch::default();
+        let mut attestation_queue = LifoQueue::new(MAX_UNAGGREGATED_ATTESTATION_QUEUE_LEN);
+        let mut attestation_debounce = TimeLatch::default();
+
+        // Using a FIFO queue for voluntary exits since it prevents exit censoring. I don't have
+        // a strong feeling about queue type for exits.
+        let mut gossip_voluntary_exit_queue = FifoQueue::new(MAX_GOSSIP_EXIT_QUEUE_LEN);
+
+        // Using a FIFO queue for slashing to prevent people from flushing their slashings from the
+        // queues with lots of junk messages.
+        let mut gossip_proposer_slashing_queue =
+            FifoQueue::new(MAX_GOSSIP_PROPOSER_SLASHING_QUEUE_LEN);
+        let mut gossip_attester_slashing_queue =
+            FifoQueue::new(MAX_GOSSIP_ATTESTER_SLASHING_QUEUE_LEN);
+
+        // Using a FIFO queue since blocks need to be imported sequentially.
+        let mut rpc_block_queue = FifoQueue::new(MAX_RPC_BLOCK_QUEUE_LEN);
+        let mut chain_segment_queue = FifoQueue::new(MAX_CHAIN_SEGMENT_QUEUE_LEN);
+        let mut gossip_block_queue = FifoQueue::new(MAX_GOSSIP_BLOCK_QUEUE_LEN);
+
+        let executor = self.executor.clone();
+
+        // The manager future will run on the core executor and delegate tasks to worker
+        // threads on the blocking executor.
+        let manager_future = async move {
+            loop {
+                // Listen to both the event and idle channels, acting on whichever is ready
+                // first.
+                //
+                // Set `work_event = Some(event)` if there is new work to be done. Otherwise sets
+                // `event = None` if it was a worker becoming idle.
+                let work_event = tokio::select! {
+                    // A worker has finished some work.
+                    new_idle_opt = idle_rx.recv() => {
+                        if new_idle_opt.is_some() {
+                            self.current_workers = self.current_workers.saturating_sub(1);
+                            None
+                        } else {
+                            // Exit if all idle senders have been dropped.
+                            //
+                            // This shouldn't happen since this function holds a sender.
+                            crit!(
+                                self.log,
+                                "Gossip processor stopped";
+                                "msg" => "all idle senders dropped"
+                            );
+                            break
+                        }
+                    },
+                    // There is a new piece of work to be handled.
+                    new_work_event_opt = event_rx.recv() => {
+                        if let Some(new_work_event) = new_work_event_opt {
+                            Some(new_work_event)
+                        } else {
+                            // Exit if all event senders have been dropped.
+                            //
+                            // This should happen when the client shuts down.
+                            debug!(
+                                self.log,
+                                "Gossip processor stopped";
+                                "msg" => "all event senders dropped"
+                            );
+                            break
+                        }
+                    }
+                };
+
+                let _event_timer =
+                    metrics::start_timer(&metrics::BEACON_PROCESSOR_EVENT_HANDLING_SECONDS);
+                if let Some(event) = &work_event {
+                    metrics::inc_counter_vec(
+                        &metrics::BEACON_PROCESSOR_WORK_EVENTS_RX_COUNT,
+                        &[event.work.str_id()],
+                    );
+                } else {
+                    metrics::inc_counter(&metrics::BEACON_PROCESSOR_IDLE_EVENTS_TOTAL);
+                }
+
+                let can_spawn = self.current_workers < self.max_workers;
+                let drop_during_sync = work_event
+                    .as_ref()
+                    .map_or(false, |event| event.drop_during_sync);
+
+                match work_event {
+                    // There is no new work event, but we are able to spawn a new worker.
+                    //
+                    // We don't check the `work.drop_during_sync` here. We assume that if it made
+                    // it into the queue at any point then we should process it.
+                    None if can_spawn => {
+                        // Check for chain segments first, they're the most efficient way to get
+                        // blocks into the system.
+                        if let Some(item) = chain_segment_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check sync blocks before gossip blocks, since we've already explicitly
+                        // requested these blocks.
+                        } else if let Some(item) = rpc_block_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check gossip blocks before gossip attestations, since a block might be
+                        // required to verify some attestations.
+                        } else if let Some(item) = gossip_block_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check the aggregates, *then* the unaggregates
+                        // since we assume that aggregates are more valuable to local validators
+                        // and effectively give us more information with less signature
+                        // verification time.
+                        } else if let Some(item) = aggregate_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        } else if let Some(item) = attestation_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check slashings after all other consensus messages so we prioritize
+                        // following head.
+                        //
+                        // Check attester slashings before proposer slashings since they have the
+                        // potential to slash multiple validators at once.
+                        } else if let Some(item) = gossip_attester_slashing_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        } else if let Some(item) = gossip_proposer_slashing_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        // Check exits last since our validators don't get rewards from them.
+                        } else if let Some(item) = gossip_voluntary_exit_queue.pop() {
+                            self.spawn_worker(idle_tx.clone(), item);
+                        }
+                    }
+                    // There is no new work event and we are unable to spawn a new worker.
+                    //
+                    // I cannot see any good reason why this would happen.
+                    None => {
+                        warn!(
+                            self.log,
+                            "Unexpected gossip processor condition";
+                            "msg" => "no new work and cannot spawn worker"
+                        );
+                    }
+                    // The chain is syncing and this event should be dropped during sync.
+                    Some(work_event)
+                        if self.network_globals.sync_state.read().is_syncing()
+                            && drop_during_sync =>
+                    {
+                        let work_id = work_event.work.str_id();
+                        metrics::inc_counter_vec(
+                            &metrics::BEACON_PROCESSOR_WORK_EVENTS_IGNORED_COUNT,
+                            &[work_id],
+                        );
+                        trace!(
+                            self.log,
+                            "Gossip processor skipping work";
+                            "msg" => "chain is syncing",
+                            "work_id" => work_id
+                        );
+                    }
+                    // There is a new work event and the chain is not syncing. Process it.
+                    Some(WorkEvent { work, .. }) => {
+                        let work_id = work.str_id();
+                        match work {
+                            _ if can_spawn => self.spawn_worker(idle_tx.clone(), work),
+                            Work::GossipAttestation { .. } => attestation_queue.push(work),
+                            Work::GossipAggregate { .. } => aggregate_queue.push(work),
+                            Work::GossipBlock { .. } => {
+                                gossip_block_queue.push(work, work_id, &self.log)
+                            }
+                            Work::GossipVoluntaryExit { .. } => {
+                                gossip_voluntary_exit_queue.push(work, work_id, &self.log)
+                            }
+                            Work::GossipProposerSlashing { .. } => {
+                                gossip_proposer_slashing_queue.push(work, work_id, &self.log)
+                            }
+                            Work::GossipAttesterSlashing { .. } => {
+                                gossip_attester_slashing_queue.push(work, work_id, &self.log)
+                            }
+                            Work::RpcBlock { .. } => rpc_block_queue.push(work, work_id, &self.log),
+                            Work::ChainSegment { .. } => {
+                                chain_segment_queue.push(work, work_id, &self.log)
+                            }
+                        }
+                    }
+                }
+
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_WORKERS_ACTIVE_TOTAL,
+                    self.current_workers as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL,
+                    attestation_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL,
+                    aggregate_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL,
+                    gossip_block_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL,
+                    rpc_block_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL,
+                    chain_segment_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_EXIT_QUEUE_TOTAL,
+                    gossip_voluntary_exit_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL,
+                    gossip_proposer_slashing_queue.len() as i64,
+                );
+                metrics::set_gauge(
+                    &metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL,
+                    gossip_attester_slashing_queue.len() as i64,
+                );
+
+                if aggregate_queue.is_full() && aggregate_debounce.elapsed() {
+                    error!(
+                        self.log,
+                        "Aggregate attestation queue full";
+                        "msg" => "the system has insufficient resources for load",
+                        "queue_len" => aggregate_queue.max_length,
+                    )
+                }
+
+                if attestation_queue.is_full() && attestation_debounce.elapsed() {
+                    error!(
+                        self.log,
+                        "Attestation queue full";
+                        "msg" => "the system has insufficient resources for load",
+                        "queue_len" => attestation_queue.max_length,
+                    )
+                }
+            }
+        };
+
+        // Spawn on the core executor.
+        executor.spawn(manager_future, MANAGER_TASK_NAME);
+    }
+
+    /// Spawns a blocking worker thread to process some `Work`.
+    ///
+    /// Sends an message on `idle_tx` when the work is complete and the task is stopping.
+    fn spawn_worker(&mut self, mut idle_tx: mpsc::Sender<()>, work: Work<T::EthSpec>) {
+        let work_id = work.str_id();
+        let worker_timer =
+            metrics::start_timer_vec(&metrics::BEACON_PROCESSOR_WORKER_TIME, &[work_id]);
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_WORKERS_SPAWNED_TOTAL);
+        metrics::inc_counter_vec(
+            &metrics::BEACON_PROCESSOR_WORK_EVENTS_STARTED_COUNT,
+            &[work.str_id()],
+        );
+
+        let worker_id = self.current_workers;
+        self.current_workers = self.current_workers.saturating_add(1);
+
+        let chain = if let Some(chain) = self.beacon_chain.upgrade() {
+            chain
+        } else {
+            debug!(
+                self.log,
+                "Beacon chain dropped, shutting down";
+            );
+            return;
+        };
+
+        let log = self.log.clone();
+        let executor = self.executor.clone();
+
+        let worker = Worker {
+            chain,
+            network_tx: self.network_tx.clone(),
+            sync_tx: self.sync_tx.clone(),
+            log: self.log.clone(),
+        };
+
+        trace!(
+            self.log,
+            "Spawning beacon processor worker";
+            "work" => work_id,
+            "worker" => worker_id,
+        );
+
+        executor.spawn_blocking(
+            move || {
+                let _worker_timer = worker_timer;
+
+                match work {
+                    /*
+                     * Unaggregated attestation verification.
+                     */
+                    Work::GossipAttestation {
+                        message_id,
+                        peer_id,
+                        attestation,
+                        subnet_id,
+                        should_import,
+                    } => worker.process_gossip_attestation(
+                        message_id,
+                        peer_id,
+                        *attestation,
+                        subnet_id,
+                        should_import,
+                    ),
+                    /*
+                     * Aggregated attestation verification.
+                     */
+                    Work::GossipAggregate {
+                        message_id,
+                        peer_id,
+                        aggregate,
+                    } => worker.process_gossip_aggregate(message_id, peer_id, *aggregate),
+                    /*
+                     * Verification for beacon blocks received on gossip.
+                     */
+                    Work::GossipBlock {
+                        message_id,
+                        peer_id,
+                        block,
+                    } => worker.process_gossip_block(message_id, peer_id, *block),
+                    /*
+                     * Voluntary exits received on gossip.
+                     */
+                    Work::GossipVoluntaryExit {
+                        message_id,
+                        peer_id,
+                        voluntary_exit,
+                    } => worker.process_gossip_voluntary_exit(message_id, peer_id, *voluntary_exit),
+                    /*
+                     * Proposer slashings received on gossip.
+                     */
+                    Work::GossipProposerSlashing {
+                        message_id,
+                        peer_id,
+                        proposer_slashing,
+                    } => worker.process_gossip_proposer_slashing(
+                        message_id,
+                        peer_id,
+                        *proposer_slashing,
+                    ),
+                    /*
+                     * Attester slashings received on gossip.
+                     */
+                    Work::GossipAttesterSlashing {
+                        message_id,
+                        peer_id,
+                        attester_slashing,
+                    } => worker.process_gossip_attester_slashing(
+                        message_id,
+                        peer_id,
+                        *attester_slashing,
+                    ),
+                    /*
+                     * Verification for beacon blocks received during syncing via RPC.
+                     */
+                    Work::RpcBlock { block, result_tx } => {
+                        worker.process_rpc_block(*block, result_tx)
+                    }
+                    /*
+                     * Verification for a chain segment (multiple blocks).
+                     */
+                    Work::ChainSegment { process_id, blocks } => {
+                        worker.process_chain_segment(process_id, blocks)
+                    }
+                };
+
+                trace!(
+                    log,
+                    "Beacon processor worker done";
+                    "work" => work_id,
+                    "worker" => worker_id,
+                );
+
+                idle_tx.try_send(()).unwrap_or_else(|e| {
+                    crit!(
+                        log,
+                        "Unable to free worker";
+                        "msg" => "failed to send idle_tx message",
+                        "error" => e.to_string()
+                    )
+                });
+            },
+            WORKER_TASK_NAME,
+        );
+    }
+}

beacon_node/network/src/beacon_processor/worker.rs

@@ -0,0 +1,726 @@
+use super::{
+    chain_segment::{handle_chain_segment, ProcessId},
+    BlockResultSender,
+};
+use crate::{metrics, service::NetworkMessage, sync::SyncMessage};
+use beacon_chain::{
+    attestation_verification::Error as AttnError, observed_operations::ObservationOutcome,
+    BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError,
+};
+use eth2_libp2p::{MessageId, PeerId};
+use slog::{crit, debug, error, info, trace, warn, Logger};
+use ssz::Encode;
+use std::sync::Arc;
+use tokio::sync::mpsc;
+use types::{
+    Attestation, AttesterSlashing, Hash256, ProposerSlashing, SignedAggregateAndProof,
+    SignedBeaconBlock, SignedVoluntaryExit, SubnetId,
+};
+
+/// Contains the context necessary to import blocks, attestations, etc to the beacon chain.
+pub struct Worker<T: BeaconChainTypes> {
+    pub chain: Arc<BeaconChain<T>>,
+    pub network_tx: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
+    pub sync_tx: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    pub log: Logger,
+}
+
+impl<T: BeaconChainTypes> Worker<T> {
+    /// Process the unaggregated attestation received from the gossip network and:
+    ///
+    /// - If it passes gossip propagation criteria, tell the network thread to forward it.
+    /// - Attempt to apply it to fork choice.
+    /// - Attempt to add it to the naive aggregation pool.
+    ///
+    /// Raises a log if there are errors.
+    pub fn process_gossip_attestation(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attestation: Attestation<T::EthSpec>,
+        subnet_id: SubnetId,
+        should_import: bool,
+    ) {
+        let beacon_block_root = attestation.data.beacon_block_root;
+
+        let attestation = match self
+            .chain
+            .verify_unaggregated_attestation_for_gossip(attestation, subnet_id)
+        {
+            Ok(attestation) => attestation,
+            Err(e) => {
+                self.handle_attestation_verification_failure(
+                    peer_id,
+                    beacon_block_root,
+                    "unaggregated",
+                    e,
+                );
+                return;
+            }
+        };
+
+        // Indicate to the `Network` service that this message is valid and can be
+        // propagated on the gossip network.
+        self.propagate_gossip_message(message_id, peer_id.clone());
+
+        if !should_import {
+            return;
+        }
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL);
+
+        if let Err(e) = self.chain.apply_attestation_to_fork_choice(&attestation) {
+            match e {
+                BeaconChainError::ForkChoiceError(ForkChoiceError::InvalidAttestation(e)) => {
+                    debug!(
+                        self.log,
+                        "Attestation invalid for fork choice";
+                        "reason" => format!("{:?}", e),
+                        "peer" => peer_id.to_string(),
+                        "beacon_block_root" => format!("{:?}", beacon_block_root)
+                    )
+                }
+                e => error!(
+                    self.log,
+                    "Error applying attestation to fork choice";
+                    "reason" => format!("{:?}", e),
+                    "peer" => peer_id.to_string(),
+                    "beacon_block_root" => format!("{:?}", beacon_block_root)
+                ),
+            }
+        }
+
+        if let Err(e) = self.chain.add_to_naive_aggregation_pool(attestation) {
+            debug!(
+                self.log,
+                "Attestation invalid for agg pool";
+                "reason" => format!("{:?}", e),
+                "peer" => peer_id.to_string(),
+                "beacon_block_root" => format!("{:?}", beacon_block_root)
+            )
+        }
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_IMPORTED_TOTAL);
+    }
+
+    /// Process the aggregated attestation received from the gossip network and:
+    ///
+    /// - If it passes gossip propagation criteria, tell the network thread to forward it.
+    /// - Attempt to apply it to fork choice.
+    /// - Attempt to add it to the block inclusion pool.
+    ///
+    /// Raises a log if there are errors.
+    pub fn process_gossip_aggregate(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        aggregate: SignedAggregateAndProof<T::EthSpec>,
+    ) {
+        let beacon_block_root = aggregate.message.aggregate.data.beacon_block_root;
+
+        let aggregate = match self
+            .chain
+            .verify_aggregated_attestation_for_gossip(aggregate)
+        {
+            Ok(aggregate) => aggregate,
+            Err(e) => {
+                self.handle_attestation_verification_failure(
+                    peer_id,
+                    beacon_block_root,
+                    "aggregated",
+                    e,
+                );
+                return;
+            }
+        };
+
+        // Indicate to the `Network` service that this message is valid and can be
+        // propagated on the gossip network.
+        self.propagate_gossip_message(message_id, peer_id.clone());
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL);
+
+        if let Err(e) = self.chain.apply_attestation_to_fork_choice(&aggregate) {
+            match e {
+                BeaconChainError::ForkChoiceError(ForkChoiceError::InvalidAttestation(e)) => {
+                    debug!(
+                        self.log,
+                        "Aggregate invalid for fork choice";
+                        "reason" => format!("{:?}", e),
+                        "peer" => peer_id.to_string(),
+                        "beacon_block_root" => format!("{:?}", beacon_block_root)
+                    )
+                }
+                e => error!(
+                    self.log,
+                    "Error applying aggregate to fork choice";
+                    "reason" => format!("{:?}", e),
+                    "peer" => peer_id.to_string(),
+                    "beacon_block_root" => format!("{:?}", beacon_block_root)
+                ),
+            }
+        }
+
+        if let Err(e) = self.chain.add_to_block_inclusion_pool(aggregate) {
+            debug!(
+                self.log,
+                "Attestation invalid for op pool";
+                "reason" => format!("{:?}", e),
+                "peer" => peer_id.to_string(),
+                "beacon_block_root" => format!("{:?}", beacon_block_root)
+            )
+        }
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_IMPORTED_TOTAL);
+    }
+
+    /// Process the beacon block received from the gossip network and:
+    ///
+    /// - If it passes gossip propagation criteria, tell the network thread to forward it.
+    /// - Attempt to add it to the beacon chain, informing the sync thread if more blocks need to
+    ///   be downloaded.
+    ///
+    /// Raises a log if there are errors.
+    pub fn process_gossip_block(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        block: SignedBeaconBlock<T::EthSpec>,
+    ) {
+        let verified_block = match self.chain.verify_block_for_gossip(block) {
+            Ok(verified_block) => {
+                info!(
+                    self.log,
+                    "New block received";
+                    "slot" => verified_block.block.slot(),
+                    "hash" => verified_block.block_root.to_string()
+                );
+                self.propagate_gossip_message(message_id, peer_id.clone());
+                verified_block
+            }
+            Err(BlockError::ParentUnknown(block)) => {
+                self.send_sync_message(SyncMessage::UnknownBlock(peer_id, block));
+                return;
+            }
+            Err(BlockError::BlockIsAlreadyKnown) => {
+                debug!(
+                    self.log,
+                    "Gossip block is already known";
+                );
+                return;
+            }
+            Err(e) => {
+                warn!(
+                    self.log,
+                    "Could not verify block for gossip";
+                    "error" => format!("{:?}", e)
+                );
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL);
+
+        let block = Box::new(verified_block.block.clone());
+        match self.chain.process_block(verified_block) {
+            Ok(_block_root) => {
+                metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL);
+
+                trace!(
+                    self.log,
+                    "Gossipsub block processed";
+                    "peer_id" => peer_id.to_string()
+                );
+
+                // TODO: It would be better if we can run this _after_ we publish the block to
+                // reduce block propagation latency.
+                //
+                // The `MessageHandler` would be the place to put this, however it doesn't seem
+                // to have a reference to the `BeaconChain`. I will leave this for future
+                // works.
+                match self.chain.fork_choice() {
+                    Ok(()) => trace!(
+                        self.log,
+                        "Fork choice success";
+                        "location" => "block gossip"
+                    ),
+                    Err(e) => error!(
+                        self.log,
+                        "Fork choice failed";
+                        "error" => format!("{:?}", e),
+                        "location" => "block gossip"
+                    ),
+                }
+            }
+            Err(BlockError::ParentUnknown { .. }) => {
+                // Inform the sync manager to find parents for this block
+                // This should not occur. It should be checked by `should_forward_block`
+                error!(
+                    self.log,
+                    "Block with unknown parent attempted to be processed";
+                    "peer_id" => peer_id.to_string()
+                );
+                self.send_sync_message(SyncMessage::UnknownBlock(peer_id, block));
+            }
+            other => {
+                debug!(
+                    self.log,
+                    "Invalid gossip beacon block";
+                    "outcome" => format!("{:?}", other),
+                    "block root" => format!("{}", block.canonical_root()),
+                    "block slot" => block.slot()
+                );
+                trace!(
+                    self.log,
+                    "Invalid gossip beacon block ssz";
+                    "ssz" => format!("0x{}", hex::encode(block.as_ssz_bytes())),
+                );
+            }
+        };
+    }
+
+    pub fn process_gossip_voluntary_exit(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: SignedVoluntaryExit,
+    ) {
+        let validator_index = voluntary_exit.message.validator_index;
+
+        let exit = match self.chain.verify_voluntary_exit_for_gossip(voluntary_exit) {
+            Ok(ObservationOutcome::New(exit)) => exit,
+            Ok(ObservationOutcome::AlreadyKnown) => {
+                debug!(
+                    self.log,
+                    "Dropping exit for already exiting validator";
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string()
+                );
+                return;
+            }
+            Err(e) => {
+                debug!(
+                    self.log,
+                    "Dropping invalid exit";
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string(),
+                    "error" => format!("{:?}", e)
+                );
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL);
+
+        self.propagate_gossip_message(message_id, peer_id);
+
+        self.chain.import_voluntary_exit(exit);
+        debug!(self.log, "Successfully imported voluntary exit");
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_EXIT_IMPORTED_TOTAL);
+    }
+
+    pub fn process_gossip_proposer_slashing(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: ProposerSlashing,
+    ) {
+        let validator_index = proposer_slashing.signed_header_1.message.proposer_index;
+
+        let slashing = match self
+            .chain
+            .verify_proposer_slashing_for_gossip(proposer_slashing)
+        {
+            Ok(ObservationOutcome::New(slashing)) => slashing,
+            Ok(ObservationOutcome::AlreadyKnown) => {
+                debug!(
+                    self.log,
+                    "Dropping proposer slashing";
+                    "reason" => "Already seen a proposer slashing for that validator",
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string()
+                );
+                return;
+            }
+            Err(e) => {
+                debug!(
+                    self.log,
+                    "Dropping invalid proposer slashing";
+                    "validator_index" => validator_index,
+                    "peer" => peer_id.to_string(),
+                    "error" => format!("{:?}", e)
+                );
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL);
+
+        self.propagate_gossip_message(message_id, peer_id);
+
+        self.chain.import_proposer_slashing(slashing);
+        debug!(self.log, "Successfully imported proposer slashing");
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_IMPORTED_TOTAL);
+    }
+
+    pub fn process_gossip_attester_slashing(
+        self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: AttesterSlashing<T::EthSpec>,
+    ) {
+        let slashing = match self
+            .chain
+            .verify_attester_slashing_for_gossip(attester_slashing)
+        {
+            Ok(ObservationOutcome::New(slashing)) => slashing,
+            Ok(ObservationOutcome::AlreadyKnown) => {
+                debug!(
+                    self.log,
+                    "Dropping attester slashing";
+                    "reason" => "Slashings already known for all slashed validators",
+                    "peer" => peer_id.to_string()
+                );
+                return;
+            }
+            Err(e) => {
+                debug!(
+                    self.log,
+                    "Dropping invalid attester slashing";
+                    "peer" => peer_id.to_string(),
+                    "error" => format!("{:?}", e)
+                );
+                return;
+            }
+        };
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL);
+
+        self.propagate_gossip_message(message_id, peer_id);
+
+        if let Err(e) = self.chain.import_attester_slashing(slashing) {
+            debug!(self.log, "Error importing attester slashing"; "error" => format!("{:?}", e));
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_ERROR_TOTAL);
+        } else {
+            debug!(self.log, "Successfully imported attester slashing");
+            metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_IMPORTED_TOTAL);
+        }
+    }
+
+    /// Attempt to process a block received from a direct RPC request, returning the processing
+    /// result on the `result_tx` channel.
+    ///
+    /// Raises a log if there are errors publishing the result to the channel.
+    pub fn process_rpc_block(
+        self,
+        block: SignedBeaconBlock<T::EthSpec>,
+        result_tx: BlockResultSender<T::EthSpec>,
+    ) {
+        let block_result = self.chain.process_block(block);
+
+        metrics::inc_counter(&metrics::BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL);
+
+        if result_tx.send(block_result).is_err() {
+            crit!(self.log, "Failed return sync block result");
+        }
+    }
+
+    /// Attempt to import the chain segment (`blocks`) to the beacon chain, informing the sync
+    /// thread if more blocks are needed to process it.
+    pub fn process_chain_segment(
+        self,
+        process_id: ProcessId,
+        blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
+    ) {
+        handle_chain_segment(self.chain, process_id, blocks, self.sync_tx, self.log)
+    }
+
+    /// Send a message on `message_tx` that the `message_id` sent by `peer_id` should be propagated on
+    /// the gossip network.
+    ///
+    /// Creates a log if there is an interal error.
+    fn propagate_gossip_message(&self, message_id: MessageId, peer_id: PeerId) {
+        self.network_tx
+            .send(NetworkMessage::Validate {
+                propagation_source: peer_id,
+                message_id,
+            })
+            .unwrap_or_else(|_| {
+                warn!(
+                    self.log,
+                    "Could not send propagation request to the network service"
+                )
+            });
+    }
+
+    /// Send a message to `sync_tx`.
+    ///
+    /// Creates a log if there is an interal error.
+    fn send_sync_message(&self, message: SyncMessage<T::EthSpec>) {
+        self.sync_tx
+            .send(message)
+            .unwrap_or_else(|_| error!(self.log, "Could not send message to the sync service"));
+    }
+
+    /// Handle an error whilst verifying an `Attestation` or `SignedAggregateAndProof` from the
+    /// network.
+    pub fn handle_attestation_verification_failure(
+        &self,
+        peer_id: PeerId,
+        beacon_block_root: Hash256,
+        attestation_type: &str,
+        error: AttnError,
+    ) {
+        metrics::register_attestation_error(&error);
+        match &error {
+            AttnError::FutureEpoch { .. }
+            | AttnError::PastEpoch { .. }
+            | AttnError::FutureSlot { .. }
+            | AttnError::PastSlot { .. } => {
+                /*
+                 * These errors can be triggered by a mismatch between our slot and the peer.
+                 *
+                 *
+                 * The peer has published an invalid consensus message, _only_ if we trust our own clock.
+                 */
+            }
+            AttnError::InvalidSelectionProof { .. } | AttnError::InvalidSignature => {
+                /*
+                 * These errors are caused by invalid signatures.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+            AttnError::EmptyAggregationBitfield => {
+                /*
+                 * The aggregate had no signatures and is therefore worthless.
+                 *
+                 * Whilst we don't gossip this attestation, this act is **not** a clear
+                 * violation of the spec nor indication of fault.
+                 *
+                 * This may change soon. Reference:
+                 *
+                 * https://github.com/ethereum/eth2.0-specs/pull/1732
+                 */
+            }
+            AttnError::AggregatorPubkeyUnknown(_) => {
+                /*
+                 * The aggregator index was higher than any known validator index. This is
+                 * possible in two cases:
+                 *
+                 * 1. The attestation is malformed
+                 * 2. The attestation attests to a beacon_block_root that we do not know.
+                 *
+                 * It should be impossible to reach (2) without triggering
+                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
+                 * faulty.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+            AttnError::AggregatorNotInCommittee { .. } => {
+                /*
+                 * The aggregator index was higher than any known validator index. This is
+                 * possible in two cases:
+                 *
+                 * 1. The attestation is malformed
+                 * 2. The attestation attests to a beacon_block_root that we do not know.
+                 *
+                 * It should be impossible to reach (2) without triggering
+                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
+                 * faulty.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+            AttnError::AttestationAlreadyKnown { .. } => {
+                /*
+                 * The aggregate attestation has already been observed on the network or in
+                 * a block.
+                 *
+                 * The peer is not necessarily faulty.
+                 */
+                trace!(
+                    self.log,
+                    "Attestation already known";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                return;
+            }
+            AttnError::AggregatorAlreadyKnown(_) => {
+                /*
+                 * There has already been an aggregate attestation seen from this
+                 * aggregator index.
+                 *
+                 * The peer is not necessarily faulty.
+                 */
+                trace!(
+                    self.log,
+                    "Aggregator already known";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                return;
+            }
+            AttnError::PriorAttestationKnown { .. } => {
+                /*
+                 * We have already seen an attestation from this validator for this epoch.
+                 *
+                 * The peer is not necessarily faulty.
+                 */
+                trace!(
+                    self.log,
+                    "Prior attestation known";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root),
+                    "type" => format!("{:?}", attestation_type),
+                );
+                return;
+            }
+            AttnError::ValidatorIndexTooHigh(_) => {
+                /*
+                 * The aggregator index (or similar field) was higher than the maximum
+                 * possible number of validators.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+            AttnError::UnknownHeadBlock { beacon_block_root } => {
+                // Note: its a little bit unclear as to whether or not this block is unknown or
+                // just old. See:
+                //
+                // https://github.com/sigp/lighthouse/issues/1039
+
+                // TODO: Maintain this attestation and re-process once sync completes
+                debug!(
+                    self.log,
+                    "Attestation for unknown block";
+                    "peer_id" => peer_id.to_string(),
+                    "block" => format!("{}", beacon_block_root)
+                );
+                // we don't know the block, get the sync manager to handle the block lookup
+                self.sync_tx
+                    .send(SyncMessage::UnknownBlockHash(peer_id, *beacon_block_root))
+                    .unwrap_or_else(|_| {
+                        warn!(
+                            self.log,
+                            "Failed to send to sync service";
+                            "msg" => "UnknownBlockHash"
+                        )
+                    });
+                return;
+            }
+            AttnError::UnknownTargetRoot(_) => {
+                /*
+                 * The block indicated by the target root is not known to us.
+                 *
+                 * We should always get `AttnError::UnknwonHeadBlock` before we get this
+                 * error, so this means we can get this error if:
+                 *
+                 * 1. The target root does not represent a valid block.
+                 * 2. We do not have the target root in our DB.
+                 *
+                 * For (2), we should only be processing attestations when we should have
+                 * all the available information. Note: if we do a weak-subjectivity sync
+                 * it's possible that this situation could occur, but I think it's
+                 * unlikely. For now, we will declare this to be an invalid message>
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+            AttnError::BadTargetEpoch => {
+                /*
+                 * The aggregator index (or similar field) was higher than the maximum
+                 * possible number of validators.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+            AttnError::NoCommitteeForSlotAndIndex { .. } => {
+                /*
+                 * It is not possible to attest this the given committee in the given slot.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+            AttnError::NotExactlyOneAggregationBitSet(_) => {
+                /*
+                 * The unaggregated attestation doesn't have only one signature.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+            AttnError::AttestsToFutureBlock { .. } => {
+                /*
+                 * The beacon_block_root is from a higher slot than the attestation.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+
+            AttnError::InvalidSubnetId { received, expected } => {
+                /*
+                 * The attestation was received on an incorrect subnet id.
+                 */
+                debug!(
+                    self.log,
+                    "Received attestation on incorrect subnet";
+                    "expected" => format!("{:?}", expected),
+                    "received" => format!("{:?}", received),
+                )
+            }
+            AttnError::Invalid(_) => {
+                /*
+                 * The attestation failed the state_processing verification.
+                 *
+                 * The peer has published an invalid consensus message.
+                 */
+            }
+            AttnError::TooManySkippedSlots {
+                head_block_slot,
+                attestation_slot,
+            } => {
+                /*
+                 * The attestation references a head block that is too far behind the attestation slot.
+                 *
+                 * The message is not necessarily invalid, but we choose to ignore it.
+                 */
+                debug!(
+                    self.log,
+                    "Rejected long skip slot attestation";
+                    "head_block_slot" => head_block_slot,
+                    "attestation_slot" => attestation_slot,
+                )
+            }
+            AttnError::BeaconChainError(e) => {
+                /*
+                 * Lighthouse hit an unexpected error whilst processing the attestation. It
+                 * should be impossible to trigger a `BeaconChainError` from the network,
+                 * so we have a bug.
+                 *
+                 * It's not clear if the message is invalid/malicious.
+                 */
+                error!(
+                    self.log,
+                    "Unable to validate aggregate";
+                    "peer_id" => peer_id.to_string(),
+                    "error" => format!("{:?}", e),
+                );
+            }
+        }
+
+        debug!(
+            self.log,
+            "Invalid attestation from network";
+            "reason" => format!("{:?}", error),
+            "block" => format!("{}", beacon_block_root),
+            "peer_id" => peer_id.to_string(),
+            "type" => format!("{:?}", attestation_type),
+        );
+    }
+}

beacon_node/network/src/lib.rs

@@ -6,6 +6,7 @@ pub mod error;
 pub mod service;
 
 mod attestation_service;
+mod beacon_processor;
 mod metrics;
 mod persisted_dht;
 mod router;

beacon_node/network/src/metrics.rs

@@ -1,3 +1,4 @@
+use beacon_chain::attestation_verification::Error as AttnError;
 pub use lighthouse_metrics::*;
 
 lazy_static! {
@@ -44,4 +45,302 @@ lazy_static! {
         "network_subnet_subscriptions_aggregator_total",
         "Count of validator subscription requests where the subscriber is an aggregator."
     );
+
+    /*
+     * Gossip processor
+     */
+    pub static ref BEACON_PROCESSOR_WORK_EVENTS_RX_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_processor_work_events_rx_count",
+        "Count of work events received (but not necessarily processed)",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORK_EVENTS_IGNORED_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_processor_work_events_ignored_count",
+        "Count of work events purposefully ignored",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORK_EVENTS_STARTED_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_processor_work_events_started_count",
+        "Count of work events which have been started by a worker",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORKER_TIME: Result<HistogramVec> = try_create_histogram_vec(
+        "beacon_processor_worker_time",
+        "Time taken for a worker to fully process some parcel of work.",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORKERS_SPAWNED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_workers_spawned_total",
+        "The number of workers ever spawned by the gossip processing pool."
+    );
+    pub static ref BEACON_PROCESSOR_WORKERS_ACTIVE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_workers_active_total",
+        "Count of active workers in the gossip processing pool."
+    );
+    pub static ref BEACON_PROCESSOR_IDLE_EVENTS_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_idle_events_total",
+        "Count of idle events processed by the gossip processor manager."
+    );
+    pub static ref BEACON_PROCESSOR_EVENT_HANDLING_SECONDS: Result<Histogram> = try_create_histogram(
+        "beacon_processor_event_handling_seconds",
+        "Time spent handling a new message and allocating it to a queue or worker."
+    );
+    // Gossip blocks.
+    pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_gossip_block_queue_total",
+        "Count of blocks from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_gossip_block_verified_total",
+        "Total number of gossip blocks verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_gossip_block_imported_total",
+        "Total number of gossip blocks imported to fork choice, etc."
+    );
+    // Gossip Exits.
+    pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_exit_queue_total",
+        "Count of exits from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_exit_verified_total",
+        "Total number of voluntary exits verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_EXIT_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_exit_imported_total",
+        "Total number of voluntary exits imported to the op pool."
+    );
+    // Gossip proposer slashings.
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_proposer_slashing_queue_total",
+        "Count of proposer slashings from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_proposer_slashing_verified_total",
+        "Total number of proposer slashings verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_proposer_slashing_imported_total",
+        "Total number of proposer slashings imported to the op pool."
+    );
+    // Gossip attester slashings.
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_attester_slashing_queue_total",
+        "Count of attester slashings from gossip waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_attester_slashing_verified_total",
+        "Total number of attester slashings verified for propagation."
+    );
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_attester_slashing_imported_total",
+        "Total number of attester slashings imported to the op pool."
+    );
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_ERROR_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_attester_slashing_error_total",
+        "Total number of attester slashings that raised an error during processing."
+    );
+    // Rpc blocks.
+    pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_rpc_block_queue_total",
+        "Count of blocks from the rpc waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_rpc_block_imported_total",
+        "Total number of gossip blocks imported to fork choice, etc."
+    );
+    // Chain segments.
+    pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_chain_segment_queue_total",
+        "Count of chain segments from the rpc waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_SUCCESS_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_chain_segment_success_total",
+        "Total number of chain segments successfully processed."
+    );
+    pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_FAILED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_chain_segment_failed_total",
+        "Total number of chain segments that failed processing."
+    );
+    // Unaggregated attestations.
+    pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_unaggregated_attestation_queue_total",
+        "Count of unagg. attestations waiting to be processed."
+    );
+    pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_unaggregated_attestation_verified_total",
+        "Total number of unaggregated attestations verified for gossip."
+    );
+    pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_unaggregated_attestation_imported_total",
+        "Total number of unaggregated attestations imported to fork choice, etc."
+    );
+    // Aggregated attestations.
+    pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_aggregated_attestation_queue_total",
+        "Count of agg. attestations waiting to be processed."
+    );
+    pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_aggregated_attestation_verified_total",
+        "Total number of aggregated attestations verified for gossip."
+    );
+    pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_aggregated_attestation_imported_total",
+        "Total number of aggregated attestations imported to fork choice, etc."
+    );
+
+    /*
+     * Attestation Errors
+     */
+    pub static ref GOSSIP_ATTESTATION_ERROR_FUTURE_EPOCH: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_future_epoch",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_PAST_EPOCH: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_past_epoch",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_FUTURE_SLOT: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_future_slot",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_PAST_SLOT: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_past_slot",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_SELECTION_PROOF: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_invalid_selection_proof",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_SIGNATURE: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_invalid_signature",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_EMPTY_AGGREGATION_BITFIELD: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_empty_aggregation_bitfield",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_AGGREGATOR_PUBKEY_UNKNOWN: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_aggregator_pubkey_unknown",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_AGGREGATOR_NOT_IN_COMMITTEE: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_aggregator_not_in_committee",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_ATTESTATION_ALREADY_KNOWN: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_attestation_already_known",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_AGGREGATOR_ALREADY_KNOWN: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_aggregator_already_known",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_PRIOR_ATTESTATION_KNOWN: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_prior_attestation_known",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_VALIDATOR_INDEX_TOO_HIGH: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_validator_index_too_high",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_UNKNOWN_HEAD_BLOCK: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_unknown_head_block",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_UNKNOWN_TARGET_ROOT: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_unknown_target_root",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_BAD_TARGET_EPOCH: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_bad_target_epoch",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_NO_COMMITTEE_FOR_SLOT_AND_INDEX: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_no_committee_for_slot_and_index",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_NOT_EXACTLY_ONE_AGGREGATION_BIT_SET: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_not_exactly_one_aggregation_bit_set",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_ATTESTS_TO_FUTURE_BLOCK: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_attests_to_future_block",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_SUBNET_ID: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_invalid_subnet_id",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_STATE_PROCESSING: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_invalid_state_processing",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_INVALID_TOO_MANY_SKIPPED_SLOTS: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_invalid_too_many_skipped_slots",
+        "Count of a specific error type (see metric name)"
+    );
+    pub static ref GOSSIP_ATTESTATION_ERROR_BEACON_CHAIN_ERROR: Result<IntCounter> = try_create_int_counter(
+        "gossip_attestation_error_beacon_chain_error",
+        "Count of a specific error type (see metric name)"
+    );
+}
+
+pub fn register_attestation_error(error: &AttnError) {
+    match error {
+        AttnError::FutureEpoch { .. } => inc_counter(&GOSSIP_ATTESTATION_ERROR_FUTURE_EPOCH),
+        AttnError::PastEpoch { .. } => inc_counter(&GOSSIP_ATTESTATION_ERROR_PAST_EPOCH),
+        AttnError::FutureSlot { .. } => inc_counter(&GOSSIP_ATTESTATION_ERROR_FUTURE_SLOT),
+        AttnError::PastSlot { .. } => inc_counter(&GOSSIP_ATTESTATION_ERROR_PAST_SLOT),
+        AttnError::InvalidSelectionProof { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_SELECTION_PROOF)
+        }
+        AttnError::InvalidSignature => inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_SIGNATURE),
+        AttnError::EmptyAggregationBitfield => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_EMPTY_AGGREGATION_BITFIELD)
+        }
+        AttnError::AggregatorPubkeyUnknown(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_AGGREGATOR_PUBKEY_UNKNOWN)
+        }
+        AttnError::AggregatorNotInCommittee { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_AGGREGATOR_NOT_IN_COMMITTEE)
+        }
+        AttnError::AttestationAlreadyKnown { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_ATTESTATION_ALREADY_KNOWN)
+        }
+        AttnError::AggregatorAlreadyKnown(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_AGGREGATOR_ALREADY_KNOWN)
+        }
+        AttnError::PriorAttestationKnown { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_PRIOR_ATTESTATION_KNOWN)
+        }
+        AttnError::ValidatorIndexTooHigh(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_VALIDATOR_INDEX_TOO_HIGH)
+        }
+        AttnError::UnknownHeadBlock { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_UNKNOWN_HEAD_BLOCK)
+        }
+        AttnError::UnknownTargetRoot(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_UNKNOWN_TARGET_ROOT)
+        }
+        AttnError::BadTargetEpoch => inc_counter(&GOSSIP_ATTESTATION_ERROR_BAD_TARGET_EPOCH),
+        AttnError::NoCommitteeForSlotAndIndex { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_NO_COMMITTEE_FOR_SLOT_AND_INDEX)
+        }
+        AttnError::NotExactlyOneAggregationBitSet(_) => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_NOT_EXACTLY_ONE_AGGREGATION_BIT_SET)
+        }
+        AttnError::AttestsToFutureBlock { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_ATTESTS_TO_FUTURE_BLOCK)
+        }
+        AttnError::InvalidSubnetId { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_SUBNET_ID)
+        }
+        AttnError::Invalid(_) => inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_STATE_PROCESSING),
+        AttnError::TooManySkippedSlots { .. } => {
+            inc_counter(&GOSSIP_ATTESTATION_ERROR_INVALID_TOO_MANY_SKIPPED_SLOTS)
+        }
+        AttnError::BeaconChainError(_) => inc_counter(&GOSSIP_ATTESTATION_ERROR_BEACON_CHAIN_ERROR),
+    }
 }

beacon_node/network/src/router/mod.rs

@@ -9,14 +9,14 @@ pub mod processor;
 
 use crate::error;
 use crate::service::NetworkMessage;
-use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError};
+use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::{
     rpc::{RPCError, RequestId},
     MessageId, NetworkGlobals, PeerId, PeerRequestId, PubsubMessage, Request, Response,
 };
 use futures::prelude::*;
 use processor::Processor;
-use slog::{debug, info, o, trace, warn};
+use slog::{debug, o, trace};
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use types::EthSpec;
@@ -26,8 +26,6 @@ use types::EthSpec;
 /// passing them to the internal message processor. The message processor spawns a syncing thread
 /// which manages which blocks need to be requested and processed.
 pub struct Router<T: BeaconChainTypes> {
-    /// A channel to the network service to allow for gossip propagation.
-    network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
     /// Access to the peer db for logging.
     network_globals: Arc<NetworkGlobals<T::EthSpec>>,
     /// Processes validated and decoded messages from the network. Has direct access to the
@@ -89,13 +87,12 @@ impl<T: BeaconChainTypes> Router<T> {
             executor.clone(),
             beacon_chain,
             network_globals.clone(),
-            network_send.clone(),
+            network_send,
             &log,
         );
 
         // generate the Message handler
         let mut handler = Router {
-            network_send,
             network_globals,
             processor,
             log: message_handler_log,
@@ -215,54 +212,24 @@ impl<T: BeaconChainTypes> Router<T> {
         match gossip_message {
             // Attestations should never reach the router.
             PubsubMessage::AggregateAndProofAttestation(aggregate_and_proof) => {
-                if let Some(gossip_verified) = self
-                    .processor
-                    .verify_aggregated_attestation_for_gossip(peer_id.clone(), *aggregate_and_proof)
-                {
-                    self.propagate_message(id, peer_id.clone());
-                    self.processor
-                        .import_aggregated_attestation(peer_id, gossip_verified);
-                }
+                self.processor
+                    .on_aggregated_attestation_gossip(id, peer_id, *aggregate_and_proof);
             }
             PubsubMessage::Attestation(subnet_attestation) => {
-                if let Some(gossip_verified) =
-                    self.processor.verify_unaggregated_attestation_for_gossip(
-                        peer_id.clone(),
-                        subnet_attestation.1.clone(),
-                        subnet_attestation.0,
-                    )
-                {
-                    self.propagate_message(id, peer_id.clone());
-                    if should_process {
-                        self.processor
-                            .import_unaggregated_attestation(peer_id, gossip_verified);
-                    }
-                }
+                self.processor.on_unaggregated_attestation_gossip(
+                    id,
+                    peer_id,
+                    subnet_attestation.1.clone(),
+                    subnet_attestation.0,
+                    should_process,
+                );
             }
             PubsubMessage::BeaconBlock(block) => {
-                match self.processor.should_forward_block(&peer_id, block) {
-                    Ok(verified_block) => {
-                        info!(self.log, "New block received"; "slot" => verified_block.block.slot(), "hash" => verified_block.block_root.to_string());
-                        self.propagate_message(id, peer_id.clone());
-                        self.processor.on_block_gossip(peer_id, verified_block);
-                    }
-                    Err(BlockError::ParentUnknown { .. }) => {} // performing a parent lookup
-                    Err(e) => {
-                        // performing a parent lookup
-                        warn!(self.log, "Could not verify block for gossip";
-                            "error" => format!("{:?}", e));
-                    }
-                }
+                self.processor.on_block_gossip(id, peer_id, block);
             }
             PubsubMessage::VoluntaryExit(exit) => {
                 debug!(self.log, "Received a voluntary exit"; "peer_id" => format!("{}", peer_id));
-                if let Some(verified_exit) = self
-                    .processor
-                    .verify_voluntary_exit_for_gossip(&peer_id, *exit)
-                {
-                    self.propagate_message(id, peer_id.clone());
-                    self.processor.import_verified_voluntary_exit(verified_exit);
-                }
+                self.processor.on_voluntary_exit_gossip(id, peer_id, exit);
             }
             PubsubMessage::ProposerSlashing(proposer_slashing) => {
                 debug!(
@@ -270,14 +237,8 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received a proposer slashing";
                     "peer_id" => format!("{}", peer_id)
                 );
-                if let Some(verified_proposer_slashing) = self
-                    .processor
-                    .verify_proposer_slashing_for_gossip(&peer_id, *proposer_slashing)
-                {
-                    self.propagate_message(id, peer_id.clone());
-                    self.processor
-                        .import_verified_proposer_slashing(verified_proposer_slashing);
-                }
+                self.processor
+                    .on_proposer_slashing_gossip(id, peer_id, proposer_slashing);
             }
             PubsubMessage::AttesterSlashing(attester_slashing) => {
                 debug!(
@@ -285,30 +246,9 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received a attester slashing";
                     "peer_id" => format!("{}", peer_id)
                 );
-                if let Some(verified_attester_slashing) = self
-                    .processor
-                    .verify_attester_slashing_for_gossip(&peer_id, *attester_slashing)
-                {
-                    self.propagate_message(id, peer_id.clone());
-                    self.processor
-                        .import_verified_attester_slashing(verified_attester_slashing);
-                }
+                self.processor
+                    .on_attester_slashing_gossip(id, peer_id, attester_slashing);
             }
         }
     }
-
-    /// Informs the network service that the message should be forwarded to other peers (is valid).
-    fn propagate_message(&mut self, message_id: MessageId, propagation_source: PeerId) {
-        self.network_send
-            .send(NetworkMessage::Validate {
-                propagation_source,
-                message_id,
-            })
-            .unwrap_or_else(|_| {
-                warn!(
-                    self.log,
-                    "Could not send propagation request to the network service"
-                )
-            });
-    }
 }

beacon_node/network/src/router/processor.rs

@@ -1,20 +1,16 @@
+use crate::beacon_processor::{
+    BeaconProcessor, WorkEvent as BeaconWorkEvent, MAX_WORK_EVENT_QUEUE_LEN,
+};
 use crate::service::NetworkMessage;
 use crate::sync::{PeerSyncInfo, SyncMessage};
-use beacon_chain::{
-    attestation_verification::{
-        Error as AttnError, SignatureVerifiedAttestation, VerifiedAggregatedAttestation,
-        VerifiedUnaggregatedAttestation,
-    },
-    observed_operations::ObservationOutcome,
-    BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError,
-    GossipVerifiedBlock,
-};
+use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::rpc::*;
-use eth2_libp2p::{NetworkGlobals, PeerAction, PeerId, PeerRequestId, Request, Response};
+use eth2_libp2p::{
+    MessageId, NetworkGlobals, PeerAction, PeerId, PeerRequestId, Request, Response,
+};
 use itertools::process_results;
 use slog::{debug, error, o, trace, warn};
-use ssz::Encode;
-use state_processing::SigVerifiedOp;
+use std::cmp;
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use types::{
@@ -37,6 +33,8 @@ pub struct Processor<T: BeaconChainTypes> {
     sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
     /// A network context to return and handle RPC requests.
     network: HandlerNetworkContext<T::EthSpec>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     /// The `RPCHandler` logger.
     log: slog::Logger,
 }
@@ -51,20 +49,36 @@ impl<T: BeaconChainTypes> Processor<T> {
         log: &slog::Logger,
     ) -> Self {
         let sync_logger = log.new(o!("service"=> "sync"));
+        let (beacon_processor_send, beacon_processor_receive) =
+            mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN);
 
         // spawn the sync thread
         let sync_send = crate::sync::manager::spawn(
-            executor,
+            executor.clone(),
             beacon_chain.clone(),
-            network_globals,
+            network_globals.clone(),
             network_send.clone(),
+            beacon_processor_send.clone(),
             sync_logger,
         );
 
+        BeaconProcessor {
+            beacon_chain: Arc::downgrade(&beacon_chain),
+            network_tx: network_send.clone(),
+            sync_tx: sync_send.clone(),
+            network_globals,
+            executor,
+            max_workers: cmp::max(1, num_cpus::get()),
+            current_workers: 0,
+            log: log.clone(),
+        }
+        .spawn_manager(beacon_processor_receive);
+
         Processor {
             chain: beacon_chain,
             sync_send,
             network: HandlerNetworkContext::new(network_send, log.clone()),
+            beacon_processor_send,
             log: log.clone(),
         }
     }
@@ -499,23 +513,6 @@ impl<T: BeaconChainTypes> Processor<T> {
         }
     }
 
-    /// Template function to be called on a block to determine if the block should be propagated
-    /// across the network.
-    pub fn should_forward_block(
-        &mut self,
-        peer_id: &PeerId,
-        block: Box<SignedBeaconBlock<T::EthSpec>>,
-    ) -> Result<GossipVerifiedBlock<T>, BlockError> {
-        let result = self.chain.verify_block_for_gossip(*block.clone());
-
-        if let Err(BlockError::ParentUnknown(_)) = result {
-            // if we don't know the parent, start a parent lookup
-            // TODO: Modify the return to avoid the block clone.
-            self.send_to_sync(SyncMessage::UnknownBlock(peer_id.clone(), block));
-        }
-        result
-    }
-
     /// Process a gossip message declaring a new block.
     ///
     /// Attempts to apply to block to the beacon chain. May queue the block for later processing.
@@ -523,543 +520,134 @@ impl<T: BeaconChainTypes> Processor<T> {
     /// Returns a `bool` which, if `true`, indicates we should forward the block to our peers.
     pub fn on_block_gossip(
         &mut self,
+        message_id: MessageId,
         peer_id: PeerId,
-        verified_block: GossipVerifiedBlock<T>,
-    ) -> bool {
-        let block = Box::new(verified_block.block.clone());
-        match self.chain.process_block(verified_block) {
-            Ok(_block_root) => {
-                trace!(
-                    self.log,
-                    "Gossipsub block processed";
-                    "peer_id" => peer_id.to_string()
-                );
-
-                // TODO: It would be better if we can run this _after_ we publish the block to
-                // reduce block propagation latency.
-                //
-                // The `MessageHandler` would be the place to put this, however it doesn't seem
-                // to have a reference to the `BeaconChain`. I will leave this for future
-                // works.
-                match self.chain.fork_choice() {
-                    Ok(()) => trace!(
-                        self.log,
-                        "Fork choice success";
-                        "location" => "block gossip"
-                    ),
-                    Err(e) => error!(
-                        self.log,
-                        "Fork choice failed";
-                        "error" => format!("{:?}", e),
-                        "location" => "block gossip"
-                    ),
-                }
-            }
-            Err(BlockError::ParentUnknown { .. }) => {
-                // Inform the sync manager to find parents for this block
-                // This should not occur. It should be checked by `should_forward_block`
-                error!(
-                    self.log,
-                    "Block with unknown parent attempted to be processed";
-                    "peer_id" => peer_id.to_string()
-                );
-                self.send_to_sync(SyncMessage::UnknownBlock(peer_id, block));
-            }
-            other => {
-                warn!(
-                    self.log,
-                    "Invalid gossip beacon block";
-                    "outcome" => format!("{:?}", other),
-                    "block root" => format!("{}", block.canonical_root()),
-                    "block slot" => block.slot()
-                );
-                trace!(
-                    self.log,
-                    "Invalid gossip beacon block ssz";
-                    "ssz" => format!("0x{}", hex::encode(block.as_ssz_bytes())),
-                );
-            }
-        }
-        // TODO: Update with correct block gossip checking
-        true
-    }
-
-    /// Handle an error whilst verifying an `Attestation` or `SignedAggregateAndProof` from the
-    /// network.
-    pub fn handle_attestation_verification_failure(
-        &mut self,
-        peer_id: PeerId,
-        beacon_block_root: Hash256,
-        attestation_type: &str,
-        error: AttnError,
+        block: Box<SignedBeaconBlock<T::EthSpec>>,
     ) {
-        debug!(
-            self.log,
-            "Invalid attestation from network";
-            "block" => format!("{}", beacon_block_root),
-            "peer_id" => peer_id.to_string(),
-            "type" => format!("{:?}", attestation_type),
-        );
-
-        match error {
-            AttnError::FutureEpoch { .. }
-            | AttnError::PastEpoch { .. }
-            | AttnError::FutureSlot { .. }
-            | AttnError::PastSlot { .. } => {
-                /*
-                 * These errors can be triggered by a mismatch between our slot and the peer.
-                 *
-                 *
-                 * The peer has published an invalid consensus message, _only_ if we trust our own clock.
-                 */
-            }
-            AttnError::InvalidSelectionProof { .. } | AttnError::InvalidSignature => {
-                /*
-                 * These errors are caused by invalid signatures.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::EmptyAggregationBitfield => {
-                /*
-                 * The aggregate had no signatures and is therefore worthless.
-                 *
-                 * Whilst we don't gossip this attestation, this act is **not** a clear
-                 * violation of the spec nor indication of fault.
-                 *
-                 * This may change soon. Reference:
-                 *
-                 * https://github.com/ethereum/eth2.0-specs/pull/1732
-                 */
-            }
-            AttnError::AggregatorPubkeyUnknown(_) => {
-                /*
-                 * The aggregator index was higher than any known validator index. This is
-                 * possible in two cases:
-                 *
-                 * 1. The attestation is malformed
-                 * 2. The attestation attests to a beacon_block_root that we do not know.
-                 *
-                 * It should be impossible to reach (2) without triggering
-                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
-                 * faulty.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::AggregatorNotInCommittee { .. } => {
-                /*
-                 * The aggregator index was higher than any known validator index. This is
-                 * possible in two cases:
-                 *
-                 * 1. The attestation is malformed
-                 * 2. The attestation attests to a beacon_block_root that we do not know.
-                 *
-                 * It should be impossible to reach (2) without triggering
-                 * `AttnError::UnknownHeadBlock`, so we can safely assume the peer is
-                 * faulty.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::AttestationAlreadyKnown { .. } => {
-                /*
-                 * The aggregate attestation has already been observed on the network or in
-                 * a block.
-                 *
-                 * The peer is not necessarily faulty.
-                 */
-            }
-            AttnError::AggregatorAlreadyKnown(_) => {
-                /*
-                 * There has already been an aggregate attestation seen from this
-                 * aggregator index.
-                 *
-                 * The peer is not necessarily faulty.
-                 */
-            }
-            AttnError::PriorAttestationKnown { .. } => {
-                /*
-                 * We have already seen an attestation from this validator for this epoch.
-                 *
-                 * The peer is not necessarily faulty.
-                 */
-            }
-            AttnError::ValidatorIndexTooHigh(_) => {
-                /*
-                 * The aggregator index (or similar field) was higher than the maximum
-                 * possible number of validators.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::UnknownHeadBlock { beacon_block_root } => {
-                // Note: its a little bit unclear as to whether or not this block is unknown or
-                // just old. See:
-                //
-                // https://github.com/sigp/lighthouse/issues/1039
-
-                // TODO: Maintain this attestation and re-process once sync completes
-                debug!(
-                    self.log,
-                    "Attestation for unknown block";
-                    "peer_id" => peer_id.to_string(),
-                    "block" => format!("{}", beacon_block_root)
-                );
-                // we don't know the block, get the sync manager to handle the block lookup
-                self.send_to_sync(SyncMessage::UnknownBlockHash(peer_id, beacon_block_root));
-            }
-            AttnError::UnknownTargetRoot(_) => {
-                /*
-                 * The block indicated by the target root is not known to us.
-                 *
-                 * We should always get `AttnError::UnknwonHeadBlock` before we get this
-                 * error, so this means we can get this error if:
-                 *
-                 * 1. The target root does not represent a valid block.
-                 * 2. We do not have the target root in our DB.
-                 *
-                 * For (2), we should only be processing attestations when we should have
-                 * all the available information. Note: if we do a weak-subjectivity sync
-                 * it's possible that this situation could occur, but I think it's
-                 * unlikely. For now, we will declare this to be an invalid message>
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::BadTargetEpoch => {
-                /*
-                 * The aggregator index (or similar field) was higher than the maximum
-                 * possible number of validators.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::NoCommitteeForSlotAndIndex { .. } => {
-                /*
-                 * It is not possible to attest this the given committee in the given slot.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::NotExactlyOneAggregationBitSet(_) => {
-                /*
-                 * The unaggregated attestation doesn't have only one signature.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::AttestsToFutureBlock { .. } => {
-                /*
-                 * The beacon_block_root is from a higher slot than the attestation.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-
-            AttnError::InvalidSubnetId { received, expected } => {
-                /*
-                 * The attestation was received on an incorrect subnet id.
-                 */
-                debug!(
-                    self.log,
-                    "Received attestation on incorrect subnet";
-                    "expected" => format!("{:?}", expected),
-                    "received" => format!("{:?}", received),
-                )
-            }
-            AttnError::Invalid(_) => {
-                /*
-                 * The attestation failed the state_processing verification.
-                 *
-                 * The peer has published an invalid consensus message.
-                 */
-            }
-            AttnError::BeaconChainError(e) => {
-                /*
-                 * Lighthouse hit an unexpected error whilst processing the attestation. It
-                 * should be impossible to trigger a `BeaconChainError` from the network,
-                 * so we have a bug.
-                 *
-                 * It's not clear if the message is invalid/malicious.
-                 */
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_beacon_block(
+                message_id, peer_id, block,
+            ))
+            .unwrap_or_else(|e| {
                 error!(
-                    self.log,
-                    "Unable to validate aggregate";
-                    "peer_id" => peer_id.to_string(),
-                    "error" => format!("{:?}", e),
-                );
-            }
-        }
-    }
-
-    pub fn verify_aggregated_attestation_for_gossip(
-        &mut self,
-        peer_id: PeerId,
-        aggregate_and_proof: SignedAggregateAndProof<T::EthSpec>,
-    ) -> Option<VerifiedAggregatedAttestation<T>> {
-        // This is provided to the error handling function to assist with debugging.
-        let beacon_block_root = aggregate_and_proof.message.aggregate.data.beacon_block_root;
-
-        self.chain
-            .verify_aggregated_attestation_for_gossip(aggregate_and_proof)
-            .map_err(|e| {
-                self.handle_attestation_verification_failure(
-                    peer_id,
-                    beacon_block_root,
-                    "aggregated",
-                    e,
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "block gossip",
+                    "error" => e.to_string(),
                 )
             })
-            .ok()
     }
 
-    pub fn import_aggregated_attestation(
-        &mut self,
-        peer_id: PeerId,
-        verified_attestation: VerifiedAggregatedAttestation<T>,
-    ) {
-        // This is provided to the error handling function to assist with debugging.
-        let beacon_block_root = verified_attestation.attestation().data.beacon_block_root;
-
-        self.apply_attestation_to_fork_choice(
-            peer_id.clone(),
-            beacon_block_root,
-            &verified_attestation,
-        );
-
-        if let Err(e) = self.chain.add_to_block_inclusion_pool(verified_attestation) {
-            debug!(
-                self.log,
-                "Attestation invalid for op pool";
-                "reason" => format!("{:?}", e),
-                "peer" => peer_id.to_string(),
-                "beacon_block_root" => format!("{:?}", beacon_block_root)
-            )
-        }
-    }
-
-    pub fn verify_unaggregated_attestation_for_gossip(
+    pub fn on_unaggregated_attestation_gossip(
         &mut self,
+        message_id: MessageId,
         peer_id: PeerId,
         unaggregated_attestation: Attestation<T::EthSpec>,
         subnet_id: SubnetId,
-    ) -> Option<VerifiedUnaggregatedAttestation<T>> {
-        // This is provided to the error handling function to assist with debugging.
-        let beacon_block_root = unaggregated_attestation.data.beacon_block_root;
-
-        self.chain
-            .verify_unaggregated_attestation_for_gossip(unaggregated_attestation, subnet_id)
-            .map_err(|e| {
-                self.handle_attestation_verification_failure(
-                    peer_id,
-                    beacon_block_root,
-                    "unaggregated",
-                    e,
+        should_process: bool,
+    ) {
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::unaggregated_attestation(
+                message_id,
+                peer_id,
+                unaggregated_attestation,
+                subnet_id,
+                should_process,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "unaggregated attestation gossip",
+                    "error" => e.to_string(),
                 )
             })
-            .ok()
     }
 
-    pub fn import_unaggregated_attestation(
+    pub fn on_aggregated_attestation_gossip(
         &mut self,
+        message_id: MessageId,
         peer_id: PeerId,
-        verified_attestation: VerifiedUnaggregatedAttestation<T>,
+        aggregate: SignedAggregateAndProof<T::EthSpec>,
     ) {
-        // This is provided to the error handling function to assist with debugging.
-        let beacon_block_root = verified_attestation.attestation().data.beacon_block_root;
-
-        self.apply_attestation_to_fork_choice(
-            peer_id.clone(),
-            beacon_block_root,
-            &verified_attestation,
-        );
-
-        if let Err(e) = self
-            .chain
-            .add_to_naive_aggregation_pool(verified_attestation)
-        {
-            debug!(
-                self.log,
-                "Attestation invalid for agg pool";
-                "reason" => format!("{:?}", e),
-                "peer" => peer_id.to_string(),
-                "beacon_block_root" => format!("{:?}", beacon_block_root)
-            )
-        }
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::aggregated_attestation(
+                message_id, peer_id, aggregate,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "aggregated attestation gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 
-    /// Apply the attestation to fork choice, suppressing errors.
-    ///
-    /// We suppress the errors when adding an attestation to fork choice since the spec
-    /// permits gossiping attestations that are invalid to be applied to fork choice.
-    ///
-    /// An attestation that is invalid for fork choice can still be included in a block.
-    ///
-    /// Reference:
-    /// https://github.com/ethereum/eth2.0-specs/issues/1408#issuecomment-617599260
-    fn apply_attestation_to_fork_choice<'a>(
-        &self,
+    pub fn on_voluntary_exit_gossip(
+        &mut self,
+        message_id: MessageId,
         peer_id: PeerId,
-        beacon_block_root: Hash256,
-        attestation: &'a impl SignatureVerifiedAttestation<T>,
+        voluntary_exit: Box<SignedVoluntaryExit>,
     ) {
-        if let Err(e) = self.chain.apply_attestation_to_fork_choice(attestation) {
-            match e {
-                BeaconChainError::ForkChoiceError(ForkChoiceError::InvalidAttestation(e)) => {
-                    debug!(
-                        self.log,
-                        "Attestation invalid for fork choice";
-                        "reason" => format!("{:?}", e),
-                        "peer" => peer_id.to_string(),
-                        "beacon_block_root" => format!("{:?}", beacon_block_root)
-                    )
-                }
-                e => error!(
-                    self.log,
-                    "Error applying attestation to fork choice";
-                    "reason" => format!("{:?}", e),
-                    "peer" => peer_id.to_string(),
-                    "beacon_block_root" => format!("{:?}", beacon_block_root)
-                ),
-            }
-        }
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_voluntary_exit(
+                message_id,
+                peer_id,
+                voluntary_exit,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "voluntary exit gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 
-    /// Verify a voluntary exit before gossiping or processing it.
-    ///
-    /// Errors are logged at debug level.
-    pub fn verify_voluntary_exit_for_gossip(
-        &self,
-        peer_id: &PeerId,
-        voluntary_exit: SignedVoluntaryExit,
-    ) -> Option<SigVerifiedOp<SignedVoluntaryExit>> {
-        let validator_index = voluntary_exit.message.validator_index;
-
-        match self.chain.verify_voluntary_exit_for_gossip(voluntary_exit) {
-            Ok(ObservationOutcome::New(sig_verified_exit)) => Some(sig_verified_exit),
-            Ok(ObservationOutcome::AlreadyKnown) => {
-                debug!(
-                    self.log,
-                    "Dropping exit for already exiting validator";
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string()
-                );
-                None
-            }
-            Err(e) => {
-                debug!(
-                    self.log,
-                    "Dropping invalid exit";
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string(),
-                    "error" => format!("{:?}", e)
-                );
-                None
-            }
-        }
-    }
-
-    /// Import a verified exit into the op pool.
-    pub fn import_verified_voluntary_exit(
-        &self,
-        verified_voluntary_exit: SigVerifiedOp<SignedVoluntaryExit>,
+    pub fn on_proposer_slashing_gossip(
+        &mut self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
     ) {
-        self.chain.import_voluntary_exit(verified_voluntary_exit);
-        debug!(self.log, "Successfully imported voluntary exit");
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_proposer_slashing(
+                message_id,
+                peer_id,
+                proposer_slashing,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "proposer slashing gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 
-    /// Verify a proposer slashing before gossiping or processing it.
-    ///
-    /// Errors are logged at debug level.
-    pub fn verify_proposer_slashing_for_gossip(
-        &self,
-        peer_id: &PeerId,
-        proposer_slashing: ProposerSlashing,
-    ) -> Option<SigVerifiedOp<ProposerSlashing>> {
-        let validator_index = proposer_slashing.signed_header_1.message.proposer_index;
-
-        match self
-            .chain
-            .verify_proposer_slashing_for_gossip(proposer_slashing)
-        {
-            Ok(ObservationOutcome::New(verified_slashing)) => Some(verified_slashing),
-            Ok(ObservationOutcome::AlreadyKnown) => {
-                debug!(
-                    self.log,
-                    "Dropping proposer slashing";
-                    "reason" => "Already seen a proposer slashing for that validator",
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string()
-                );
-                None
-            }
-            Err(e) => {
-                debug!(
-                    self.log,
-                    "Dropping invalid proposer slashing";
-                    "validator_index" => validator_index,
-                    "peer" => peer_id.to_string(),
-                    "error" => format!("{:?}", e)
-                );
-                None
-            }
-        }
-    }
-
-    /// Import a verified proposer slashing into the op pool.
-    pub fn import_verified_proposer_slashing(
-        &self,
-        proposer_slashing: SigVerifiedOp<ProposerSlashing>,
+    pub fn on_attester_slashing_gossip(
+        &mut self,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<T::EthSpec>>,
     ) {
-        self.chain.import_proposer_slashing(proposer_slashing);
-        debug!(self.log, "Successfully imported proposer slashing");
-    }
-
-    /// Verify an attester slashing before gossiping or processing it.
-    ///
-    /// Errors are logged at debug level.
-    pub fn verify_attester_slashing_for_gossip(
-        &self,
-        peer_id: &PeerId,
-        attester_slashing: AttesterSlashing<T::EthSpec>,
-    ) -> Option<SigVerifiedOp<AttesterSlashing<T::EthSpec>>> {
-        match self
-            .chain
-            .verify_attester_slashing_for_gossip(attester_slashing)
-        {
-            Ok(ObservationOutcome::New(verified_slashing)) => Some(verified_slashing),
-            Ok(ObservationOutcome::AlreadyKnown) => {
-                debug!(
-                    self.log,
-                    "Dropping attester slashing";
-                    "reason" => "Slashings already known for all slashed validators",
-                    "peer" => peer_id.to_string()
-                );
-                None
-            }
-            Err(e) => {
-                debug!(
-                    self.log,
-                    "Dropping invalid attester slashing";
-                    "peer" => peer_id.to_string(),
-                    "error" => format!("{:?}", e)
-                );
-                None
-            }
-        }
-    }
-
-    /// Import a verified attester slashing into the op pool.
-    pub fn import_verified_attester_slashing(
-        &self,
-        attester_slashing: SigVerifiedOp<AttesterSlashing<T::EthSpec>>,
-    ) {
-        if let Err(e) = self.chain.import_attester_slashing(attester_slashing) {
-            debug!(self.log, "Error importing attester slashing"; "error" => format!("{:?}", e));
-        } else {
-            debug!(self.log, "Successfully imported attester slashing");
-        }
+        self.beacon_processor_send
+            .try_send(BeaconWorkEvent::gossip_attester_slashing(
+                message_id,
+                peer_id,
+                attester_slashing,
+            ))
+            .unwrap_or_else(|e| {
+                error!(
+                    &self.log,
+                    "Unable to send to gossip processor";
+                    "type" => "attester slashing gossip",
+                    "error" => e.to_string(),
+                )
+            })
     }
 }
 

beacon_node/network/src/service.rs

@@ -95,7 +95,7 @@ pub struct NetworkService<T: BeaconChainTypes> {
 
 impl<T: BeaconChainTypes> NetworkService<T> {
     #[allow(clippy::type_complexity)]
-    pub fn start(
+    pub async fn start(
         beacon_chain: Arc<BeaconChain<T>>,
         config: &NetworkConfig,
         executor: environment::TaskExecutor,
@@ -117,7 +117,7 @@ impl<T: BeaconChainTypes> NetworkService<T> {
 
         // launch libp2p service
         let (network_globals, mut libp2p) =
-            LibP2PService::new(executor.clone(), config, enr_fork_id, &network_log)?;
+            LibP2PService::new(executor.clone(), config, enr_fork_id, &network_log).await?;
 
         // Repopulate the DHT with stored ENR's.
         let enrs_to_load = load_dht::<T::EthSpec, T::HotStore, T::ColdStore>(store.clone());
@@ -126,7 +126,7 @@ impl<T: BeaconChainTypes> NetworkService<T> {
             "Loading peers into the routing table"; "peers" => enrs_to_load.len()
         );
         for enr in enrs_to_load {
-            libp2p.swarm.add_enr(enr.clone());
+            libp2p.swarm.add_enr(enr.clone()); //TODO change?
         }
 
         // launch derived network services
@@ -145,7 +145,7 @@ impl<T: BeaconChainTypes> NetworkService<T> {
             AttestationService::new(beacon_chain.clone(), network_globals.clone(), &network_log);
 
         // create the network service and spawn the task
-        let network_log = network_log.new(o!("service"=> "network"));
+        let network_log = network_log.new(o!("service" => "network"));
         let network_service = NetworkService {
             beacon_chain,
             libp2p,
@@ -169,6 +169,7 @@ fn spawn_service<T: BeaconChainTypes>(
     mut service: NetworkService<T>,
 ) -> error::Result<()> {
     let mut exit_rx = executor.exit();
+    let mut shutdown_sender = executor.shutdown_sender();
 
     // spawn on the current executor
     executor.spawn_without_exit(async move {
@@ -271,8 +272,8 @@ fn spawn_service<T: BeaconChainTypes>(
                         AttServiceMessage::EnrRemove(subnet_id) => {
                             service.libp2p.swarm.update_enr_subnet(subnet_id, false);
                         }
-                        AttServiceMessage::DiscoverPeers{subnet_id, min_ttl} => {
-                            service.libp2p.swarm.discover_subnet_peers(subnet_id, min_ttl);
+                        AttServiceMessage::DiscoverPeers(subnets_to_discover) => {
+                            service.libp2p.swarm.discover_subnet_peers(subnets_to_discover);
                         }
                     }
                 }
@@ -376,6 +377,12 @@ fn spawn_service<T: BeaconChainTypes>(
                         Libp2pEvent::NewListenAddr(multiaddr) => {
                             service.network_globals.listen_multiaddrs.write().push(multiaddr);
                         }
+                        Libp2pEvent::ZeroListeners => {
+                            let _ = shutdown_sender.send("All listeners are closed. Unable to listen").await.map_err(|e| {
+                                warn!(service.log, "failed to send a shutdown signal"; "error" => e.to_string()
+                                )
+                            });
+                        }
                     }
                 }
             }

beacon_node/network/src/service/tests.rs

@@ -40,17 +40,25 @@ mod tests {
         let runtime = Runtime::new().unwrap();
 
         let (signal, exit) = exit_future::signal();
-        let executor = environment::TaskExecutor::new(runtime.handle().clone(), exit, log.clone());
+        let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+        let executor = environment::TaskExecutor::new(
+            runtime.handle().clone(),
+            exit,
+            log.clone(),
+            shutdown_tx,
+        );
 
         let mut config = NetworkConfig::default();
         config.libp2p_port = 21212;
         config.discovery_port = 21212;
-        config.boot_nodes = enrs.clone();
+        config.boot_nodes_enr = enrs.clone();
         runtime.spawn(async move {
             // Create a new network service which implicitly gets dropped at the
             // end of the block.
 
-            let _ = NetworkService::start(beacon_chain.clone(), &config, executor).unwrap();
+            let _ = NetworkService::start(beacon_chain.clone(), &config, executor)
+                .await
+                .unwrap();
             drop(signal);
         });
         runtime.shutdown_timeout(tokio::time::Duration::from_millis(300));

beacon_node/network/src/sync/block_processor.rs

@@ -1,249 +0,0 @@
-use crate::router::processor::FUTURE_SLOT_TOLERANCE;
-use crate::sync::manager::SyncMessage;
-use crate::sync::range_sync::{BatchId, ChainId};
-use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, ChainSegmentResult};
-use eth2_libp2p::PeerId;
-use slog::{debug, error, trace, warn};
-use std::sync::{Arc, Weak};
-use tokio::sync::mpsc;
-use types::SignedBeaconBlock;
-
-/// Id associated to a block processing request, either a batch or a single block.
-#[derive(Clone, Debug, PartialEq)]
-pub enum ProcessId {
-    /// Processing Id of a range syncing batch.
-    RangeBatchId(ChainId, BatchId),
-    /// Processing Id of the parent lookup of a block
-    ParentLookup(PeerId),
-}
-
-/// The result of a block processing request.
-// TODO: When correct batch error handling occurs, we will include an error type.
-#[derive(Debug)]
-pub enum BatchProcessResult {
-    /// The batch was completed successfully.
-    Success,
-    /// The batch processing failed.
-    Failed,
-    /// The batch processing failed but managed to import at least one block.
-    Partial,
-}
-
-/// Spawns a thread handling the block processing of a request: range syncing or parent lookup.
-pub fn spawn_block_processor<T: BeaconChainTypes>(
-    chain: Weak<BeaconChain<T>>,
-    process_id: ProcessId,
-    downloaded_blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
-    log: slog::Logger,
-) {
-    std::thread::spawn(move || {
-        match process_id {
-            // this a request from the range sync
-            ProcessId::RangeBatchId(chain_id, batch_id) => {
-                let len = downloaded_blocks.len();
-                let start_slot = if len > 0 {
-                    downloaded_blocks[0].message.slot.as_u64()
-                } else {
-                    0
-                };
-                let end_slot = if len > 0 {
-                    downloaded_blocks[len - 1].message.slot.as_u64()
-                } else {
-                    0
-                };
-
-                debug!(log, "Processing batch"; "id" => *batch_id, "blocks" => downloaded_blocks.len(),  "start_slot" => start_slot, "end_slot" => end_slot);
-                let result = match process_blocks(chain, downloaded_blocks.iter(), &log) {
-                    (_, Ok(_)) => {
-                        debug!(log, "Batch processed"; "id" => *batch_id , "start_slot" => start_slot, "end_slot" => end_slot);
-                        BatchProcessResult::Success
-                    }
-                    (imported_blocks, Err(e)) if imported_blocks > 0 => {
-                        debug!(log, "Batch processing failed but imported some blocks";
-                            "id" => *batch_id, "error" => e, "imported_blocks"=> imported_blocks);
-                        BatchProcessResult::Partial
-                    }
-                    (_, Err(e)) => {
-                        debug!(log, "Batch processing failed"; "id" => *batch_id, "error" => e);
-                        BatchProcessResult::Failed
-                    }
-                };
-
-                let msg = SyncMessage::BatchProcessed {
-                    chain_id,
-                    batch_id,
-                    downloaded_blocks,
-                    result,
-                };
-                sync_send.send(msg).unwrap_or_else(|_| {
-                    debug!(
-                        log,
-                        "Block processor could not inform range sync result. Likely shutting down."
-                    );
-                });
-            }
-            // this a parent lookup request from the sync manager
-            ProcessId::ParentLookup(peer_id) => {
-                debug!(
-                    log, "Processing parent lookup";
-                    "last_peer_id" => format!("{}", peer_id),
-                    "blocks" => downloaded_blocks.len()
-                );
-                // parent blocks are ordered from highest slot to lowest, so we need to process in
-                // reverse
-                match process_blocks(chain, downloaded_blocks.iter().rev(), &log) {
-                    (_, Err(e)) => {
-                        warn!(log, "Parent lookup failed"; "last_peer_id" => format!("{}", peer_id), "error" => e);
-                        sync_send
-                        .send(SyncMessage::ParentLookupFailed(peer_id))
-                        .unwrap_or_else(|_| {
-                            // on failure, inform to downvote the peer
-                            debug!(
-                                log,
-                                "Block processor could not inform parent lookup result. Likely shutting down."
-                            );
-                        });
-                    }
-                    (_, Ok(_)) => {
-                        debug!(log, "Parent lookup processed successfully");
-                    }
-                }
-            }
-        }
-    });
-}
-
-/// Helper function to process blocks batches which only consumes the chain and blocks to process.
-fn process_blocks<
-    'a,
-    T: BeaconChainTypes,
-    I: Iterator<Item = &'a SignedBeaconBlock<T::EthSpec>>,
->(
-    chain: Weak<BeaconChain<T>>,
-    downloaded_blocks: I,
-    log: &slog::Logger,
-) -> (usize, Result<(), String>) {
-    if let Some(chain) = chain.upgrade() {
-        let blocks = downloaded_blocks.cloned().collect::<Vec<_>>();
-        let (imported_blocks, r) = match chain.process_chain_segment(blocks) {
-            ChainSegmentResult::Successful { imported_blocks } => {
-                if imported_blocks == 0 {
-                    debug!(log, "All blocks already known");
-                } else {
-                    debug!(
-                        log, "Imported blocks from network";
-                        "count" => imported_blocks,
-                    );
-                    // Batch completed successfully with at least one block, run fork choice.
-                    run_fork_choice(chain, log);
-                }
-
-                (imported_blocks, Ok(()))
-            }
-            ChainSegmentResult::Failed {
-                imported_blocks,
-                error,
-            } => {
-                let r = handle_failed_chain_segment(error, log);
-                if imported_blocks > 0 {
-                    run_fork_choice(chain, log);
-                }
-                (imported_blocks, r)
-            }
-        };
-
-        return (imported_blocks, r);
-    }
-
-    (0, Ok(()))
-}
-
-/// Runs fork-choice on a given chain. This is used during block processing after one successful
-/// block import.
-fn run_fork_choice<T: BeaconChainTypes>(chain: Arc<BeaconChain<T>>, log: &slog::Logger) {
-    match chain.fork_choice() {
-        Ok(()) => trace!(
-            log,
-            "Fork choice success";
-            "location" => "batch processing"
-        ),
-        Err(e) => error!(
-            log,
-            "Fork choice failed";
-            "error" => format!("{:?}", e),
-            "location" => "batch import error"
-        ),
-    }
-}
-
-/// Helper function to handle a `BlockError` from `process_chain_segment`
-fn handle_failed_chain_segment(error: BlockError, log: &slog::Logger) -> Result<(), String> {
-    match error {
-        BlockError::ParentUnknown(parent) => {
-            // blocks should be sequential and all parents should exist
-
-            Err(format!("Block has an unknown parent: {}", parent))
-        }
-        BlockError::BlockIsAlreadyKnown => {
-            // This can happen for many reasons. Head sync's can download multiples and parent
-            // lookups can download blocks before range sync
-            Ok(())
-        }
-        BlockError::FutureSlot {
-            present_slot,
-            block_slot,
-        } => {
-            if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot {
-                // The block is too far in the future, drop it.
-                warn!(
-                    log, "Block is ahead of our slot clock";
-                    "msg" => "block for future slot rejected, check your time",
-                    "present_slot" => present_slot,
-                    "block_slot" => block_slot,
-                    "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE,
-                );
-            } else {
-                // The block is in the future, but not too far.
-                debug!(
-                    log, "Block is slightly ahead of our slot clock, ignoring.";
-                    "present_slot" => present_slot,
-                    "block_slot" => block_slot,
-                    "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE,
-                );
-            }
-
-            Err(format!(
-                "Block with slot {} is higher than the current slot {}",
-                block_slot, present_slot
-            ))
-        }
-        BlockError::WouldRevertFinalizedSlot { .. } => {
-            debug!( log, "Finalized or earlier block processed";);
-
-            Ok(())
-        }
-        BlockError::GenesisBlock => {
-            debug!(log, "Genesis block was processed");
-            Ok(())
-        }
-        BlockError::BeaconChainError(e) => {
-            warn!(
-                log, "BlockProcessingFailure";
-                "msg" => "unexpected condition in processing block.",
-                "outcome" => format!("{:?}", e)
-            );
-
-            Err(format!("Internal error whilst processing block: {:?}", e))
-        }
-        other => {
-            debug!(
-                log, "Invalid block received";
-                "msg" => "peer sent invalid block",
-                "outcome" => format!("{:?}", other),
-            );
-
-            Err(format!("Peer sent invalid block. Reason: {:?}", other))
-        }
-    }
-}

beacon_node/network/src/sync/manager.rs

@@ -33,17 +33,18 @@
 //! if an attestation references an unknown block) this manager can search for the block and
 //! subsequently search for parents if needed.
 
-use super::block_processor::{spawn_block_processor, BatchProcessResult, ProcessId};
 use super::network_context::SyncNetworkContext;
 use super::peer_sync_info::{PeerSyncInfo, PeerSyncType};
-use super::range_sync::{BatchId, ChainId, RangeSync, EPOCHS_PER_BATCH};
+use super::range_sync::{ChainId, RangeSync, EPOCHS_PER_BATCH};
 use super::RequestId;
+use crate::beacon_processor::{ProcessId, WorkEvent as BeaconWorkEvent};
 use crate::service::NetworkMessage;
 use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError};
 use eth2_libp2p::rpc::{methods::MAX_REQUEST_BLOCKS, BlocksByRootRequest, GoodbyeReason};
 use eth2_libp2p::types::NetworkGlobals;
 use eth2_libp2p::{PeerAction, PeerId};
 use fnv::FnvHashMap;
+use lru_cache::LRUCache;
 use slog::{crit, debug, error, info, trace, warn, Logger};
 use smallvec::SmallVec;
 use ssz_types::VariableList;
@@ -51,7 +52,7 @@ use std::boxed::Box;
 use std::ops::Sub;
 use std::sync::Arc;
 use tokio::sync::mpsc;
-use types::{EthSpec, Hash256, SignedBeaconBlock, Slot};
+use types::{Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot};
 
 /// The number of slots ahead of us that is allowed before requesting a long-range (batch)  Sync
 /// from a peer. If a peer is within this tolerance (forwards or backwards), it is treated as a
@@ -100,13 +101,30 @@ pub enum SyncMessage<T: EthSpec> {
     /// A batch has been processed by the block processor thread.
     BatchProcessed {
         chain_id: ChainId,
-        batch_id: BatchId,
+        epoch: Epoch,
         downloaded_blocks: Vec<SignedBeaconBlock<T>>,
         result: BatchProcessResult,
     },
 
-    /// A parent lookup has failed for a block given by this `peer_id`.
-    ParentLookupFailed(PeerId),
+    /// A parent lookup has failed.
+    ParentLookupFailed {
+        /// The head of the chain of blocks that failed to process.
+        chain_head: Hash256,
+        /// The peer that instigated the chain lookup.
+        peer_id: PeerId,
+    },
+}
+
+/// The result of processing a multiple blocks (a chain segment).
+// TODO: When correct batch error handling occurs, we will include an error type.
+#[derive(Debug)]
+pub enum BatchProcessResult {
+    /// The batch was completed successfully.
+    Success,
+    /// The batch processing failed.
+    Failed,
+    /// The batch processing failed but managed to import at least one block.
+    Partial,
 }
 
 /// Maintains a sequential list of parents to lookup and the lookup's current state.
@@ -149,6 +167,9 @@ pub struct SyncManager<T: BeaconChainTypes> {
     /// A collection of parent block lookups.
     parent_queue: SmallVec<[ParentRequests<T::EthSpec>; 3]>,
 
+    /// A cache of failed chain lookups to prevent duplicate searches.
+    failed_chains: LRUCache<Hash256>,
+
     /// A collection of block hashes being searched for and a flag indicating if a result has been
     /// received or not.
     ///
@@ -158,8 +179,8 @@ pub struct SyncManager<T: BeaconChainTypes> {
     /// The logger for the import manager.
     log: Logger,
 
-    /// The sending part of input_channel
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
 }
 
 /// Object representing a single block lookup request.
@@ -187,6 +208,7 @@ pub fn spawn<T: BeaconChainTypes>(
     beacon_chain: Arc<BeaconChain<T>>,
     network_globals: Arc<NetworkGlobals<T::EthSpec>>,
     network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     log: slog::Logger,
 ) -> mpsc::UnboundedSender<SyncMessage<T::EthSpec>> {
     assert!(
@@ -201,7 +223,7 @@ pub fn spawn<T: BeaconChainTypes>(
         range_sync: RangeSync::new(
             beacon_chain.clone(),
             network_globals.clone(),
-            sync_send.clone(),
+            beacon_processor_send.clone(),
             log.clone(),
         ),
         network: SyncNetworkContext::new(network_send, network_globals.clone(), log.clone()),
@@ -209,9 +231,10 @@ pub fn spawn<T: BeaconChainTypes>(
         network_globals,
         input_channel: sync_recv,
         parent_queue: SmallVec::new(),
+        failed_chains: LRUCache::new(500),
         single_block_lookups: FnvHashMap::default(),
         log: log.clone(),
-        sync_send: sync_send.clone(),
+        beacon_processor_send,
     };
 
     // spawn the sync manager thread
@@ -300,7 +323,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
     /// There are two reasons we could have received a BlocksByRoot response
     /// - We requested a single hash and have received a response for the single_block_lookup
     /// - We are looking up parent blocks in parent lookup search
-    fn blocks_by_root_response(
+    async fn blocks_by_root_response(
         &mut self,
         peer_id: PeerId,
         request_id: RequestId,
@@ -318,7 +341,8 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                     single_block_hash = Some(block_request.hash);
                 }
                 if let Some(block_hash) = single_block_hash {
-                    self.single_block_lookup_response(peer_id, block, block_hash);
+                    self.single_block_lookup_response(peer_id, block, block_hash)
+                        .await;
                     return;
                 }
 
@@ -337,10 +361,26 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                         return;
                     }
                 };
+
+                // check if the parent of this block isn't in our failed cache. If it is, this
+                // chain should be dropped and the peer downscored.
+                if self.failed_chains.contains(&block.message.parent_root) {
+                    debug!(self.log, "Parent chain ignored due to past failure"; "block" => format!("{:?}", block.message.parent_root), "slot" => block.message.slot);
+                    if !parent_request.downloaded_blocks.is_empty() {
+                        // Add the root block to failed chains
+                        self.failed_chains
+                            .insert(parent_request.downloaded_blocks[0].canonical_root());
+                    } else {
+                        crit!(self.log, "Parent chain has no blocks");
+                    }
+                    self.network
+                        .report_peer(peer_id, PeerAction::MidToleranceError);
+                    return;
+                }
                 // add the block to response
                 parent_request.downloaded_blocks.push(block);
                 // queue for processing
-                self.process_parent_request(parent_request);
+                self.process_parent_request(parent_request).await;
             }
             None => {
                 // this is a stream termination
@@ -381,10 +421,40 @@ impl<T: BeaconChainTypes> SyncManager<T> {
         }
     }
 
+    async fn process_block_async(
+        &mut self,
+        block: SignedBeaconBlock<T::EthSpec>,
+    ) -> Option<Result<Hash256, BlockError<T::EthSpec>>> {
+        let (event, rx) = BeaconWorkEvent::rpc_beacon_block(Box::new(block));
+        match self.beacon_processor_send.try_send(event) {
+            Ok(_) => {}
+            Err(e) => {
+                error!(
+                    self.log,
+                    "Failed to send sync block to processor";
+                    "error" => format!("{:?}", e)
+                );
+                return None;
+            }
+        }
+
+        match rx.await {
+            Ok(block_result) => Some(block_result),
+            Err(_) => {
+                warn!(
+                    self.log,
+                    "Sync block not processed";
+                    "msg" => "likely due to system resource exhaustion"
+                );
+                None
+            }
+        }
+    }
+
     /// Processes the response obtained from a single block lookup search. If the block is
     /// processed or errors, the search ends. If the blocks parent is unknown, a block parent
     /// lookup search is started.
-    fn single_block_lookup_response(
+    async fn single_block_lookup_response(
         &mut self,
         peer_id: PeerId,
         block: SignedBeaconBlock<T::EthSpec>,
@@ -399,8 +469,13 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             return;
         }
 
+        let block_result = match self.process_block_async(block.clone()).await {
+            Some(block_result) => block_result,
+            None => return,
+        };
+
         // we have the correct block, try and process it
-        match self.chain.process_block(block.clone()) {
+        match block_result {
             Ok(block_root) => {
                 info!(self.log, "Processed block"; "block" => format!("{}", block_root));
 
@@ -461,6 +536,15 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             }
         }
 
+        let block_root = block.canonical_root();
+        // If this block or it's parent is part of a known failed chain, ignore it.
+        if self.failed_chains.contains(&block.message.parent_root)
+            || self.failed_chains.contains(&block_root)
+        {
+            debug!(self.log, "Block is from a past failed chain. Dropping"; "block_root" => format!("{:?}", block_root), "block_slot" => block.message.slot);
+            return;
+        }
+
         // Make sure this block is not already being searched for
         // NOTE: Potentially store a hashset of blocks for O(1) lookups
         for parent_req in self.parent_queue.iter() {
@@ -599,7 +683,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
     // manager
 
     /// A new block has been received for a parent lookup query, process it.
-    fn process_parent_request(&mut self, mut parent_request: ParentRequests<T::EthSpec>) {
+    async fn process_parent_request(&mut self, mut parent_request: ParentRequests<T::EthSpec>) {
         // verify the last added block is the parent of the last requested block
 
         if parent_request.downloaded_blocks.len() < 2 {
@@ -648,11 +732,19 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             // If the last block in the queue has an unknown parent, we continue the parent
             // lookup-search.
 
+            let chain_block_hash = parent_request.downloaded_blocks[0].canonical_root();
+
             let newest_block = parent_request
                 .downloaded_blocks
                 .pop()
                 .expect("There is always at least one block in the queue");
-            match self.chain.process_block(newest_block.clone()) {
+
+            let block_result = match self.process_block_async(newest_block.clone()).await {
+                Some(block_result) => block_result,
+                None => return,
+            };
+
+            match block_result {
                 Err(BlockError::ParentUnknown { .. }) => {
                     // need to keep looking for parents
                     // add the block back to the queue and continue the search
@@ -660,13 +752,25 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                     self.request_parent(parent_request);
                 }
                 Ok(_) | Err(BlockError::BlockIsAlreadyKnown { .. }) => {
-                    spawn_block_processor(
-                        Arc::downgrade(&self.chain),
-                        ProcessId::ParentLookup(parent_request.last_submitted_peer.clone()),
-                        parent_request.downloaded_blocks,
-                        self.sync_send.clone(),
-                        self.log.clone(),
+                    let process_id = ProcessId::ParentLookup(
+                        parent_request.last_submitted_peer.clone(),
+                        chain_block_hash,
                     );
+                    let blocks = parent_request.downloaded_blocks;
+
+                    match self
+                        .beacon_processor_send
+                        .try_send(BeaconWorkEvent::chain_segment(process_id, blocks))
+                    {
+                        Ok(_) => {}
+                        Err(e) => {
+                            error!(
+                                self.log,
+                                "Failed to send chain segment to processor";
+                                "error" => format!("{:?}", e)
+                            );
+                        }
+                    }
                 }
                 Err(outcome) => {
                     // all else we consider the chain a failure and downvote the peer that sent
@@ -677,6 +781,10 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                         "outcome" => format!("{:?}", outcome),
                         "last_peer" => parent_request.last_submitted_peer.to_string(),
                     );
+
+                    // Add this chain to cache of failed chains
+                    self.failed_chains.insert(chain_block_hash);
+
                     // This currently can be a host of errors. We permit this due to the partial
                     // ambiguity.
                     // TODO: Refine the error types and score the peer appropriately.
@@ -699,8 +807,17 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             || parent_request.downloaded_blocks.len() >= PARENT_DEPTH_TOLERANCE
         {
             let error = if parent_request.failed_attempts >= PARENT_FAIL_TOLERANCE {
+                // This is a peer-specific error and the chain could be continued with another
+                // peer. We don't consider this chain a failure and prevent retries with another
+                // peer.
                 "too many failed attempts"
             } else {
+                if !parent_request.downloaded_blocks.is_empty() {
+                    self.failed_chains
+                        .insert(parent_request.downloaded_blocks[0].canonical_root());
+                } else {
+                    crit!(self.log, "Parent lookup has no blocks");
+                }
                 "reached maximum lookup-depth"
             };
 
@@ -709,6 +826,11 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             "ancestors_found" => parent_request.downloaded_blocks.len(),
             "reason" => error
             );
+            // Downscore the peer.
+            self.network.report_peer(
+                parent_request.last_submitted_peer,
+                PeerAction::LowToleranceError,
+            );
             return; // drop the request
         }
 
@@ -760,7 +882,8 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                         request_id,
                         beacon_block,
                     } => {
-                        self.blocks_by_root_response(peer_id, request_id, beacon_block.map(|b| *b));
+                        self.blocks_by_root_response(peer_id, request_id, beacon_block.map(|b| *b))
+                            .await;
                     }
                     SyncMessage::UnknownBlock(peer_id, block) => {
                         self.add_unknown_block(peer_id, *block);
@@ -776,24 +899,25 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                     }
                     SyncMessage::BatchProcessed {
                         chain_id,
-                        batch_id,
+                        epoch,
                         downloaded_blocks,
                         result,
                     } => {
                         self.range_sync.handle_block_process_result(
                             &mut self.network,
                             chain_id,
-                            batch_id,
+                            epoch,
                             downloaded_blocks,
                             result,
                         );
                     }
-                    SyncMessage::ParentLookupFailed(peer_id) => {
+                    SyncMessage::ParentLookupFailed {
+                        chain_head,
+                        peer_id,
+                    } => {
                         // A peer sent an object (block or attestation) that referenced a parent.
-                        // On request for this parent the peer indicated it did not have this
-                        // block.
-                        // This is not fatal. Peer's could prune old blocks so we moderately
-                        // tolerate this behaviour.
+                        // The processing of this chain failed.
+                        self.failed_chains.insert(chain_head);
                         self.network
                             .report_peer(peer_id, PeerAction::MidToleranceError);
                     }

beacon_node/network/src/sync/mod.rs

@@ -1,14 +1,14 @@
 //! Syncing for lighthouse.
 //!
 //! Stores the various syncing methods for the beacon chain.
-mod block_processor;
 pub mod manager;
 mod network_context;
 mod peer_sync_info;
 mod range_sync;
 
-pub use manager::SyncMessage;
+pub use manager::{BatchProcessResult, SyncMessage};
 pub use peer_sync_info::PeerSyncInfo;
+pub use range_sync::ChainId;
 
 /// Type of id of rpc requests sent by sync
 pub type RequestId = usize;

beacon_node/network/src/sync/network_context.rs

@@ -110,7 +110,7 @@ impl<T: EthSpec> SyncNetworkContext<T> {
     }
 
     pub fn report_peer(&mut self, peer_id: PeerId, action: PeerAction) {
-        debug!(self.log, "Sync reporting peer"; "peer_id" => peer_id.to_string(), "action"=> action.to_string());
+        debug!(self.log, "Sync reporting peer"; "peer_id" => peer_id.to_string(), "action" => action.to_string());
         self.network_send
             .send(NetworkMessage::ReportPeer { peer_id, action })
             .unwrap_or_else(|_| {

beacon_node/network/src/sync/peer_sync_info.rs

@@ -86,8 +86,8 @@ impl PeerSyncInfo {
             // that we are within SLOT_IMPORT_TOLERANCE of our two heads
             if (self.head_slot >= remote.head_slot
                 && self.head_slot.sub(remote.head_slot).as_usize() <= SLOT_IMPORT_TOLERANCE)
-                || (self.head_slot < remote.head_slot)
-                    && remote.head_slot.sub(self.head_slot).as_usize() <= SLOT_IMPORT_TOLERANCE
+                || (self.head_slot < remote.head_slot
+                    && remote.head_slot.sub(self.head_slot).as_usize() <= SLOT_IMPORT_TOLERANCE)
             {
                 return true;
             }

beacon_node/network/src/sync/range_sync/batch.rs

@@ -9,37 +9,14 @@ use std::collections::hash_map::Entry;
 use std::collections::{HashMap, HashSet};
 use std::hash::{Hash, Hasher};
 use std::ops::Sub;
-use types::{EthSpec, SignedBeaconBlock, Slot};
-
-#[derive(Copy, Clone, Debug, PartialEq)]
-pub struct BatchId(pub u64);
-
-impl std::ops::Deref for BatchId {
-    type Target = u64;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-impl std::ops::DerefMut for BatchId {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-impl std::convert::From<u64> for BatchId {
-    fn from(id: u64) -> Self {
-        BatchId(id)
-    }
-}
+use types::{Epoch, EthSpec, SignedBeaconBlock, Slot};
 
 /// A collection of sequential blocks that are requested from peers in a single RPC request.
 #[derive(PartialEq, Debug)]
 pub struct Batch<T: EthSpec> {
-    /// The ID of the batch, these are sequential.
-    pub id: BatchId,
-    /// The requested start slot of the batch, inclusive.
-    pub start_slot: Slot,
-    /// The requested end slot of batch, exlcusive.
+    /// The requested start epoch of the batch.
+    pub start_epoch: Epoch,
+    /// The requested end slot of batch, exclusive.
     pub end_slot: Slot,
     /// The `Attempts` that have been made to send us this batch.
     pub attempts: Vec<Attempt>,
@@ -69,10 +46,9 @@ pub struct Attempt {
 impl<T: EthSpec> Eq for Batch<T> {}
 
 impl<T: EthSpec> Batch<T> {
-    pub fn new(id: BatchId, start_slot: Slot, end_slot: Slot, peer_id: PeerId) -> Self {
+    pub fn new(start_epoch: Epoch, end_slot: Slot, peer_id: PeerId) -> Self {
         Batch {
-            id,
-            start_slot,
+            start_epoch,
             end_slot,
             attempts: Vec::new(),
             current_peer: peer_id,
@@ -82,12 +58,21 @@ impl<T: EthSpec> Batch<T> {
         }
     }
 
+    pub fn start_slot(&self) -> Slot {
+        // batches are shifted by 1
+        self.start_epoch.start_slot(T::slots_per_epoch()) + 1
+    }
+
+    pub fn end_slot(&self) -> Slot {
+        self.end_slot
+    }
     pub fn to_blocks_by_range_request(&self) -> BlocksByRangeRequest {
+        let start_slot = self.start_slot();
         BlocksByRangeRequest {
-            start_slot: self.start_slot.into(),
+            start_slot: start_slot.into(),
             count: min(
                 T::slots_per_epoch() * EPOCHS_PER_BATCH,
-                self.end_slot.sub(self.start_slot).into(),
+                self.end_slot.sub(start_slot).into(),
             ),
             step: 1,
         }
@@ -105,7 +90,7 @@ impl<T: EthSpec> Batch<T> {
 
 impl<T: EthSpec> Ord for Batch<T> {
     fn cmp(&self, other: &Self) -> Ordering {
-        self.id.0.cmp(&other.id.0)
+        self.start_epoch.cmp(&other.start_epoch)
     }
 }
 

beacon_node/network/src/sync/range_sync/chain.rs

@@ -1,11 +1,12 @@
-use super::batch::{Batch, BatchId, PendingBatches};
-use crate::sync::block_processor::{spawn_block_processor, BatchProcessResult, ProcessId};
-use crate::sync::network_context::SyncNetworkContext;
-use crate::sync::{RequestId, SyncMessage};
+use super::batch::{Batch, PendingBatches};
+use crate::beacon_processor::ProcessId;
+use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
+use crate::sync::RequestId;
+use crate::sync::{network_context::SyncNetworkContext, BatchProcessResult};
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::{PeerAction, PeerId};
 use rand::prelude::*;
-use slog::{crit, debug, warn};
+use slog::{crit, debug, error, warn};
 use std::collections::HashSet;
 use std::sync::Arc;
 use tokio::sync::mpsc;
@@ -72,11 +73,12 @@ pub struct SyncingChain<T: BeaconChainTypes> {
     /// and thus available to download this chain from.
     pub peer_pool: HashSet<PeerId>,
 
-    /// The next batch_id that needs to be downloaded.
-    to_be_downloaded_id: BatchId,
+    /// Starting epoch of the next batch that needs to be downloaded.
+    to_be_downloaded: Epoch,
 
-    /// The next batch id that needs to be processed.
-    to_be_processed_id: BatchId,
+    /// Starting epoch of the batch that needs to be processed next.
+    /// This is incremented as the chain advances.
+    processing_target: Epoch,
 
     /// The current state of the chain.
     pub state: ChainSyncingState,
@@ -84,14 +86,13 @@ pub struct SyncingChain<T: BeaconChainTypes> {
     /// The current processing batch, if any.
     current_processing_batch: Option<Batch<T::EthSpec>>,
 
-    /// A send channel to the sync manager. This is given to the batch processor thread to report
-    /// back once batch processing has completed.
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
 
     /// A reference to the underlying beacon chain.
     chain: Arc<BeaconChain<T>>,
 
-    /// A reference to the sync logger.
+    /// The chain's log.
     log: slog::Logger,
 }
 
@@ -111,7 +112,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         target_head_slot: Slot,
         target_head_root: Hash256,
         peer_id: PeerId,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
         chain: Arc<BeaconChain<T>>,
         log: slog::Logger,
     ) -> Self {
@@ -127,11 +128,11 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             completed_batches: Vec::new(),
             processed_batches: Vec::new(),
             peer_pool,
-            to_be_downloaded_id: BatchId(1),
-            to_be_processed_id: BatchId(1),
+            to_be_downloaded: start_epoch,
+            processing_target: start_epoch,
             state: ChainSyncingState::Stopped,
             current_processing_batch: None,
-            sync_send,
+            beacon_processor_send,
             chain,
             log,
         }
@@ -139,13 +140,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
 
     /// Returns the latest slot number that has been processed.
     fn current_processed_slot(&self) -> Slot {
-        self.start_epoch
+        // the last slot we processed was included in the previous batch, and corresponds to the
+        // first slot of the current target epoch
+        self.processing_target
             .start_slot(T::EthSpec::slots_per_epoch())
-            .saturating_add(
-                self.to_be_processed_id.saturating_sub(1u64)
-                    * T::EthSpec::slots_per_epoch()
-                    * EPOCHS_PER_BATCH,
-            )
     }
 
     /// A batch of blocks has been received. This function gets run on all chains and should
@@ -182,21 +180,19 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         // An entire batch of blocks has been received. This functions checks to see if it can be processed,
         // remove any batches waiting to be verified and if this chain is syncing, request new
         // blocks for the peer.
-        debug!(self.log, "Completed batch received"; "id"=> *batch.id, "blocks" => &batch.downloaded_blocks.len(), "awaiting_batches" => self.completed_batches.len());
+        debug!(self.log, "Completed batch received"; "epoch" => batch.start_epoch, "blocks" => &batch.downloaded_blocks.len(), "awaiting_batches" => self.completed_batches.len());
 
         // verify the range of received blocks
         // Note that the order of blocks is verified in block processing
         if let Some(last_slot) = batch.downloaded_blocks.last().map(|b| b.slot()) {
             // the batch is non-empty
             let first_slot = batch.downloaded_blocks[0].slot();
-            if batch.start_slot > first_slot || batch.end_slot < last_slot {
+            if batch.start_slot() > first_slot || batch.end_slot() < last_slot {
                 warn!(self.log, "BlocksByRange response returned out of range blocks";
-                          "response_initial_slot" => first_slot,
-                          "requested_initial_slot" => batch.start_slot);
-                // This is a pretty bad error. We don't consider this fatal, but we don't tolerate
-                // this much either.
-                network.report_peer(batch.current_peer, PeerAction::LowToleranceError);
-                self.to_be_processed_id = batch.id; // reset the id back to here, when incrementing, it will check against completed batches
+                    "response_initial_slot" => first_slot,
+                    "requested_initial_slot" => batch.start_slot());
+                // this batch can't be used, so we need to request it again.
+                self.failed_batch(network, batch);
                 return;
             }
         }
@@ -242,7 +238,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
 
         // Check if there is a batch ready to be processed
         if !self.completed_batches.is_empty()
-            && self.completed_batches[0].id == self.to_be_processed_id
+            && self.completed_batches[0].start_epoch == self.processing_target
         {
             let batch = self.completed_batches.remove(0);
 
@@ -255,18 +251,23 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         }
     }
 
-    /// Sends a batch to the batch processor.
+    /// Sends a batch to the beacon processor for async processing in a queue.
     fn process_batch(&mut self, mut batch: Batch<T::EthSpec>) {
-        let downloaded_blocks = std::mem::replace(&mut batch.downloaded_blocks, Vec::new());
-        let process_id = ProcessId::RangeBatchId(self.id, batch.id);
+        let blocks = std::mem::replace(&mut batch.downloaded_blocks, Vec::new());
+        let process_id = ProcessId::RangeBatchId(self.id, batch.start_epoch);
         self.current_processing_batch = Some(batch);
-        spawn_block_processor(
-            Arc::downgrade(&self.chain.clone()),
-            process_id,
-            downloaded_blocks,
-            self.sync_send.clone(),
-            self.log.clone(),
-        );
+
+        if let Err(e) = self
+            .beacon_processor_send
+            .try_send(BeaconWorkEvent::chain_segment(process_id, blocks))
+        {
+            error!(
+                self.log,
+                "Failed to send chain segment to processor";
+                "msg" => "process_batch",
+                "error" => format!("{:?}", e)
+            );
+        }
     }
 
     /// The block processor has completed processing a batch. This function handles the result
@@ -275,7 +276,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         &mut self,
         network: &mut SyncNetworkContext<T::EthSpec>,
         chain_id: ChainId,
-        batch_id: BatchId,
+        batch_start_epoch: Epoch,
         downloaded_blocks: &mut Option<Vec<SignedBeaconBlock<T::EthSpec>>>,
         result: &BatchProcessResult,
     ) -> Option<ProcessingResult> {
@@ -284,14 +285,14 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             return None;
         }
         match &self.current_processing_batch {
-            Some(current_batch) if current_batch.id != batch_id => {
+            Some(current_batch) if current_batch.start_epoch != batch_start_epoch => {
                 debug!(self.log, "Unexpected batch result";
-                    "chain_id" => self.id, "batch_id" => *batch_id, "expected_batch_id" => *current_batch.id);
+                    "batch_epoch" => batch_start_epoch, "expected_batch_epoch" => current_batch.start_epoch);
                 return None;
             }
             None => {
                 debug!(self.log, "Chain was not expecting a batch result";
-                    "chain_id" => self.id, "batch_id" => *batch_id);
+                    "batch_epoch" => batch_start_epoch);
                 return None;
             }
             _ => {
@@ -303,7 +304,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         let downloaded_blocks = downloaded_blocks.take().or_else(|| {
             // if taken by another chain, we are no longer waiting on a result.
             self.current_processing_batch = None;
-            crit!(self.log, "Processed batch taken by another chain"; "chain_id" => self.id);
+            crit!(self.log, "Processed batch taken by another chain");
             None
         })?;
 
@@ -313,16 +314,15 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         batch.downloaded_blocks = downloaded_blocks;
 
         // double check batches are processed in order TODO: Remove for prod
-        if batch.id != self.to_be_processed_id {
+        if batch.start_epoch != self.processing_target {
             crit!(self.log, "Batch processed out of order";
-                "chain_id" => self.id,
-                "processed_batch_id" => *batch.id,
-                "expected_id" => *self.to_be_processed_id);
+                "processed_starting_epoch" => batch.start_epoch,
+                "expected_epoch" => self.processing_target);
         }
 
         let res = match result {
             BatchProcessResult::Success => {
-                *self.to_be_processed_id += 1;
+                self.processing_target += EPOCHS_PER_BATCH;
 
                 // If the processed batch was not empty, we can validate previous invalidated
                 // blocks including the current batch.
@@ -352,7 +352,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             }
             BatchProcessResult::Partial => {
                 warn!(self.log, "Batch processing failed but at least one block was imported";
-                    "chain_id" => self.id, "id" => *batch.id, "peer" => format!("{}", batch.current_peer)
+                    "batch_epoch" => batch.start_epoch, "peer" => batch.current_peer.to_string()
                 );
                 // At least one block was successfully verified and imported, so we can be sure all
                 // previous batches are valid and we only need to download the current failed
@@ -370,7 +370,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                     let action = PeerAction::LowToleranceError;
                     warn!(self.log, "Batch failed to download. Dropping chain scoring peers";
                         "score_adjustment" => action.to_string(),
-                        "chain_id" => self.id, "id"=> *batch.id);
+                        "batch_epoch"=> batch.start_epoch);
                     for peer_id in self.peer_pool.drain() {
                         network.report_peer(peer_id, action);
                     }
@@ -383,7 +383,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             }
             BatchProcessResult::Failed => {
                 debug!(self.log, "Batch processing failed";
-                    "chain_id" => self.id,"id" => *batch.id, "peer" => batch.current_peer.to_string(), "client" => network.client_type(&batch.current_peer).to_string());
+                    "batch_epoch" => batch.start_epoch, "peer" => batch.current_peer.to_string(), "client" => network.client_type(&batch.current_peer).to_string());
                 // The batch processing failed
                 // This could be because this batch is invalid, or a previous invalidated batch
                 // is invalid. We need to find out which and downvote the peer that has sent us
@@ -398,7 +398,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                     let action = PeerAction::LowToleranceError;
                     warn!(self.log, "Batch failed to download. Dropping chain scoring peers";
                         "score_adjustment" => action.to_string(),
-                        "chain_id" => self.id, "id"=> *batch.id);
+                        "batch_epoch" => batch.start_epoch);
                     for peer_id in self.peer_pool.drain() {
                         network.report_peer(peer_id, action);
                     }
@@ -428,11 +428,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
     ) {
         while !self.processed_batches.is_empty() {
             let mut processed_batch = self.processed_batches.remove(0);
-            if *processed_batch.id >= *last_batch.id {
+            if processed_batch.start_epoch >= last_batch.start_epoch {
                 crit!(self.log, "A processed batch had a greater id than the current process id";
-                    "chain_id" => self.id,
-                    "processed_id" => *processed_batch.id,
-                    "current_id" => *last_batch.id);
+                    "processed_start_epoch" => processed_batch.start_epoch,
+                    "current_start_epoch" => last_batch.start_epoch);
             }
 
             // Go through passed attempts and downscore peers that returned invalid batches
@@ -447,11 +446,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                         let action = PeerAction::LowToleranceError;
                         debug!(
                             self.log, "Re-processed batch validated. Scoring original peer";
-                                "chain_id" => self.id,
-                                "batch_id" => *processed_batch.id,
-                                "score_adjustment" => action.to_string(),
-                                "original_peer" => format!("{}",attempt.peer_id),
-                                "new_peer" => format!("{}", processed_batch.current_peer)
+                            "batch_epoch" => processed_batch.start_epoch,
+                            "score_adjustment" => action.to_string(),
+                            "original_peer" => format!("{}",attempt.peer_id),
+                            "new_peer" => format!("{}", processed_batch.current_peer)
                         );
                         network.report_peer(attempt.peer_id, action);
                     } else {
@@ -460,11 +458,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                         let action = PeerAction::MidToleranceError;
                         debug!(
                             self.log, "Re-processed batch validated by the same peer.";
-                                "chain_id" => self.id,
-                                "batch_id" => *processed_batch.id,
-                                "score_adjustment" => action.to_string(),
-                                "original_peer" => format!("{}",attempt.peer_id),
-                                "new_peer" => format!("{}", processed_batch.current_peer)
+                            "batch_epoch" => processed_batch.start_epoch,
+                            "score_adjustment" => action.to_string(),
+                            "original_peer" => format!("{}",attempt.peer_id),
+                            "new_peer" => format!("{}", processed_batch.current_peer)
                         );
                         network.report_peer(attempt.peer_id, action);
                     }
@@ -503,7 +500,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         // Find any pre-processed batches awaiting validation
         while !self.processed_batches.is_empty() {
             let past_batch = self.processed_batches.remove(0);
-            *self.to_be_processed_id = std::cmp::min(*self.to_be_processed_id, *past_batch.id);
+            self.processing_target = std::cmp::min(self.processing_target, past_batch.start_epoch);
             self.reprocess_batch(network, past_batch);
         }
 
@@ -547,11 +544,10 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         batch.current_peer = new_peer.clone();
 
         debug!(self.log, "Re-requesting batch";
-            "chain_id" => self.id,
-            "start_slot" => batch.start_slot,
+            "start_slot" => batch.start_slot(),
             "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
-            "id" => *batch.id,
-            "peer" => format!("{}", batch.current_peer),
+            "batch_epoch" => batch.start_epoch,
+            "peer" => batch.current_peer.to_string(),
             "retries" => batch.retries,
             "re-processes" =>  batch.reprocess_retries);
         self.send_batch(network, batch);
@@ -587,12 +583,11 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             self.start_epoch = local_finalized_epoch;
 
             debug!(self.log, "Updating chain's progress";
-                "chain_id" => self.id,
                 "prev_completed_slot" => current_processed_slot,
                 "new_completed_slot" => self.current_processed_slot());
             // Re-index batches
-            *self.to_be_downloaded_id = 1;
-            *self.to_be_processed_id = 1;
+            self.to_be_downloaded = local_finalized_epoch;
+            self.processing_target = local_finalized_epoch;
 
             // remove any completed or processed batches
             self.completed_batches.clear();
@@ -616,7 +611,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         // do not request blocks if the chain is not syncing
         if let ChainSyncingState::Stopped = self.state {
             debug!(self.log, "Peer added to a non-syncing chain";
-                "chain_id" => self.id, "peer_id" => format!("{}", peer_id));
+                "peer_id" => format!("{}", peer_id));
             return;
         }
 
@@ -645,8 +640,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
     ) -> Option<ProcessingResult> {
         if let Some(batch) = self.pending_batches.remove(request_id) {
             debug!(self.log, "Batch failed. RPC Error";
-                "chain_id" => self.id,
-                "id" => *batch.id,
+                "batch_epoch" => batch.start_epoch,
                 "retries" => batch.retries,
                 "peer" => format!("{:?}", peer_id));
 
@@ -683,17 +677,23 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
 
             batch.current_peer = new_peer.clone();
             debug!(self.log, "Re-Requesting batch";
-                "chain_id" => self.id,
-                "start_slot" => batch.start_slot,
+                "start_slot" => batch.start_slot(),
                 "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
-
-                "id" => *batch.id,
-                "peer" => format!("{:?}", batch.current_peer));
+                "batch_epoch" => batch.start_epoch,
+                "peer" => batch.current_peer.to_string());
             self.send_batch(network, batch);
             ProcessingResult::KeepChain
         }
     }
 
+    /// Returns true if this chain is currently syncing.
+    pub fn is_syncing(&self) -> bool {
+        match self.state {
+            ChainSyncingState::Syncing => true,
+            ChainSyncingState::Stopped => false,
+        }
+    }
+
     /// Attempts to request the next required batches from the peer pool if the chain is syncing. It will exhaust the peer
     /// pool and left over batches until the batch buffer is reached or all peers are exhausted.
     fn request_batches(&mut self, network: &mut SyncNetworkContext<T::EthSpec>) {
@@ -709,10 +709,9 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         if let Some(peer_id) = self.get_next_peer() {
             if let Some(batch) = self.get_next_batch(peer_id) {
                 debug!(self.log, "Requesting batch";
-                    "chain_id" => self.id,
-                    "start_slot" => batch.start_slot,
-                    "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks 
-                    "id" => *batch.id,
+                    "start_slot" => batch.start_slot(),
+                    "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
+                    "batch_epoch" => batch.start_epoch,
                     "peer" => format!("{}", batch.current_peer));
                 // send the batch
                 self.send_batch(network, batch);
@@ -765,22 +764,15 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             return None;
         }
 
-        // One is added to the start slot to begin one slot after the epoch boundary
-        let batch_start_slot = self
-            .start_epoch
-            .start_slot(slots_per_epoch)
-            .saturating_add(1u64)
-            + self.to_be_downloaded_id.saturating_sub(1) * blocks_per_batch;
-
         // don't request batches beyond the target head slot
-        if batch_start_slot > self.target_head_slot {
+        if self.to_be_downloaded.start_slot(slots_per_epoch) > self.target_head_slot {
             return None;
         }
 
         // truncate the batch to the epoch containing the target head of the chain
         let batch_end_slot = std::cmp::min(
             // request either a batch containing the max number of blocks per batch
-            batch_start_slot + blocks_per_batch,
+            self.to_be_downloaded.start_slot(slots_per_epoch) + blocks_per_batch + 1,
             // or a batch of one epoch of blocks, which contains the `target_head_slot`
             self.target_head_slot
                 .saturating_add(slots_per_epoch)
@@ -788,28 +780,9 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                 .start_slot(slots_per_epoch),
         );
 
-        let batch_id = self.to_be_downloaded_id;
-
-        // Find the next batch id. The largest of the next sequential id, or the next uncompleted
-        // id
-        let max_completed_id = self
-            .completed_batches
-            .iter()
-            .last()
-            .map(|x| x.id.0)
-            .unwrap_or_else(|| 0);
-        // TODO: Check if this is necessary
-        self.to_be_downloaded_id = BatchId(std::cmp::max(
-            self.to_be_downloaded_id.0 + 1,
-            max_completed_id + 1,
-        ));
-
-        Some(Batch::new(
-            batch_id,
-            batch_start_slot,
-            batch_end_slot,
-            peer_id,
-        ))
+        let batch = Some(Batch::new(self.to_be_downloaded, batch_end_slot, peer_id));
+        self.to_be_downloaded += EPOCHS_PER_BATCH;
+        batch
     }
 
     /// Requests the provided batch from the provided peer.
@@ -827,14 +800,13 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             }
             Err(e) => {
                 warn!(self.log, "Batch request failed";
-                "chain_id" => self.id,
-                "start_slot" => batch.start_slot,
-                "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
-                "id" => *batch.id,
-                "peer" => format!("{}", batch.current_peer),
-                "retries" => batch.retries,
-                "error" => e,
-                "re-processes" =>  batch.reprocess_retries);
+                    "start_slot" => batch.start_slot(),
+                    "end_slot" => batch.end_slot -1, // The -1 shows inclusive blocks
+                    "start_epoch" => batch.start_epoch,
+                    "peer" => batch.current_peer.to_string(),
+                    "retries" => batch.retries,
+                    "error" => e,
+                    "re-processes" =>  batch.reprocess_retries);
                 self.failed_batch(network, batch);
             }
         }

beacon_node/network/src/sync/range_sync/chain_collection.rs

@@ -4,17 +4,20 @@
 //! with this struct to to simplify the logic of the other layers of sync.
 
 use super::chain::{ChainSyncingState, SyncingChain};
-use crate::sync::manager::SyncMessage;
+use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
 use crate::sync::network_context::SyncNetworkContext;
 use crate::sync::PeerSyncInfo;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::{types::SyncState, NetworkGlobals, PeerId};
-use slog::{debug, error, info};
+use slog::{debug, error, info, o};
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use types::EthSpec;
 use types::{Epoch, Hash256, Slot};
 
+/// The number of head syncing chains to sync at a time.
+const PARALLEL_HEAD_CHAINS: usize = 2;
+
 /// The state of the long range/batch sync.
 #[derive(Clone)]
 pub enum RangeSyncState {
@@ -205,8 +208,9 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
     /// Updates the state of the chain collection.
     ///
     /// This removes any out-dated chains, swaps to any higher priority finalized chains and
-    /// updates the state of the collection.
-    pub fn update_finalized(&mut self, network: &mut SyncNetworkContext<T::EthSpec>) {
+    /// updates the state of the collection. This starts head chains syncing if any are required to
+    /// do so.
+    pub fn update(&mut self, network: &mut SyncNetworkContext<T::EthSpec>) {
         let local_epoch = {
             let local = match PeerSyncInfo::from_chain(&self.beacon_chain) {
                 Some(local) => local,
@@ -222,9 +226,25 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
             local.finalized_epoch
         };
 
-        // Remove any outdated finalized chains
+        // Remove any outdated finalized/head chains
         self.purge_outdated_chains(network);
 
+        // Choose the best finalized chain if one needs to be selected.
+        self.update_finalized_chains(network, local_epoch);
+
+        if self.finalized_syncing_index().is_none() {
+            // Handle head syncing chains if there are no finalized chains left.
+            self.update_head_chains(network, local_epoch);
+        }
+    }
+
+    /// This looks at all current finalized chains and decides if a new chain should be prioritised
+    /// or not.
+    fn update_finalized_chains(
+        &mut self,
+        network: &mut SyncNetworkContext<T::EthSpec>,
+        local_epoch: Epoch,
+    ) {
         // Check if any chains become the new syncing chain
         if let Some(index) = self.finalized_syncing_index() {
             // There is a current finalized chain syncing
@@ -269,32 +289,92 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
                 head_root: chain.target_head_root,
             };
             self.state = state;
-        } else {
-            // There are no finalized chains, update the state.
-            if self.head_chains.is_empty() {
-                self.state = RangeSyncState::Idle;
-            } else {
-                // for the syncing API, we find the minimal start_slot and the maximum
-                // target_slot of all head chains to report back.
-
-                let (min_epoch, max_slot) = self.head_chains.iter().fold(
-                    (Epoch::from(0u64), Slot::from(0u64)),
-                    |(min, max), chain| {
-                        (
-                            std::cmp::min(min, chain.start_epoch),
-                            std::cmp::max(max, chain.target_head_slot),
-                        )
-                    },
-                );
-                let head_state = RangeSyncState::Head {
-                    start_slot: min_epoch.start_slot(T::EthSpec::slots_per_epoch()),
-                    head_slot: max_slot,
-                };
-                self.state = head_state;
-            }
         }
     }
 
+    /// Start syncing any head chains if required.
+    fn update_head_chains(
+        &mut self,
+        network: &mut SyncNetworkContext<T::EthSpec>,
+        local_epoch: Epoch,
+    ) {
+        // There are no finalized chains, update the state.
+        if self.head_chains.is_empty() {
+            self.state = RangeSyncState::Idle;
+            return;
+        }
+
+        let mut currently_syncing = self
+            .head_chains
+            .iter()
+            .filter(|chain| chain.is_syncing())
+            .count();
+        let mut not_syncing = self.head_chains.len() - currently_syncing;
+
+        // Find all head chains that are not currently syncing ordered by peer count.
+        while currently_syncing <= PARALLEL_HEAD_CHAINS && not_syncing > 0 {
+            // Find the chain with the most peers and start syncing
+            if let Some((_index, chain)) = self
+                .head_chains
+                .iter_mut()
+                .filter(|chain| !chain.is_syncing())
+                .enumerate()
+                .max_by_key(|(_index, chain)| chain.peer_pool.len())
+            {
+                // start syncing this chain
+                debug!(self.log, "New head chain started syncing"; "new_target_root" => format!("{}", chain.target_head_root), "new_end_slot" => chain.target_head_slot, "new_start_epoch"=> chain.start_epoch);
+                chain.start_syncing(network, local_epoch);
+            }
+
+            // update variables
+            currently_syncing = self
+                .head_chains
+                .iter()
+                .filter(|chain| chain.is_syncing())
+                .count();
+            not_syncing = self.head_chains.len() - currently_syncing;
+        }
+
+        // Start
+        // for the syncing API, we find the minimal start_slot and the maximum
+        // target_slot of all head chains to report back.
+
+        let (min_epoch, max_slot) = self
+            .head_chains
+            .iter()
+            .filter(|chain| chain.is_syncing())
+            .fold(
+                (Epoch::from(0u64), Slot::from(0u64)),
+                |(min, max), chain| {
+                    (
+                        std::cmp::min(min, chain.start_epoch),
+                        std::cmp::max(max, chain.target_head_slot),
+                    )
+                },
+            );
+        let head_state = RangeSyncState::Head {
+            start_slot: min_epoch.start_slot(T::EthSpec::slots_per_epoch()),
+            head_slot: max_slot,
+        };
+        self.state = head_state;
+    }
+
+    /// This is called once a head chain has completed syncing. It removes all non-syncing head
+    /// chains and re-status their peers.
+    pub fn clear_head_chains(&mut self, network: &mut SyncNetworkContext<T::EthSpec>) {
+        let log_ref = &self.log;
+        self.head_chains.retain(|chain| {
+            if !chain.is_syncing()
+                {
+                debug!(log_ref, "Removing old head chain"; "start_epoch" => chain.start_epoch, "end_slot" => chain.target_head_slot);
+                chain.status_peers(network);
+                false
+            } else {
+                true
+            }
+        });
+    }
+
     /// Add a new finalized chain to the collection.
     pub fn new_finalized_chain(
         &mut self,
@@ -302,7 +382,7 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         target_head: Hash256,
         target_slot: Slot,
         peer_id: PeerId,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     ) {
         let chain_id = rand::random();
         self.finalized_chains.push(SyncingChain::new(
@@ -311,9 +391,9 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
             target_slot,
             target_head,
             peer_id,
-            sync_send,
+            beacon_processor_send,
             self.beacon_chain.clone(),
-            self.log.clone(),
+            self.log.new(o!("chain" => chain_id)),
         ));
     }
 
@@ -321,12 +401,11 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
     #[allow(clippy::too_many_arguments)]
     pub fn new_head_chain(
         &mut self,
-        network: &mut SyncNetworkContext<T::EthSpec>,
         remote_finalized_epoch: Epoch,
         target_head: Hash256,
         target_slot: Slot,
         peer_id: PeerId,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     ) {
         // remove the peer from any other head chains
 
@@ -336,18 +415,16 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         self.head_chains.retain(|chain| !chain.peer_pool.is_empty());
 
         let chain_id = rand::random();
-        let mut new_head_chain = SyncingChain::new(
+        let new_head_chain = SyncingChain::new(
             chain_id,
             remote_finalized_epoch,
             target_slot,
             target_head,
             peer_id,
-            sync_send,
+            beacon_processor_send,
             self.beacon_chain.clone(),
             self.log.clone(),
         );
-        // All head chains can sync simultaneously
-        new_head_chain.start_syncing(network, remote_finalized_epoch);
         self.head_chains.push(new_head_chain);
     }
 
@@ -511,7 +588,7 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         debug!(self.log, "Chain was removed"; "start_epoch" => chain.start_epoch, "end_slot" => chain.target_head_slot);
 
         // update the state
-        self.update_finalized(network);
+        self.update(network);
     }
 
     /// Returns the index of finalized chain that is currently syncing. Returns `None` if no

beacon_node/network/src/sync/range_sync/mod.rs

@@ -8,6 +8,5 @@ mod range;
 mod sync_type;
 
 pub use batch::Batch;
-pub use batch::BatchId;
 pub use chain::{ChainId, EPOCHS_PER_BATCH};
 pub use range::RangeSync;

beacon_node/network/src/sync/range_sync/range.rs

@@ -42,10 +42,9 @@
 use super::chain::{ChainId, ProcessingResult};
 use super::chain_collection::{ChainCollection, RangeSyncState};
 use super::sync_type::RangeSyncType;
-use super::BatchId;
-use crate::sync::block_processor::BatchProcessResult;
-use crate::sync::manager::SyncMessage;
+use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
 use crate::sync::network_context::SyncNetworkContext;
+use crate::sync::BatchProcessResult;
 use crate::sync::PeerSyncInfo;
 use crate::sync::RequestId;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
@@ -54,7 +53,7 @@ use slog::{debug, error, trace};
 use std::collections::HashSet;
 use std::sync::Arc;
 use tokio::sync::mpsc;
-use types::{EthSpec, SignedBeaconBlock};
+use types::{Epoch, EthSpec, SignedBeaconBlock};
 
 /// The primary object dealing with long range/batch syncing. This contains all the active and
 /// non-active chains that need to be processed before the syncing is considered complete. This
@@ -69,9 +68,8 @@ pub struct RangeSync<T: BeaconChainTypes> {
     /// finalized chain(s) complete, these peer's get STATUS'ed to update their head slot before
     /// the head chains are formed and downloaded.
     awaiting_head_peers: HashSet<PeerId>,
-    /// The sync manager channel, allowing the batch processor thread to callback the sync task
-    /// once complete.
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     /// The syncing logger.
     log: slog::Logger,
 }
@@ -80,14 +78,14 @@ impl<T: BeaconChainTypes> RangeSync<T> {
     pub fn new(
         beacon_chain: Arc<BeaconChain<T>>,
         network_globals: Arc<NetworkGlobals<T::EthSpec>>,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
         log: slog::Logger,
     ) -> Self {
         RangeSync {
             beacon_chain: beacon_chain.clone(),
             chains: ChainCollection::new(beacon_chain, network_globals, log.clone()),
             awaiting_head_peers: HashSet::new(),
-            sync_send,
+            beacon_processor_send,
             log,
         }
     }
@@ -162,13 +160,13 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                     .chains
                     .get_finalized_mut(remote_info.finalized_root, remote_finalized_slot)
                 {
-                    debug!(self.log, "Finalized chain exists, adding peer"; "peer_id" => format!("{:?}", peer_id), "target_root" => format!("{}", chain.target_head_root), "end_slot" => chain.target_head_slot, "start_epoch"=> chain.start_epoch);
+                    debug!(self.log, "Finalized chain exists, adding peer"; "peer_id" => peer_id.to_string(), "target_root" => chain.target_head_root.to_string(), "targe_slot" => chain.target_head_slot);
 
                     // add the peer to the chain's peer pool
                     chain.add_peer(network, peer_id);
 
                     // check if the new peer's addition will favour a new syncing chain.
-                    self.chains.update_finalized(network);
+                    self.chains.update(network);
                     // update the global sync state if necessary
                     self.chains.update_sync_state();
                 } else {
@@ -181,9 +179,9 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                         remote_info.finalized_root,
                         remote_finalized_slot,
                         peer_id,
-                        self.sync_send.clone(),
+                        self.beacon_processor_send.clone(),
                     );
-                    self.chains.update_finalized(network);
+                    self.chains.update(network);
                     // update the global sync state
                     self.chains.update_sync_state();
                 }
@@ -223,15 +221,14 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                     debug!(self.log, "Creating a new syncing head chain"; "head_root" => format!("{}",remote_info.head_root), "start_epoch" => start_epoch, "head_slot" => remote_info.head_slot, "peer_id" => format!("{:?}", peer_id));
 
                     self.chains.new_head_chain(
-                        network,
                         start_epoch,
                         remote_info.head_root,
                         remote_info.head_slot,
                         peer_id,
-                        self.sync_send.clone(),
+                        self.beacon_processor_send.clone(),
                     );
                 }
-                self.chains.update_finalized(network);
+                self.chains.update(network);
                 self.chains.update_sync_state();
             }
         }
@@ -272,7 +269,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         &mut self,
         network: &mut SyncNetworkContext<T::EthSpec>,
         chain_id: ChainId,
-        batch_id: BatchId,
+        epoch: Epoch,
         downloaded_blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
         result: BatchProcessResult,
     ) {
@@ -280,19 +277,13 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         let mut downloaded_blocks = Some(downloaded_blocks);
 
         match self.chains.finalized_request(|chain| {
-            chain.on_batch_process_result(
-                network,
-                chain_id,
-                batch_id,
-                &mut downloaded_blocks,
-                &result,
-            )
+            chain.on_batch_process_result(network, chain_id, epoch, &mut downloaded_blocks, &result)
         }) {
             Some((index, ProcessingResult::RemoveChain)) => {
                 let chain = self.chains.remove_finalized_chain(index);
                 debug!(self.log, "Finalized chain removed"; "start_epoch" => chain.start_epoch, "end_slot" => chain.target_head_slot);
                 // update the state of the collection
-                self.chains.update_finalized(network);
+                self.chains.update(network);
 
                 // the chain is complete, re-status it's peers
                 chain.status_peers(network);
@@ -320,7 +311,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                     chain.on_batch_process_result(
                         network,
                         chain_id,
-                        batch_id,
+                        epoch,
                         &mut downloaded_blocks,
                         &result,
                     )
@@ -331,8 +322,12 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                         // the chain is complete, re-status it's peers and remove it
                         chain.status_peers(network);
 
+                        // Remove non-syncing head chains and re-status the peers
+                        // This removes a build-up of potentially duplicate head chains. Any
+                        // legitimate head chains will be re-established
+                        self.chains.clear_head_chains(network);
                         // update the state of the collection
-                        self.chains.update_finalized(network);
+                        self.chains.update(network);
                         // update the global state and log any change
                         self.chains.update_sync_state();
                     }
@@ -340,7 +335,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                     None => {
                         // This can happen if a chain gets purged due to being out of date whilst a
                         // batch process is in progress.
-                        debug!(self.log, "No chains match the block processing id"; "id" => *batch_id);
+                        debug!(self.log, "No chains match the block processing id"; "batch_epoch" => epoch, "chain_id" => chain_id);
                     }
                 }
             }
@@ -361,7 +356,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         self.remove_peer(network, peer_id);
 
         // update the state of the collection
-        self.chains.update_finalized(network);
+        self.chains.update(network);
         // update the global state and inform the user
         self.chains.update_sync_state();
     }

beacon_node/operation_pool/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "operation_pool"
-version = "0.1.0"
+version = "0.2.0"
 authors = ["Michael Sproul <michael@sigmaprime.io>"]
 edition = "2018"
 

beacon_node/rest_api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rest_api"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com>", "Luke Anderson <luke@sigmaprime.io>"]
 edition = "2018"
 
@@ -34,7 +34,6 @@ hex = "0.4.2"
 parking_lot = "0.11.0"
 futures = "0.3.5"
 operation_pool = { path = "../operation_pool" }
-rayon = "1.3.0"
 environment = { path = "../../lighthouse/environment" }
 uhttp_sse = "0.5.1"
 bus = "2.2.3"

beacon_node/rest_api/src/advanced.rs

@@ -1,34 +0,0 @@
-use crate::response_builder::ResponseBuilder;
-use crate::ApiResult;
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use hyper::{Body, Request};
-use operation_pool::PersistedOperationPool;
-use std::sync::Arc;
-
-/// Returns the `proto_array` fork choice struct, encoded as JSON.
-///
-/// Useful for debugging or advanced inspection of the chain.
-pub fn get_fork_choice<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(
-        &*beacon_chain
-            .fork_choice
-            .read()
-            .proto_array()
-            .core_proto_array(),
-    )
-}
-
-/// Returns the `PersistedOperationPool` struct.
-///
-/// Useful for debugging or advanced inspection of the stored operations.
-pub fn get_operation_pool<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&PersistedOperationPool::from_operation_pool(
-        &beacon_chain.op_pool,
-    ))
-}

beacon_node/rest_api/src/beacon.rs

@@ -1,14 +1,13 @@
 use crate::helpers::*;
-use crate::response_builder::ResponseBuilder;
 use crate::validator::get_state_for_epoch;
-use crate::{ApiError, ApiResult, UrlQuery};
+use crate::Context;
+use crate::{ApiError, UrlQuery};
 use beacon_chain::{
     observed_operations::ObservationOutcome, BeaconChain, BeaconChainTypes, StateSkipConfig,
 };
-use bus::BusReader;
 use futures::executor::block_on;
 use hyper::body::Bytes;
-use hyper::{Body, Request, Response};
+use hyper::{Body, Request};
 use rest_types::{
     BlockResponse, CanonicalHeadResponse, Committee, HeadBeaconBlock, StateResponse,
     ValidatorRequest, ValidatorResponse,
@@ -16,20 +15,20 @@ use rest_types::{
 use std::io::Write;
 use std::sync::Arc;
 
-use slog::{error, Logger};
+use slog::error;
 use types::{
     AttesterSlashing, BeaconState, EthSpec, Hash256, ProposerSlashing, PublicKeyBytes,
     RelativeEpoch, SignedBeaconBlockHash, Slot,
 };
 
-/// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`.
+/// Returns a summary of the head of the beacon chain.
 pub fn get_head<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    ctx: Arc<Context<T>>,
+) -> Result<CanonicalHeadResponse, ApiError> {
+    let beacon_chain = &ctx.beacon_chain;
     let chain_head = beacon_chain.head()?;
 
-    let head = CanonicalHeadResponse {
+    Ok(CanonicalHeadResponse {
         slot: chain_head.beacon_state.slot,
         block_root: chain_head.beacon_block_root,
         state_root: chain_head.beacon_state_root,
@@ -51,33 +50,27 @@ pub fn get_head<T: BeaconChainTypes>(
             .epoch
             .start_slot(T::EthSpec::slots_per_epoch()),
         previous_justified_block_root: chain_head.beacon_state.previous_justified_checkpoint.root,
-    };
-
-    ResponseBuilder::new(&req)?.body(&head)
+    })
 }
 
-/// HTTP handler to return a list of head BeaconBlocks.
-pub fn get_heads<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let heads = beacon_chain
+/// Return the list of heads of the beacon chain.
+pub fn get_heads<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Vec<HeadBeaconBlock> {
+    ctx.beacon_chain
         .heads()
         .into_iter()
         .map(|(beacon_block_root, beacon_block_slot)| HeadBeaconBlock {
             beacon_block_root,
             beacon_block_slot,
         })
-        .collect::<Vec<_>>();
-
-    ResponseBuilder::new(&req)?.body(&heads)
+        .collect()
 }
 
 /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`.
 pub fn get_block<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<BlockResponse<T::EthSpec>, ApiError> {
+    let beacon_chain = &ctx.beacon_chain;
     let query_params = ["root", "slot"];
     let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?;
 
@@ -85,7 +78,7 @@ pub fn get_block<T: BeaconChainTypes>(
         ("slot", value) => {
             let target = parse_slot(&value)?;
 
-            block_root_at_slot(&beacon_chain, target)?.ok_or_else(|| {
+            block_root_at_slot(beacon_chain, target)?.ok_or_else(|| {
                 ApiError::NotFound(format!(
                     "Unable to find SignedBeaconBlock for slot {:?}",
                     target
@@ -103,30 +96,26 @@ pub fn get_block<T: BeaconChainTypes>(
         ))
     })?;
 
-    let response = BlockResponse {
+    Ok(BlockResponse {
         root: block_root,
         beacon_block: block,
-    };
-
-    ResponseBuilder::new(&req)?.body(&response)
+    })
 }
 
 /// HTTP handler to return a `SignedBeaconBlock` root at a given `slot`.
 pub fn get_block_root<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Hash256, ApiError> {
     let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?;
     let target = parse_slot(&slot_string)?;
 
-    let root = block_root_at_slot(&beacon_chain, target)?.ok_or_else(|| {
+    block_root_at_slot(&ctx.beacon_chain, target)?.ok_or_else(|| {
         ApiError::NotFound(format!(
             "Unable to find SignedBeaconBlock for slot {:?}",
             target
         ))
-    })?;
-
-    ResponseBuilder::new(&req)?.body(&root)
+    })
 }
 
 fn make_sse_response_chunk(new_head_hash: SignedBeaconBlockHash) -> std::io::Result<Bytes> {
@@ -140,43 +129,27 @@ fn make_sse_response_chunk(new_head_hash: SignedBeaconBlockHash) -> std::io::Res
     Ok(bytes)
 }
 
-pub fn stream_forks<T: BeaconChainTypes>(
-    log: Logger,
-    mut events: BusReader<SignedBeaconBlockHash>,
-) -> ApiResult {
+pub fn stream_forks<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Result<Body, ApiError> {
+    let mut events = ctx.events.lock().add_rx();
     let (mut sender, body) = Body::channel();
     std::thread::spawn(move || {
         while let Ok(new_head_hash) = events.recv() {
             let chunk = match make_sse_response_chunk(new_head_hash) {
                 Ok(chunk) => chunk,
                 Err(e) => {
-                    error!(log, "Failed to make SSE chunk"; "error" => e.to_string());
+                    error!(ctx.log, "Failed to make SSE chunk"; "error" => e.to_string());
                     sender.abort();
                     break;
                 }
             };
-            if let Err(bytes) = block_on(sender.send_data(chunk)) {
-                error!(log, "Couldn't stream piece {:?}", bytes);
+            match block_on(sender.send_data(chunk)) {
+                Err(e) if e.is_closed() => break,
+                Err(e) => error!(ctx.log, "Couldn't stream piece {:?}", e),
+                Ok(_) => (),
             }
         }
     });
-    let response = Response::builder()
-        .status(200)
-        .header("Content-Type", "text/event-stream")
-        .header("Connection", "Keep-Alive")
-        .header("Cache-Control", "no-cache")
-        .header("Access-Control-Allow-Origin", "*")
-        .body(body)
-        .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))?;
-    Ok(response)
-}
-
-/// HTTP handler to return the `Fork` of the current head.
-pub fn get_fork<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&beacon_chain.head()?.beacon_state.fork)
+    Ok(body)
 }
 
 /// HTTP handler to which accepts a query string of a list of validator pubkeys and maps it to a
@@ -185,9 +158,9 @@ pub fn get_fork<T: BeaconChainTypes>(
 /// This method is limited to as many `pubkeys` that can fit in a URL. See `post_validators` for
 /// doing bulk requests.
 pub fn get_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let validator_pubkeys = query
@@ -202,17 +175,14 @@ pub fn get_validators<T: BeaconChainTypes>(
         None
     };
 
-    let validators =
-        validator_responses_by_pubkey(beacon_chain, state_root_opt, validator_pubkeys)?;
-
-    ResponseBuilder::new(&req)?.body(&validators)
+    validator_responses_by_pubkey(&ctx.beacon_chain, state_root_opt, validator_pubkeys)
 }
 
 /// HTTP handler to return all validators, each as a `ValidatorResponse`.
 pub fn get_all_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let state_root_opt = if let Some((_key, value)) = query.first_of_opt(&["state_root"]) {
@@ -221,23 +191,21 @@ pub fn get_all_validators<T: BeaconChainTypes>(
         None
     };
 
-    let mut state = get_state_from_root_opt(&beacon_chain, state_root_opt)?;
+    let mut state = get_state_from_root_opt(&ctx.beacon_chain, state_root_opt)?;
     state.update_pubkey_cache()?;
 
-    let validators = state
+    state
         .validators
         .iter()
         .map(|validator| validator_response_by_pubkey(&state, validator.pubkey.clone()))
-        .collect::<Result<Vec<_>, _>>()?;
-
-    ResponseBuilder::new(&req)?.body(&validators)
+        .collect::<Result<Vec<_>, _>>()
 }
 
 /// HTTP handler to return all active validators, each as a `ValidatorResponse`.
 pub fn get_active_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let state_root_opt = if let Some((_key, value)) = query.first_of_opt(&["state_root"]) {
@@ -246,17 +214,15 @@ pub fn get_active_validators<T: BeaconChainTypes>(
         None
     };
 
-    let mut state = get_state_from_root_opt(&beacon_chain, state_root_opt)?;
+    let mut state = get_state_from_root_opt(&ctx.beacon_chain, state_root_opt)?;
     state.update_pubkey_cache()?;
 
-    let validators = state
+    state
         .validators
         .iter()
         .filter(|validator| validator.is_active_at(state.current_epoch()))
         .map(|validator| validator_response_by_pubkey(&state, validator.pubkey.clone()))
-        .collect::<Result<Vec<_>, _>>()?;
-
-    ResponseBuilder::new(&req)?.body(&validators)
+        .collect::<Result<Vec<_>, _>>()
 }
 
 /// HTTP handler to which accepts a `ValidatorRequest` and returns a `ValidatorResponse` for
@@ -264,17 +230,11 @@ pub fn get_active_validators<T: BeaconChainTypes>(
 ///
 /// This method allows for a basically unbounded list of `pubkeys`, where as the `get_validators`
 /// request is limited by the max number of pubkeys you can fit in a URL.
-pub async fn post_validators<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
-    let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-    serde_json::from_slice::<ValidatorRequest>(&chunks)
+pub fn post_validators<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorResponse>, ApiError> {
+    serde_json::from_slice::<ValidatorRequest>(&req.into_body())
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorRequest: {:?}",
@@ -283,12 +243,11 @@ pub async fn post_validators<T: BeaconChainTypes>(
         })
         .and_then(|bulk_request| {
             validator_responses_by_pubkey(
-                beacon_chain,
+                &ctx.beacon_chain,
                 bulk_request.state_root,
                 bulk_request.pubkeys,
             )
         })
-        .and_then(|validators| response_builder?.body(&validators))
 }
 
 /// Returns either the state given by `state_root_opt`, or the canonical head state if it is
@@ -315,11 +274,11 @@ fn get_state_from_root_opt<T: BeaconChainTypes>(
 /// Maps a vec of `validator_pubkey` to a vec of `ValidatorResponse`, using the state at the given
 /// `state_root`. If `state_root.is_none()`, uses the canonial head state.
 fn validator_responses_by_pubkey<T: BeaconChainTypes>(
-    beacon_chain: Arc<BeaconChain<T>>,
+    beacon_chain: &BeaconChain<T>,
     state_root_opt: Option<Hash256>,
     validator_pubkeys: Vec<PublicKeyBytes>,
 ) -> Result<Vec<ValidatorResponse>, ApiError> {
-    let mut state = get_state_from_root_opt(&beacon_chain, state_root_opt)?;
+    let mut state = get_state_from_root_opt(beacon_chain, state_root_opt)?;
     state.update_pubkey_cache()?;
 
     validator_pubkeys
@@ -370,24 +329,25 @@ fn validator_response_by_pubkey<E: EthSpec>(
 
 /// HTTP handler
 pub fn get_committees<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<Committee>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
 
-    let mut state = get_state_for_epoch(&beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
+    let mut state =
+        get_state_for_epoch(&ctx.beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
 
     let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), epoch).map_err(|e| {
         ApiError::ServerError(format!("Failed to get state suitable for epoch: {:?}", e))
     })?;
 
     state
-        .build_committee_cache(relative_epoch, &beacon_chain.spec)
+        .build_committee_cache(relative_epoch, &ctx.beacon_chain.spec)
         .map_err(|e| ApiError::ServerError(format!("Unable to build committee cache: {:?}", e)))?;
 
-    let committees = state
+    Ok(state
         .get_beacon_committees_at_epoch(relative_epoch)
         .map_err(|e| ApiError::ServerError(format!("Unable to get all committees: {:?}", e)))?
         .into_iter()
@@ -396,9 +356,7 @@ pub fn get_committees<T: BeaconChainTypes>(
             index: c.index,
             committee: c.committee.to_vec(),
         })
-        .collect::<Vec<_>>();
-
-    ResponseBuilder::new(&req)?.body(&committees)
+        .collect::<Vec<_>>())
 }
 
 /// HTTP handler to return a `BeaconState` at a given `root` or `slot`.
@@ -406,10 +364,10 @@ pub fn get_committees<T: BeaconChainTypes>(
 /// Will not return a state if the request slot is in the future. Will return states higher than
 /// the current head by skipping slots.
 pub fn get_state<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let head_state = beacon_chain.head()?.beacon_state;
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<StateResponse<T::EthSpec>, ApiError> {
+    let head_state = ctx.beacon_chain.head()?.beacon_state;
 
     let (key, value) = match UrlQuery::from_request(&req) {
         Ok(query) => {
@@ -427,11 +385,12 @@ pub fn get_state<T: BeaconChainTypes>(
     };
 
     let (root, state): (Hash256, BeaconState<T::EthSpec>) = match (key.as_ref(), value) {
-        ("slot", value) => state_at_slot(&beacon_chain, parse_slot(&value)?)?,
+        ("slot", value) => state_at_slot(&ctx.beacon_chain, parse_slot(&value)?)?,
         ("root", value) => {
             let root = &parse_root(&value)?;
 
-            let state = beacon_chain
+            let state = ctx
+                .beacon_chain
                 .store
                 .get_state(root, None)?
                 .ok_or_else(|| ApiError::NotFound(format!("No state for root: {:?}", root)))?;
@@ -441,12 +400,10 @@ pub fn get_state<T: BeaconChainTypes>(
         _ => return Err(ApiError::ServerError("Unexpected query parameter".into())),
     };
 
-    let response = StateResponse {
+    Ok(StateResponse {
         root,
         beacon_state: state,
-    };
-
-    ResponseBuilder::new(&req)?.body(&response)
+    })
 }
 
 /// HTTP handler to return a `BeaconState` root at a given `slot`.
@@ -454,15 +411,13 @@ pub fn get_state<T: BeaconChainTypes>(
 /// Will not return a state if the request slot is in the future. Will return states higher than
 /// the current head by skipping slots.
 pub fn get_state_root<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Hash256, ApiError> {
     let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?;
     let slot = parse_slot(&slot_string)?;
 
-    let root = state_root_at_slot(&beacon_chain, slot, StateSkipConfig::WithStateRoots)?;
-
-    ResponseBuilder::new(&req)?.body(&root)
+    state_root_at_slot(&ctx.beacon_chain, slot, StateSkipConfig::WithStateRoots)
 }
 
 /// HTTP handler to return a `BeaconState` at the genesis block.
@@ -470,50 +425,28 @@ pub fn get_state_root<T: BeaconChainTypes>(
 /// This is an undocumented convenience method used during testing. For production, simply do a
 /// state request at slot 0.
 pub fn get_genesis_state<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?;
-
-    ResponseBuilder::new(&req)?.body(&state)
+    ctx: Arc<Context<T>>,
+) -> Result<BeaconState<T::EthSpec>, ApiError> {
+    state_at_slot(&ctx.beacon_chain, Slot::new(0)).map(|(_root, state)| state)
 }
 
-/// Read the genesis time from the current beacon chain state.
-pub fn get_genesis_time<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&beacon_chain.head_info()?.genesis_time)
-}
-
-/// Read the `genesis_validators_root` from the current beacon chain state.
-pub fn get_genesis_validators_root<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&beacon_chain.head_info()?.genesis_validators_root)
-}
-
-pub async fn proposer_slashing<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn proposer_slashing<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<bool, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice::<ProposerSlashing>(&chunks)
+    serde_json::from_slice::<ProposerSlashing>(&body)
         .map_err(|e| format!("Unable to parse JSON into ProposerSlashing: {:?}", e))
         .and_then(move |proposer_slashing| {
-            if beacon_chain.eth1_chain.is_some() {
-                let obs_outcome = beacon_chain
+            if ctx.beacon_chain.eth1_chain.is_some() {
+                let obs_outcome = ctx
+                    .beacon_chain
                     .verify_proposer_slashing_for_gossip(proposer_slashing)
                     .map_err(|e| format!("Error while verifying proposer slashing: {:?}", e))?;
                 if let ObservationOutcome::New(verified_proposer_slashing) = obs_outcome {
-                    beacon_chain.import_proposer_slashing(verified_proposer_slashing);
+                    ctx.beacon_chain
+                        .import_proposer_slashing(verified_proposer_slashing);
                     Ok(())
                 } else {
                     Err("Proposer slashing for that validator index already known".into())
@@ -522,22 +455,17 @@ pub async fn proposer_slashing<T: BeaconChainTypes>(
                 Err("Cannot insert proposer slashing on node without Eth1 connection.".to_string())
             }
         })
-        .map_err(ApiError::BadRequest)
-        .and_then(|_| response_builder?.body(&true))
+        .map_err(ApiError::BadRequest)?;
+
+    Ok(true)
 }
 
-pub async fn attester_slashing<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn attester_slashing<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<bool, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-
-    serde_json::from_slice::<AttesterSlashing<T::EthSpec>>(&chunks)
+    serde_json::from_slice::<AttesterSlashing<T::EthSpec>>(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into AttesterSlashing: {:?}",
@@ -545,13 +473,13 @@ pub async fn attester_slashing<T: BeaconChainTypes>(
             ))
         })
         .and_then(move |attester_slashing| {
-            if beacon_chain.eth1_chain.is_some() {
-                beacon_chain
+            if ctx.beacon_chain.eth1_chain.is_some() {
+                ctx.beacon_chain
                     .verify_attester_slashing_for_gossip(attester_slashing)
                     .map_err(|e| format!("Error while verifying attester slashing: {:?}", e))
                     .and_then(|outcome| {
                         if let ObservationOutcome::New(verified_attester_slashing) = outcome {
-                            beacon_chain
+                            ctx.beacon_chain
                                 .import_attester_slashing(verified_attester_slashing)
                                 .map_err(|e| {
                                     format!("Error while importing attester slashing: {:?}", e)
@@ -566,6 +494,7 @@ pub async fn attester_slashing<T: BeaconChainTypes>(
                     "Cannot insert attester slashing on node without Eth1 connection.".to_string(),
                 ))
             }
-        })
-        .and_then(|_| response_builder?.body(&true))
+        })?;
+
+    Ok(true)
 }

beacon_node/rest_api/src/consensus.rs

@@ -1,8 +1,7 @@
 use crate::helpers::*;
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult, UrlQuery};
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use hyper::{Body, Request};
+use crate::{ApiError, Context, UrlQuery};
+use beacon_chain::BeaconChainTypes;
+use hyper::Request;
 use rest_types::{IndividualVotesRequest, IndividualVotesResponse};
 use serde::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -50,38 +49,31 @@ impl Into<VoteCount> for TotalBalances {
 
 /// HTTP handler return a `VoteCount` for some given `Epoch`.
 pub fn get_vote_count<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<VoteCount, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
     // This is the last slot of the given epoch (one prior to the first slot of the next epoch).
     let target_slot = (epoch + 1).start_slot(T::EthSpec::slots_per_epoch()) - 1;
 
-    let (_root, state) = state_at_slot(&beacon_chain, target_slot)?;
-    let spec = &beacon_chain.spec;
+    let (_root, state) = state_at_slot(&ctx.beacon_chain, target_slot)?;
+    let spec = &ctx.beacon_chain.spec;
 
     let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
     validator_statuses.process_attestations(&state, spec)?;
 
-    let report: VoteCount = validator_statuses.total_balances.into();
-
-    ResponseBuilder::new(&req)?.body(&report)
+    Ok(validator_statuses.total_balances.into())
 }
 
-pub async fn post_individual_votes<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn post_individual_votes<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<IndividualVotesResponse>, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice::<IndividualVotesRequest>(&chunks)
+    serde_json::from_slice::<IndividualVotesRequest>(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorDutiesRequest: {:?}",
@@ -94,8 +86,8 @@ pub async fn post_individual_votes<T: BeaconChainTypes>(
             // This is the last slot of the given epoch (one prior to the first slot of the next epoch).
             let target_slot = (epoch + 1).start_slot(T::EthSpec::slots_per_epoch()) - 1;
 
-            let (_root, mut state) = state_at_slot(&beacon_chain, target_slot)?;
-            let spec = &beacon_chain.spec;
+            let (_root, mut state) = state_at_slot(&ctx.beacon_chain, target_slot)?;
+            let spec = &ctx.beacon_chain.spec;
 
             let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
             validator_statuses.process_attestations(&state, spec)?;
@@ -135,5 +127,4 @@ pub async fn post_individual_votes<T: BeaconChainTypes>(
                 })
                 .collect::<Result<Vec<_>, _>>()
         })
-        .and_then(|votes| response_builder?.body_no_ssz(&votes))
 }

beacon_node/rest_api/src/helpers.rs

@@ -1,9 +1,7 @@
-use crate::{ApiError, ApiResult, NetworkChannel};
+use crate::{ApiError, NetworkChannel};
 use beacon_chain::{BeaconChain, BeaconChainTypes, StateSkipConfig};
 use bls::PublicKeyBytes;
 use eth2_libp2p::PubsubMessage;
-use http::header;
-use hyper::{Body, Request};
 use itertools::process_results;
 use network::NetworkMessage;
 use ssz::Decode;
@@ -41,21 +39,6 @@ pub fn parse_committee_index(string: &str) -> Result<CommitteeIndex, ApiError> {
         .map_err(|e| ApiError::BadRequest(format!("Unable to parse committee index: {:?}", e)))
 }
 
-/// Checks the provided request to ensure that the `content-type` header.
-///
-/// The content-type header should either be omitted, in which case JSON is assumed, or it should
-/// explicitly specify `application/json`. If anything else is provided, an error is returned.
-pub fn check_content_type_for_json(req: &Request<Body>) -> Result<(), ApiError> {
-    match req.headers().get(header::CONTENT_TYPE) {
-        Some(h) if h == "application/json" => Ok(()),
-        Some(h) => Err(ApiError::BadRequest(format!(
-            "The provided content-type {:?} is not available, this endpoint only supports json.",
-            h
-        ))),
-        _ => Ok(()),
-    }
-}
-
 /// Parse an SSZ object from some hex-encoded bytes.
 ///
 /// E.g., A signature is `"0x0000000000000000000000000000000000000000000000000000000000000000"`
@@ -228,14 +211,8 @@ pub fn state_root_at_slot<T: BeaconChainTypes>(
     }
 }
 
-pub fn implementation_pending_response(_req: Request<Body>) -> ApiResult {
-    Err(ApiError::NotImplemented(
-        "API endpoint has not yet been implemented, but is planned to be soon.".to_owned(),
-    ))
-}
-
 pub fn publish_beacon_block_to_network<T: BeaconChainTypes + 'static>(
-    chan: NetworkChannel<T::EthSpec>,
+    chan: &NetworkChannel<T::EthSpec>,
     block: SignedBeaconBlock<T::EthSpec>,
 ) -> Result<(), ApiError> {
     // send the block via SSZ encoding

beacon_node/rest_api/src/lib.rs

@@ -1,22 +1,15 @@
 #[macro_use]
-mod macros;
-#[macro_use]
 extern crate lazy_static;
+mod router;
 extern crate network as client_network;
 
-mod advanced;
 mod beacon;
 pub mod config;
 mod consensus;
-mod error;
 mod helpers;
 mod lighthouse;
 mod metrics;
-mod network;
 mod node;
-mod response_builder;
-mod router;
-mod spec;
 mod url_query;
 mod validator;
 
@@ -24,7 +17,6 @@ use beacon_chain::{BeaconChain, BeaconChainTypes};
 use bus::Bus;
 use client_network::NetworkMessage;
 pub use config::ApiEncodingFormat;
-use error::{ApiError, ApiResult};
 use eth2_config::Eth2Config;
 use eth2_libp2p::NetworkGlobals;
 use futures::future::TryFutureExt;
@@ -32,6 +24,7 @@ use hyper::server::conn::AddrStream;
 use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Request, Server};
 use parking_lot::Mutex;
+use rest_types::ApiError;
 use slog::{info, warn};
 use std::net::SocketAddr;
 use std::path::PathBuf;
@@ -42,6 +35,7 @@ use url_query::UrlQuery;
 
 pub use crate::helpers::parse_pubkey_bytes;
 pub use config::Config;
+pub use router::Context;
 
 pub type NetworkChannel<T> = mpsc::UnboundedSender<NetworkMessage<T>>;
 
@@ -63,36 +57,28 @@ pub fn start_server<T: BeaconChainTypes>(
     events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
 ) -> Result<SocketAddr, hyper::Error> {
     let log = executor.log();
-    let inner_log = log.clone();
-    let rest_api_config = Arc::new(config.clone());
     let eth2_config = Arc::new(eth2_config);
 
+    let context = Arc::new(Context {
+        executor: executor.clone(),
+        config: config.clone(),
+        beacon_chain,
+        network_globals: network_info.network_globals.clone(),
+        network_chan: network_info.network_chan,
+        eth2_config,
+        log: log.clone(),
+        db_path,
+        freezer_db_path,
+        events,
+    });
+
     // Define the function that will build the request handler.
     let make_service = make_service_fn(move |_socket: &AddrStream| {
-        let beacon_chain = beacon_chain.clone();
-        let log = inner_log.clone();
-        let rest_api_config = rest_api_config.clone();
-        let eth2_config = eth2_config.clone();
-        let network_globals = network_info.network_globals.clone();
-        let network_channel = network_info.network_chan.clone();
-        let db_path = db_path.clone();
-        let freezer_db_path = freezer_db_path.clone();
-        let events = events.clone();
+        let ctx = context.clone();
 
         async move {
             Ok::<_, hyper::Error>(service_fn(move |req: Request<Body>| {
-                router::route(
-                    req,
-                    beacon_chain.clone(),
-                    network_globals.clone(),
-                    network_channel.clone(),
-                    rest_api_config.clone(),
-                    eth2_config.clone(),
-                    log.clone(),
-                    db_path.clone(),
-                    freezer_db_path.clone(),
-                    events.clone(),
-                )
+                router::on_http_request(req, ctx.clone())
             }))
         }
     });

beacon_node/rest_api/src/lighthouse.rs

@@ -1,24 +1,16 @@
 //! This contains a collection of lighthouse specific HTTP endpoints.
 
-use crate::response_builder::ResponseBuilder;
-use crate::ApiResult;
-use eth2_libp2p::{NetworkGlobals, PeerInfo};
-use hyper::{Body, Request};
+use crate::{ApiError, Context};
+use beacon_chain::BeaconChainTypes;
+use eth2_libp2p::PeerInfo;
 use serde::Serialize;
 use std::sync::Arc;
 use types::EthSpec;
 
-/// The syncing state of the beacon node.
-pub fn syncing<T: EthSpec>(
-    req: Request<Body>,
-    network_globals: Arc<NetworkGlobals<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&network_globals.sync_state())
-}
-
 /// Returns all known peers and corresponding information
-pub fn peers<T: EthSpec>(req: Request<Body>, network_globals: Arc<NetworkGlobals<T>>) -> ApiResult {
-    let peers: Vec<Peer<T>> = network_globals
+pub fn peers<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Result<Vec<Peer<T::EthSpec>>, ApiError> {
+    Ok(ctx
+        .network_globals
         .peers
         .read()
         .peers()
@@ -26,16 +18,15 @@ pub fn peers<T: EthSpec>(req: Request<Body>, network_globals: Arc<NetworkGlobals
             peer_id: peer_id.to_string(),
             peer_info: peer_info.clone(),
         })
-        .collect();
-    ResponseBuilder::new(&req)?.body_no_ssz(&peers)
+        .collect())
 }
 
 /// Returns all known connected peers and their corresponding information
-pub fn connected_peers<T: EthSpec>(
-    req: Request<Body>,
-    network_globals: Arc<NetworkGlobals<T>>,
-) -> ApiResult {
-    let peers: Vec<Peer<T>> = network_globals
+pub fn connected_peers<T: BeaconChainTypes>(
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<Peer<T::EthSpec>>, ApiError> {
+    Ok(ctx
+        .network_globals
         .peers
         .read()
         .connected_peers()
@@ -43,14 +34,13 @@ pub fn connected_peers<T: EthSpec>(
             peer_id: peer_id.to_string(),
             peer_info: peer_info.clone(),
         })
-        .collect();
-    ResponseBuilder::new(&req)?.body_no_ssz(&peers)
+        .collect())
 }
 
 /// Information returned by `peers` and `connected_peers`.
 #[derive(Clone, Debug, Serialize)]
 #[serde(bound = "T: EthSpec")]
-struct Peer<T: EthSpec> {
+pub struct Peer<T: EthSpec> {
     /// The Peer's ID
     peer_id: String,
     /// The PeerInfo associated with the peer.

beacon_node/rest_api/src/macros.rs

@@ -1,11 +0,0 @@
-macro_rules! try_future {
-    ($expr:expr) => {
-        match $expr {
-            core::result::Result::Ok(val) => val,
-            core::result::Result::Err(err) => return Err(std::convert::From::from(err)),
-        }
-    };
-    ($expr:expr,) => {
-        $crate::try_future!($expr)
-    };
-}

beacon_node/rest_api/src/metrics.rs

@@ -1,42 +1,38 @@
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult};
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use hyper::{Body, Request};
+use crate::{ApiError, Context};
+use beacon_chain::BeaconChainTypes;
 use lighthouse_metrics::{Encoder, TextEncoder};
 use rest_types::Health;
-use std::path::PathBuf;
 use std::sync::Arc;
 
 pub use lighthouse_metrics::*;
 
 lazy_static! {
+    pub static ref BEACON_HTTP_API_REQUESTS_TOTAL: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "beacon_http_api_requests_total",
+            "Count of HTTP requests received",
+            &["endpoint"]
+        );
+    pub static ref BEACON_HTTP_API_SUCCESS_TOTAL: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "beacon_http_api_success_total",
+            "Count of HTTP requests that returned 200 OK",
+            &["endpoint"]
+        );
+    pub static ref BEACON_HTTP_API_ERROR_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_http_api_error_total",
+        "Count of HTTP that did not return 200 OK",
+        &["endpoint"]
+    );
+    pub static ref BEACON_HTTP_API_TIMES_TOTAL: Result<HistogramVec> = try_create_histogram_vec(
+        "beacon_http_api_times_total",
+        "Duration to process HTTP requests",
+        &["endpoint"]
+    );
     pub static ref REQUEST_RESPONSE_TIME: Result<Histogram> = try_create_histogram(
         "http_server_request_duration_seconds",
         "Time taken to build a response to a HTTP request"
     );
-    pub static ref REQUEST_COUNT: Result<IntCounter> = try_create_int_counter(
-        "http_server_request_total",
-        "Total count of HTTP requests received"
-    );
-    pub static ref SUCCESS_COUNT: Result<IntCounter> = try_create_int_counter(
-        "http_server_success_total",
-        "Total count of HTTP 200 responses sent"
-    );
-    pub static ref VALIDATOR_GET_BLOCK_REQUEST_RESPONSE_TIME: Result<Histogram> =
-        try_create_histogram(
-            "http_server_validator_block_get_request_duration_seconds",
-            "Time taken to respond to GET /validator/block"
-        );
-    pub static ref VALIDATOR_GET_ATTESTATION_REQUEST_RESPONSE_TIME: Result<Histogram> =
-        try_create_histogram(
-            "http_server_validator_attestation_get_request_duration_seconds",
-            "Time taken to respond to GET /validator/attestation"
-        );
-    pub static ref VALIDATOR_GET_DUTIES_REQUEST_RESPONSE_TIME: Result<Histogram> =
-        try_create_histogram(
-            "http_server_validator_duties_get_request_duration_seconds",
-            "Time taken to respond to GET /validator/duties"
-        );
     pub static ref PROCESS_NUM_THREADS: Result<IntGauge> = try_create_int_gauge(
         "process_num_threads",
         "Number of threads used by the current process"
@@ -77,11 +73,8 @@ lazy_static! {
 ///
 /// This is a HTTP handler method.
 pub fn get_prometheus<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    db_path: PathBuf,
-    freezer_db_path: PathBuf,
-) -> ApiResult {
+    ctx: Arc<Context<T>>,
+) -> std::result::Result<String, ApiError> {
     let mut buffer = vec![];
     let encoder = TextEncoder::new();
 
@@ -101,9 +94,9 @@ pub fn get_prometheus<T: BeaconChainTypes>(
     // using `lighthouse_metrics::gather(..)` to collect the global `DEFAULT_REGISTRY` metrics into
     // a string that can be returned via HTTP.
 
-    slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&beacon_chain.slot_clock);
-    store::scrape_for_metrics(&db_path, &freezer_db_path);
-    beacon_chain::scrape_for_metrics(&beacon_chain);
+    slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&ctx.beacon_chain.slot_clock);
+    store::scrape_for_metrics(&ctx.db_path, &ctx.freezer_db_path);
+    beacon_chain::scrape_for_metrics(&ctx.beacon_chain);
     eth2_libp2p::scrape_discovery_metrics();
 
     // This will silently fail if we are unable to observe the health. This is desired behaviour
@@ -133,6 +126,5 @@ pub fn get_prometheus<T: BeaconChainTypes>(
         .unwrap();
 
     String::from_utf8(buffer)
-        .map(|string| ResponseBuilder::new(&req)?.body_text(string))
-        .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e)))?
+        .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e)))
 }

beacon_node/rest_api/src/network.rs

@@ -1,72 +0,0 @@
-use crate::error::ApiResult;
-use crate::response_builder::ResponseBuilder;
-use crate::NetworkGlobals;
-use beacon_chain::BeaconChainTypes;
-use eth2_libp2p::{Multiaddr, PeerId};
-use hyper::{Body, Request};
-use std::sync::Arc;
-
-/// HTTP handler to return the list of libp2p multiaddr the client is listening on.
-///
-/// Returns a list of `Multiaddr`, serialized according to their `serde` impl.
-pub fn get_listen_addresses<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    let multiaddresses: Vec<Multiaddr> = network.listen_multiaddrs();
-    ResponseBuilder::new(&req)?.body_no_ssz(&multiaddresses)
-}
-
-/// HTTP handler to return the network port the client is listening on.
-///
-/// Returns the TCP port number in its plain form (which is also valid JSON serialization)
-pub fn get_listen_port<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&network.listen_port_tcp())
-}
-
-/// HTTP handler to return the Discv5 ENR from the client's libp2p service.
-///
-/// ENR is encoded as base64 string.
-pub fn get_enr<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&network.local_enr().to_base64())
-}
-
-/// HTTP handler to return the `PeerId` from the client's libp2p service.
-///
-/// PeerId is encoded as base58 string.
-pub fn get_peer_id<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&network.local_peer_id().to_base58())
-}
-
-/// HTTP handler to return the number of peers connected in the client's libp2p service.
-pub fn get_peer_count<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&network.connected_peers())
-}
-
-/// HTTP handler to return the list of peers connected to the client's libp2p service.
-///
-/// Peers are presented as a list of `PeerId::to_string()`.
-pub fn get_peer_list<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T::EthSpec>>,
-) -> ApiResult {
-    let connected_peers: Vec<String> = network
-        .peers
-        .read()
-        .connected_peer_ids()
-        .map(PeerId::to_string)
-        .collect();
-    ResponseBuilder::new(&req)?.body_no_ssz(&connected_peers)
-}

beacon_node/rest_api/src/node.rs

@@ -1,23 +1,19 @@
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult};
-use eth2_libp2p::{types::SyncState, NetworkGlobals};
-use hyper::{Body, Request};
-use lighthouse_version::version_with_platform;
-use rest_types::{Health, SyncingResponse, SyncingStatus};
+use crate::{ApiError, Context};
+use beacon_chain::BeaconChainTypes;
+use eth2_libp2p::types::SyncState;
+use rest_types::{SyncingResponse, SyncingStatus};
 use std::sync::Arc;
-use types::{EthSpec, Slot};
+use types::Slot;
 
-/// Read the version string from the current Lighthouse build.
-pub fn get_version(req: Request<Body>) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&version_with_platform())
-}
+/// Returns a syncing status.
+pub fn syncing<T: BeaconChainTypes>(ctx: Arc<Context<T>>) -> Result<SyncingResponse, ApiError> {
+    let current_slot = ctx
+        .beacon_chain
+        .head_info()
+        .map_err(|e| ApiError::ServerError(format!("Unable to read head slot: {:?}", e)))?
+        .slot;
 
-pub fn syncing<T: EthSpec>(
-    req: Request<Body>,
-    network: Arc<NetworkGlobals<T>>,
-    current_slot: Slot,
-) -> ApiResult {
-    let (starting_slot, highest_slot) = match network.sync_state() {
+    let (starting_slot, highest_slot) = match ctx.network_globals.sync_state() {
         SyncState::SyncingFinalized {
             start_slot,
             head_slot,
@@ -36,14 +32,8 @@ pub fn syncing<T: EthSpec>(
         highest_slot,
     };
 
-    ResponseBuilder::new(&req)?.body(&SyncingResponse {
-        is_syncing: network.is_syncing(),
+    Ok(SyncingResponse {
+        is_syncing: ctx.network_globals.is_syncing(),
         sync_status,
     })
 }
-
-pub fn get_health(req: Request<Body>) -> ApiResult {
-    let health = Health::observe().map_err(ApiError::ServerError)?;
-
-    ResponseBuilder::new(&req)?.body_no_ssz(&health)
-}

beacon_node/rest_api/src/response_builder.rs

@@ -1,83 +0,0 @@
-use super::{ApiError, ApiResult};
-use crate::config::ApiEncodingFormat;
-use hyper::header;
-use hyper::{Body, Request, Response, StatusCode};
-use serde::Serialize;
-use ssz::Encode;
-
-pub struct ResponseBuilder {
-    encoding: ApiEncodingFormat,
-}
-
-impl ResponseBuilder {
-    pub fn new(req: &Request<Body>) -> Result<Self, ApiError> {
-        let accept_header: String = req
-            .headers()
-            .get(header::ACCEPT)
-            .map_or(Ok(""), |h| h.to_str())
-            .map_err(|e| {
-                ApiError::BadRequest(format!(
-                    "The Accept header contains invalid characters: {:?}",
-                    e
-                ))
-            })
-            .map(String::from)?;
-
-        // JSON is our default encoding, unless something else is requested.
-        let encoding = ApiEncodingFormat::from(accept_header.as_str());
-        Ok(Self { encoding })
-    }
-
-    pub fn body<T: Serialize + Encode>(self, item: &T) -> ApiResult {
-        match self.encoding {
-            ApiEncodingFormat::SSZ => Response::builder()
-                .status(StatusCode::OK)
-                .header("content-type", "application/ssz")
-                .body(Body::from(item.as_ssz_bytes()))
-                .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))),
-            _ => self.body_no_ssz(item),
-        }
-    }
-
-    pub fn body_no_ssz<T: Serialize>(self, item: &T) -> ApiResult {
-        let (body, content_type) = match self.encoding {
-            ApiEncodingFormat::JSON => (
-                Body::from(serde_json::to_string(&item).map_err(|e| {
-                    ApiError::ServerError(format!(
-                        "Unable to serialize response body as JSON: {:?}",
-                        e
-                    ))
-                })?),
-                "application/json",
-            ),
-            ApiEncodingFormat::SSZ => {
-                return Err(ApiError::UnsupportedType(
-                    "Response cannot be encoded as SSZ.".into(),
-                ));
-            }
-            ApiEncodingFormat::YAML => (
-                Body::from(serde_yaml::to_string(&item).map_err(|e| {
-                    ApiError::ServerError(format!(
-                        "Unable to serialize response body as YAML: {:?}",
-                        e
-                    ))
-                })?),
-                "application/yaml",
-            ),
-        };
-
-        Response::builder()
-            .status(StatusCode::OK)
-            .header("content-type", content_type)
-            .body(body)
-            .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))
-    }
-
-    pub fn body_text(self, text: String) -> ApiResult {
-        Response::builder()
-            .status(StatusCode::OK)
-            .header("content-type", "text/plain; charset=utf-8")
-            .body(Body::from(text))
-            .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))
-    }
-}

beacon_node/rest_api/src/router.rs

@@ -1,241 +1,322 @@
 use crate::{
-    advanced, beacon, config::Config, consensus, error::ApiError, helpers, lighthouse, metrics,
-    network, node, spec, validator, NetworkChannel,
+    beacon, config::Config, consensus, lighthouse, metrics, node, validator, NetworkChannel,
 };
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use bus::Bus;
+use environment::TaskExecutor;
 use eth2_config::Eth2Config;
-use eth2_libp2p::NetworkGlobals;
+use eth2_libp2p::{NetworkGlobals, PeerId};
 use hyper::header::HeaderValue;
 use hyper::{Body, Method, Request, Response};
+use lighthouse_version::version_with_platform;
+use operation_pool::PersistedOperationPool;
 use parking_lot::Mutex;
+use rest_types::{ApiError, Handler, Health};
 use slog::debug;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
-use types::{SignedBeaconBlockHash, Slot};
+use types::{EthSpec, SignedBeaconBlockHash};
 
-// Allowing more than 7 arguments.
-#[allow(clippy::too_many_arguments)]
-pub async fn route<T: BeaconChainTypes>(
+pub struct Context<T: BeaconChainTypes> {
+    pub executor: TaskExecutor,
+    pub config: Config,
+    pub beacon_chain: Arc<BeaconChain<T>>,
+    pub network_globals: Arc<NetworkGlobals<T::EthSpec>>,
+    pub network_chan: NetworkChannel<T::EthSpec>,
+    pub eth2_config: Arc<Eth2Config>,
+    pub log: slog::Logger,
+    pub db_path: PathBuf,
+    pub freezer_db_path: PathBuf,
+    pub events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
+}
+
+pub async fn on_http_request<T: BeaconChainTypes>(
     req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    network_globals: Arc<NetworkGlobals<T::EthSpec>>,
-    network_channel: NetworkChannel<T::EthSpec>,
-    rest_api_config: Arc<Config>,
-    eth2_config: Arc<Eth2Config>,
-    local_log: slog::Logger,
-    db_path: PathBuf,
-    freezer_db_path: PathBuf,
-    events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
+    ctx: Arc<Context<T>>,
 ) -> Result<Response<Body>, ApiError> {
-    metrics::inc_counter(&metrics::REQUEST_COUNT);
-    let received_instant = Instant::now();
-
     let path = req.uri().path().to_string();
 
-    let log = local_log.clone();
-    let result = {
-        let _timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME);
+    let _timer = metrics::start_timer_vec(&metrics::BEACON_HTTP_API_TIMES_TOTAL, &[&path]);
+    metrics::inc_counter_vec(&metrics::BEACON_HTTP_API_REQUESTS_TOTAL, &[&path]);
 
-        match (req.method(), path.as_ref()) {
-            // Methods for Client
-            (&Method::GET, "/node/health") => node::get_health(req),
-            (&Method::GET, "/node/version") => node::get_version(req),
-            (&Method::GET, "/node/syncing") => {
-                // inform the current slot, or set to 0
-                let current_slot = beacon_chain
-                    .head_info()
-                    .map(|info| info.slot)
-                    .unwrap_or_else(|_| Slot::from(0u64));
+    let received_instant = Instant::now();
+    let log = ctx.log.clone();
+    let allow_origin = ctx.config.allow_origin.clone();
 
-                node::syncing::<T::EthSpec>(req, network_globals, current_slot)
-            }
-
-            // Methods for Network
-            (&Method::GET, "/network/enr") => network::get_enr::<T>(req, network_globals),
-            (&Method::GET, "/network/peer_count") => {
-                network::get_peer_count::<T>(req, network_globals)
-            }
-            (&Method::GET, "/network/peer_id") => network::get_peer_id::<T>(req, network_globals),
-            (&Method::GET, "/network/peers") => network::get_peer_list::<T>(req, network_globals),
-            (&Method::GET, "/network/listen_port") => {
-                network::get_listen_port::<T>(req, network_globals)
-            }
-            (&Method::GET, "/network/listen_addresses") => {
-                network::get_listen_addresses::<T>(req, network_globals)
-            }
-
-            // Methods for Beacon Node
-            (&Method::GET, "/beacon/head") => beacon::get_head::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/heads") => beacon::get_heads::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/block") => beacon::get_block::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/block_root") => beacon::get_block_root::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/fork") => beacon::get_fork::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/fork/stream") => {
-                let reader = events.lock().add_rx();
-                beacon::stream_forks::<T>(log, reader)
-            }
-            (&Method::GET, "/beacon/genesis_time") => {
-                beacon::get_genesis_time::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/genesis_validators_root") => {
-                beacon::get_genesis_validators_root::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/validators") => beacon::get_validators::<T>(req, beacon_chain),
-            (&Method::POST, "/beacon/validators") => {
-                beacon::post_validators::<T>(req, beacon_chain).await
-            }
-            (&Method::GET, "/beacon/validators/all") => {
-                beacon::get_all_validators::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/validators/active") => {
-                beacon::get_active_validators::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/state") => beacon::get_state::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/state_root") => beacon::get_state_root::<T>(req, beacon_chain),
-            (&Method::GET, "/beacon/state/genesis") => {
-                beacon::get_genesis_state::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/beacon/committees") => beacon::get_committees::<T>(req, beacon_chain),
-            (&Method::POST, "/beacon/proposer_slashing") => {
-                beacon::proposer_slashing::<T>(req, beacon_chain).await
-            }
-            (&Method::POST, "/beacon/attester_slashing") => {
-                beacon::attester_slashing::<T>(req, beacon_chain).await
-            }
-
-            // Methods for Validator
-            (&Method::POST, "/validator/duties") => {
-                let timer =
-                    metrics::start_timer(&metrics::VALIDATOR_GET_DUTIES_REQUEST_RESPONSE_TIME);
-                let response = validator::post_validator_duties::<T>(req, beacon_chain);
-                drop(timer);
-                response.await
-            }
-            (&Method::POST, "/validator/subscribe") => {
-                validator::post_validator_subscriptions::<T>(req, network_channel).await
-            }
-            (&Method::GET, "/validator/duties/all") => {
-                validator::get_all_validator_duties::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/validator/duties/active") => {
-                validator::get_active_validator_duties::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/validator/block") => {
-                let timer =
-                    metrics::start_timer(&metrics::VALIDATOR_GET_BLOCK_REQUEST_RESPONSE_TIME);
-                let response = validator::get_new_beacon_block::<T>(req, beacon_chain, log);
-                drop(timer);
-                response
-            }
-            (&Method::POST, "/validator/block") => {
-                validator::publish_beacon_block::<T>(req, beacon_chain, network_channel, log).await
-            }
-            (&Method::GET, "/validator/attestation") => {
-                let timer =
-                    metrics::start_timer(&metrics::VALIDATOR_GET_ATTESTATION_REQUEST_RESPONSE_TIME);
-                let response = validator::get_new_attestation::<T>(req, beacon_chain);
-                drop(timer);
-                response
-            }
-            (&Method::GET, "/validator/aggregate_attestation") => {
-                validator::get_aggregate_attestation::<T>(req, beacon_chain)
-            }
-            (&Method::POST, "/validator/attestations") => {
-                validator::publish_attestations::<T>(req, beacon_chain, network_channel, log).await
-            }
-            (&Method::POST, "/validator/aggregate_and_proofs") => {
-                validator::publish_aggregate_and_proofs::<T>(
-                    req,
-                    beacon_chain,
-                    network_channel,
-                    log,
-                )
-                .await
-            }
-
-            // Methods for consensus
-            (&Method::GET, "/consensus/global_votes") => {
-                consensus::get_vote_count::<T>(req, beacon_chain)
-            }
-            (&Method::POST, "/consensus/individual_votes") => {
-                consensus::post_individual_votes::<T>(req, beacon_chain).await
-            }
-
-            // Methods for bootstrap and checking configuration
-            (&Method::GET, "/spec") => spec::get_spec::<T>(req, beacon_chain),
-            (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::<T>(req),
-            (&Method::GET, "/spec/deposit_contract") => {
-                helpers::implementation_pending_response(req)
-            }
-            (&Method::GET, "/spec/eth2_config") => spec::get_eth2_config::<T>(req, eth2_config),
-
-            // Methods for advanced parameters
-            (&Method::GET, "/advanced/fork_choice") => {
-                advanced::get_fork_choice::<T>(req, beacon_chain)
-            }
-            (&Method::GET, "/advanced/operation_pool") => {
-                advanced::get_operation_pool::<T>(req, beacon_chain)
-            }
-
-            (&Method::GET, "/metrics") => {
-                metrics::get_prometheus::<T>(req, beacon_chain, db_path, freezer_db_path)
-            }
-
-            // Lighthouse specific
-            (&Method::GET, "/lighthouse/syncing") => {
-                lighthouse::syncing::<T::EthSpec>(req, network_globals)
-            }
-
-            (&Method::GET, "/lighthouse/peers") => {
-                lighthouse::peers::<T::EthSpec>(req, network_globals)
-            }
-
-            (&Method::GET, "/lighthouse/connected_peers") => {
-                lighthouse::connected_peers::<T::EthSpec>(req, network_globals)
-            }
-            _ => Err(ApiError::NotFound(
-                "Request path and/or method not found.".to_owned(),
-            )),
-        }
-    };
-
-    let request_processing_duration = Instant::now().duration_since(received_instant);
-
-    // Map the Rust-friendly `Result` in to a http-friendly response. In effect, this ensures that
-    // any `Err` returned from our response handlers becomes a valid http response to the client
-    // (e.g., a response with a 404 or 500 status).
-
-    match result {
+    match route(req, ctx).await {
         Ok(mut response) => {
-            if rest_api_config.allow_origin != "" {
+            metrics::inc_counter_vec(&metrics::BEACON_HTTP_API_SUCCESS_TOTAL, &[&path]);
+
+            if allow_origin != "" {
                 let headers = response.headers_mut();
                 headers.insert(
                     hyper::header::ACCESS_CONTROL_ALLOW_ORIGIN,
-                    HeaderValue::from_str(&rest_api_config.allow_origin)?,
+                    HeaderValue::from_str(&allow_origin)?,
                 );
                 headers.insert(hyper::header::VARY, HeaderValue::from_static("Origin"));
             }
 
             debug!(
-                local_log,
+                log,
                 "HTTP API request successful";
                 "path" => path,
-                "duration_ms" => request_processing_duration.as_millis()
+                "duration_ms" => Instant::now().duration_since(received_instant).as_millis()
             );
-            metrics::inc_counter(&metrics::SUCCESS_COUNT);
             Ok(response)
         }
 
         Err(error) => {
+            metrics::inc_counter_vec(&metrics::BEACON_HTTP_API_ERROR_TOTAL, &[&path]);
+
             debug!(
-                local_log,
+                log,
                 "HTTP API request failure";
                 "path" => path,
-                "duration_ms" => request_processing_duration.as_millis()
+                "duration_ms" => Instant::now().duration_since(received_instant).as_millis()
             );
             Ok(error.into())
         }
     }
 }
+
+async fn route<T: BeaconChainTypes>(
+    req: Request<Body>,
+    ctx: Arc<Context<T>>,
+) -> Result<Response<Body>, ApiError> {
+    let path = req.uri().path().to_string();
+    let ctx = ctx.clone();
+    let method = req.method().clone();
+    let executor = ctx.executor.clone();
+    let handler = Handler::new(req, ctx, executor)?;
+
+    match (method, path.as_ref()) {
+        (Method::GET, "/node/version") => handler
+            .static_value(version_with_platform())
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/node/health") => handler
+            .static_value(Health::observe().map_err(ApiError::ServerError)?)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/node/syncing") => handler
+            .allow_body()
+            .in_blocking_task(|_, ctx| node::syncing(ctx))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/enr") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.local_enr().to_base64()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/peer_count") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.connected_peers()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/peer_id") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.local_peer_id().to_base58()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/peers") => handler
+            .in_blocking_task(|_, ctx| {
+                Ok(ctx
+                    .network_globals
+                    .peers
+                    .read()
+                    .connected_peer_ids()
+                    .map(PeerId::to_string)
+                    .collect::<Vec<_>>())
+            })
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/listen_port") => handler
+            .in_core_task(|_, ctx| Ok(ctx.network_globals.listen_port_tcp()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/network/listen_addresses") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.network_globals.listen_multiaddrs()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/beacon/head") => handler
+            .in_blocking_task(|_, ctx| beacon::get_head(ctx))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/heads") => handler
+            .in_blocking_task(|_, ctx| Ok(beacon::get_heads(ctx)))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/block") => handler
+            .in_blocking_task(beacon::get_block)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/block_root") => handler
+            .in_blocking_task(beacon::get_block_root)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/fork") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.head_info()?.fork))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/fork/stream") => {
+            handler.sse_stream(|_, ctx| beacon::stream_forks(ctx)).await
+        }
+        (Method::GET, "/beacon/genesis_time") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.head_info()?.genesis_time))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/genesis_validators_root") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.head_info()?.genesis_validators_root))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/validators") => handler
+            .in_blocking_task(beacon::get_validators)
+            .await?
+            .all_encodings(),
+        (Method::POST, "/beacon/validators") => handler
+            .allow_body()
+            .in_blocking_task(beacon::post_validators)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/validators/all") => handler
+            .in_blocking_task(beacon::get_all_validators)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/validators/active") => handler
+            .in_blocking_task(beacon::get_active_validators)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/state") => handler
+            .in_blocking_task(beacon::get_state)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/state_root") => handler
+            .in_blocking_task(beacon::get_state_root)
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/state/genesis") => handler
+            .in_blocking_task(|_, ctx| beacon::get_genesis_state(ctx))
+            .await?
+            .all_encodings(),
+        (Method::GET, "/beacon/committees") => handler
+            .in_blocking_task(beacon::get_committees)
+            .await?
+            .all_encodings(),
+        (Method::POST, "/beacon/proposer_slashing") => handler
+            .allow_body()
+            .in_blocking_task(beacon::proposer_slashing)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/beacon/attester_slashing") => handler
+            .allow_body()
+            .in_blocking_task(beacon::attester_slashing)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/duties") => handler
+            .allow_body()
+            .in_blocking_task(validator::post_validator_duties)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/subscribe") => handler
+            .allow_body()
+            .in_blocking_task(validator::post_validator_subscriptions)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/duties/all") => handler
+            .in_blocking_task(validator::get_all_validator_duties)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/duties/active") => handler
+            .in_blocking_task(validator::get_active_validator_duties)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/block") => handler
+            .in_blocking_task(validator::get_new_beacon_block)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/block") => handler
+            .allow_body()
+            .in_blocking_task(validator::publish_beacon_block)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/attestation") => handler
+            .in_blocking_task(validator::get_new_attestation)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/validator/aggregate_attestation") => handler
+            .in_blocking_task(validator::get_aggregate_attestation)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/attestations") => handler
+            .allow_body()
+            .in_blocking_task(validator::publish_attestations)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/validator/aggregate_and_proofs") => handler
+            .allow_body()
+            .in_blocking_task(validator::publish_aggregate_and_proofs)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/consensus/global_votes") => handler
+            .allow_body()
+            .in_blocking_task(consensus::get_vote_count)
+            .await?
+            .serde_encodings(),
+        (Method::POST, "/consensus/individual_votes") => handler
+            .allow_body()
+            .in_blocking_task(consensus::post_individual_votes)
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/spec") => handler
+            // TODO: this clone is not ideal.
+            .in_blocking_task(|_, ctx| Ok(ctx.beacon_chain.spec.clone()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/spec/slots_per_epoch") => handler
+            .static_value(T::EthSpec::slots_per_epoch())
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/spec/eth2_config") => handler
+            // TODO: this clone is not ideal.
+            .in_blocking_task(|_, ctx| Ok(ctx.eth2_config.as_ref().clone()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/advanced/fork_choice") => handler
+            .in_blocking_task(|_, ctx| {
+                Ok(ctx
+                    .beacon_chain
+                    .fork_choice
+                    .read()
+                    .proto_array()
+                    .core_proto_array()
+                    .clone())
+            })
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/advanced/operation_pool") => handler
+            .in_blocking_task(|_, ctx| {
+                Ok(PersistedOperationPool::from_operation_pool(
+                    &ctx.beacon_chain.op_pool,
+                ))
+            })
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/metrics") => handler
+            .in_blocking_task(|_, ctx| metrics::get_prometheus(ctx))
+            .await?
+            .text_encoding(),
+        (Method::GET, "/lighthouse/syncing") => handler
+            .in_blocking_task(|_, ctx| Ok(ctx.network_globals.sync_state()))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/lighthouse/peers") => handler
+            .in_blocking_task(|_, ctx| lighthouse::peers(ctx))
+            .await?
+            .serde_encodings(),
+        (Method::GET, "/lighthouse/connected_peers") => handler
+            .in_blocking_task(|_, ctx| lighthouse::connected_peers(ctx))
+            .await?
+            .serde_encodings(),
+        _ => Err(ApiError::NotFound(
+            "Request path and/or method not found.".to_owned(),
+        )),
+    }
+}

beacon_node/rest_api/src/spec.rs

@@ -1,28 +0,0 @@
-use super::ApiResult;
-use crate::response_builder::ResponseBuilder;
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use eth2_config::Eth2Config;
-use hyper::{Body, Request};
-use std::sync::Arc;
-use types::EthSpec;
-
-/// HTTP handler to return the full spec object.
-pub fn get_spec<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(&beacon_chain.spec)
-}
-
-/// HTTP handler to return the full Eth2Config object.
-pub fn get_eth2_config<T: BeaconChainTypes>(
-    req: Request<Body>,
-    eth2_config: Arc<Eth2Config>,
-) -> ApiResult {
-    ResponseBuilder::new(&req)?.body_no_ssz(eth2_config.as_ref())
-}
-
-/// HTTP handler to return the full spec object.
-pub fn get_slots_per_epoch<T: BeaconChainTypes>(req: Request<Body>) -> ApiResult {
-    ResponseBuilder::new(&req)?.body(&T::EthSpec::slots_per_epoch())
-}

beacon_node/rest_api/src/validator.rs

@@ -1,39 +1,32 @@
-use crate::helpers::{check_content_type_for_json, publish_beacon_block_to_network};
-use crate::response_builder::ResponseBuilder;
-use crate::{ApiError, ApiResult, NetworkChannel, UrlQuery};
+use crate::helpers::{parse_hex_ssz_bytes, publish_beacon_block_to_network};
+use crate::{ApiError, Context, NetworkChannel, UrlQuery};
 use beacon_chain::{
     attestation_verification::Error as AttnError, BeaconChain, BeaconChainError, BeaconChainTypes,
     BlockError, ForkChoiceError, StateSkipConfig,
 };
 use bls::PublicKeyBytes;
 use eth2_libp2p::PubsubMessage;
-use hyper::{Body, Request};
+use hyper::Request;
 use network::NetworkMessage;
-use rayon::prelude::*;
 use rest_types::{ValidatorDutiesRequest, ValidatorDutyBytes, ValidatorSubscription};
 use slog::{error, info, trace, warn, Logger};
 use std::sync::Arc;
 use types::beacon_state::EthSpec;
 use types::{
-    Attestation, AttestationData, BeaconState, Epoch, RelativeEpoch, SelectionProof,
+    Attestation, AttestationData, BeaconBlock, BeaconState, Epoch, RelativeEpoch, SelectionProof,
     SignedAggregateAndProof, SignedBeaconBlock, SubnetId,
 };
 
 /// HTTP Handler to retrieve the duties for a set of validators during a particular epoch. This
 /// method allows for collecting bulk sets of validator duties without risking exceeding the max
 /// URL length with query pairs.
-pub async fn post_validator_duties<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn post_validator_duties<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice::<ValidatorDutiesRequest>(&chunks)
+    serde_json::from_slice::<ValidatorDutiesRequest>(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorDutiesRequest: {:?}",
@@ -42,29 +35,22 @@ pub async fn post_validator_duties<T: BeaconChainTypes>(
         })
         .and_then(|bulk_request| {
             return_validator_duties(
-                beacon_chain,
+                &ctx.beacon_chain.clone(),
                 bulk_request.epoch,
                 bulk_request.pubkeys.into_iter().map(Into::into).collect(),
             )
         })
-        .and_then(|duties| response_builder?.body_no_ssz(&duties))
 }
 
 /// HTTP Handler to retrieve subscriptions for a set of validators. This allows the node to
 /// organise peer discovery and topic subscription for known validators.
-pub async fn post_validator_subscriptions<T: BeaconChainTypes>(
-    req: Request<Body>,
-    network_chan: NetworkChannel<T::EthSpec>,
-) -> ApiResult {
-    try_future!(check_content_type_for_json(&req));
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn post_validator_subscriptions<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<(), ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice(&chunks)
+    serde_json::from_slice(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to parse JSON into ValidatorSubscriptions: {:?}",
@@ -72,7 +58,7 @@ pub async fn post_validator_subscriptions<T: BeaconChainTypes>(
             ))
         })
         .and_then(move |subscriptions: Vec<ValidatorSubscription>| {
-            network_chan
+            ctx.network_chan
                 .send(NetworkMessage::Subscribe { subscriptions })
                 .map_err(|e| {
                     ApiError::ServerError(format!(
@@ -82,19 +68,18 @@ pub async fn post_validator_subscriptions<T: BeaconChainTypes>(
                 })?;
             Ok(())
         })
-        .and_then(|_| response_builder?.body_no_ssz(&()))
 }
 
 /// HTTP Handler to retrieve all validator duties for the given epoch.
 pub fn get_all_validator_duties<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
 
-    let state = get_state_for_epoch(&beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
+    let state = get_state_for_epoch(&ctx.beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
 
     let validator_pubkeys = state
         .validators
@@ -102,21 +87,19 @@ pub fn get_all_validator_duties<T: BeaconChainTypes>(
         .map(|validator| validator.pubkey.clone())
         .collect();
 
-    let duties = return_validator_duties(beacon_chain, epoch, validator_pubkeys)?;
-
-    ResponseBuilder::new(&req)?.body_no_ssz(&duties)
+    return_validator_duties(&ctx.beacon_chain, epoch, validator_pubkeys)
 }
 
 /// HTTP Handler to retrieve all active validator duties for the given epoch.
 pub fn get_active_validator_duties<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let epoch = query.epoch()?;
 
-    let state = get_state_for_epoch(&beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
+    let state = get_state_for_epoch(&ctx.beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
 
     let validator_pubkeys = state
         .validators
@@ -125,9 +108,7 @@ pub fn get_active_validator_duties<T: BeaconChainTypes>(
         .map(|validator| validator.pubkey.clone())
         .collect();
 
-    let duties = return_validator_duties(beacon_chain, epoch, validator_pubkeys)?;
-
-    ResponseBuilder::new(&req)?.body_no_ssz(&duties)
+    return_validator_duties(&ctx.beacon_chain, epoch, validator_pubkeys)
 }
 
 /// Helper function to return the state that can be used to determine the duties for some `epoch`.
@@ -163,7 +144,7 @@ pub fn get_state_for_epoch<T: BeaconChainTypes>(
 
 /// Helper function to get the duties for some `validator_pubkeys` in some `epoch`.
 fn return_validator_duties<T: BeaconChainTypes>(
-    beacon_chain: Arc<BeaconChain<T>>,
+    beacon_chain: &BeaconChain<T>,
     epoch: Epoch,
     validator_pubkeys: Vec<PublicKeyBytes>,
 ) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
@@ -279,20 +260,26 @@ fn return_validator_duties<T: BeaconChainTypes>(
 
 /// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator.
 pub fn get_new_beacon_block<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    log: Logger,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<BeaconBlock<T::EthSpec>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let slot = query.slot()?;
     let randao_reveal = query.randao_reveal()?;
 
-    let (new_block, _state) = beacon_chain
-        .produce_block(randao_reveal, slot)
+    let validator_graffiti = if let Some((_key, value)) = query.first_of_opt(&["graffiti"]) {
+        Some(parse_hex_ssz_bytes(&value)?)
+    } else {
+        None
+    };
+
+    let (new_block, _state) = ctx
+        .beacon_chain
+        .produce_block(randao_reveal, slot, validator_graffiti)
         .map_err(|e| {
             error!(
-                log,
+                ctx.log,
                 "Error whilst producing block";
                 "error" => format!("{:?}", e)
             );
@@ -303,48 +290,40 @@ pub fn get_new_beacon_block<T: BeaconChainTypes>(
             ))
         })?;
 
-    ResponseBuilder::new(&req)?.body(&new_block)
+    Ok(new_block)
 }
 
 /// HTTP Handler to publish a SignedBeaconBlock, which has been signed by a validator.
-pub async fn publish_beacon_block<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    network_chan: NetworkChannel<T::EthSpec>,
-    log: Logger,
-) -> ApiResult {
-    try_future!(check_content_type_for_json(&req));
-    let response_builder = ResponseBuilder::new(&req);
-
+pub fn publish_beacon_block<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<(), ApiError> {
     let body = req.into_body();
-    let chunks = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
 
-    serde_json::from_slice(&chunks).map_err(|e| {
+    serde_json::from_slice(&body).map_err(|e| {
                     ApiError::BadRequest(format!("Unable to parse JSON into SignedBeaconBlock: {:?}", e))
                 })
             .and_then(move |block: SignedBeaconBlock<T::EthSpec>| {
                 let slot = block.slot();
-                match beacon_chain.process_block(block.clone()) {
+                match ctx.beacon_chain.process_block(block.clone()) {
                     Ok(block_root) => {
                         // Block was processed, publish via gossipsub
                         info!(
-                            log,
+                            ctx.log,
                             "Block from local validator";
                             "block_root" => format!("{}", block_root),
                             "block_slot" => slot,
                         );
 
-                        publish_beacon_block_to_network::<T>(network_chan, block)?;
+                        publish_beacon_block_to_network::<T>(&ctx.network_chan, block)?;
 
                         // Run the fork choice algorithm and enshrine a new canonical head, if
                         // found.
                         //
                         // The new head may or may not be the block we just received.
-                        if let Err(e) = beacon_chain.fork_choice() {
+                        if let Err(e) = ctx.beacon_chain.fork_choice() {
                             error!(
-                                log,
+                                ctx.log,
                                 "Failed to find beacon chain head";
                                 "error" => format!("{:?}", e)
                             );
@@ -358,9 +337,9 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
                             // - Excessive time between block produce and publish.
                             // - A validator is using another beacon node to produce blocks and
                             // submitting them here.
-                            if beacon_chain.head()?.beacon_block_root != block_root {
+                            if ctx.beacon_chain.head()?.beacon_block_root != block_root {
                                 warn!(
-                                    log,
+                                    ctx.log,
                                     "Block from validator is not head";
                                     "desc" => "potential re-org",
                                 );
@@ -372,7 +351,7 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
                     }
                     Err(BlockError::BeaconChainError(e)) => {
                         error!(
-                            log,
+                            ctx.log,
                             "Error whilst processing block";
                             "error" => format!("{:?}", e)
                         );
@@ -384,7 +363,7 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
                     }
                     Err(other) => {
                         warn!(
-                            log,
+                            ctx.log,
                             "Invalid block from local validator";
                             "outcome" => format!("{:?}", other)
                         );
@@ -396,41 +375,41 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
                     }
                 }
         })
-        .and_then(|_| response_builder?.body_no_ssz(&()))
 }
 
 /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator.
 pub fn get_new_attestation<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Attestation<T::EthSpec>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let slot = query.slot()?;
     let index = query.committee_index()?;
 
-    let attestation = beacon_chain
+    ctx.beacon_chain
         .produce_unaggregated_attestation(slot, index)
-        .map_err(|e| ApiError::BadRequest(format!("Unable to produce attestation: {:?}", e)))?;
-
-    ResponseBuilder::new(&req)?.body(&attestation)
+        .map_err(|e| ApiError::BadRequest(format!("Unable to produce attestation: {:?}", e)))
 }
 
 /// HTTP Handler to retrieve the aggregate attestation for a slot
 pub fn get_aggregate_attestation<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-) -> ApiResult {
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<Attestation<T::EthSpec>, ApiError> {
     let query = UrlQuery::from_request(&req)?;
 
     let attestation_data = query.attestation_data()?;
 
-    match beacon_chain.get_aggregated_attestation(&attestation_data) {
-        Ok(Some(attestation)) => ResponseBuilder::new(&req)?.body(&attestation),
+    match ctx
+        .beacon_chain
+        .get_aggregated_attestation(&attestation_data)
+    {
+        Ok(Some(attestation)) => Ok(attestation),
         Ok(None) => Err(ApiError::NotFound(format!(
             "No matching aggregate attestation for slot {:?} is known in slot {:?}",
             attestation_data.slot,
-            beacon_chain.slot()
+            ctx.beacon_chain.slot()
         ))),
         Err(e) => Err(ApiError::ServerError(format!(
             "Unable to obtain attestation: {:?}",
@@ -440,22 +419,13 @@ pub fn get_aggregate_attestation<T: BeaconChainTypes>(
 }
 
 /// HTTP Handler to publish a list of Attestations, which have been signed by a number of validators.
-pub async fn publish_attestations<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    network_chan: NetworkChannel<T::EthSpec>,
-    log: Logger,
-) -> ApiResult {
-    try_future!(check_content_type_for_json(&req));
-    let response_builder = ResponseBuilder::new(&req);
+pub fn publish_attestations<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<(), ApiError> {
+    let bytes = req.into_body();
 
-    let body = req.into_body();
-    let chunk = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-
-    let chunks = chunk.iter().cloned().collect::<Vec<u8>>();
-    serde_json::from_slice(&chunks.as_slice())
+    serde_json::from_slice(&bytes)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to deserialize JSON into a list of attestations: {:?}",
@@ -466,16 +436,16 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
         .map(
             move |attestations: Vec<(Attestation<T::EthSpec>, SubnetId)>| {
                 attestations
-                    .into_par_iter()
+                    .into_iter()
                     .enumerate()
                     .map(|(i, (attestation, subnet_id))| {
                         process_unaggregated_attestation(
-                            &beacon_chain,
-                            network_chan.clone(),
+                            &ctx.beacon_chain,
+                            ctx.network_chan.clone(),
                             attestation,
                             subnet_id,
                             i,
-                            &log,
+                            &ctx.log,
                         )
                     })
                     .collect::<Vec<Result<_, _>>>()
@@ -485,7 +455,7 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
         //
         // Note: this will only provide info about the _first_ failure, not all failures.
         .and_then(|processing_results| processing_results.into_iter().try_for_each(|result| result))
-        .and_then(|_| response_builder?.body_no_ssz(&()))
+        .map(|_| ())
 }
 
 /// Processes an unaggregrated attestation that was included in a list of attestations with the
@@ -558,21 +528,13 @@ fn process_unaggregated_attestation<T: BeaconChainTypes>(
 }
 
 /// HTTP Handler to publish an Attestation, which has been signed by a validator.
-#[allow(clippy::redundant_clone)] // false positives in this function.
-pub async fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
-    req: Request<Body>,
-    beacon_chain: Arc<BeaconChain<T>>,
-    network_chan: NetworkChannel<T::EthSpec>,
-    log: Logger,
-) -> ApiResult {
-    try_future!(check_content_type_for_json(&req));
-    let response_builder = ResponseBuilder::new(&req);
+pub fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
+    req: Request<Vec<u8>>,
+    ctx: Arc<Context<T>>,
+) -> Result<(), ApiError> {
     let body = req.into_body();
-    let chunk = hyper::body::to_bytes(body)
-        .await
-        .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
-    let chunks = chunk.iter().cloned().collect::<Vec<u8>>();
-    serde_json::from_slice(&chunks.as_slice())
+
+    serde_json::from_slice(&body)
         .map_err(|e| {
             ApiError::BadRequest(format!(
                 "Unable to deserialize JSON into a list of SignedAggregateAndProof: {:?}",
@@ -583,15 +545,15 @@ pub async fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
         .map(
             move |signed_aggregates: Vec<SignedAggregateAndProof<T::EthSpec>>| {
                 signed_aggregates
-                    .into_par_iter()
+                    .into_iter()
                     .enumerate()
                     .map(|(i, signed_aggregate)| {
                         process_aggregated_attestation(
-                            &beacon_chain,
-                            network_chan.clone(),
+                            &ctx.beacon_chain,
+                            ctx.network_chan.clone(),
                             signed_aggregate,
                             i,
-                            &log,
+                            &ctx.log,
                         )
                     })
                     .collect::<Vec<Result<_, _>>>()
@@ -601,7 +563,6 @@ pub async fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
         //
         // Note: this will only provide info about the _first_ failure, not all failures.
         .and_then(|processing_results| processing_results.into_iter().try_for_each(|result| result))
-        .and_then(|_| response_builder?.body_no_ssz(&()))
 }
 
 /// Processes an aggregrated attestation that was included in a list of attestations with the index

beacon_node/rest_api/tests/test.rs

@@ -460,7 +460,7 @@ fn validator_block_post() {
             remote_node
                 .http
                 .validator()
-                .produce_block(slot, randao_reveal),
+                .produce_block(slot, randao_reveal, None),
         )
         .expect("should fetch block from http api");
 
@@ -547,7 +547,7 @@ fn validator_block_get() {
             remote_node
                 .http
                 .validator()
-                .produce_block(slot, randao_reveal.clone()),
+                .produce_block(slot, randao_reveal.clone(), None),
         )
         .expect("should fetch block from http api");
 
@@ -555,7 +555,50 @@ fn validator_block_get() {
         .client
         .beacon_chain()
         .expect("client should have beacon chain")
-        .produce_block(randao_reveal, slot)
+        .produce_block(randao_reveal, slot, None)
+        .expect("should produce block");
+
+    assert_eq!(
+        block, expected_block,
+        "the block returned from the API should be as expected"
+    );
+}
+
+#[test]
+fn validator_block_get_with_graffiti() {
+    let mut env = build_env();
+
+    let spec = &E::default_spec();
+
+    let node = build_node(&mut env, testing_client_config());
+    let remote_node = node.remote_node().expect("should produce remote node");
+
+    let beacon_chain = node
+        .client
+        .beacon_chain()
+        .expect("client should have beacon chain");
+
+    let slot = Slot::new(1);
+    let randao_reveal = get_randao_reveal(beacon_chain, slot, spec);
+
+    let block = env
+        .runtime()
+        .block_on(remote_node.http.validator().produce_block(
+            slot,
+            randao_reveal.clone(),
+            Some(*b"test-graffiti-test-graffiti-test"),
+        ))
+        .expect("should fetch block from http api");
+
+    let (expected_block, _state) = node
+        .client
+        .beacon_chain()
+        .expect("client should have beacon chain")
+        .produce_block(
+            randao_reveal,
+            slot,
+            Some(*b"test-graffiti-test-graffiti-test"),
+        )
         .expect("should produce block");
 
     assert_eq!(
@@ -761,7 +804,7 @@ fn get_version() {
     let version = env
         .runtime()
         .block_on(remote_node.http.node().get_version())
-        .expect("should fetch eth2 config from http api");
+        .expect("should fetch version from http api");
 
     assert_eq!(
         lighthouse_version::version_with_platform(),

beacon_node/src/cli.rs

@@ -71,8 +71,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
             Arg::with_name("boot-nodes")
                 .long("boot-nodes")
                 .allow_hyphen_values(true)
-                .value_name("ENR-LIST")
-                .help("One or more comma-delimited base64-encoded ENR's to bootstrap the p2p network.")
+                .value_name("ENR/MULTIADDR LIST")
+                .help("One or more comma-delimited base64-encoded ENR's to bootstrap the p2p network. Multiaddr is also supported.")
                 .takes_value(true),
         )
         .arg(
@@ -249,4 +249,17 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .value_name("GRAFFITI")
                 .takes_value(true)
         )
+        .arg(
+            Arg::with_name("max-skip-slots")
+                .long("max-skip-slots")
+                .help(
+                    "Refuse to skip more than this many slots when processing a block or attestation. \
+                    This prevents nodes on minority forks from wasting our time and RAM, \
+                    but might need to be raised or set to 'none' in times of extreme network \
+                    outage."
+                )
+                .value_name("NUM_SLOTS")
+                .takes_value(true)
+                .default_value("700")
+        )
 }

beacon_node/src/config.rs

@@ -2,7 +2,7 @@ use beacon_chain::builder::PUBKEY_CACHE_FILENAME;
 use clap::ArgMatches;
 use clap_utils::BAD_TESTNET_DIR_MESSAGE;
 use client::{config::DEFAULT_DATADIR, ClientConfig, ClientGenesis};
-use eth2_libp2p::{Enr, Multiaddr};
+use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig};
 use eth2_testnet_config::Eth2TestnetConfig;
 use slog::{crit, info, Logger};
 use ssz::Encode;
@@ -75,130 +75,13 @@ pub fn get_config<E: EthSpec>(
     /*
      * Networking
      */
-    // If a network dir has been specified, override the `datadir` definition.
-    if let Some(dir) = cli_args.value_of("network-dir") {
-        client_config.network.network_dir = PathBuf::from(dir);
-    } else {
-        client_config.network.network_dir = client_config.data_dir.join(NETWORK_DIR);
-    };
-
-    if let Some(listen_address_str) = cli_args.value_of("listen-address") {
-        let listen_address = listen_address_str
-            .parse()
-            .map_err(|_| format!("Invalid listen address: {:?}", listen_address_str))?;
-        client_config.network.listen_address = listen_address;
-    }
-
-    if let Some(target_peers_str) = cli_args.value_of("target-peers") {
-        client_config.network.target_peers = target_peers_str
-            .parse::<usize>()
-            .map_err(|_| format!("Invalid number of target peers: {}", target_peers_str))?;
-    }
-
-    if let Some(port_str) = cli_args.value_of("port") {
-        let port = port_str
-            .parse::<u16>()
-            .map_err(|_| format!("Invalid port: {}", port_str))?;
-        client_config.network.libp2p_port = port;
-        client_config.network.discovery_port = port;
-    }
-
-    if let Some(port_str) = cli_args.value_of("discovery-port") {
-        let port = port_str
-            .parse::<u16>()
-            .map_err(|_| format!("Invalid port: {}", port_str))?;
-        client_config.network.discovery_port = port;
-    }
-
-    if let Some(boot_enr_str) = cli_args.value_of("boot-nodes") {
-        client_config.network.boot_nodes = boot_enr_str
-            .split(',')
-            .map(|enr| enr.parse().map_err(|_| format!("Invalid ENR: {}", enr)))
-            .collect::<Result<Vec<Enr>, _>>()?;
-    }
-
-    if let Some(libp2p_addresses_str) = cli_args.value_of("libp2p-addresses") {
-        client_config.network.libp2p_nodes = libp2p_addresses_str
-            .split(',')
-            .map(|multiaddr| {
-                multiaddr
-                    .parse()
-                    .map_err(|_| format!("Invalid Multiaddr: {}", multiaddr))
-            })
-            .collect::<Result<Vec<Multiaddr>, _>>()?;
-    }
-
-    if let Some(enr_udp_port_str) = cli_args.value_of("enr-udp-port") {
-        client_config.network.enr_udp_port = Some(
-            enr_udp_port_str
-                .parse::<u16>()
-                .map_err(|_| format!("Invalid discovery port: {}", enr_udp_port_str))?,
-        );
-    }
-
-    if let Some(enr_tcp_port_str) = cli_args.value_of("enr-tcp-port") {
-        client_config.network.enr_tcp_port = Some(
-            enr_tcp_port_str
-                .parse::<u16>()
-                .map_err(|_| format!("Invalid ENR TCP port: {}", enr_tcp_port_str))?,
-        );
-    }
-
-    if cli_args.is_present("enr-match") {
-        // set the enr address to localhost if the address is 0.0.0.0
-        if client_config.network.listen_address
-            == "0.0.0.0".parse::<IpAddr>().expect("valid ip addr")
-        {
-            client_config.network.enr_address =
-                Some("127.0.0.1".parse::<IpAddr>().expect("valid ip addr"));
-        } else {
-            client_config.network.enr_address = Some(client_config.network.listen_address);
-        }
-        client_config.network.enr_udp_port = Some(client_config.network.discovery_port);
-    }
-
-    if let Some(enr_address) = cli_args.value_of("enr-address") {
-        let resolved_addr = match enr_address.parse::<IpAddr>() {
-            Ok(addr) => addr, // // Input is an IpAddr
-            Err(_) => {
-                let mut addr = enr_address.to_string();
-                // Appending enr-port to the dns hostname to appease `to_socket_addrs()` parsing.
-                // Since enr-update is disabled with a dns address, not setting the enr-udp-port
-                // will make the node undiscoverable.
-                if let Some(enr_udp_port) = client_config.network.enr_udp_port {
-                    addr.push_str(&format!(":{}", enr_udp_port.to_string()));
-                } else {
-                    return Err(
-                        "enr-udp-port must be set for node to be discoverable with dns address"
-                            .into(),
-                    );
-                }
-                // `to_socket_addr()` does the dns resolution
-                // Note: `to_socket_addrs()` is a blocking call
-                let resolved_addr = if let Ok(mut resolved_addrs) = addr.to_socket_addrs() {
-                    // Pick the first ip from the list of resolved addresses
-                    resolved_addrs
-                        .next()
-                        .map(|a| a.ip())
-                        .ok_or_else(|| "Resolved dns addr contains no entries".to_string())?
-                } else {
-                    return Err(format!("Failed to parse enr-address: {}", enr_address));
-                };
-                client_config.network.discv5_config.enr_update = false;
-                resolved_addr
-            }
-        };
-        client_config.network.enr_address = Some(resolved_addr);
-    }
-
-    if cli_args.is_present("disable_enr_auto_update") {
-        client_config.network.discv5_config.enr_update = false;
-    }
-
-    if cli_args.is_present("disable-discovery") {
-        client_config.network.disable_discovery = true;
-        slog::warn!(log, "Discovery is disabled. New peers will not be found");
-    }
+    set_network_config(
+        &mut client_config.network,
+        cli_args,
+        &client_config.data_dir,
+        &log,
+        false,
+    )?;
 
     /*
      * Http server
@@ -337,7 +220,7 @@ pub fn get_config<E: EthSpec>(
     client_config.eth1.follow_distance = spec.eth1_follow_distance;
 
     if let Some(mut boot_nodes) = eth2_testnet_config.boot_enr {
-        client_config.network.boot_nodes.append(&mut boot_nodes)
+        client_config.network.boot_nodes_enr.append(&mut boot_nodes)
     }
 
     if let Some(genesis_state) = eth2_testnet_config.genesis_state {
@@ -368,9 +251,176 @@ pub fn get_config<E: EthSpec>(
     client_config.graffiti[..trimmed_graffiti_len]
         .copy_from_slice(&raw_graffiti[..trimmed_graffiti_len]);
 
+    if let Some(max_skip_slots) = cli_args.value_of("max-skip-slots") {
+        client_config.chain.import_max_skip_slots = match max_skip_slots {
+            "none" => None,
+            n => Some(
+                n.parse()
+                    .map_err(|_| "Invalid max-skip-slots".to_string())?,
+            ),
+        };
+    }
+
     Ok(client_config)
 }
 
+/// Sets the network config from the command line arguments
+pub fn set_network_config(
+    config: &mut NetworkConfig,
+    cli_args: &ArgMatches,
+    data_dir: &PathBuf,
+    log: &Logger,
+    use_listening_port_as_enr_port_by_default: bool,
+) -> Result<(), String> {
+    // If a network dir has been specified, override the `datadir` definition.
+    if let Some(dir) = cli_args.value_of("network-dir") {
+        config.network_dir = PathBuf::from(dir);
+    } else {
+        config.network_dir = data_dir.join(NETWORK_DIR);
+    };
+
+    if let Some(listen_address_str) = cli_args.value_of("listen-address") {
+        let listen_address = listen_address_str
+            .parse()
+            .map_err(|_| format!("Invalid listen address: {:?}", listen_address_str))?;
+        config.listen_address = listen_address;
+    }
+
+    if let Some(target_peers_str) = cli_args.value_of("target-peers") {
+        config.target_peers = target_peers_str
+            .parse::<usize>()
+            .map_err(|_| format!("Invalid number of target peers: {}", target_peers_str))?;
+    }
+
+    if let Some(port_str) = cli_args.value_of("port") {
+        let port = port_str
+            .parse::<u16>()
+            .map_err(|_| format!("Invalid port: {}", port_str))?;
+        config.libp2p_port = port;
+        config.discovery_port = port;
+    }
+
+    if let Some(port_str) = cli_args.value_of("discovery-port") {
+        let port = port_str
+            .parse::<u16>()
+            .map_err(|_| format!("Invalid port: {}", port_str))?;
+        config.discovery_port = port;
+    }
+
+    if let Some(boot_enr_str) = cli_args.value_of("boot-nodes") {
+        let mut enrs: Vec<Enr> = vec![];
+        let mut multiaddrs: Vec<Multiaddr> = vec![];
+        for addr in boot_enr_str.split(',') {
+            match addr.parse() {
+                Ok(enr) => enrs.push(enr),
+                Err(_) => {
+                    // parsing as ENR failed, try as Multiaddr
+                    let multi: Multiaddr = addr
+                        .parse()
+                        .map_err(|_| format!("Not valid as ENR nor Multiaddr: {}", addr))?;
+                    if !multi.iter().any(|proto| matches!(proto, Protocol::Udp(_))) {
+                        slog::error!(log, "Missing UDP in Multiaddr {}", multi.to_string());
+                    }
+                    if !multi.iter().any(|proto| matches!(proto, Protocol::P2p(_))) {
+                        slog::error!(log, "Missing P2P in Multiaddr {}", multi.to_string());
+                    }
+                    multiaddrs.push(multi);
+                }
+            }
+        }
+        config.boot_nodes_enr = enrs;
+        config.boot_nodes_multiaddr = multiaddrs;
+    }
+
+    if let Some(libp2p_addresses_str) = cli_args.value_of("libp2p-addresses") {
+        config.libp2p_nodes = libp2p_addresses_str
+            .split(',')
+            .map(|multiaddr| {
+                multiaddr
+                    .parse()
+                    .map_err(|_| format!("Invalid Multiaddr: {}", multiaddr))
+            })
+            .collect::<Result<Vec<Multiaddr>, _>>()?;
+    }
+
+    if let Some(enr_udp_port_str) = cli_args.value_of("enr-udp-port") {
+        config.enr_udp_port = Some(
+            enr_udp_port_str
+                .parse::<u16>()
+                .map_err(|_| format!("Invalid discovery port: {}", enr_udp_port_str))?,
+        );
+    }
+
+    if let Some(enr_tcp_port_str) = cli_args.value_of("enr-tcp-port") {
+        config.enr_tcp_port = Some(
+            enr_tcp_port_str
+                .parse::<u16>()
+                .map_err(|_| format!("Invalid ENR TCP port: {}", enr_tcp_port_str))?,
+        );
+    }
+
+    if cli_args.is_present("enr-match") {
+        // set the enr address to localhost if the address is 0.0.0.0
+        if config.listen_address == "0.0.0.0".parse::<IpAddr>().expect("valid ip addr") {
+            config.enr_address = Some("127.0.0.1".parse::<IpAddr>().expect("valid ip addr"));
+        } else {
+            config.enr_address = Some(config.listen_address);
+        }
+        config.enr_udp_port = Some(config.discovery_port);
+    }
+
+    if let Some(enr_address) = cli_args.value_of("enr-address") {
+        let resolved_addr = match enr_address.parse::<IpAddr>() {
+            Ok(addr) => addr, // // Input is an IpAddr
+            Err(_) => {
+                let mut addr = enr_address.to_string();
+                // Appending enr-port to the dns hostname to appease `to_socket_addrs()` parsing.
+                // Since enr-update is disabled with a dns address, not setting the enr-udp-port
+                // will make the node undiscoverable.
+                if let Some(enr_udp_port) = config.enr_udp_port.or_else(|| {
+                    if use_listening_port_as_enr_port_by_default {
+                        Some(config.discovery_port)
+                    } else {
+                        None
+                    }
+                }) {
+                    addr.push_str(&format!(":{}", enr_udp_port.to_string()));
+                } else {
+                    return Err(
+                        "enr-udp-port must be set for node to be discoverable with dns address"
+                            .into(),
+                    );
+                }
+                // `to_socket_addr()` does the dns resolution
+                // Note: `to_socket_addrs()` is a blocking call
+                let resolved_addr = if let Ok(mut resolved_addrs) = addr.to_socket_addrs() {
+                    // Pick the first ip from the list of resolved addresses
+                    resolved_addrs
+                        .next()
+                        .map(|a| a.ip())
+                        .ok_or_else(|| "Resolved dns addr contains no entries".to_string())?
+                } else {
+                    return Err(format!("Failed to parse enr-address: {}", enr_address));
+                };
+                config.discv5_config.enr_update = false;
+                resolved_addr
+            }
+        };
+        config.enr_address = Some(resolved_addr);
+    }
+
+    if cli_args.is_present("disable_enr_auto_update") {
+        config.discv5_config.enr_update = false;
+    }
+
+    if cli_args.is_present("disable-discovery") {
+        config.disable_discovery = true;
+        slog::warn!(log, "Discovery is disabled. New peers will not be found");
+    }
+
+    Ok(())
+}
+
 /// Gets the datadir which should be used.
 pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf {
     // Read the `--datadir` flag.

beacon_node/src/lib.rs

@@ -7,7 +7,7 @@ mod config;
 pub use beacon_chain;
 pub use cli::cli_app;
 pub use client::{Client, ClientBuilder, ClientConfig, ClientGenesis};
-pub use config::{get_data_dir, get_eth2_testnet_config};
+pub use config::{get_data_dir, get_eth2_testnet_config, set_network_config};
 pub use eth2_config::Eth2Config;
 
 use beacon_chain::events::TeeEventHandler;
@@ -126,7 +126,8 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
 
         let builder = builder
             .build_beacon_chain()?
-            .network(&client_config.network)?
+            .network(&client_config.network)
+            .await?
             .notifier()?;
 
         let builder = if client_config.rest_api.enabled {

beacon_node/store/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "store"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 

beacon_node/store/src/hot_cold_store.rs

@@ -30,6 +30,16 @@ use types::*;
 /// 32-byte key for accessing the `split` of the freezer DB.
 pub const SPLIT_DB_KEY: &str = "FREEZERDBSPLITFREEZERDBSPLITFREE";
 
+/// Defines how blocks should be replayed on states.
+#[derive(PartialEq)]
+pub enum BlockReplay {
+    /// Perform all transitions faithfully to the specification.
+    Accurate,
+    /// Don't compute state roots, eventually computing an invalid beacon state that can only be
+    /// used for obtaining shuffling.
+    InconsistentStateRoots,
+}
+
 /// On-disk database that stores finalized states efficiently.
 ///
 /// Stores vector fields like the `block_roots` and `state_roots` separately, and only stores
@@ -208,6 +218,13 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
     }
 
     /// Fetch a state from the store.
+    ///
+    /// If `slot` is provided then it will be used as a hint as to which database should
+    /// be checked. Importantly, if the slot hint is provided and indicates a slot that lies
+    /// in the freezer database, then only the freezer database will be accessed and `Ok(None)`
+    /// will be returned if the provided `state_root` doesn't match the state root of the
+    /// frozen state at `slot`. Consequently, if a state from a non-canonical chain is desired, it's
+    /// best to set `slot` to `None`, or call `load_hot_state` directly.
     pub fn get_state(
         &self,
         state_root: &Hash256,
@@ -217,18 +234,46 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
 
         if let Some(slot) = slot {
             if slot < self.get_split_slot() {
-                self.load_cold_state_by_slot(slot).map(Some)
+                // Although we could avoid a DB lookup by shooting straight for the
+                // frozen state using `load_cold_state_by_slot`, that would be incorrect
+                // in the case where the caller provides a `state_root` that's off the canonical
+                // chain. This way we avoid returning a state that doesn't match `state_root`.
+                self.load_cold_state(state_root)
             } else {
-                self.load_hot_state(state_root)
+                self.load_hot_state(state_root, BlockReplay::Accurate)
             }
         } else {
-            match self.load_hot_state(state_root)? {
+            match self.load_hot_state(state_root, BlockReplay::Accurate)? {
                 Some(state) => Ok(Some(state)),
                 None => self.load_cold_state(state_root),
             }
         }
     }
 
+    /// Fetch a state from the store, but don't compute all of the values when replaying blocks
+    /// upon that state (e.g., state roots). Additionally, only states from the hot store are
+    /// returned.
+    ///
+    /// See `Self::get_state` for information about `slot`.
+    ///
+    /// ## Warning
+    ///
+    /// The returned state **is not a valid beacon state**, it can only be used for obtaining
+    /// shuffling to process attestations.
+    pub fn get_inconsistent_state_for_attestation_verification_only(
+        &self,
+        state_root: &Hash256,
+        slot: Option<Slot>,
+    ) -> Result<Option<BeaconState<E>>, Error> {
+        metrics::inc_counter(&metrics::BEACON_STATE_GET_COUNT);
+
+        if slot.map_or(false, |slot| slot < self.get_split_slot()) {
+            Ok(None)
+        } else {
+            self.load_hot_state(state_root, BlockReplay::InconsistentStateRoots)
+        }
+    }
+
     /// Delete a state, ensuring it is removed from the LRU cache, as well as from on-disk.
     ///
     /// It is assumed that all states being deleted reside in the hot DB, even if their slot is less
@@ -272,8 +317,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
         }) = self.load_hot_state_summary(state_root)?
         {
             // NOTE: minor inefficiency here because we load an unnecessary hot state summary
+            //
+            // `BlockReplay` should be irrelevant here since we never replay blocks for an epoch
+            // boundary state in the hot DB.
             let state = self
-                .load_hot_state(&epoch_boundary_state_root)?
+                .load_hot_state(&epoch_boundary_state_root, BlockReplay::Accurate)?
                 .ok_or_else(|| {
                     HotColdDBError::MissingEpochBoundaryState(epoch_boundary_state_root)
                 })?;
@@ -404,7 +452,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
     /// Load a post-finalization state from the hot database.
     ///
     /// Will replay blocks from the nearest epoch boundary.
-    pub fn load_hot_state(&self, state_root: &Hash256) -> Result<Option<BeaconState<E>>, Error> {
+    pub fn load_hot_state(
+        &self,
+        state_root: &Hash256,
+        block_replay: BlockReplay,
+    ) -> Result<Option<BeaconState<E>>, Error> {
         metrics::inc_counter(&metrics::BEACON_STATE_HOT_GET_COUNT);
 
         if let Some(HotStateSummary {
@@ -425,7 +477,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
             } else {
                 let blocks =
                     self.load_blocks_to_replay(boundary_state.slot, slot, latest_block_root)?;
-                self.replay_blocks(boundary_state, blocks, slot)?
+                self.replay_blocks(boundary_state, blocks, slot, block_replay)?
             };
 
             Ok(Some(state))
@@ -556,7 +608,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
         )?;
 
         // 3. Replay the blocks on top of the low restore point.
-        self.replay_blocks(low_restore_point, blocks, slot)
+        self.replay_blocks(low_restore_point, blocks, slot, BlockReplay::Accurate)
     }
 
     /// Get a suitable block root for backtracking from `high_restore_point` to the state at `slot`.
@@ -613,9 +665,19 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
     fn replay_blocks(
         &self,
         mut state: BeaconState<E>,
-        blocks: Vec<SignedBeaconBlock<E>>,
+        mut blocks: Vec<SignedBeaconBlock<E>>,
         target_slot: Slot,
+        block_replay: BlockReplay,
     ) -> Result<BeaconState<E>, Error> {
+        if block_replay == BlockReplay::InconsistentStateRoots {
+            for i in 0..blocks.len() {
+                blocks[i].message.state_root = Hash256::zero();
+                if i > 0 {
+                    blocks[i].message.parent_root = blocks[i - 1].canonical_root()
+                }
+            }
+        }
+
         let state_root_from_prev_block = |i: usize, state: &BeaconState<E>| {
             if i > 0 {
                 let prev_block = &blocks[i - 1].message;
@@ -635,10 +697,14 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
             }
 
             while state.slot < block.message.slot {
-                let state_root = state_root_from_prev_block(i, &state);
+                let state_root = match block_replay {
+                    BlockReplay::Accurate => state_root_from_prev_block(i, &state),
+                    BlockReplay::InconsistentStateRoots => Some(Hash256::zero()),
+                };
                 per_slot_processing(&mut state, state_root, &self.spec)
                     .map_err(HotColdDBError::BlockReplaySlotError)?;
             }
+
             per_block_processing(
                 &mut state,
                 &block,
@@ -650,7 +716,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
         }
 
         while state.slot < target_slot {
-            let state_root = state_root_from_prev_block(blocks.len(), &state);
+            let state_root = match block_replay {
+                BlockReplay::Accurate => state_root_from_prev_block(blocks.len(), &state),
+                BlockReplay::InconsistentStateRoots => Some(Hash256::zero()),
+            };
             per_slot_processing(&mut state, state_root, &self.spec)
                 .map_err(HotColdDBError::BlockReplaySlotError)?;
         }

beacon_node/timer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "timer"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 

beacon_node/websocket_server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "websocket_server"
-version = "0.1.2"
+version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 

book/src/SUMMARY.md

@@ -7,7 +7,8 @@
 * [Installation](./installation.md)
     * [Docker](./docker.md)
     * [Raspberry Pi 4](./pi.md)
-* [Key Management](./key-managment.md)
+    * [Cross-Compiling](./cross-compiling.md)
+* [Key Management](./key-management.md)
     * [Create a wallet](./wallet-create.md)
     * [Create a validator](./validator-create.md)
 * [Validator Management](./validator-management.md)

book/src/become-a-validator-source.md

@@ -3,7 +3,9 @@
 ## 0. Install Rust
 If you don't have Rust installed already, visit [rustup.rs](https://rustup.rs/) to install it.
 
-> Note:  if you're not familiar with Rust or you'd like more detailed instructions, see our  [installation guide](./installation.md).
+> Notes:
+>   - If you're not familiar with Rust or you'd like more detailed instructions, see our  [installation guide](./installation.md).
+>   - Windows is presently only supported via [WSL](https://docs.microsoft.com/en-us/windows/wsl/about).
 
 
 ## 1. Download and install Lighthouse

book/src/cross-compiling.md

@@ -0,0 +1,41 @@
+# Cross-compiling
+
+Lighthouse supports cross-compiling, allowing users to run a binary on one
+platform (e.g., `aarch64`) that was compiled on another platform (e.g.,
+`x86_64`).
+
+
+## Instructions
+
+Cross-compiling requires [`Docker`](https://docs.docker.com/engine/install/),
+[`rustembedded/cross`](https://github.com/rust-embedded/cross) and for the
+current user to be in the `docker` group.
+
+The binaries will be created in the `target/` directory of the Lighthouse
+project.
+
+### Targets
+
+The `Makefile` in the project contains four targets for cross-compiling:
+
+- `build-x86_64`: builds an optimized version for x86_64 processors (suitable
+	for most users).
+- `build-x86_64-portable`: builds a version x86_64 processors which avoids
+	using some modern CPU instructions that might cause an "illegal
+	instruction" error on older CPUs.
+- `build-aarch64`: builds an optimized version for 64bit ARM processors
+	(suitable for Raspberry Pi 4).
+- `build-aarch64-portable`: builds a version 64 bit ARM processors which avoids
+	using some modern CPU instructions that might cause an "illegal
+	instruction" error on older CPUs.
+
+
+### Example
+
+```bash
+cd lighthouse
+make build-aarch64
+```
+
+The `lighthouse` binary will be compiled inside a Docker container and placed
+in `lighthouse/target/aarch64-unknown-linux-gnu/release`.