Process gossip blocks on the GossipProcessor (#1523)

## Issue Addressed

NA

## Proposed Changes

Moves beacon block processing over to the newly-added `GossipProcessor`. This moves the task off the core executor onto the blocking one.

## Additional Info

- With this PR, gossip blocks are being ignored during sync.

beacon_node/network/src/sync/range_sync/chain.rs

@@ -1,11 +1,12 @@
 use super::batch::{Batch, BatchId, PendingBatches};
-use crate::sync::block_processor::{spawn_block_processor, BatchProcessResult, ProcessId};
-use crate::sync::network_context::SyncNetworkContext;
-use crate::sync::{RequestId, SyncMessage};
+use crate::beacon_processor::ProcessId;
+use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
+use crate::sync::RequestId;
+use crate::sync::{network_context::SyncNetworkContext, BatchProcessResult};
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::{PeerAction, PeerId};
 use rand::prelude::*;
-use slog::{crit, debug, warn};
+use slog::{crit, debug, error, warn};
 use std::collections::HashSet;
 use std::sync::Arc;
 use tokio::sync::mpsc;
@@ -84,9 +85,8 @@ pub struct SyncingChain<T: BeaconChainTypes> {
     /// The current processing batch, if any.
     current_processing_batch: Option<Batch<T::EthSpec>>,
 
-    /// A send channel to the sync manager. This is given to the batch processor thread to report
-    /// back once batch processing has completed.
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
 
     /// A reference to the underlying beacon chain.
     chain: Arc<BeaconChain<T>>,
@@ -111,7 +111,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         target_head_slot: Slot,
         target_head_root: Hash256,
         peer_id: PeerId,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
         chain: Arc<BeaconChain<T>>,
         log: slog::Logger,
     ) -> Self {
@@ -131,7 +131,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             to_be_processed_id: BatchId(1),
             state: ChainSyncingState::Stopped,
             current_processing_batch: None,
-            sync_send,
+            beacon_processor_send,
             chain,
             log,
         }
@@ -255,18 +255,23 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         }
     }
 
-    /// Sends a batch to the batch processor.
+    /// Sends a batch to the beacon processor for async processing in a queue.
     fn process_batch(&mut self, mut batch: Batch<T::EthSpec>) {
-        let downloaded_blocks = std::mem::replace(&mut batch.downloaded_blocks, Vec::new());
+        let blocks = std::mem::replace(&mut batch.downloaded_blocks, Vec::new());
         let process_id = ProcessId::RangeBatchId(self.id, batch.id);
         self.current_processing_batch = Some(batch);
-        spawn_block_processor(
-            Arc::downgrade(&self.chain.clone()),
-            process_id,
-            downloaded_blocks,
-            self.sync_send.clone(),
-            self.log.clone(),
-        );
+
+        if let Err(e) = self
+            .beacon_processor_send
+            .try_send(BeaconWorkEvent::chain_segment(process_id, blocks))
+        {
+            error!(
+                self.log,
+                "Failed to send chain segment to processor";
+                "msg" => "process_batch",
+                "error" => format!("{:?}", e)
+            );
+        }
     }
 
     /// The block processor has completed processing a batch. This function handles the result

beacon_node/network/src/sync/range_sync/chain_collection.rs

@@ -4,7 +4,7 @@
 //! with this struct to to simplify the logic of the other layers of sync.
 
 use super::chain::{ChainSyncingState, SyncingChain};
-use crate::sync::manager::SyncMessage;
+use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
 use crate::sync::network_context::SyncNetworkContext;
 use crate::sync::PeerSyncInfo;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
@@ -302,7 +302,7 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         target_head: Hash256,
         target_slot: Slot,
         peer_id: PeerId,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     ) {
         let chain_id = rand::random();
         self.finalized_chains.push(SyncingChain::new(
@@ -311,7 +311,7 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
             target_slot,
             target_head,
             peer_id,
-            sync_send,
+            beacon_processor_send,
             self.beacon_chain.clone(),
             self.log.clone(),
         ));
@@ -326,7 +326,7 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         target_head: Hash256,
         target_slot: Slot,
         peer_id: PeerId,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     ) {
         // remove the peer from any other head chains
 
@@ -342,7 +342,7 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
             target_slot,
             target_head,
             peer_id,
-            sync_send,
+            beacon_processor_send,
             self.beacon_chain.clone(),
             self.log.clone(),
         );

beacon_node/network/src/sync/range_sync/range.rs

@@ -43,9 +43,9 @@ use super::chain::{ChainId, ProcessingResult};
 use super::chain_collection::{ChainCollection, RangeSyncState};
 use super::sync_type::RangeSyncType;
 use super::BatchId;
-use crate::sync::block_processor::BatchProcessResult;
-use crate::sync::manager::SyncMessage;
+use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
 use crate::sync::network_context::SyncNetworkContext;
+use crate::sync::BatchProcessResult;
 use crate::sync::PeerSyncInfo;
 use crate::sync::RequestId;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
@@ -69,9 +69,8 @@ pub struct RangeSync<T: BeaconChainTypes> {
     /// finalized chain(s) complete, these peer's get STATUS'ed to update their head slot before
     /// the head chains are formed and downloaded.
     awaiting_head_peers: HashSet<PeerId>,
-    /// The sync manager channel, allowing the batch processor thread to callback the sync task
-    /// once complete.
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
+    beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
     /// The syncing logger.
     log: slog::Logger,
 }
@@ -80,14 +79,14 @@ impl<T: BeaconChainTypes> RangeSync<T> {
     pub fn new(
         beacon_chain: Arc<BeaconChain<T>>,
         network_globals: Arc<NetworkGlobals<T::EthSpec>>,
-        sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+        beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T::EthSpec>>,
         log: slog::Logger,
     ) -> Self {
         RangeSync {
             beacon_chain: beacon_chain.clone(),
             chains: ChainCollection::new(beacon_chain, network_globals, log.clone()),
             awaiting_head_peers: HashSet::new(),
-            sync_send,
+            beacon_processor_send,
             log,
         }
     }
@@ -181,7 +180,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                         remote_info.finalized_root,
                         remote_finalized_slot,
                         peer_id,
-                        self.sync_send.clone(),
+                        self.beacon_processor_send.clone(),
                     );
                     self.chains.update_finalized(network);
                     // update the global sync state
@@ -228,7 +227,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
                         remote_info.head_root,
                         remote_info.head_slot,
                         peer_id,
-                        self.sync_send.clone(),
+                        self.beacon_processor_send.clone(),
                     );
                 }
                 self.chains.update_finalized(network);