Move the BeaconProcessor into a new crate (#4435)

*Replaces #4434. It is identical, but this PR has a smaller diff due to a curated commit history.*

## Issue Addressed

NA

## Proposed Changes

This PR moves the scheduling logic for the `BeaconProcessor` into a new crate in `beacon_node/beacon_processor`. Previously it existed in the `beacon_node/network` crate.

This addresses a circular-dependency problem where it's not possible to use the `BeaconProcessor` from the `beacon_chain` crate. The `network` crate depends on the `beacon_chain` crate (`network -> beacon_chain`), but importing the `BeaconProcessor` into the `beacon_chain` crate would create a circular dependancy of `beacon_chain -> network`.

The `BeaconProcessor` was designed to provide queuing and prioritized scheduling for messages from the network. It has proven to be quite valuable and I believe we'd make Lighthouse more stable and effective by using it elsewhere. In particular, I think we should use the `BeaconProcessor` for:

1. HTTP API requests.
1. Scheduled tasks in the `BeaconChain` (e.g., state advance).

Using the `BeaconProcessor` for these tasks would help prevent the BN from becoming overwhelmed and would also help it to prioritize operations (e.g., choosing to process blocks from gossip before responding to low-priority HTTP API requests).

## Additional Info

This PR is intended to have zero impact on runtime behaviour. It aims to simply separate the *scheduling* code (i.e., the `BeaconProcessor`) from the *business logic* in the `network` crate (i.e., the `Worker` impls). Future PRs (see #4462) can build upon these works to actually use the `BeaconProcessor` for more operations.

I've gone to some effort to use `git mv` to make the diff look more like "file was moved and modified" rather than "file was deleted and a new one added". This should reduce review burden and help maintain commit attribution.

beacon_node/network/src/sync/block_lookups/mod.rs

@@ -2,6 +2,7 @@ use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::time::Duration;
 
+use crate::network_beacon_processor::ChainSegmentProcessId;
 use beacon_chain::{BeaconChainTypes, BlockError};
 use fnv::FnvHashMap;
 use lighthouse_network::{PeerAction, PeerId};
@@ -11,7 +12,6 @@ use smallvec::SmallVec;
 use std::sync::Arc;
 use store::{Hash256, SignedBeaconBlock};
 
-use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent};
 use crate::metrics;
 
 use self::parent_lookup::PARENT_FAIL_TOLERANCE;
@@ -542,8 +542,8 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
             BlockProcessResult::Ok
             | BlockProcessResult::Err(BlockError::BlockIsAlreadyKnown { .. }) => {
                 // Check if the beacon processor is available
-                let beacon_processor_send = match cx.processor_channel_if_enabled() {
-                    Some(channel) => channel,
+                let beacon_processor = match cx.beacon_processor_if_enabled() {
+                    Some(beacon_processor) => beacon_processor,
                     None => {
                         return trace!(
                             self.log,
@@ -555,7 +555,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
                 let (chain_hash, blocks, hashes, request) = parent_lookup.parts_for_processing();
                 let process_id = ChainSegmentProcessId::ParentLookup(chain_hash);
 
-                match beacon_processor_send.try_send(WorkEvent::chain_segment(process_id, blocks)) {
+                match beacon_processor.send_chain_segment(process_id, blocks) {
                     Ok(_) => {
                         self.processing_parent_lookups
                             .insert(chain_hash, (hashes, request));
@@ -664,11 +664,15 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
         process_type: BlockProcessType,
         cx: &mut SyncNetworkContext<T>,
     ) -> Result<(), ()> {
-        match cx.processor_channel_if_enabled() {
-            Some(beacon_processor_send) => {
+        match cx.beacon_processor_if_enabled() {
+            Some(beacon_processor) => {
                 trace!(self.log, "Sending block for processing"; "block" => ?block_root, "process" => ?process_type);
-                let event = WorkEvent::rpc_beacon_block(block_root, block, duration, process_type);
-                if let Err(e) = beacon_processor_send.try_send(event) {
+                if let Err(e) = beacon_processor.send_rpc_beacon_block(
+                    block_root,
+                    block,
+                    duration,
+                    process_type,
+                ) {
                     error!(
                         self.log,
                         "Failed to send sync block to processor";

beacon_node/network/src/sync/block_lookups/tests.rs

@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use crate::beacon_processor::BeaconProcessorSend;
+use crate::network_beacon_processor::NetworkBeaconProcessor;
 use crate::service::RequestId;
 use crate::sync::manager::RequestId as SyncId;
 use crate::NetworkMessage;
@@ -9,18 +9,19 @@ use super::*;
 
 use beacon_chain::builder::Witness;
 use beacon_chain::eth1_chain::CachingEth1Backend;
+use beacon_processor::WorkEvent;
 use lighthouse_network::{NetworkGlobals, Request};
 use slog::{Drain, Level};
-use slot_clock::SystemTimeSlotClock;
+use slot_clock::ManualSlotClock;
 use store::MemoryStore;
 use tokio::sync::mpsc;
 use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
 use types::MinimalEthSpec as E;
 
-type T = Witness<SystemTimeSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
+type T = Witness<ManualSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
 
 struct TestRig {
-    beacon_processor_rx: mpsc::Receiver<WorkEvent<T>>,
+    beacon_processor_rx: mpsc::Receiver<WorkEvent<E>>,
     network_rx: mpsc::UnboundedReceiver<NetworkMessage<E>>,
     rng: XorShiftRng,
 }
@@ -41,8 +42,10 @@ impl TestRig {
             }
         };
 
-        let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(100);
         let (network_tx, network_rx) = mpsc::unbounded_channel();
+        let globals = Arc::new(NetworkGlobals::new_test_globals(Vec::new(), &log));
+        let (network_beacon_processor, beacon_processor_rx) =
+            NetworkBeaconProcessor::null_for_testing(globals);
         let rng = XorShiftRng::from_seed([42; 16]);
         let rig = TestRig {
             beacon_processor_rx,
@@ -51,11 +54,9 @@ impl TestRig {
         };
         let bl = BlockLookups::new(log.new(slog::o!("component" => "block_lookups")));
         let cx = {
-            let globals = Arc::new(NetworkGlobals::new_test_globals(Vec::new(), &log));
             SyncNetworkContext::new(
                 network_tx,
-                globals,
-                BeaconProcessorSend(beacon_processor_tx),
+                Arc::new(network_beacon_processor),
                 log.new(slog::o!("component" => "network_context")),
             )
         };
@@ -102,7 +103,7 @@ impl TestRig {
     fn expect_block_process(&mut self) {
         match self.beacon_processor_rx.try_recv() {
             Ok(work) => {
-                assert_eq!(work.work_type(), crate::beacon_processor::RPC_BLOCK);
+                assert_eq!(work.work_type(), beacon_processor::RPC_BLOCK);
             }
             other => panic!("Expected block process, found {:?}", other),
         }
@@ -112,7 +113,7 @@ impl TestRig {
     fn expect_parent_chain_process(&mut self) {
         match self.beacon_processor_rx.try_recv() {
             Ok(work) => {
-                assert_eq!(work.work_type(), crate::beacon_processor::CHAIN_SEGMENT);
+                assert_eq!(work.work_type(), beacon_processor::CHAIN_SEGMENT);
             }
             other => panic!("Expected chain segment process, found {:?}", other),
         }