Remove reprocess channel (#7437)

Partially https://github.com/sigp/lighthouse/issues/6291


  This PR removes the reprocess event channel from being externally exposed. All work events are now sent through the single `BeaconProcessorSend` channel. I've introduced a new `Work::Reprocess` enum variant which we then use to schedule jobs for reprocess. I've also created a new scheduler module which will eventually house the different scheduler impls.

This is all needed as an initial step to generalize the beacon processor

A "full" implementation for the generalized beacon processor can be found here
https://github.com/sigp/lighthouse/pull/6448

I'm going to try to break up the full implementation into smaller PR's so it can actually be reviewed

beacon_node/http_api/src/lib.rs

@@ -40,7 +40,7 @@ use beacon_chain::{
     validator_monitor::timestamp_now, AttestationError as AttnError, BeaconChain, BeaconChainError,
     BeaconChainTypes, WhenSlotSkipped,
 };
-use beacon_processor::{work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend};
+use beacon_processor::BeaconProcessorSend;
 pub use block_id::BlockId;
 use builder_states::get_next_withdrawals;
 use bytes::Bytes;
@@ -130,7 +130,6 @@ pub struct Context<T: BeaconChainTypes> {
     pub network_senders: Option<NetworkSenders<T::EthSpec>>,
     pub network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
     pub beacon_processor_send: Option<BeaconProcessorSend<T::EthSpec>>,
-    pub beacon_processor_reprocess_send: Option<Sender<ReprocessQueueMessage>>,
     pub eth1_service: Option<eth1::Service>,
     pub sse_logging_components: Option<SSELoggingComponents>,
 }
@@ -554,11 +553,6 @@ pub fn serve<T: BeaconChainTypes>(
         .filter(|_| config.enable_beacon_processor);
     let task_spawner_filter =
         warp::any().map(move || TaskSpawner::new(beacon_processor_send.clone()));
-    let beacon_processor_reprocess_send = ctx
-        .beacon_processor_reprocess_send
-        .clone()
-        .filter(|_| config.enable_beacon_processor);
-    let reprocess_send_filter = warp::any().map(move || beacon_processor_reprocess_send.clone());
 
     let duplicate_block_status_code = ctx.config.duplicate_block_status_code;
 
@@ -1986,20 +1980,18 @@ pub fn serve<T: BeaconChainTypes>(
         .and(warp_utils::json::json::<Vec<SingleAttestation>>())
         .and(optional_consensus_version_header_filter)
         .and(network_tx_filter.clone())
-        .and(reprocess_send_filter.clone())
         .then(
             |task_spawner: TaskSpawner<T::EthSpec>,
              chain: Arc<BeaconChain<T>>,
              attestations: Vec<SingleAttestation>,
              _fork_name: Option<ForkName>,
-             network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>,
-             reprocess_tx: Option<Sender<ReprocessQueueMessage>>| async move {
+             network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>| async move {
                 let result = crate::publish_attestations::publish_attestations(
                     task_spawner,
                     chain,
                     attestations,
                     network_tx,
-                    reprocess_tx,
+                    true,
                 )
                 .await
                 .map(|()| warp::reply::json(&()));

beacon_node/http_api/src/publish_attestations.rs

@@ -40,15 +40,13 @@ use beacon_chain::{
     BeaconChainTypes,
 };
 use beacon_processor::work_reprocessing_queue::{QueuedUnaggregate, ReprocessQueueMessage};
+use beacon_processor::{Work, WorkEvent};
 use eth2::types::Failure;
 use lighthouse_network::PubsubMessage;
 use network::NetworkMessage;
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::{
-    mpsc::{Sender, UnboundedSender},
-    oneshot,
-};
+use tokio::sync::{mpsc::UnboundedSender, oneshot};
 use tracing::{debug, error, warn};
 use types::SingleAttestation;
 
@@ -130,7 +128,7 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
     chain: Arc<BeaconChain<T>>,
     attestations: Vec<SingleAttestation>,
     network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>,
-    reprocess_send: Option<Sender<ReprocessQueueMessage>>,
+    allow_reprocess: bool,
 ) -> Result<(), warp::Rejection> {
     // Collect metadata about attestations which we'll use to report failures. We need to
     // move the `attestations` vec into the blocking task, so this small overhead is unavoidable.
@@ -142,6 +140,7 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
     // Gossip validate and publish attestations that can be immediately processed.
     let seen_timestamp = timestamp_now();
     let mut prelim_results = task_spawner
+        .clone()
         .blocking_task(Priority::P0, move || {
             Ok(attestations
                 .into_iter()
@@ -156,7 +155,7 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
                         Err(Error::Validation(AttestationError::UnknownHeadBlock {
                             beacon_block_root,
                         })) => {
-                            let Some(reprocess_tx) = &reprocess_send else {
+                            if !allow_reprocess {
                                 return PublishAttestationResult::Failure(Error::ReprocessDisabled);
                             };
                             // Re-process.
@@ -180,7 +179,13 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
                                     beacon_block_root,
                                     process_fn: Box::new(reprocess_fn),
                                 });
-                            if reprocess_tx.try_send(reprocess_msg).is_err() {
+                            if task_spawner
+                                .try_send(WorkEvent {
+                                    drop_during_sync: false,
+                                    work: Work::Reprocess(reprocess_msg),
+                                })
+                                .is_err()
+                            {
                                 PublishAttestationResult::Failure(Error::ReprocessFull)
                             } else {
                                 PublishAttestationResult::Reprocessing(rx)

beacon_node/http_api/src/task_spawner.rs

@@ -30,6 +30,7 @@ impl Priority {
 }
 
 /// Spawns tasks on the `BeaconProcessor` or directly on the tokio executor.
+#[derive(Clone)]
 pub struct TaskSpawner<E: EthSpec> {
     /// Used to send tasks to the `BeaconProcessor`. The tokio executor will be
     /// used if this is `None`.
@@ -155,6 +156,32 @@ impl<E: EthSpec> TaskSpawner<E> {
                 .and_then(|x| x)
         }
     }
+
+    pub fn try_send(&self, work_event: WorkEvent<E>) -> Result<(), warp::Rejection> {
+        if let Some(beacon_processor_send) = &self.beacon_processor_send {
+            let error_message = match beacon_processor_send.try_send(work_event) {
+                Ok(()) => None,
+                Err(TrySendError::Full(_)) => {
+                    Some("The task was dropped. The server is overloaded.")
+                }
+                Err(TrySendError::Closed(_)) => {
+                    Some("The task was dropped. The server is shutting down.")
+                }
+            };
+
+            if let Some(error_message) = error_message {
+                return Err(warp_utils::reject::custom_server_error(
+                    error_message.to_string(),
+                ));
+            };
+
+            Ok(())
+        } else {
+            Err(warp_utils::reject::custom_server_error(
+                "The beacon processor is unavailable".to_string(),
+            ))
+        }
+    }
 }
 
 /// Send a task to the beacon processor and await execution.

beacon_node/http_api/src/test_utils.rs

@@ -201,12 +201,9 @@ pub async fn create_api_server_with_config<T: BeaconChainTypes>(
     let BeaconProcessorChannels {
         beacon_processor_tx,
         beacon_processor_rx,
-        work_reprocessing_tx,
-        work_reprocessing_rx,
     } = BeaconProcessorChannels::new(&beacon_processor_config);
 
     let beacon_processor_send = beacon_processor_tx;
-    let reprocess_send = work_reprocessing_tx.clone();
     BeaconProcessor {
         network_globals: network_globals.clone(),
         executor: test_runtime.task_executor.clone(),
@@ -215,8 +212,6 @@ pub async fn create_api_server_with_config<T: BeaconChainTypes>(
     }
     .spawn_manager(
         beacon_processor_rx,
-        work_reprocessing_tx,
-        work_reprocessing_rx,
         None,
         chain.slot_clock.clone(),
         chain.spec.maximum_gossip_clock_disparity(),
@@ -241,7 +236,6 @@ pub async fn create_api_server_with_config<T: BeaconChainTypes>(
         network_senders: Some(network_senders),
         network_globals: Some(network_globals),
         beacon_processor_send: Some(beacon_processor_send),
-        beacon_processor_reprocess_send: Some(reprocess_send),
         eth1_service: Some(eth1_service),
         sse_logging_components: None,
     });