Merge branch 'unstable' of https://github.com/sigp/lighthouse into merge-unstable-deneb-aug-9

beacon_node/http_api/Cargo.toml

@@ -3,12 +3,12 @@ name = "http_api"
 version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2021"
-autotests = false # using a single test binary compiles faster
+autotests = false                               # using a single test binary compiles faster
 
 [dependencies]
 warp = { version = "0.3.2", features = ["tls"] }
 serde = { version = "1.0.116", features = ["derive"] }
-tokio = { version = "1.14.0", features = ["macros","sync"] }
+tokio = { version = "1.14.0", features = ["macros", "sync"] }
 tokio-stream = { version = "0.1.3", features = ["sync"] }
 types = { path = "../../consensus/types" }
 hex = "0.4.2"
@@ -27,9 +27,9 @@ slot_clock = { path = "../../common/slot_clock" }
 ethereum_ssz = "0.5.0"
 bs58 = "0.4.0"
 futures = "0.3.8"
-execution_layer = {path = "../execution_layer"}
+execution_layer = { path = "../execution_layer" }
 parking_lot = "0.12.0"
-safe_arith = {path = "../../consensus/safe_arith"}
+safe_arith = { path = "../../consensus/safe_arith" }
 task_executor = { path = "../../common/task_executor" }
 lru = "0.7.7"
 tree_hash = "0.5.2"
@@ -40,9 +40,10 @@ logging = { path = "../../common/logging" }
 ethereum_serde_utils = "0.5.0"
 operation_pool = { path = "../operation_pool" }
 sensitive_url = { path = "../../common/sensitive_url" }
-unused_port = {path = "../../common/unused_port"}
+unused_port = { path = "../../common/unused_port" }
 store = { path = "../store" }
 bytes = "1.1.0"
+beacon_processor = { path = "../beacon_processor" }
 
 [dev-dependencies]
 environment = { path = "../../lighthouse/environment" }
@@ -52,4 +53,4 @@ genesis = { path = "../genesis" }
 
 [[test]]
 name = "bn_http_api_tests"
-path = "tests/main.rs"
+path = "tests/main.rs"

beacon_node/http_api/src/task_spawner.rs

@@ -0,0 +1,214 @@
+use beacon_processor::{BeaconProcessorSend, BlockingOrAsync, Work, WorkEvent};
+use serde::Serialize;
+use std::future::Future;
+use tokio::sync::{mpsc::error::TrySendError, oneshot};
+use types::EthSpec;
+use warp::reply::{Reply, Response};
+
+/// Maps a request to a queue in the `BeaconProcessor`.
+#[derive(Clone, Copy)]
+pub enum Priority {
+    /// The highest priority.
+    P0,
+    /// The lowest priority.
+    P1,
+}
+
+impl Priority {
+    /// Wrap `self` in a `WorkEvent` with an appropriate priority.
+    fn work_event<E: EthSpec>(&self, process_fn: BlockingOrAsync) -> WorkEvent<E> {
+        let work = match self {
+            Priority::P0 => Work::ApiRequestP0(process_fn),
+            Priority::P1 => Work::ApiRequestP1(process_fn),
+        };
+        WorkEvent {
+            drop_during_sync: false,
+            work,
+        }
+    }
+}
+
+/// Spawns tasks on the `BeaconProcessor` or directly on the tokio executor.
+pub struct TaskSpawner<E: EthSpec> {
+    /// Used to send tasks to the `BeaconProcessor`. The tokio executor will be
+    /// used if this is `None`.
+    beacon_processor_send: Option<BeaconProcessorSend<E>>,
+}
+
+impl<E: EthSpec> TaskSpawner<E> {
+    pub fn new(beacon_processor_send: Option<BeaconProcessorSend<E>>) -> Self {
+        Self {
+            beacon_processor_send,
+        }
+    }
+
+    /// Executes a "blocking" (non-async) task which returns a `Response`.
+    pub async fn blocking_response_task<F, T>(
+        self,
+        priority: Priority,
+        func: F,
+    ) -> Result<Response, warp::Rejection>
+    where
+        F: FnOnce() -> Result<T, warp::Rejection> + Send + Sync + 'static,
+        T: Reply + Send + 'static,
+    {
+        if let Some(beacon_processor_send) = &self.beacon_processor_send {
+            // Create a closure that will execute `func` and send the result to
+            // a channel held by this thread.
+            let (tx, rx) = oneshot::channel();
+            let process_fn = move || {
+                // Execute the function, collect the return value.
+                let func_result = func();
+                // Send the result down the channel. Ignore any failures; the
+                // send can only fail if the receiver is dropped.
+                let _ = tx.send(func_result);
+            };
+
+            // Send the function to the beacon processor for execution at some arbitrary time.
+            match send_to_beacon_processor(
+                beacon_processor_send,
+                priority,
+                BlockingOrAsync::Blocking(Box::new(process_fn)),
+                rx,
+            )
+            .await
+            {
+                Ok(result) => result.map(Reply::into_response),
+                Err(error_response) => Ok(error_response),
+            }
+        } else {
+            // There is no beacon processor so spawn a task directly on the
+            // tokio executor.
+            warp_utils::task::blocking_response_task(func).await
+        }
+    }
+
+    /// Executes a "blocking" (non-async) task which returns a JSON-serializable
+    /// object.
+    pub async fn blocking_json_task<F, T>(
+        self,
+        priority: Priority,
+        func: F,
+    ) -> Result<Response, warp::Rejection>
+    where
+        F: FnOnce() -> Result<T, warp::Rejection> + Send + Sync + 'static,
+        T: Serialize + Send + 'static,
+    {
+        let func = || func().map(|t| warp::reply::json(&t).into_response());
+        self.blocking_response_task(priority, func).await
+    }
+
+    /// Executes an async task which may return a `warp::Rejection`.
+    pub async fn spawn_async_with_rejection(
+        self,
+        priority: Priority,
+        func: impl Future<Output = Result<Response, warp::Rejection>> + Send + Sync + 'static,
+    ) -> Result<Response, warp::Rejection> {
+        if let Some(beacon_processor_send) = &self.beacon_processor_send {
+            // Create a wrapper future that will execute `func` and send the
+            // result to a channel held by this thread.
+            let (tx, rx) = oneshot::channel();
+            let process_fn = async move {
+                // Await the future, collect the return value.
+                let func_result = func.await;
+                // Send the result down the channel. Ignore any failures; the
+                // send can only fail if the receiver is dropped.
+                let _ = tx.send(func_result);
+            };
+
+            // Send the function to the beacon processor for execution at some arbitrary time.
+            send_to_beacon_processor(
+                beacon_processor_send,
+                priority,
+                BlockingOrAsync::Async(Box::pin(process_fn)),
+                rx,
+            )
+            .await
+            .unwrap_or_else(Result::Ok)
+        } else {
+            // There is no beacon processor so spawn a task directly on the
+            // tokio executor.
+            tokio::task::spawn(func).await.unwrap_or_else(|e| {
+                let response = warp::reply::with_status(
+                    warp::reply::json(&format!("Tokio did not execute task: {e:?}")),
+                    eth2::StatusCode::INTERNAL_SERVER_ERROR,
+                )
+                .into_response();
+                Ok(response)
+            })
+        }
+    }
+
+    /// Executes an async task which always returns a `Response`.
+    pub async fn spawn_async(
+        self,
+        priority: Priority,
+        func: impl Future<Output = Response> + Send + Sync + 'static,
+    ) -> Response {
+        if let Some(beacon_processor_send) = &self.beacon_processor_send {
+            // Create a wrapper future that will execute `func` and send the
+            // result to a channel held by this thread.
+            let (tx, rx) = oneshot::channel();
+            let process_fn = async move {
+                // Await the future, collect the return value.
+                let func_result = func.await;
+                // Send the result down the channel. Ignore any failures; the
+                // send can only fail if the receiver is dropped.
+                let _ = tx.send(func_result);
+            };
+
+            // Send the function to the beacon processor for execution at some arbitrary time.
+            send_to_beacon_processor(
+                beacon_processor_send,
+                priority,
+                BlockingOrAsync::Async(Box::pin(process_fn)),
+                rx,
+            )
+            .await
+            .unwrap_or_else(|error_response| error_response)
+        } else {
+            // There is no beacon processor so spawn a task directly on the
+            // tokio executor.
+            tokio::task::spawn(func).await.unwrap_or_else(|e| {
+                warp::reply::with_status(
+                    warp::reply::json(&format!("Tokio did not execute task: {e:?}")),
+                    eth2::StatusCode::INTERNAL_SERVER_ERROR,
+                )
+                .into_response()
+            })
+        }
+    }
+}
+
+/// Send a task to the beacon processor and await execution.
+///
+/// If the task is not executed, return an `Err(response)` with an error message
+/// for the API consumer.
+async fn send_to_beacon_processor<E: EthSpec, T>(
+    beacon_processor_send: &BeaconProcessorSend<E>,
+    priority: Priority,
+    process_fn: BlockingOrAsync,
+    rx: oneshot::Receiver<T>,
+) -> Result<T, Response> {
+    let error_message = match beacon_processor_send.try_send(priority.work_event(process_fn)) {
+        Ok(()) => {
+            match rx.await {
+                // The beacon processor executed the task and sent a result.
+                Ok(func_result) => return Ok(func_result),
+                // The beacon processor dropped the channel without sending a
+                // result. The beacon processor dropped this task because its
+                // queues are full or it's shutting down.
+                Err(_) => "The task did not execute. The server is overloaded or shutting down.",
+            }
+        }
+        Err(TrySendError::Full(_)) => "The task was dropped. The server is overloaded.",
+        Err(TrySendError::Closed(_)) => "The task was dropped. The server is shutting down.",
+    };
+
+    let error_response = warp::reply::with_status(
+        warp::reply::json(&error_message),
+        eth2::StatusCode::INTERNAL_SERVER_ERROR,
+    )
+    .into_response();
+    Err(error_response)
+}

beacon_node/http_api/src/test_utils.rs

@@ -3,6 +3,7 @@ use beacon_chain::{
     test_utils::{BeaconChainHarness, BoxedMutator, Builder, EphemeralHarnessType},
     BeaconChain, BeaconChainTypes,
 };
+use beacon_processor::{BeaconProcessor, BeaconProcessorChannels, BeaconProcessorConfig};
 use directory::DEFAULT_ROOT_DIR;
 use eth2::{BeaconNodeHttpClient, Timeouts};
 use lighthouse_network::{
@@ -24,7 +25,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use std::sync::Arc;
 use std::time::Duration;
 use store::MemoryStore;
-use tokio::sync::oneshot;
+use task_executor::test_utils::TestRuntime;
 use types::{ChainSpec, EthSpec};
 
 pub const TCP_PORT: u16 = 42;
@@ -37,7 +38,6 @@ pub struct InteractiveTester<E: EthSpec> {
     pub harness: BeaconChainHarness<EphemeralHarnessType<E>>,
     pub client: BeaconNodeHttpClient,
     pub network_rx: NetworkReceivers<E>,
-    _server_shutdown: oneshot::Sender<()>,
 }
 
 /// The result of calling `create_api_server`.
@@ -46,7 +46,6 @@ pub struct InteractiveTester<E: EthSpec> {
 pub struct ApiServer<E: EthSpec, SFut: Future<Output = ()>> {
     pub server: SFut,
     pub listening_socket: SocketAddr,
-    pub shutdown_tx: oneshot::Sender<()>,
     pub network_rx: NetworkReceivers<E>,
     pub local_enr: Enr,
     pub external_peer_id: PeerId,
@@ -93,10 +92,14 @@ impl<E: EthSpec> InteractiveTester<E> {
         let ApiServer {
             server,
             listening_socket,
-            shutdown_tx: _server_shutdown,
             network_rx,
             ..
-        } = create_api_server(harness.chain.clone(), harness.logger().clone()).await;
+        } = create_api_server(
+            harness.chain.clone(),
+            &harness.runtime,
+            harness.logger().clone(),
+        )
+        .await;
 
         tokio::spawn(server);
 
@@ -114,22 +117,23 @@ impl<E: EthSpec> InteractiveTester<E> {
             harness,
             client,
             network_rx,
-            _server_shutdown,
         }
     }
 }
 
 pub async fn create_api_server<T: BeaconChainTypes>(
     chain: Arc<BeaconChain<T>>,
+    test_runtime: &TestRuntime,
     log: Logger,
 ) -> ApiServer<T::EthSpec, impl Future<Output = ()>> {
     // Get a random unused port.
     let port = unused_port::unused_tcp4_port().unwrap();
-    create_api_server_on_port(chain, log, port).await
+    create_api_server_on_port(chain, test_runtime, log, port).await
 }
 
 pub async fn create_api_server_on_port<T: BeaconChainTypes>(
     chain: Arc<BeaconChain<T>>,
+    test_runtime: &TestRuntime,
     log: Logger,
     port: u16,
 ) -> ApiServer<T::EthSpec, impl Future<Output = ()>> {
@@ -177,6 +181,37 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
     let eth1_service =
         eth1::Service::new(eth1::Config::default(), log.clone(), chain.spec.clone()).unwrap();
 
+    let beacon_processor_config = BeaconProcessorConfig::default();
+    let BeaconProcessorChannels {
+        beacon_processor_tx,
+        beacon_processor_rx,
+        work_reprocessing_tx,
+        work_reprocessing_rx,
+    } = BeaconProcessorChannels::new(&beacon_processor_config);
+
+    let beacon_processor_send = beacon_processor_tx;
+    BeaconProcessor {
+        network_globals: network_globals.clone(),
+        executor: test_runtime.task_executor.clone(),
+        // The number of workers must be greater than one. Tests which use the
+        // builder workflow sometimes require an internal HTTP request in order
+        // to fulfill an already in-flight HTTP request, therefore having only
+        // one worker will result in a deadlock.
+        max_workers: 2,
+        current_workers: 0,
+        config: beacon_processor_config,
+        log: log.clone(),
+    }
+    .spawn_manager(
+        beacon_processor_rx,
+        work_reprocessing_tx,
+        work_reprocessing_rx,
+        None,
+        chain.slot_clock.clone(),
+        chain.spec.maximum_gossip_clock_disparity(),
+    )
+    .unwrap();
+
     let ctx = Arc::new(Context {
         config: Config {
             enabled: true,
@@ -187,26 +222,22 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
             allow_sync_stalled: false,
             data_dir: std::path::PathBuf::from(DEFAULT_ROOT_DIR),
             spec_fork_name: None,
+            enable_beacon_processor: true,
         },
         chain: Some(chain),
         network_senders: Some(network_senders),
         network_globals: Some(network_globals),
+        beacon_processor_send: Some(beacon_processor_send),
         eth1_service: Some(eth1_service),
         sse_logging_components: None,
         log,
     });
 
-    let (shutdown_tx, shutdown_rx) = oneshot::channel();
-    let server_shutdown = async {
-        // It's not really interesting why this triggered, just that it happened.
-        let _ = shutdown_rx.await;
-    };
-    let (listening_socket, server) = crate::serve(ctx, server_shutdown).unwrap();
+    let (listening_socket, server) = crate::serve(ctx, test_runtime.task_executor.exit()).unwrap();
 
     ApiServer {
         server,
         listening_socket,
-        shutdown_tx,
         network_rx: network_receivers,
         local_enr: enr,
         external_peer_id: peer_id,

beacon_node/http_api/src/validator.rs

@@ -0,0 +1,21 @@
+use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use types::*;
+
+/// Uses the `chain.validator_pubkey_cache` to resolve a pubkey to a validator
+/// index and then ensures that the validator exists in the given `state`.
+pub fn pubkey_to_validator_index<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    state: &BeaconState<T::EthSpec>,
+    pubkey: &PublicKeyBytes,
+) -> Result<Option<usize>, BeaconChainError> {
+    chain
+        .validator_index(pubkey)?
+        .filter(|&index| {
+            state
+                .validators()
+                .get(index)
+                .map_or(false, |v| v.pubkey == *pubkey)
+        })
+        .map(Result::Ok)
+        .transpose()
+}

beacon_node/http_api/tests/tests.rs

@@ -30,7 +30,6 @@ use state_processing::per_block_processing::get_expected_withdrawals;
 use state_processing::per_slot_processing;
 use std::convert::TryInto;
 use std::sync::Arc;
-use tokio::sync::oneshot;
 use tokio::time::Duration;
 use tree_hash::TreeHash;
 use types::application_domain::ApplicationDomain;
@@ -70,7 +69,6 @@ struct ApiTester {
     attester_slashing: AttesterSlashing<E>,
     proposer_slashing: ProposerSlashing,
     voluntary_exit: SignedVoluntaryExit,
-    _server_shutdown: oneshot::Sender<()>,
     network_rx: NetworkReceivers<E>,
     local_enr: Enr,
     external_peer_id: PeerId,
@@ -236,11 +234,10 @@ impl ApiTester {
         let ApiServer {
             server,
             listening_socket: _,
-            shutdown_tx,
             network_rx,
             local_enr,
             external_peer_id,
-        } = create_api_server_on_port(chain.clone(), log, port).await;
+        } = create_api_server_on_port(chain.clone(), &harness.runtime, log, port).await;
 
         harness.runtime.task_executor.spawn(server, "api_server");
 
@@ -268,7 +265,6 @@ impl ApiTester {
             attester_slashing,
             proposer_slashing,
             voluntary_exit,
-            _server_shutdown: shutdown_tx,
             network_rx,
             local_enr,
             external_peer_id,
@@ -324,11 +320,10 @@ impl ApiTester {
         let ApiServer {
             server,
             listening_socket,
-            shutdown_tx,
             network_rx,
             local_enr,
             external_peer_id,
-        } = create_api_server(chain.clone(), log).await;
+        } = create_api_server(chain.clone(), &harness.runtime, log).await;
 
         harness.runtime.task_executor.spawn(server, "api_server");
 
@@ -353,7 +348,6 @@ impl ApiTester {
             attester_slashing,
             proposer_slashing,
             voluntary_exit,
-            _server_shutdown: shutdown_tx,
             network_rx,
             local_enr,
             external_peer_id,