Merged with unstable

beacon_node/http_api/Cargo.toml

@@ -36,6 +36,9 @@ safe_arith = {path = "../../consensus/safe_arith"}
 task_executor = { path = "../../common/task_executor" }
 lru = "0.7.7"
 tree_hash = "0.4.1"
+sysinfo = "0.26.5"
+system_health = { path = "../../common/system_health" }
+directory = { path = "../../common/directory" }
 
 [dev-dependencies]
 store = { path = "../store" }

beacon_node/http_api/src/lib.rs

@@ -26,12 +26,14 @@ use beacon_chain::{
     BeaconChainTypes, ProduceBlockVerification, WhenSlotSkipped,
 };
 pub use block_id::BlockId;
+use directory::DEFAULT_ROOT_DIR;
 use eth2::types::{
     self as api_types, EndpointVersion, SkipRandaoVerification, ValidatorId, ValidatorStatus,
 };
 use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage};
 use lighthouse_version::version_with_platform;
 use network::{NetworkMessage, NetworkSenders, ValidatorSubscriptionMessage};
+use parking_lot::RwLock;
 use serde::{Deserialize, Serialize};
 use slog::{crit, debug, error, info, warn, Logger};
 use slot_clock::SlotClock;
@@ -43,6 +45,8 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use std::path::PathBuf;
 use std::pin::Pin;
 use std::sync::Arc;
+use sysinfo::{System, SystemExt};
+use system_health::observe_system_health_bn;
 use tokio::sync::mpsc::{Sender, UnboundedSender};
 use tokio_stream::{wrappers::BroadcastStream, StreamExt};
 use types::{
@@ -110,6 +114,7 @@ pub struct Config {
     pub tls_config: Option<TlsConfig>,
     pub allow_sync_stalled: bool,
     pub spec_fork_name: Option<ForkName>,
+    pub data_dir: PathBuf,
 }
 
 impl Default for Config {
@@ -122,6 +127,7 @@ impl Default for Config {
             tls_config: None,
             allow_sync_stalled: false,
             spec_fork_name: None,
+            data_dir: PathBuf::from(DEFAULT_ROOT_DIR),
         }
     }
 }
@@ -323,6 +329,10 @@ pub fn serve<T: BeaconChainTypes>(
             }
         });
 
+    // Create a `warp` filter for the data_dir.
+    let inner_data_dir = ctx.config.data_dir.clone();
+    let data_dir_filter = warp::any().map(move || inner_data_dir.clone());
+
     // Create a `warp` filter that provides access to the beacon chain.
     let inner_ctx = ctx.clone();
     let chain_filter =
@@ -431,6 +441,37 @@ pub fn serve<T: BeaconChainTypes>(
     let inner_ctx = ctx.clone();
     let log_filter = warp::any().map(move || inner_ctx.log.clone());
 
+    // Create a `warp` filter that provides access to local system information.
+    let system_info = Arc::new(RwLock::new(sysinfo::System::new()));
+    {
+        // grab write access for initialisation
+        let mut system_info = system_info.write();
+        system_info.refresh_disks_list();
+        system_info.refresh_networks_list();
+        system_info.refresh_cpu_specifics(sysinfo::CpuRefreshKind::everything());
+        system_info.refresh_cpu();
+    } // end lock
+
+    let system_info_filter =
+        warp::any()
+            .map(move || system_info.clone())
+            .map(|sysinfo: Arc<RwLock<System>>| {
+                {
+                    // refresh stats
+                    let mut sysinfo_lock = sysinfo.write();
+                    sysinfo_lock.refresh_memory();
+                    sysinfo_lock.refresh_cpu_specifics(sysinfo::CpuRefreshKind::everything());
+                    sysinfo_lock.refresh_cpu();
+                    sysinfo_lock.refresh_system();
+                    sysinfo_lock.refresh_networks();
+                    sysinfo_lock.refresh_disks();
+                } // end lock
+                sysinfo
+            });
+
+    let app_start = std::time::Instant::now();
+    let app_start_filter = warp::any().map(move || app_start);
+
     /*
      *
      * Start of HTTP method definitions.
@@ -891,6 +932,37 @@ pub fn serve<T: BeaconChainTypes>(
             },
         );
 
+    // GET beacon/states/{state_id}/randao?epoch
+    let get_beacon_state_randao = beacon_states_path
+        .clone()
+        .and(warp::path("randao"))
+        .and(warp::query::<api_types::RandaoQuery>())
+        .and(warp::path::end())
+        .and_then(
+            |state_id: StateId, chain: Arc<BeaconChain<T>>, query: api_types::RandaoQuery| {
+                blocking_json_task(move || {
+                    let (randao, execution_optimistic) = state_id
+                        .map_state_and_execution_optimistic(
+                            &chain,
+                            |state, execution_optimistic| {
+                                let epoch = query.epoch.unwrap_or_else(|| state.current_epoch());
+                                let randao = *state.get_randao_mix(epoch).map_err(|e| {
+                                    warp_utils::reject::custom_bad_request(format!(
+                                        "epoch out of range: {e:?}"
+                                    ))
+                                })?;
+                                Ok((randao, execution_optimistic))
+                            },
+                        )?;
+
+                    Ok(
+                        api_types::GenericResponse::from(api_types::RandaoMix { randao })
+                            .add_execution_optimistic(execution_optimistic),
+                    )
+                })
+            },
+        );
+
     // GET beacon/headers
     //
     // Note: this endpoint only returns information about blocks in the canonical chain. Given that
@@ -1169,6 +1241,51 @@ pub fn serve<T: BeaconChainTypes>(
             })
         });
 
+    // GET beacon/blinded_blocks/{block_id}
+    let get_beacon_blinded_block = eth_v1
+        .and(warp::path("beacon"))
+        .and(warp::path("blinded_blocks"))
+        .and(block_id_or_err)
+        .and(chain_filter.clone())
+        .and(warp::path::end())
+        .and(warp::header::optional::<api_types::Accept>("accept"))
+        .and_then(
+            |block_id: BlockId,
+             chain: Arc<BeaconChain<T>>,
+             accept_header: Option<api_types::Accept>| {
+                blocking_task(move || {
+                    let (block, execution_optimistic) = block_id.blinded_block(&chain)?;
+                    let fork_name = block
+                        .fork_name(&chain.spec)
+                        .map_err(inconsistent_fork_rejection)?;
+
+                    match accept_header {
+                        Some(api_types::Accept::Ssz) => Response::builder()
+                            .status(200)
+                            .header("Content-Type", "application/octet-stream")
+                            .body(block.as_ssz_bytes().into())
+                            .map_err(|e| {
+                                warp_utils::reject::custom_server_error(format!(
+                                    "failed to create response: {}",
+                                    e
+                                ))
+                            }),
+                        _ => {
+                            // Post as a V2 endpoint so we return the fork version.
+                            execution_optimistic_fork_versioned_response(
+                                V2,
+                                fork_name,
+                                execution_optimistic,
+                                block,
+                            )
+                            .map(|res| warp::reply::json(&res).into_response())
+                        }
+                    }
+                    .map(|resp| add_consensus_version_header(resp, fork_name))
+                })
+            },
+        );
+
     /*
      * beacon/pool
      */
@@ -2682,7 +2799,12 @@ pub fn serve<T: BeaconChainTypes>(
                     .await
                     .map(|resp| warp::reply::json(&resp))
                     .map_err(|e| {
-                        error!(log, "Error from connected relay"; "error" => ?e);
+                        error!(
+                            log,
+                            "Relay error when registering validator(s)";
+                            "num_registrations" => filtered_registration_data.len(),
+                            "error" => ?e
+                        );
                         // Forward the HTTP status code if we are able to, otherwise fall back
                         // to a server error.
                         if let eth2::Error::ServerMessage(message) = e {
@@ -2796,6 +2918,29 @@ pub fn serve<T: BeaconChainTypes>(
             })
         });
 
+    // GET lighthouse/ui/health
+    let get_lighthouse_ui_health = warp::path("lighthouse")
+        .and(warp::path("ui"))
+        .and(warp::path("health"))
+        .and(warp::path::end())
+        .and(system_info_filter)
+        .and(app_start_filter)
+        .and(data_dir_filter)
+        .and(network_globals.clone())
+        .and_then(
+            |sysinfo, app_start: std::time::Instant, data_dir, network_globals| {
+                blocking_json_task(move || {
+                    let app_uptime = app_start.elapsed().as_secs() as u64;
+                    Ok(api_types::GenericResponse::from(observe_system_health_bn(
+                        sysinfo,
+                        data_dir,
+                        app_uptime,
+                        network_globals,
+                    )))
+                })
+            },
+        );
+
     // GET lighthouse/syncing
     let get_lighthouse_syncing = warp::path("lighthouse")
         .and(warp::path("syncing"))
@@ -3214,10 +3359,12 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(get_beacon_state_validators.boxed())
                 .or(get_beacon_state_committees.boxed())
                 .or(get_beacon_state_sync_committees.boxed())
+                .or(get_beacon_state_randao.boxed())
                 .or(get_beacon_headers.boxed())
                 .or(get_beacon_headers_block_id.boxed())
                 .or(get_beacon_block.boxed())
                 .or(get_beacon_block_attestations.boxed())
+                .or(get_beacon_blinded_block.boxed())
                 .or(get_beacon_block_root.boxed())
                 .or(get_beacon_pool_attestations.boxed())
                 .or(get_beacon_pool_attester_slashings.boxed())
@@ -3244,6 +3391,7 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(get_validator_aggregate_attestation.boxed())
                 .or(get_validator_sync_committee_contribution.boxed())
                 .or(get_lighthouse_health.boxed())
+                .or(get_lighthouse_ui_health.boxed())
                 .or(get_lighthouse_syncing.boxed())
                 .or(get_lighthouse_nat.boxed())
                 .or(get_lighthouse_peers.boxed())
@@ -3263,6 +3411,7 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(get_lighthouse_merge_readiness.boxed())
                 .or(get_events.boxed()),
         )
+        .boxed()
         .or(warp::post().and(
             post_beacon_blocks
                 .boxed()

beacon_node/http_api/src/publish_blocks.rs

@@ -1,6 +1,8 @@
 use crate::metrics;
 use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now};
-use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, CountUnrealized};
+use beacon_chain::{
+    BeaconChain, BeaconChainTypes, BlockError, CountUnrealized, NotifyExecutionLayer,
+};
 use lighthouse_network::{PubsubMessage, SignedBeaconBlockAndBlobsSidecar};
 use network::NetworkMessage;
 use slog::{crit, error, info, warn, Logger};
@@ -53,7 +55,12 @@ pub async fn publish_block<T: BeaconChainTypes>(
     let block_root = block_root.unwrap_or_else(|| block.canonical_root());
 
     match chain
-        .process_block(block_root, block.clone(), CountUnrealized::True)
+        .process_block(
+            block_root,
+            block.clone(),
+            CountUnrealized::True,
+            NotifyExecutionLayer::Yes,
+        )
         .await
     {
         Ok(root) => {

beacon_node/http_api/tests/common.rs

@@ -2,6 +2,7 @@ use beacon_chain::{
     test_utils::{BeaconChainHarness, EphemeralHarnessType},
     BeaconChain, BeaconChainTypes,
 };
+use directory::DEFAULT_ROOT_DIR;
 use eth2::{BeaconNodeHttpClient, Timeouts};
 use http_api::{Config, Context};
 use lighthouse_network::{
@@ -142,6 +143,7 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
             allow_origin: None,
             tls_config: None,
             allow_sync_stalled: false,
+            data_dir: std::path::PathBuf::from(DEFAULT_ROOT_DIR),
             spec_fork_name: None,
         },
         chain: Some(chain.clone()),

beacon_node/http_api/tests/tests.rs

@@ -745,6 +745,36 @@ impl ApiTester {
         self
     }
 
+    pub async fn test_beacon_states_randao(self) -> Self {
+        for state_id in self.interesting_state_ids() {
+            let mut state_opt = state_id
+                .state(&self.chain)
+                .ok()
+                .map(|(state, _execution_optimistic)| state);
+
+            let epoch_opt = state_opt.as_ref().map(|state| state.current_epoch());
+            let result = self
+                .client
+                .get_beacon_states_randao(state_id.0, epoch_opt)
+                .await
+                .unwrap()
+                .map(|res| res.data);
+
+            if result.is_none() && state_opt.is_none() {
+                continue;
+            }
+
+            let state = state_opt.as_mut().expect("result should be none");
+            let randao_mix = state
+                .get_randao_mix(state.slot().epoch(E::slots_per_epoch()))
+                .unwrap();
+
+            assert_eq!(result.unwrap().randao, *randao_mix);
+        }
+
+        self
+    }
+
     pub async fn test_beacon_headers_all_slots(self) -> Self {
         for slot in 0..CHAIN_LENGTH {
             let slot = Slot::from(slot);
@@ -1016,6 +1046,82 @@ impl ApiTester {
         self
     }
 
+    pub async fn test_beacon_blinded_blocks(self) -> Self {
+        for block_id in self.interesting_block_ids() {
+            let expected = block_id
+                .blinded_block(&self.chain)
+                .ok()
+                .map(|(block, _execution_optimistic)| block);
+
+            if let CoreBlockId::Slot(slot) = block_id.0 {
+                if expected.is_none() {
+                    assert!(SKIPPED_SLOTS.contains(&slot.as_u64()));
+                } else {
+                    assert!(!SKIPPED_SLOTS.contains(&slot.as_u64()));
+                }
+            }
+
+            // Check the JSON endpoint.
+            let json_result = self
+                .client
+                .get_beacon_blinded_blocks(block_id.0)
+                .await
+                .unwrap();
+
+            if let (Some(json), Some(expected)) = (&json_result, &expected) {
+                assert_eq!(&json.data, expected, "{:?}", block_id);
+                assert_eq!(
+                    json.version,
+                    Some(expected.fork_name(&self.chain.spec).unwrap())
+                );
+            } else {
+                assert_eq!(json_result, None);
+                assert_eq!(expected, None);
+            }
+
+            // Check the SSZ endpoint.
+            let ssz_result = self
+                .client
+                .get_beacon_blinded_blocks_ssz(block_id.0, &self.chain.spec)
+                .await
+                .unwrap();
+            assert_eq!(ssz_result.as_ref(), expected.as_ref(), "{:?}", block_id);
+
+            // Check that version headers are provided.
+            let url = self
+                .client
+                .get_beacon_blinded_blocks_path(block_id.0)
+                .unwrap();
+
+            let builders: Vec<fn(RequestBuilder) -> RequestBuilder> = vec![
+                |b| b,
+                |b| b.accept(Accept::Ssz),
+                |b| b.accept(Accept::Json),
+                |b| b.accept(Accept::Any),
+            ];
+
+            for req_builder in builders {
+                let raw_res = self
+                    .client
+                    .get_response(url.clone(), req_builder)
+                    .await
+                    .optional()
+                    .unwrap();
+                if let (Some(raw_res), Some(expected)) = (&raw_res, &expected) {
+                    assert_eq!(
+                        raw_res.fork_name_from_header().unwrap(),
+                        Some(expected.fork_name(&self.chain.spec).unwrap())
+                    );
+                } else {
+                    assert!(raw_res.is_none());
+                    assert_eq!(expected, None);
+                }
+            }
+        }
+
+        self
+    }
+
     pub async fn test_beacon_blocks_attestations(self) -> Self {
         for block_id in self.interesting_block_ids() {
             let result = self
@@ -3696,6 +3802,8 @@ async fn beacon_get() {
         .await
         .test_beacon_states_validator_id()
         .await
+        .test_beacon_states_randao()
+        .await
         .test_beacon_headers_all_slots()
         .await
         .test_beacon_headers_all_parents()
@@ -3704,6 +3812,8 @@ async fn beacon_get() {
         .await
         .test_beacon_blocks()
         .await
+        .test_beacon_blinded_blocks()
+        .await
         .test_beacon_blocks_attestations()
         .await
         .test_beacon_blocks_root()