gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-30 04:37:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'deneb-free-blobs' of https://github.com/sigp/lighthouse into some-blob-reprocessing-work

Browse Source
This commit is contained in:
realbigsean
2023-06-02 08:19:58 -04:00
parent 28b92cde12 e8f1d533fb
commit e503c8e002
333 changed files with 5934 additions and 13398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
beacon_node/http_api/Cargo.toml
View File

@@ -24,7 +24,7 @@ lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
lazy_static = "1.4.0"
warp_utils = { path = "../../common/warp_utils" }
slot_clock = { path = "../../common/slot_clock" }
eth2_ssz = "0.4.1"
ethereum_ssz = "0.5.0"
bs58 = "0.4.0"
futures = "0.3.8"
execution_layer = {path = "../execution_layer"}
@@ -32,15 +32,15 @@ parking_lot = "0.12.0"
safe_arith = {path = "../../consensus/safe_arith"}
task_executor = { path = "../../common/task_executor" }
lru = "0.7.7"
tree_hash = "0.4.1"
tree_hash = "0.5.0"
sysinfo = "0.26.5"
system_health = { path = "../../common/system_health" }
directory = { path = "../../common/directory" }
eth2_serde_utils = "0.1.1"
logging = { path = "../../common/logging" }
ethereum_serde_utils = "0.5.0"
operation_pool = { path = "../operation_pool" }
sensitive_url = { path = "../../common/sensitive_url" }
unused_port = {path = "../../common/unused_port"}
logging = { path = "../../common/logging" }
store = { path = "../store" }
[dev-dependencies]
@@ -51,4 +51,4 @@ genesis = { path = "../genesis" }
[[test]]
name = "bn_http_api_tests"
path = "tests/main.rs"
path = "tests/main.rs"

104
beacon_node/http_api/src/lib.rs
View File

@@ -37,6 +37,7 @@ use eth2::types::{
};
use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage};
use lighthouse_version::version_with_platform;
use logging::SSELoggingComponents;
use network::{NetworkMessage, NetworkSenders, ValidatorSubscriptionMessage};
use operation_pool::ReceivedPreCapella;
use parking_lot::RwLock;
@@ -109,6 +110,7 @@ pub struct Context<T: BeaconChainTypes> {
pub network_senders: Option<NetworkSenders<T::EthSpec>>,
pub network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
pub eth1_service: Option<eth1::Service>,
pub sse_logging_components: Option<SSELoggingComponents>,
pub log: Logger,
}
@@ -449,6 +451,9 @@ pub fn serve<T: BeaconChainTypes>(
let inner_ctx = ctx.clone();
let log_filter = warp::any().map(move || inner_ctx.log.clone());
let inner_components = ctx.sse_logging_components.clone();
let sse_component_filter = warp::any().map(move || inner_components.clone());
// Create a `warp` filter that provides access to local system information.
let system_info = Arc::new(RwLock::new(sysinfo::System::new()));
{
@@ -2238,12 +2243,8 @@ pub fn serve<T: BeaconChainTypes>(
.parent
.and_then(|index| proto_array.nodes.get(index))
.map(|parent| parent.root),
justified_epoch: node
.justified_checkpoint
.map(|checkpoint| checkpoint.epoch),
finalized_epoch: node
.finalized_checkpoint
.map(|checkpoint| checkpoint.epoch),
justified_epoch: node.justified_checkpoint.epoch,
finalized_epoch: node.finalized_checkpoint.epoch,
weight: node.weight,
validity: execution_status,
execution_block_hash: node
@@ -2325,28 +2326,40 @@ pub fn serve<T: BeaconChainTypes>(
.and(chain_filter.clone())
.and_then(
|network_globals: Arc<NetworkGlobals<T::EthSpec>>, chain: Arc<BeaconChain<T>>| {
blocking_json_task(move || {
let head_slot = chain.canonical_head.cached_head().head_slot();
let current_slot = chain.slot_clock.now_or_genesis().ok_or_else(|| {
warp_utils::reject::custom_server_error("Unable to read slot clock".into())
})?;
// Taking advantage of saturating subtraction on slot.
let sync_distance = current_slot - head_slot;
let is_optimistic = chain
.is_optimistic_or_invalid_head()
.map_err(warp_utils::reject::beacon_chain_error)?;
let syncing_data = api_types::SyncingData {
is_syncing: network_globals.sync_state.read().is_syncing(),
is_optimistic: Some(is_optimistic),
head_slot,
sync_distance,
async move {
let el_offline = if let Some(el) = &chain.execution_layer {
el.is_offline_or_erroring().await
} else {
true
};
Ok(api_types::GenericResponse::from(syncing_data))
})
blocking_json_task(move || {
let head_slot = chain.canonical_head.cached_head().head_slot();
let current_slot = chain.slot_clock.now_or_genesis().ok_or_else(|| {
warp_utils::reject::custom_server_error(
"Unable to read slot clock".into(),
)
})?;
// Taking advantage of saturating subtraction on slot.
let sync_distance = current_slot - head_slot;
let is_optimistic = chain
.is_optimistic_or_invalid_head()
.map_err(warp_utils::reject::beacon_chain_error)?;
let syncing_data = api_types::SyncingData {
is_syncing: network_globals.sync_state.read().is_syncing(),
is_optimistic: Some(is_optimistic),
el_offline: Some(el_offline),
head_slot,
sync_distance,
};
Ok(api_types::GenericResponse::from(syncing_data))
})
.await
}
},
);
@@ -3760,6 +3773,44 @@ pub fn serve<T: BeaconChainTypes>(
},
);
// Subscribe to logs via Server Side Events
// /lighthouse/logs
let lighthouse_log_events = warp::path("lighthouse")
.and(warp::path("logs"))
.and(warp::path::end())
.and(sse_component_filter)
.and_then(|sse_component: Option<SSELoggingComponents>| {
blocking_response_task(move || {
if let Some(logging_components) = sse_component {
// Build a JSON stream
let s =
BroadcastStream::new(logging_components.sender.subscribe()).map(|msg| {
match msg {
Ok(data) => {
// Serialize to json
match data.to_json_string() {
// Send the json as a Server Side Event
Ok(json) => Ok(Event::default().data(json)),
Err(e) => Err(warp_utils::reject::server_sent_event_error(
format!("Unable to serialize to JSON {}", e),
)),
}
}
Err(e) => Err(warp_utils::reject::server_sent_event_error(
format!("Unable to receive event {}", e),
)),
}
});
Ok::<_, warp::Rejection>(warp::sse::reply(warp::sse::keep_alive().stream(s)))
} else {
Err(warp_utils::reject::custom_server_error(
"SSE Logging is not enabled".to_string(),
))
}
})
});
// Define the ultimate set of routes that will be provided to the server.
// Use `uor` rather than `or` in order to simplify types (see `UnifyingOrFilter`).
let routes = warp::get()
@@ -3828,6 +3879,7 @@ pub fn serve<T: BeaconChainTypes>(
.uor(get_lighthouse_block_packing_efficiency)
.uor(get_lighthouse_merge_readiness)
.uor(get_events)
.uor(lighthouse_log_events.boxed())
.recover(warp_utils::reject::handle_rejection),
)
.boxed()

4
beacon_node/http_api/src/sync_committees.rs
View File

@@ -199,10 +199,14 @@ pub fn process_sync_committee_signatures<T: BeaconChainTypes>(
Err(SyncVerificationError::PriorSyncCommitteeMessageKnown {
validator_index,
slot,
prev_root,
new_root,
}) => {
debug!(
log,
"Ignoring already-known sync message";
"new_root" => ?new_root,
"prev_root" => ?prev_root,
"slot" => slot,
"validator_index" => validator_index,
);

1
beacon_node/http_api/src/test_utils.rs
View File

@@ -195,6 +195,7 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
network_senders: Some(network_senders),
network_globals: Some(network_globals),
eth1_service: Some(eth1_service),
sse_logging_components: None,
log,
});

10
beacon_node/http_api/src/ui.rs
View File

@@ -75,15 +75,15 @@ pub fn get_validator_count<T: BeaconChainTypes>(
#[derive(PartialEq, Serialize, Deserialize)]
pub struct ValidatorInfoRequestData {
#[serde(with = "eth2_serde_utils::quoted_u64_vec")]
#[serde(with = "serde_utils::quoted_u64_vec")]
indices: Vec<u64>,
}
#[derive(PartialEq, Serialize, Deserialize)]
pub struct ValidatorInfoValues {
#[serde(with = "eth2_serde_utils::quoted_u64")]
#[serde(with = "serde_utils::quoted_u64")]
epoch: u64,
#[serde(with = "eth2_serde_utils::quoted_u64")]
#[serde(with = "serde_utils::quoted_u64")]
total_balance: u64,
}
@@ -165,6 +165,7 @@ pub struct ValidatorMetrics {
attestation_target_hits: u64,
attestation_target_misses: u64,
attestation_target_hit_percentage: f64,
latest_attestation_inclusion_distance: u64,
}
#[derive(PartialEq, Serialize, Deserialize)]
@@ -210,6 +211,8 @@ pub fn post_validator_monitor_metrics<T: BeaconChainTypes>(
let attestation_head_misses = val_metrics.attestation_head_misses;
let attestation_target_hits = val_metrics.attestation_target_hits;
let attestation_target_misses = val_metrics.attestation_target_misses;
let latest_attestation_inclusion_distance =
val_metrics.latest_attestation_inclusion_distance;
drop(val_metrics);
let attestations = attestation_hits + attestation_misses;
@@ -242,6 +245,7 @@ pub fn post_validator_monitor_metrics<T: BeaconChainTypes>(
attestation_target_hits,
attestation_target_misses,
attestation_target_hit_percentage,
latest_attestation_inclusion_distance,
};
validators.insert(id.clone(), metrics);

1
beacon_node/http_api/tests/main.rs
View File

@@ -2,4 +2,5 @@
pub mod fork_tests;
pub mod interactive_tests;
pub mod status_tests;
pub mod tests;

151
beacon_node/http_api/tests/status_tests.rs Normal file
View File

@@ -0,0 +1,151 @@
//! Tests related to the beacon node's sync status
use beacon_chain::{
test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy},
BlockError,
};
use execution_layer::{PayloadStatusV1, PayloadStatusV1Status};
use http_api::test_utils::InteractiveTester;
use types::{EthSpec, ExecPayload, ForkName, MinimalEthSpec, Slot};
type E = MinimalEthSpec;
/// Create a new test environment that is post-merge with `chain_depth` blocks.
async fn post_merge_tester(chain_depth: u64, validator_count: u64) -> InteractiveTester<E> {
// Test using latest fork so that we simulate conditions as similar to mainnet as possible.
// TODO(jimmy): We should change this back to `latest()`. These tests currently fail on Deneb because:
// 1. KZG library doesn't support Minimal spec, changing to Mainnet spec fixes some tests; BUT
// 2. `harness.process_block_result` in the test below panics due to
// `AvailabilityProcessingStatus::PendingBlobs`, and there seems to be some race
// condition going on, because the test passes if I step through the code in debug.
let mut spec = ForkName::Capella.make_genesis_spec(E::default_spec());
spec.terminal_total_difficulty = 1.into();
let tester = InteractiveTester::<E>::new(Some(spec), validator_count as usize).await;
let harness = &tester.harness;
let mock_el = harness.mock_execution_layer.as_ref().unwrap();
let execution_ctx = mock_el.server.ctx.clone();
// Move to terminal block.
mock_el.server.all_payloads_valid();
execution_ctx
.execution_block_generator
.write()
.move_to_terminal_block()
.unwrap();
// Create some chain depth.
harness.advance_slot();
harness
.extend_chain_with_sync(
chain_depth as usize,
BlockStrategy::OnCanonicalHead,
AttestationStrategy::AllValidators,
SyncCommitteeStrategy::AllValidators,
)
.await;
tester
}
/// Check `syncing` endpoint when the EL is syncing.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn el_syncing_then_synced() {
let num_blocks = E::slots_per_epoch() / 2;
let num_validators = E::slots_per_epoch();
let tester = post_merge_tester(num_blocks, num_validators).await;
let harness = &tester.harness;
let mock_el = harness.mock_execution_layer.as_ref().unwrap();
// EL syncing
mock_el.server.set_syncing_response(Ok(true));
mock_el.el.upcheck().await;
let api_response = tester.client.get_node_syncing().await.unwrap().data;
assert_eq!(api_response.el_offline, Some(false));
assert_eq!(api_response.is_optimistic, Some(false));
assert_eq!(api_response.is_syncing, false);
// EL synced
mock_el.server.set_syncing_response(Ok(false));
mock_el.el.upcheck().await;
let api_response = tester.client.get_node_syncing().await.unwrap().data;
assert_eq!(api_response.el_offline, Some(false));
assert_eq!(api_response.is_optimistic, Some(false));
assert_eq!(api_response.is_syncing, false);
}
/// Check `syncing` endpoint when the EL is offline (errors on upcheck).
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn el_offline() {
let num_blocks = E::slots_per_epoch() / 2;
let num_validators = E::slots_per_epoch();
let tester = post_merge_tester(num_blocks, num_validators).await;
let harness = &tester.harness;
let mock_el = harness.mock_execution_layer.as_ref().unwrap();
// EL offline
mock_el.server.set_syncing_response(Err("offline".into()));
mock_el.el.upcheck().await;
let api_response = tester.client.get_node_syncing().await.unwrap().data;
assert_eq!(api_response.el_offline, Some(true));
assert_eq!(api_response.is_optimistic, Some(false));
assert_eq!(api_response.is_syncing, false);
}
/// Check `syncing` endpoint when the EL errors on newPaylod but is not fully offline.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn el_error_on_new_payload() {
let num_blocks = E::slots_per_epoch() / 2;
let num_validators = E::slots_per_epoch();
let tester = post_merge_tester(num_blocks, num_validators).await;
let harness = &tester.harness;
let mock_el = harness.mock_execution_layer.as_ref().unwrap();
// Make a block.
let pre_state = harness.get_current_state();
let (block_contents, _) = harness
.make_block(pre_state, Slot::new(num_blocks + 1))
.await;
let block = block_contents.0;
let block_hash = block
.message()
.body()
.execution_payload()
.unwrap()
.block_hash();
// Make sure `newPayload` errors for the new block.
mock_el
.server
.set_new_payload_error(block_hash, "error".into());
// Attempt to process the block, which should error.
harness.advance_slot();
assert!(matches!(
harness.process_block_result(block.clone()).await,
Err(BlockError::ExecutionPayloadError(_))
));
// The EL should now be *offline* according to the API.
let api_response = tester.client.get_node_syncing().await.unwrap().data;
assert_eq!(api_response.el_offline, Some(true));
assert_eq!(api_response.is_optimistic, Some(false));
assert_eq!(api_response.is_syncing, false);
// Processing a block successfully should remove the status.
mock_el.server.set_new_payload_status(
block_hash,
PayloadStatusV1 {
status: PayloadStatusV1Status::Valid,
latest_valid_hash: Some(block_hash),
validation_error: None,
},
);
harness.process_block_result(block).await.unwrap();
let api_response = tester.client.get_node_syncing().await.unwrap().data;
assert_eq!(api_response.el_offline, Some(false));
assert_eq!(api_response.is_optimistic, Some(false));
assert_eq!(api_response.is_syncing, false);
}

6
beacon_node/http_api/tests/tests.rs
View File

@@ -1729,6 +1729,8 @@ impl ApiTester {
let expected = SyncingData {
is_syncing: false,
is_optimistic: Some(false),
// these tests run without the Bellatrix fork enabled
el_offline: Some(true),
head_slot,
sync_distance,
};
@@ -1964,8 +1966,8 @@ impl ApiTester {
.parent
.and_then(|index| expected_proto_array.nodes.get(index))
.map(|parent| parent.root),
justified_epoch: node.justified_checkpoint.map(|checkpoint| checkpoint.epoch),
finalized_epoch: node.finalized_checkpoint.map(|checkpoint| checkpoint.epoch),
justified_epoch: node.justified_checkpoint.epoch,
finalized_epoch: node.finalized_checkpoint.epoch,
weight: node.weight,
validity: execution_status,
execution_block_hash: node