gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-10 12:11:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Integrate tracing (#6339)

Browse Source 
Tracing Integration
- [reference](5bbf1859e9/projects/project-ideas.md (L297))


  - [x] replace slog & log with tracing throughout the codebase
- [x] implement custom crit log
- [x] make relevant changes in the formatter
- [x] replace sloggers
- [x] re-write SSE logging components

cc: @macladson @eserilev
This commit is contained in:
ThreeHrSleep
2025-03-13 04:01:05 +05:30
committed by GitHub
parent f23f984f85
commit d60c24ef1c
241 changed files with 9485 additions and 9328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

389
beacon_node/client/src/notifier.rs
View File

@@ -8,12 +8,13 @@ use beacon_chain::{
BeaconChain, BeaconChainTypes, ExecutionStatus,
};
use lighthouse_network::{types::SyncState, NetworkGlobals};
use slog::{crit, debug, error, info, warn, Logger};
use logging::crit;
use slot_clock::SlotClock;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
use tokio::time::sleep;
use tracing::{debug, error, info, warn};
use types::*;
/// Create a warning log whenever the peer count is at or below this value.
@@ -39,7 +40,6 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
let slot_duration = Duration::from_secs(seconds_per_slot);
let speedo = Mutex::new(Speedo::default());
let log = executor.log().clone();
// Keep track of sync state and reset the speedo on specific sync state changes.
// Specifically, if we switch between a sync and a backfill sync, reset the speedo.
@@ -56,15 +56,14 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
// waiting for genesis.
Some(next_slot) if next_slot > slot_duration => {
info!(
log,
"Waiting for genesis";
"peers" => peer_count_pretty(network.connected_peers()),
"wait_time" => estimated_time_pretty(Some(next_slot.as_secs() as f64)),
peers = peer_count_pretty(network.connected_peers()),
wait_time = estimated_time_pretty(Some(next_slot.as_secs() as f64)),
"Waiting for genesis"
);
eth1_logging(&beacon_chain, &log);
bellatrix_readiness_logging(Slot::new(0), &beacon_chain, &log).await;
capella_readiness_logging(Slot::new(0), &beacon_chain, &log).await;
genesis_execution_payload_logging(&beacon_chain, &log).await;
eth1_logging(&beacon_chain);
bellatrix_readiness_logging(Slot::new(0), &beacon_chain).await;
capella_readiness_logging(Slot::new(0), &beacon_chain).await;
genesis_execution_payload_logging(&beacon_chain).await;
sleep(slot_duration).await;
}
_ => break,
@@ -82,7 +81,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
let wait = match beacon_chain.slot_clock.duration_to_next_slot() {
Some(duration) => duration + slot_duration / 2,
None => {
warn!(log, "Unable to read current slot");
warn!("Unable to read current slot");
sleep(slot_duration).await;
continue;
}
@@ -120,11 +119,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
let current_slot = match beacon_chain.slot() {
Ok(slot) => slot,
Err(e) => {
error!(
log,
"Unable to read current slot";
"error" => format!("{:?}", e)
);
error!(error = ?e, "Unable to read current slot");
break;
}
};
@@ -168,19 +163,21 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
);
if connected_peer_count <= WARN_PEER_COUNT {
warn!(log, "Low peer count"; "peer_count" => peer_count_pretty(connected_peer_count));
warn!(
peer_count = peer_count_pretty(connected_peer_count),
"Low peer count"
);
}
debug!(
log,
"Slot timer";
"peers" => peer_count_pretty(connected_peer_count),
"finalized_root" => format!("{}", finalized_checkpoint.root),
"finalized_epoch" => finalized_checkpoint.epoch,
"head_block" => format!("{}", head_root),
"head_slot" => head_slot,
"current_slot" => current_slot,
"sync_state" =>format!("{}", current_sync_state)
peers = peer_count_pretty(connected_peer_count),
finalized_root = %finalized_checkpoint.root,
finalized_epoch = %finalized_checkpoint.epoch,
head_block = %head_root,
%head_slot,
%current_slot,
sync_state = %current_sync_state,
"Slot timer"
);
// Log if we are backfilling.
@@ -202,26 +199,31 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
if display_speed {
info!(
log,
"Downloading historical blocks";
"distance" => distance,
"speed" => sync_speed_pretty(speed),
"est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_oldest_block_slot.saturating_sub(beacon_chain.genesis_backfill_slot))),
distance,
speed = sync_speed_pretty(speed),
est_time = estimated_time_pretty(
speedo.estimated_time_till_slot(
original_oldest_block_slot
.saturating_sub(beacon_chain.genesis_backfill_slot)
)
),
"Downloading historical blocks"
);
} else {
info!(
log,
"Downloading historical blocks";
"distance" => distance,
"est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_oldest_block_slot.saturating_sub(beacon_chain.genesis_backfill_slot))),
distance,
est_time = estimated_time_pretty(
speedo.estimated_time_till_slot(
original_oldest_block_slot
.saturating_sub(beacon_chain.genesis_backfill_slot)
)
),
"Downloading historical blocks"
);
}
} else if !is_backfilling && last_backfill_log_slot.is_some() {
last_backfill_log_slot = None;
info!(
log,
"Historical block download complete";
);
info!("Historical block download complete");
}
// Log if we are syncing
@@ -238,20 +240,20 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
if display_speed {
info!(
log,
"Syncing";
"peers" => peer_count_pretty(connected_peer_count),
"distance" => distance,
"speed" => sync_speed_pretty(speed),
"est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
peers = peer_count_pretty(connected_peer_count),
distance,
speed = sync_speed_pretty(speed),
est_time =
estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
"Syncing"
);
} else {
info!(
log,
"Syncing";
"peers" => peer_count_pretty(connected_peer_count),
"distance" => distance,
"est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
peers = peer_count_pretty(connected_peer_count),
distance,
est_time =
estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
"Syncing"
);
}
} else if current_sync_state.is_synced() {
@@ -267,20 +269,18 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
Ok(ExecutionStatus::Valid(hash)) => format!("{} (verified)", hash),
Ok(ExecutionStatus::Optimistic(hash)) => {
warn!(
log,
"Head is optimistic";
"info" => "chain not fully verified, \
block and attestation production disabled until execution engine syncs",
"execution_block_hash" => ?hash,
info = "chain not fully verified, \
block and attestation production disabled until execution engine syncs",
execution_block_hash = ?hash,
"Head is optimistic"
);
format!("{} (unverified)", hash)
}
Ok(ExecutionStatus::Invalid(hash)) => {
crit!(
log,
"Head execution payload is invalid";
"msg" => "this scenario may be unrecoverable",
"execution_block_hash" => ?hash,
msg = "this scenario may be unrecoverable",
execution_block_hash = ?hash,
"Head execution payload is invalid"
);
format!("{} (invalid)", hash)
}
@@ -288,35 +288,33 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
};
info!(
log,
"Synced";
"peers" => peer_count_pretty(connected_peer_count),
"exec_hash" => block_hash,
"finalized_root" => format!("{}", finalized_checkpoint.root),
"finalized_epoch" => finalized_checkpoint.epoch,
"epoch" => current_epoch,
"block" => block_info,
"slot" => current_slot,
peers = peer_count_pretty(connected_peer_count),
exec_hash = block_hash,
finalized_root = %finalized_checkpoint.root,
finalized_epoch = %finalized_checkpoint.epoch,
epoch = %current_epoch,
block = block_info,
slot = %current_slot,
"Synced"
);
} else {
metrics::set_gauge(&metrics::IS_SYNCED, 0);
info!(
log,
"Searching for peers";
"peers" => peer_count_pretty(connected_peer_count),
"finalized_root" => format!("{}", finalized_checkpoint.root),
"finalized_epoch" => finalized_checkpoint.epoch,
"head_slot" => head_slot,
"current_slot" => current_slot,
peers = peer_count_pretty(connected_peer_count),
finalized_root = %finalized_checkpoint.root,
finalized_epoch = %finalized_checkpoint.epoch,
%head_slot,
%current_slot,
"Searching for peers"
);
}
eth1_logging(&beacon_chain, &log);
bellatrix_readiness_logging(current_slot, &beacon_chain, &log).await;
capella_readiness_logging(current_slot, &beacon_chain, &log).await;
deneb_readiness_logging(current_slot, &beacon_chain, &log).await;
electra_readiness_logging(current_slot, &beacon_chain, &log).await;
fulu_readiness_logging(current_slot, &beacon_chain, &log).await;
eth1_logging(&beacon_chain);
bellatrix_readiness_logging(current_slot, &beacon_chain).await;
capella_readiness_logging(current_slot, &beacon_chain).await;
deneb_readiness_logging(current_slot, &beacon_chain).await;
electra_readiness_logging(current_slot, &beacon_chain).await;
fulu_readiness_logging(current_slot, &beacon_chain).await;
}
};
@@ -331,7 +329,6 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
async fn bellatrix_readiness_logging<T: BeaconChainTypes>(
current_slot: Slot,
beacon_chain: &BeaconChain<T>,
log: &Logger,
) {
let merge_completed = beacon_chain
.canonical_head
@@ -355,10 +352,9 @@ async fn bellatrix_readiness_logging<T: BeaconChainTypes>(
// Logging of the EE being offline is handled in the other readiness logging functions.
if !beacon_chain.is_time_to_prepare_for_capella(current_slot) {
error!(
log,
"Execution endpoint required";
"info" => "you need an execution engine to validate blocks, see: \
https://lighthouse-book.sigmaprime.io/merge-migration.html"
info = "you need an execution engine to validate blocks, see: \
https://lighthouse-book.sigmaprime.io/merge-migration.html",
"Execution endpoint required"
);
}
return;
@@ -375,12 +371,11 @@ async fn bellatrix_readiness_logging<T: BeaconChainTypes>(
terminal_block_hash_epoch: None,
} => {
info!(
log,
"Ready for Bellatrix";
"terminal_total_difficulty" => %ttd,
"current_difficulty" => current_difficulty
terminal_total_difficulty = %ttd,
current_difficulty = current_difficulty
.map(|d| d.to_string())
.unwrap_or_else(|| "??".into()),
"Ready for Bellatrix"
)
}
MergeConfig {
@@ -389,29 +384,25 @@ async fn bellatrix_readiness_logging<T: BeaconChainTypes>(
terminal_block_hash_epoch: Some(terminal_block_hash_epoch),
} => {
info!(
log,
"Ready for Bellatrix";
"info" => "you are using override parameters, please ensure that you \
understand these parameters and their implications.",
"terminal_block_hash" => ?terminal_block_hash,
"terminal_block_hash_epoch" => ?terminal_block_hash_epoch,
info = "you are using override parameters, please ensure that you \
understand these parameters and their implications.",
?terminal_block_hash,
?terminal_block_hash_epoch,
"Ready for Bellatrix"
)
}
other => error!(
log,
"Inconsistent merge configuration";
"config" => ?other
config = ?other,
"Inconsistent merge configuration"
),
},
readiness @ BellatrixReadiness::NotSynced => warn!(
log,
"Not ready Bellatrix";
"info" => %readiness,
info = %readiness,
"Not ready Bellatrix"
),
readiness @ BellatrixReadiness::NoExecutionEndpoint => warn!(
log,
"Not ready for Bellatrix";
"info" => %readiness,
info = %readiness,
"Not ready for Bellatrix"
),
}
}
@@ -420,7 +411,6 @@ async fn bellatrix_readiness_logging<T: BeaconChainTypes>(
async fn capella_readiness_logging<T: BeaconChainTypes>(
current_slot: Slot,
beacon_chain: &BeaconChain<T>,
log: &Logger,
) {
let capella_completed = beacon_chain
.canonical_head
@@ -442,10 +432,9 @@ async fn capella_readiness_logging<T: BeaconChainTypes>(
// Logging of the EE being offline is handled in the other readiness logging functions.
if !beacon_chain.is_time_to_prepare_for_deneb(current_slot) {
error!(
log,
"Execution endpoint required";
"info" => "you need a Capella enabled execution engine to validate blocks, see: \
https://lighthouse-book.sigmaprime.io/merge-migration.html"
info = "you need a Capella enabled execution engine to validate blocks, see: \
https://lighthouse-book.sigmaprime.io/merge-migration.html",
"Execution endpoint required"
);
}
return;
@@ -454,24 +443,21 @@ async fn capella_readiness_logging<T: BeaconChainTypes>(
match beacon_chain.check_capella_readiness().await {
CapellaReadiness::Ready => {
info!(
log,
"Ready for Capella";
"info" => "ensure the execution endpoint is updated to the latest Capella/Shanghai release"
info = "ensure the execution endpoint is updated to the latest Capella/Shanghai release",
"Ready for Capella"
)
}
readiness @ CapellaReadiness::ExchangeCapabilitiesFailed { error: _ } => {
error!(
log,
"Not ready for Capella";
"hint" => "the execution endpoint may be offline",
"info" => %readiness,
hint = "the execution endpoint may be offline",
info = %readiness,
"Not ready for Capella"
)
}
readiness => warn!(
log,
"Not ready for Capella";
"hint" => "try updating the execution endpoint",
"info" => %readiness,
hint = "try updating the execution endpoint",
info = %readiness,
"Not ready for Capella"
),
}
}
@@ -480,7 +466,6 @@ async fn capella_readiness_logging<T: BeaconChainTypes>(
async fn deneb_readiness_logging<T: BeaconChainTypes>(
current_slot: Slot,
beacon_chain: &BeaconChain<T>,
log: &Logger,
) {
let deneb_completed = beacon_chain
.canonical_head
@@ -500,9 +485,8 @@ async fn deneb_readiness_logging<T: BeaconChainTypes>(
if deneb_completed && !has_execution_layer {
error!(
log,
"Execution endpoint required";
"info" => "you need a Deneb enabled execution engine to validate blocks."
info = "you need a Deneb enabled execution engine to validate blocks.",
"Execution endpoint required"
);
return;
}
@@ -510,24 +494,22 @@ async fn deneb_readiness_logging<T: BeaconChainTypes>(
match beacon_chain.check_deneb_readiness().await {
DenebReadiness::Ready => {
info!(
log,
"Ready for Deneb";
"info" => "ensure the execution endpoint is updated to the latest Deneb/Cancun release"
info =
"ensure the execution endpoint is updated to the latest Deneb/Cancun release",
"Ready for Deneb"
)
}
readiness @ DenebReadiness::ExchangeCapabilitiesFailed { error: _ } => {
error!(
log,
"Not ready for Deneb";
"hint" => "the execution endpoint may be offline",
"info" => %readiness,
hint = "the execution endpoint may be offline",
info = %readiness,
"Not ready for Deneb"
)
}
readiness => warn!(
log,
"Not ready for Deneb";
"hint" => "try updating the execution endpoint",
"info" => %readiness,
hint = "try updating the execution endpoint",
info = %readiness,
"Not ready for Deneb"
),
}
}
@@ -535,7 +517,6 @@ async fn deneb_readiness_logging<T: BeaconChainTypes>(
async fn electra_readiness_logging<T: BeaconChainTypes>(
current_slot: Slot,
beacon_chain: &BeaconChain<T>,
log: &Logger,
) {
let electra_completed = beacon_chain
.canonical_head
@@ -556,9 +537,8 @@ async fn electra_readiness_logging<T: BeaconChainTypes>(
if electra_completed && !has_execution_layer {
// When adding a new fork, add a check for the next fork readiness here.
error!(
log,
"Execution endpoint required";
"info" => "you need a Electra enabled execution engine to validate blocks."
info = "you need a Electra enabled execution engine to validate blocks.",
"Execution endpoint required"
);
return;
}
@@ -566,24 +546,22 @@ async fn electra_readiness_logging<T: BeaconChainTypes>(
match beacon_chain.check_electra_readiness().await {
ElectraReadiness::Ready => {
info!(
log,
"Ready for Electra";
"info" => "ensure the execution endpoint is updated to the latest Electra/Prague release"
info =
"ensure the execution endpoint is updated to the latest Electra/Prague release",
"Ready for Electra"
)
}
readiness @ ElectraReadiness::ExchangeCapabilitiesFailed { error: _ } => {
error!(
log,
"Not ready for Electra";
"hint" => "the execution endpoint may be offline",
"info" => %readiness,
hint = "the execution endpoint may be offline",
info = %readiness,
"Not ready for Electra"
)
}
readiness => warn!(
log,
"Not ready for Electra";
"hint" => "try updating the execution endpoint",
"info" => %readiness,
hint = "try updating the execution endpoint",
info = %readiness,
"Not ready for Electra"
),
}
}
@@ -592,7 +570,6 @@ async fn electra_readiness_logging<T: BeaconChainTypes>(
async fn fulu_readiness_logging<T: BeaconChainTypes>(
current_slot: Slot,
beacon_chain: &BeaconChain<T>,
log: &Logger,
) {
let fulu_completed = beacon_chain
.canonical_head
@@ -612,9 +589,8 @@ async fn fulu_readiness_logging<T: BeaconChainTypes>(
if fulu_completed && !has_execution_layer {
error!(
log,
"Execution endpoint required";
"info" => "you need a Fulu enabled execution engine to validate blocks."
info = "you need a Fulu enabled execution engine to validate blocks.",
"Execution endpoint required"
);
return;
}
@@ -622,102 +598,86 @@ async fn fulu_readiness_logging<T: BeaconChainTypes>(
match beacon_chain.check_fulu_readiness().await {
FuluReadiness::Ready => {
info!(
log,
"Ready for Fulu";
"info" => "ensure the execution endpoint is updated to the latest Fulu release"
info = "ensure the execution endpoint is updated to the latest Fulu release",
"Ready for Fulu"
)
}
readiness @ FuluReadiness::ExchangeCapabilitiesFailed { error: _ } => {
error!(
log,
"Not ready for Fulu";
"hint" => "the execution endpoint may be offline",
"info" => %readiness,
hint = "the execution endpoint may be offline",
info = %readiness,
"Not ready for Fulu"
)
}
readiness => warn!(
log,
"Not ready for Fulu";
"hint" => "try updating the execution endpoint",
"info" => %readiness,
hint = "try updating the execution endpoint",
info = %readiness,
"Not ready for Fulu"
),
}
}
async fn genesis_execution_payload_logging<T: BeaconChainTypes>(
beacon_chain: &BeaconChain<T>,
log: &Logger,
) {
async fn genesis_execution_payload_logging<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
match beacon_chain
.check_genesis_execution_payload_is_correct()
.await
{
Ok(GenesisExecutionPayloadStatus::Correct(block_hash)) => {
info!(
log,
"Execution enabled from genesis";
"genesis_payload_block_hash" => ?block_hash,
genesis_payload_block_hash = ?block_hash,
"Execution enabled from genesis"
);
}
Ok(GenesisExecutionPayloadStatus::BlockHashMismatch { got, expected }) => {
error!(
log,
"Genesis payload block hash mismatch";
"info" => "genesis is misconfigured and likely to fail",
"consensus_node_block_hash" => ?expected,
"execution_node_block_hash" => ?got,
info = "genesis is misconfigured and likely to fail",
consensus_node_block_hash = ?expected,
execution_node_block_hash = ?got,
"Genesis payload block hash mismatch"
);
}
Ok(GenesisExecutionPayloadStatus::TransactionsRootMismatch { got, expected }) => {
error!(
log,
"Genesis payload transactions root mismatch";
"info" => "genesis is misconfigured and likely to fail",
"consensus_node_transactions_root" => ?expected,
"execution_node_transactions_root" => ?got,
info = "genesis is misconfigured and likely to fail",
consensus_node_transactions_root = ?expected,
execution_node_transactions_root = ?got,
"Genesis payload transactions root mismatch"
);
}
Ok(GenesisExecutionPayloadStatus::WithdrawalsRootMismatch { got, expected }) => {
error!(
log,
"Genesis payload withdrawals root mismatch";
"info" => "genesis is misconfigured and likely to fail",
"consensus_node_withdrawals_root" => ?expected,
"execution_node_withdrawals_root" => ?got,
info = "genesis is misconfigured and likely to fail",
consensus_node_withdrawals_root = ?expected,
execution_node_withdrawals_root = ?got,
"Genesis payload withdrawals root mismatch"
);
}
Ok(GenesisExecutionPayloadStatus::OtherMismatch) => {
error!(
log,
"Genesis payload header mismatch";
"info" => "genesis is misconfigured and likely to fail",
"detail" => "see debug logs for payload headers"
info = "genesis is misconfigured and likely to fail",
detail = "see debug logs for payload headers",
"Genesis payload header mismatch"
);
}
Ok(GenesisExecutionPayloadStatus::Irrelevant) => {
info!(
log,
"Execution is not enabled from genesis";
);
info!("Execution is not enabled from genesis");
}
Ok(GenesisExecutionPayloadStatus::AlreadyHappened) => {
warn!(
log,
"Unable to check genesis which has already occurred";
"info" => "this is probably a race condition or a bug"
info = "this is probably a race condition or a bug",
"Unable to check genesis which has already occurred"
);
}
Err(e) => {
error!(
log,
"Unable to check genesis execution payload";
"error" => ?e
error = ?e,
"Unable to check genesis execution payload"
);
}
}
}
fn eth1_logging<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>, log: &Logger) {
fn eth1_logging<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
let current_slot_opt = beacon_chain.slot().ok();
// Perform some logging about the eth1 chain
@@ -733,13 +693,12 @@ fn eth1_logging<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>, log: &Logger
&beacon_chain.spec,
) {
debug!(
log,
"Eth1 cache sync status";
"eth1_head_block" => status.head_block_number,
"latest_cached_block_number" => status.latest_cached_block_number,
"latest_cached_timestamp" => status.latest_cached_block_timestamp,
"voting_target_timestamp" => status.voting_target_timestamp,
"ready" => status.lighthouse_is_cached_and_ready
eth1_head_block = status.head_block_number,
latest_cached_block_number = status.latest_cached_block_number,
latest_cached_timestamp = status.latest_cached_block_timestamp,
voting_target_timestamp = status.voting_target_timestamp,
ready = status.lighthouse_is_cached_and_ready,
"Eth1 cache sync status"
);
if !status.lighthouse_is_cached_and_ready {
@@ -755,16 +714,12 @@ fn eth1_logging<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>, log: &Logger
.unwrap_or_else(|| "initializing deposits".to_string());
warn!(
log,
"Syncing deposit contract block cache";
"est_blocks_remaining" => distance,
est_blocks_remaining = distance,
"Syncing deposit contract block cache"
);
}
} else {
error!(
log,
"Unable to determine deposit contract sync status";
);
error!("Unable to determine deposit contract sync status");
}
}
}