Integrate tracing (#6339)

Tracing Integration
- [reference](5bbf1859e9/projects/project-ideas.md (L297))


  - [x] replace slog & log with tracing throughout the codebase
- [x] implement custom crit log
- [x] make relevant changes in the formatter
- [x] replace sloggers
- [x] re-write SSE logging components

cc: @macladson @eserilev
This commit is contained in:
ThreeHrSleep
2025-03-13 04:01:05 +05:30
committed by GitHub
parent f23f984f85
commit d60c24ef1c
241 changed files with 9485 additions and 9328 deletions

View File

@@ -15,10 +15,10 @@ eth2 = { workspace = true }
futures = { workspace = true }
itertools = { workspace = true }
serde = { workspace = true }
slog = { workspace = true }
slot_clock = { workspace = true }
strum = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
validator_metrics = { workspace = true }

View File

@@ -1,9 +1,9 @@
use super::CandidateError;
use eth2::BeaconNodeHttpClient;
use serde::{Deserialize, Serialize};
use slog::{warn, Logger};
use std::cmp::Ordering;
use std::fmt::{Debug, Display, Formatter};
use tracing::warn;
use types::Slot;
/// Sync distances between 0 and DEFAULT_SYNC_TOLERANCE are considered `synced`.
@@ -276,15 +276,13 @@ impl BeaconNodeHealth {
pub async fn check_node_health(
beacon_node: &BeaconNodeHttpClient,
log: &Logger,
) -> Result<(Slot, bool, bool), CandidateError> {
let resp = match beacon_node.get_node_syncing().await {
Ok(resp) => resp,
Err(e) => {
warn!(
log,
"Unable connect to beacon node";
"error" => %e
error = %e,
"Unable connect to beacon node"
);
return Err(CandidateError::Offline);

View File

@@ -12,7 +12,6 @@ use environment::RuntimeContext;
use eth2::BeaconNodeHttpClient;
use futures::future;
use serde::{ser::SerializeStruct, Deserialize, Serialize, Serializer};
use slog::{debug, error, warn, Logger};
use slot_clock::SlotClock;
use std::cmp::Ordering;
use std::fmt;
@@ -24,6 +23,7 @@ use std::time::{Duration, Instant};
use std::vec::Vec;
use strum::EnumVariantNames;
use tokio::{sync::RwLock, time::sleep};
use tracing::{debug, error, warn};
use types::{ChainSpec, Config as ConfigSpec, EthSpec, Slot};
use validator_metrics::{inc_counter_vec, ENDPOINT_ERRORS, ENDPOINT_REQUESTS};
@@ -222,15 +222,14 @@ impl<E: EthSpec> CandidateBeaconNode<E> {
distance_tiers: &BeaconNodeSyncDistanceTiers,
slot_clock: Option<&T>,
spec: &ChainSpec,
log: &Logger,
) -> Result<(), CandidateError> {
if let Err(e) = self.is_compatible(spec, log).await {
if let Err(e) = self.is_compatible(spec).await {
*self.health.write().await = Err(e);
return Err(e);
}
if let Some(slot_clock) = slot_clock {
match check_node_health(&self.beacon_node, log).await {
match check_node_health(&self.beacon_node).await {
Ok((head, is_optimistic, el_offline)) => {
let Some(slot_clock_head) = slot_clock.now() else {
let e = match slot_clock.is_prior_to_genesis() {
@@ -288,17 +287,16 @@ impl<E: EthSpec> CandidateBeaconNode<E> {
}
/// Checks if the node has the correct specification.
async fn is_compatible(&self, spec: &ChainSpec, log: &Logger) -> Result<(), CandidateError> {
async fn is_compatible(&self, spec: &ChainSpec) -> Result<(), CandidateError> {
let config = self
.beacon_node
.get_config_spec::<ConfigSpec>()
.await
.map_err(|e| {
error!(
log,
"Unable to read spec from beacon node";
"error" => %e,
"endpoint" => %self.beacon_node,
error = %e,
endpoint = %self.beacon_node,
"Unable to read spec from beacon node"
);
CandidateError::Offline
})?
@@ -306,71 +304,64 @@ impl<E: EthSpec> CandidateBeaconNode<E> {
let beacon_node_spec = ChainSpec::from_config::<E>(&config).ok_or_else(|| {
error!(
log,
endpoint = %self.beacon_node,
"The minimal/mainnet spec type of the beacon node does not match the validator \
client. See the --network command.";
"endpoint" => %self.beacon_node,
client. See the --network command."
);
CandidateError::Incompatible
})?;
if beacon_node_spec.genesis_fork_version != spec.genesis_fork_version {
error!(
log,
"Beacon node is configured for a different network";
"endpoint" => %self.beacon_node,
"bn_genesis_fork" => ?beacon_node_spec.genesis_fork_version,
"our_genesis_fork" => ?spec.genesis_fork_version,
endpoint = %self.beacon_node,
bn_genesis_fork = ?beacon_node_spec.genesis_fork_version,
our_genesis_fork = ?spec.genesis_fork_version,
"Beacon node is configured for a different network"
);
return Err(CandidateError::Incompatible);
} else if beacon_node_spec.altair_fork_epoch != spec.altair_fork_epoch {
warn!(
log,
"Beacon node has mismatched Altair fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_altair_fork_epoch" => ?beacon_node_spec.altair_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_altair_fork_epoch = ?beacon_node_spec.altair_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Altair fork epoch"
);
} else if beacon_node_spec.bellatrix_fork_epoch != spec.bellatrix_fork_epoch {
warn!(
log,
"Beacon node has mismatched Bellatrix fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_bellatrix_fork_epoch" => ?beacon_node_spec.bellatrix_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_bellatrix_fork_epoch = ?beacon_node_spec.bellatrix_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Bellatrix fork epoch"
);
} else if beacon_node_spec.capella_fork_epoch != spec.capella_fork_epoch {
warn!(
log,
"Beacon node has mismatched Capella fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_capella_fork_epoch" => ?beacon_node_spec.capella_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_capella_fork_epoch = ?beacon_node_spec.capella_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Capella fork epoch"
);
} else if beacon_node_spec.deneb_fork_epoch != spec.deneb_fork_epoch {
warn!(
log,
"Beacon node has mismatched Deneb fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_deneb_fork_epoch" => ?beacon_node_spec.deneb_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_deneb_fork_epoch = ?beacon_node_spec.deneb_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Deneb fork epoch"
);
} else if beacon_node_spec.electra_fork_epoch != spec.electra_fork_epoch {
warn!(
log,
"Beacon node has mismatched Electra fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_electra_fork_epoch" => ?beacon_node_spec.electra_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_electra_fork_epoch = ?beacon_node_spec.electra_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Electra fork epoch"
);
} else if beacon_node_spec.fulu_fork_epoch != spec.fulu_fork_epoch {
warn!(
log,
"Beacon node has mismatched Fulu fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_fulu_fork_epoch" => ?beacon_node_spec.fulu_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
);
endpoint = %self.beacon_node,
endpoint_fulu_fork_epoch = ?beacon_node_spec.fulu_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Fulu fork epoch"
);
}
Ok(())
@@ -387,7 +378,6 @@ pub struct BeaconNodeFallback<T, E> {
slot_clock: Option<T>,
broadcast_topics: Vec<ApiTopic>,
spec: Arc<ChainSpec>,
log: Logger,
}
impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
@@ -396,7 +386,6 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
config: Config,
broadcast_topics: Vec<ApiTopic>,
spec: Arc<ChainSpec>,
log: Logger,
) -> Self {
let distance_tiers = config.sync_tolerances;
Self {
@@ -405,7 +394,6 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
slot_clock: None,
broadcast_topics,
spec,
log,
}
}
@@ -488,7 +476,6 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
&self.distance_tiers,
self.slot_clock.as_ref(),
&self.spec,
&self.log,
));
nodes.push(candidate.beacon_node.to_string());
}
@@ -501,10 +488,9 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
if let Err(e) = result {
if *e != CandidateError::PreGenesis {
warn!(
self.log,
"A connected beacon node errored during routine health check";
"error" => ?e,
"endpoint" => node,
error = ?e,
endpoint = %node,
"A connected beacon node errored during routine health check"
);
}
}
@@ -576,11 +562,7 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
// Run `func` using a `candidate`, returning the value or capturing errors.
for candidate in candidates.iter() {
futures.push(Self::run_on_candidate(
candidate.beacon_node.clone(),
&func,
&self.log,
));
futures.push(Self::run_on_candidate(candidate.beacon_node.clone(), &func));
}
drop(candidates);
@@ -598,11 +580,7 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
// Run `func` using a `candidate`, returning the value or capturing errors.
for candidate in candidates.iter() {
futures.push(Self::run_on_candidate(
candidate.beacon_node.clone(),
&func,
&self.log,
));
futures.push(Self::run_on_candidate(candidate.beacon_node.clone(), &func));
}
drop(candidates);
@@ -621,7 +599,6 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
async fn run_on_candidate<F, R, Err, O>(
candidate: BeaconNodeHttpClient,
func: F,
log: &Logger,
) -> Result<O, (String, Error<Err>)>
where
F: Fn(BeaconNodeHttpClient) -> R,
@@ -636,10 +613,9 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
Ok(val) => Ok(val),
Err(e) => {
debug!(
log,
"Request to beacon node failed";
"node" => %candidate,
"error" => ?e,
node = %candidate,
error = ?e,
"Request to beacon node failed"
);
inc_counter_vec(&ENDPOINT_ERRORS, &[candidate.as_ref()]);
Err((candidate.to_string(), Error::RequestFailed(e)))
@@ -666,11 +642,7 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
// Run `func` using a `candidate`, returning the value or capturing errors.
for candidate in candidates.iter() {
futures.push(Self::run_on_candidate(
candidate.beacon_node.clone(),
&func,
&self.log,
));
futures.push(Self::run_on_candidate(candidate.beacon_node.clone(), &func));
}
drop(candidates);
@@ -752,7 +724,6 @@ mod tests {
use crate::beacon_node_health::BeaconNodeHealthTier;
use eth2::SensitiveUrl;
use eth2::Timeouts;
use logging::test_logger;
use slot_clock::TestingSlotClock;
use strum::VariantNames;
use types::{BeaconBlockDeneb, MainnetEthSpec, Slot};
@@ -902,10 +873,9 @@ mod tests {
candidates: Vec<CandidateBeaconNode<E>>,
topics: Vec<ApiTopic>,
spec: Arc<ChainSpec>,
log: Logger,
) -> BeaconNodeFallback<TestingSlotClock, E> {
let mut beacon_node_fallback =
BeaconNodeFallback::new(candidates, Config::default(), topics, spec, log);
BeaconNodeFallback::new(candidates, Config::default(), topics, spec);
beacon_node_fallback.set_slot_clock(TestingSlotClock::new(
Slot::new(1),
@@ -932,7 +902,6 @@ mod tests {
],
vec![],
spec.clone(),
test_logger(),
);
// BeaconNodeHealthTier 1
@@ -979,7 +948,6 @@ mod tests {
vec![beacon_node_1, beacon_node_2],
vec![ApiTopic::Blocks],
spec.clone(),
test_logger(),
);
mock_beacon_node_1.mock_post_beacon_blinded_blocks_v2_ssz(Duration::from_secs(0));
@@ -1021,7 +989,6 @@ mod tests {
vec![beacon_node_1, beacon_node_2, beacon_node_3],
vec![],
spec.clone(),
test_logger(),
);
let mock1 = mock_beacon_node_1.mock_offline_node();