gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-04 09:11:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Integrate tracing (#6339)

Browse Source 
Tracing Integration
- [reference](5bbf1859e9/projects/project-ideas.md (L297))


  - [x] replace slog & log with tracing throughout the codebase
- [x] implement custom crit log
- [x] make relevant changes in the formatter
- [x] replace sloggers
- [x] re-write SSE logging components

cc: @macladson @eserilev
This commit is contained in:
ThreeHrSleep
2025-03-13 04:01:05 +05:30
committed by GitHub
parent f23f984f85
commit d60c24ef1c
241 changed files with 9485 additions and 9328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
validator_client/src/check_synced.rs Normal file
View File

@@ -0,0 +1,25 @@
use crate::beacon_node_fallback::CandidateError;
use eth2::{types::Slot, BeaconNodeHttpClient};
use tracing::warn;
pub async fn check_node_health(
beacon_node: &BeaconNodeHttpClient,
) -> Result<(Slot, bool, bool), CandidateError> {
let resp = match beacon_node.get_node_syncing().await {
Ok(resp) => resp,
Err(e) => {
warn!(
error = %e,
"Unable connect to beacon node"
);
return Err(CandidateError::Offline);
}
};
Ok((
resp.data.head_slot,
resp.data.is_optimistic,
resp.data.el_offline,
))
}

15
validator_client/src/config.rs
View File

@@ -12,11 +12,11 @@ use graffiti_file::GraffitiFile;
use initialized_validators::Config as InitializedValidatorsConfig;
use sensitive_url::SensitiveUrl;
use serde::{Deserialize, Serialize};
use slog::{info, warn, Logger};
use std::fs;
use std::net::IpAddr;
use std::path::PathBuf;
use std::time::Duration;
use tracing::{info, warn};
use types::GRAFFITI_BYTES_LEN;
use validator_http_api::{self, PK_FILENAME};
use validator_http_metrics;
@@ -141,7 +141,6 @@ impl Config {
pub fn from_cli(
cli_args: &ArgMatches,
validator_client_config: &ValidatorClient,
log: &Logger,
) -> Result<Config, String> {
let mut config = Config::default();
@@ -207,7 +206,10 @@ impl Config {
.read_graffiti_file()
.map_err(|e| format!("Error reading graffiti file: {:?}", e))?;
config.graffiti_file = Some(graffiti_file);
info!(log, "Successfully loaded graffiti file"; "path" => graffiti_file_path.to_str());
info!(
path = graffiti_file_path.to_str(),
"Successfully loaded graffiti file"
);
}
if let Some(input_graffiti) = validator_client_config.graffiti.as_ref() {
@@ -375,10 +377,9 @@ impl Config {
config.validator_store.enable_web3signer_slashing_protection =
if validator_client_config.disable_slashing_protection_web3signer {
warn!(
log,
"Slashing protection for remote keys disabled";
"info" => "ensure slashing protection on web3signer is enabled or you WILL \
get slashed"
info = "ensure slashing protection on web3signer is enabled or you WILL \
get slashed",
"Slashing protection for remote keys disabled"
);
false
} else {

11
validator_client/src/latency.rs
View File

@@ -1,9 +1,9 @@
use beacon_node_fallback::BeaconNodeFallback;
use environment::RuntimeContext;
use slog::debug;
use slot_clock::SlotClock;
use std::sync::Arc;
use tokio::time::sleep;
use tracing::debug;
use types::EthSpec;
/// The latency service will run 11/12ths of the way through the slot.
@@ -17,8 +17,6 @@ pub fn start_latency_service<T: SlotClock + 'static, E: EthSpec>(
slot_clock: T,
beacon_nodes: Arc<BeaconNodeFallback<T, E>>,
) {
let log = context.log().clone();
let future = async move {
loop {
let sleep_time = slot_clock
@@ -39,10 +37,9 @@ pub fn start_latency_service<T: SlotClock + 'static, E: EthSpec>(
for (i, measurement) in beacon_nodes.measure_latency().await.iter().enumerate() {
if let Some(latency) = measurement.latency {
debug!(
log,
"Measured BN latency";
"node" => &measurement.beacon_node_id,
"latency" => latency.as_millis(),
node = &measurement.beacon_node_id,
latency = latency.as_millis(),
"Measured BN latency"
);
validator_metrics::observe_timer_vec(
&validator_metrics::VC_BEACON_NODE_LATENCY,

173
validator_client/src/lib.rs
View File

@@ -23,7 +23,6 @@ use initialized_validators::Error::UnableToOpenVotingKeystore;
use notifier::spawn_notifier;
use parking_lot::RwLock;
use reqwest::Certificate;
use slog::{debug, error, info, warn, Logger};
use slot_clock::SlotClock;
use slot_clock::SystemTimeSlotClock;
use std::fs::File;
@@ -37,6 +36,7 @@ use tokio::{
sync::mpsc,
time::{sleep, Duration},
};
use tracing::{debug, error, info, warn};
use types::{EthSpec, Hash256};
use validator_http_api::ApiSecret;
use validator_services::{
@@ -97,7 +97,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
cli_args: &ArgMatches,
validator_client_config: &ValidatorClient,
) -> Result<Self, String> {
let config = Config::from_cli(cli_args, validator_client_config, context.log())
let config = Config::from_cli(cli_args, validator_client_config)
.map_err(|e| format!("Unable to initialize config: {}", e))?;
Self::new(context, config).await
}
@@ -105,8 +105,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
/// Instantiates the validator client, _without_ starting the timers to trigger block
/// and attestation production.
pub async fn new(context: RuntimeContext<E>, config: Config) -> Result<Self, String> {
let log = context.log().clone();
// Attempt to raise soft fd limit. The behavior is OS specific:
// `linux` - raise soft fd limit to hard
// `macos` - raise soft fd limit to `min(kernel limit, hard fd limit)`
@@ -114,25 +112,20 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
match fdlimit::raise_fd_limit().map_err(|e| format!("Unable to raise fd limit: {}", e))? {
fdlimit::Outcome::LimitRaised { from, to } => {
debug!(
log,
"Raised soft open file descriptor resource limit";
"old_limit" => from,
"new_limit" => to
old_limit = from,
new_limit = to,
"Raised soft open file descriptor resource limit"
);
}
fdlimit::Outcome::Unsupported => {
debug!(
log,
"Raising soft open file descriptor resource limit is not supported"
);
debug!("Raising soft open file descriptor resource limit is not supported");
}
};
info!(
log,
"Starting validator client";
"beacon_nodes" => format!("{:?}", &config.beacon_nodes),
"validator_dir" => format!("{:?}", config.validator_dir),
beacon_nodes = ?config.beacon_nodes,
validator_dir = ?config.validator_dir,
"Starting validator client"
);
// Optionally start the metrics server.
@@ -147,7 +140,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
Arc::new(validator_http_metrics::Context {
config: config.http_metrics.clone(),
shared: RwLock::new(shared),
log: log.clone(),
});
let exit = context.executor.exit();
@@ -162,15 +154,14 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
Some(ctx)
} else {
info!(log, "HTTP metrics server is disabled");
info!("HTTP metrics server is disabled");
None
};
// Start the explorer client which periodically sends validator process
// and system metrics to the configured endpoint.
if let Some(monitoring_config) = &config.monitoring_api {
let monitoring_client =
MonitoringHttpClient::new(monitoring_config, context.log().clone())?;
let monitoring_client = MonitoringHttpClient::new(monitoring_config)?;
monitoring_client.auto_update(
context.executor.clone(),
vec![ProcessType::Validator, ProcessType::System],
@@ -182,7 +173,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
if !config.disable_auto_discover {
let new_validators = validator_defs
.discover_local_keystores(&config.validator_dir, &config.secrets_dir, &log)
.discover_local_keystores(&config.validator_dir, &config.secrets_dir)
.map_err(|e| format!("Unable to discover local validator keystores: {:?}", e))?;
validator_defs.save(&config.validator_dir).map_err(|e| {
format!(
@@ -190,18 +181,13 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
e
)
})?;
info!(
log,
"Completed validator discovery";
"new_validators" => new_validators,
);
info!(new_validators, "Completed validator discovery");
}
let validators = InitializedValidators::from_definitions(
validator_defs,
config.validator_dir.clone(),
config.initialized_validators.clone(),
log.clone(),
)
.await
.map_err(|e| {
@@ -218,17 +204,17 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
let voting_pubkeys: Vec<_> = validators.iter_voting_pubkeys().collect();
info!(
log,
"Initialized validators";
"disabled" => validators.num_total().saturating_sub(validators.num_enabled()),
"enabled" => validators.num_enabled(),
disabled = validators
.num_total()
.saturating_sub(validators.num_enabled()),
enabled = validators.num_enabled(),
"Initialized validators"
);
if voting_pubkeys.is_empty() {
warn!(
log,
"No enabled validators";
"hint" => "create validators via the API, or the `lighthouse account` CLI command"
hint = "create validators via the API, or the `lighthouse account` CLI command",
"No enabled validators"
);
}
@@ -303,10 +289,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
// Use quicker timeouts if a fallback beacon node exists.
let timeouts = if i < last_beacon_node_index && !config.use_long_timeouts {
info!(
log,
"Fallback endpoints are available, using optimized timeouts.";
);
info!("Fallback endpoints are available, using optimized timeouts.");
Timeouts {
attestation: slot_duration / HTTP_ATTESTATION_TIMEOUT_QUOTIENT,
attester_duties: slot_duration / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT,
@@ -389,7 +372,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
config.beacon_node_fallback,
config.broadcast_topics.clone(),
context.eth2_config.spec.clone(),
log.clone(),
);
let mut proposer_nodes: BeaconNodeFallback<_, E> = BeaconNodeFallback::new(
@@ -397,12 +379,11 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
config.beacon_node_fallback,
config.broadcast_topics.clone(),
context.eth2_config.spec.clone(),
log.clone(),
);
// Perform some potentially long-running initialization tasks.
let (genesis_time, genesis_validators_root) = tokio::select! {
tuple = init_from_beacon_node(&beacon_nodes, &proposer_nodes, &context) => tuple?,
tuple = init_from_beacon_node(&beacon_nodes, &proposer_nodes) => tuple?,
() = context.executor.exit() => return Err("Shutting down".to_string())
};
@@ -427,12 +408,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
start_fallback_updater_service(context.clone(), proposer_nodes.clone())?;
let doppelganger_service = if config.enable_doppelganger_protection {
Some(Arc::new(DoppelgangerService::new(
context
.service_context(DOPPELGANGER_SERVICE_NAME.into())
.log()
.clone(),
)))
Some(Arc::new(DoppelgangerService::default()))
} else {
None
};
@@ -446,16 +422,14 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
slot_clock.clone(),
&config.validator_store,
context.executor.clone(),
log.clone(),
));
// Ensure all validators are registered in doppelganger protection.
validator_store.register_all_in_doppelganger_protection_if_enabled()?;
info!(
log,
"Loaded validator keypair store";
"voting_validators" => validator_store.num_voting_validators()
voting_validators = validator_store.num_voting_validators(),
"Loaded validator keypair store"
);
// Perform pruning of the slashing protection database on start-up. In case the database is
@@ -551,7 +525,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
// whole epoch!
let channel_capacity = E::slots_per_epoch() as usize;
let (block_service_tx, block_service_rx) = mpsc::channel(channel_capacity);
let log = self.context.log();
let api_secret = ApiSecret::create_or_open(&self.config.http_api.http_token_path)?;
@@ -569,7 +542,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
config: self.config.http_api.clone(),
sse_logging_components: self.context.sse_logging_components.clone(),
slot_clock: self.slot_clock.clone(),
log: log.clone(),
_phantom: PhantomData,
});
@@ -585,12 +557,12 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
Some(listen_addr)
} else {
info!(log, "HTTP API server is disabled");
info!("HTTP API server is disabled");
None
};
// Wait until genesis has occurred.
wait_for_genesis(&self.beacon_nodes, self.genesis_time, &self.context).await?;
wait_for_genesis(&self.beacon_nodes, self.genesis_time).await?;
duties_service::start_update_service(self.duties_service.clone(), block_service_tx);
@@ -625,7 +597,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
)
.map_err(|e| format!("Unable to start doppelganger service: {}", e))?
} else {
info!(log, "Doppelganger protection disabled.")
info!("Doppelganger protection disabled.")
}
spawn_notifier(self).map_err(|e| format!("Failed to start notifier: {}", e))?;
@@ -645,7 +617,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
async fn init_from_beacon_node<E: EthSpec>(
beacon_nodes: &BeaconNodeFallback<SystemTimeSlotClock, E>,
proposer_nodes: &BeaconNodeFallback<SystemTimeSlotClock, E>,
context: &RuntimeContext<E>,
) -> Result<(u64, Hash256), String> {
loop {
beacon_nodes.update_all_candidates().await;
@@ -659,41 +630,37 @@ async fn init_from_beacon_node<E: EthSpec>(
if proposer_total > 0 && proposer_available == 0 {
warn!(
context.log(),
"Unable to connect to a proposer node";
"retry in" => format!("{} seconds", RETRY_DELAY.as_secs()),
"total_proposers" => proposer_total,
"available_proposers" => proposer_available,
"total_beacon_nodes" => num_total,
"available_beacon_nodes" => num_available,
retry_in = format!("{} seconds", RETRY_DELAY.as_secs()),
total_proposers = proposer_total,
available_proposers = proposer_available,
total_beacon_nodes = num_total,
available_beacon_nodes = num_available,
"Unable to connect to a proposer node"
);
}
if num_available > 0 && proposer_available == 0 {
info!(
context.log(),
"Initialized beacon node connections";
"total" => num_total,
"available" => num_available,
total = num_total,
available = num_available,
"Initialized beacon node connections"
);
break;
} else if num_available > 0 {
info!(
context.log(),
"Initialized beacon node connections";
"total" => num_total,
"available" => num_available,
"proposers_available" => proposer_available,
"proposers_total" => proposer_total,
total = num_total,
available = num_available,
proposer_available,
proposer_total,
"Initialized beacon node connections"
);
break;
} else {
warn!(
context.log(),
"Unable to connect to a beacon node";
"retry in" => format!("{} seconds", RETRY_DELAY.as_secs()),
"total" => num_total,
"available" => num_available,
retry_in = format!("{} seconds", RETRY_DELAY.as_secs()),
total = num_total,
available = num_available,
"Unable to connect to a beacon node"
);
sleep(RETRY_DELAY).await;
}
@@ -714,15 +681,11 @@ async fn init_from_beacon_node<E: EthSpec>(
.filter_map(|(_, e)| e.request_failure())
.any(|e| e.status() == Some(StatusCode::NOT_FOUND))
{
info!(
context.log(),
"Waiting for genesis";
);
info!("Waiting for genesis");
} else {
error!(
context.log(),
"Errors polling beacon node";
"error" => %errors
%errors,
"Errors polling beacon node"
);
}
}
@@ -737,7 +700,6 @@ async fn init_from_beacon_node<E: EthSpec>(
async fn wait_for_genesis<E: EthSpec>(
beacon_nodes: &BeaconNodeFallback<SystemTimeSlotClock, E>,
genesis_time: u64,
context: &RuntimeContext<E>,
) -> Result<(), String> {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -751,28 +713,25 @@ async fn wait_for_genesis<E: EthSpec>(
// the slot clock.
if now < genesis_time {
info!(
context.log(),
"Starting node prior to genesis";
"seconds_to_wait" => (genesis_time - now).as_secs()
seconds_to_wait = (genesis_time - now).as_secs(),
"Starting node prior to genesis"
);
// Start polling the node for pre-genesis information, cancelling the polling as soon as the
// timer runs out.
tokio::select! {
result = poll_whilst_waiting_for_genesis(beacon_nodes, genesis_time, context.log()) => result?,
result = poll_whilst_waiting_for_genesis(beacon_nodes, genesis_time) => result?,
() = sleep(genesis_time - now) => ()
};
info!(
context.log(),
"Genesis has occurred";
"ms_since_genesis" => (genesis_time - now).as_millis()
ms_since_genesis = (genesis_time - now).as_millis(),
"Genesis has occurred"
);
} else {
info!(
context.log(),
"Genesis has already occurred";
"seconds_ago" => (now - genesis_time).as_secs()
seconds_ago = (now - genesis_time).as_secs(),
"Genesis has already occurred"
);
}
@@ -784,7 +743,6 @@ async fn wait_for_genesis<E: EthSpec>(
async fn poll_whilst_waiting_for_genesis<E: EthSpec>(
beacon_nodes: &BeaconNodeFallback<SystemTimeSlotClock, E>,
genesis_time: Duration,
log: &Logger,
) -> Result<(), String> {
loop {
match beacon_nodes
@@ -798,19 +756,17 @@ async fn poll_whilst_waiting_for_genesis<E: EthSpec>(
if !is_staking {
error!(
log,
"Staking is disabled for beacon node";
"msg" => "this will caused missed duties",
"info" => "see the --staking CLI flag on the beacon node"
msg = "this will caused missed duties",
info = "see the --staking CLI flag on the beacon node",
"Staking is disabled for beacon node"
);
}
if now < genesis_time {
info!(
log,
"Waiting for genesis";
"bn_staking_enabled" => is_staking,
"seconds_to_wait" => (genesis_time - now).as_secs()
bn_staking_enabled = is_staking,
seconds_to_wait = (genesis_time - now).as_secs(),
"Waiting for genesis"
);
} else {
break Ok(());
@@ -818,9 +774,8 @@ async fn poll_whilst_waiting_for_genesis<E: EthSpec>(
}
Err(e) => {
error!(
log,
"Error polling beacon node";
"error" => %e
error = %e,
"Error polling beacon node"
);
}
}

106
validator_client/src/notifier.rs
View File

@@ -1,8 +1,8 @@
use crate::{DutiesService, ProductionValidatorClient};
use metrics::set_gauge;
use slog::{debug, error, info, Logger};
use slot_clock::SlotClock;
use tokio::time::{sleep, Duration};
use tracing::{debug, error, info};
use types::EthSpec;
/// Spawns a notifier service which periodically logs information about the node.
@@ -14,14 +14,12 @@ pub fn spawn_notifier<E: EthSpec>(client: &ProductionValidatorClient<E>) -> Resu
let slot_duration = Duration::from_secs(context.eth2_config.spec.seconds_per_slot);
let interval_fut = async move {
let log = context.log();
loop {
if let Some(duration_to_next_slot) = duties_service.slot_clock.duration_to_next_slot() {
sleep(duration_to_next_slot + slot_duration / 2).await;
notify(&duties_service, log).await;
notify(&duties_service).await;
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
continue;
@@ -34,10 +32,7 @@ pub fn spawn_notifier<E: EthSpec>(client: &ProductionValidatorClient<E>) -> Resu
}
/// Performs a single notification routine.
async fn notify<T: SlotClock + 'static, E: EthSpec>(
duties_service: &DutiesService<T, E>,
log: &Logger,
) {
async fn notify<T: SlotClock + 'static, E: EthSpec>(duties_service: &DutiesService<T, E>) {
let (candidate_info, num_available, num_synced) =
duties_service.beacon_nodes.get_notifier_info().await;
let num_total = candidate_info.len();
@@ -61,20 +56,18 @@ async fn notify<T: SlotClock + 'static, E: EthSpec>(
.map(|candidate| candidate.endpoint.as_str())
.unwrap_or("None");
info!(
log,
"Connected to beacon node(s)";
"primary" => primary,
"total" => num_total,
"available" => num_available,
"synced" => num_synced,
primary,
total = num_total,
available = num_available,
synced = num_synced,
"Connected to beacon node(s)"
)
} else {
error!(
log,
"No synced beacon nodes";
"total" => num_total,
"available" => num_available,
"synced" => num_synced,
total = num_total,
available = num_available,
synced = num_synced,
"No synced beacon nodes"
)
}
if num_synced_fallback > 0 {
@@ -86,23 +79,21 @@ async fn notify<T: SlotClock + 'static, E: EthSpec>(
for info in candidate_info {
if let Ok(health) = info.health {
debug!(
log,
"Beacon node info";
"status" => "Connected",
"index" => info.index,
"endpoint" => info.endpoint,
"head_slot" => %health.head,
"is_optimistic" => ?health.optimistic_status,
"execution_engine_status" => ?health.execution_status,
"health_tier" => %health.health_tier,
status = "Connected",
index = info.index,
endpoint = info.endpoint,
head_slot = %health.head,
is_optimistic = ?health.optimistic_status,
execution_engine_status = ?health.execution_status,
health_tier = %health.health_tier,
"Beacon node info"
);
} else {
debug!(
log,
"Beacon node info";
"status" => "Disconnected",
"index" => info.index,
"endpoint" => info.endpoint,
status = "Disconnected",
index = info.index,
endpoint = info.endpoint,
"Beacon node info"
);
}
}
@@ -116,45 +107,44 @@ async fn notify<T: SlotClock + 'static, E: EthSpec>(
let doppelganger_detecting_validators = duties_service.doppelganger_detecting_count();
if doppelganger_detecting_validators > 0 {
info!(log, "Listening for doppelgangers"; "doppelganger_detecting_validators" => doppelganger_detecting_validators)
info!(
doppelganger_detecting_validators,
"Listening for doppelgangers"
)
}
if total_validators == 0 {
info!(
log,
"No validators present";
"msg" => "see `lighthouse vm create --help` or the HTTP API documentation"
msg = "see `lighthouse vm create --help` or the HTTP API documentation",
"No validators present"
)
} else if total_validators == attesting_validators {
info!(
log,
"All validators active";
"current_epoch_proposers" => proposing_validators,
"active_validators" => attesting_validators,
"total_validators" => total_validators,
"epoch" => format!("{}", epoch),
"slot" => format!("{}", slot),
current_epoch_proposers = proposing_validators,
active_validators = attesting_validators,
total_validators = total_validators,
%epoch,
%slot,
"All validators active"
);
} else if attesting_validators > 0 {
info!(
log,
"Some validators active";
"current_epoch_proposers" => proposing_validators,
"active_validators" => attesting_validators,
"total_validators" => total_validators,
"epoch" => format!("{}", epoch),
"slot" => format!("{}", slot),
current_epoch_proposers = proposing_validators,
active_validators = attesting_validators,
total_validators = total_validators,
%epoch,
%slot,
"Some validators active"
);
} else {
info!(
log,
"Awaiting activation";
"validators" => total_validators,
"epoch" => format!("{}", epoch),
"slot" => format!("{}", slot),
validators = total_validators,
%epoch,
%slot,
"Awaiting activation"
);
}
} else {
error!(log, "Unable to read slot clock");
error!("Unable to read slot clock");
}
}