mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-16 03:12:41 +00:00
Integrate tracing (#6339)
Tracing Integration
- [reference](5bbf1859e9/projects/project-ideas.md (L297))
- [x] replace slog & log with tracing throughout the codebase
- [x] implement custom crit log
- [x] make relevant changes in the formatter
- [x] replace sloggers
- [x] re-write SSE logging components
cc: @macladson @eserilev
This commit is contained in:
@@ -3,12 +3,13 @@ use beacon_node_fallback::{ApiTopic, BeaconNodeFallback};
|
||||
use either::Either;
|
||||
use environment::RuntimeContext;
|
||||
use futures::future::join_all;
|
||||
use slog::{crit, debug, error, info, trace, warn};
|
||||
use logging::crit;
|
||||
use slot_clock::SlotClock;
|
||||
use std::collections::HashMap;
|
||||
use std::ops::Deref;
|
||||
use std::sync::Arc;
|
||||
use tokio::time::{sleep, sleep_until, Duration, Instant};
|
||||
use tracing::{debug, error, info, trace, warn};
|
||||
use tree_hash::TreeHash;
|
||||
use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot};
|
||||
use validator_store::{Error as ValidatorStoreError, ValidatorStore};
|
||||
@@ -128,9 +129,8 @@ impl<T, E: EthSpec> Deref for AttestationService<T, E> {
|
||||
impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
/// Starts the service which periodically produces attestations.
|
||||
pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> {
|
||||
let log = self.context.log().clone();
|
||||
if self.disable {
|
||||
info!(log, "Attestation service disabled");
|
||||
info!("Attestation service disabled");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -141,9 +141,8 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
.ok_or("Unable to determine duration to next slot")?;
|
||||
|
||||
info!(
|
||||
log,
|
||||
"Attestation production service started";
|
||||
"next_update_millis" => duration_to_next_slot.as_millis()
|
||||
next_update_millis = duration_to_next_slot.as_millis(),
|
||||
"Attestation production service started"
|
||||
);
|
||||
|
||||
let executor = self.context.executor.clone();
|
||||
@@ -152,22 +151,14 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
loop {
|
||||
if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
|
||||
sleep(duration_to_next_slot + slot_duration / 3).await;
|
||||
let log = self.context.log();
|
||||
|
||||
if let Err(e) = self.spawn_attestation_tasks(slot_duration) {
|
||||
crit!(
|
||||
log,
|
||||
"Failed to spawn attestation tasks";
|
||||
"error" => e
|
||||
)
|
||||
crit!(error = e, "Failed to spawn attestation tasks")
|
||||
} else {
|
||||
trace!(
|
||||
log,
|
||||
"Spawned attestation tasks";
|
||||
)
|
||||
trace!("Spawned attestation tasks");
|
||||
}
|
||||
} else {
|
||||
error!(log, "Failed to read slot clock");
|
||||
error!("Failed to read slot clock");
|
||||
// If we can't read the slot clock, just wait another slot.
|
||||
sleep(slot_duration).await;
|
||||
continue;
|
||||
@@ -249,7 +240,6 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
validator_duties: Vec<DutyAndProof>,
|
||||
aggregate_production_instant: Instant,
|
||||
) -> Result<(), ()> {
|
||||
let log = self.context.log();
|
||||
let attestations_timer = validator_metrics::start_timer_vec(
|
||||
&validator_metrics::ATTESTATION_SERVICE_TIMES,
|
||||
&[validator_metrics::ATTESTATIONS],
|
||||
@@ -269,11 +259,10 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
.await
|
||||
.map_err(move |e| {
|
||||
crit!(
|
||||
log,
|
||||
"Error during attestation routine";
|
||||
"error" => format!("{:?}", e),
|
||||
"committee_index" => committee_index,
|
||||
"slot" => slot.as_u64(),
|
||||
error = format!("{:?}", e),
|
||||
committee_index,
|
||||
slot = slot.as_u64(),
|
||||
"Error during attestation routine"
|
||||
)
|
||||
})?;
|
||||
|
||||
@@ -306,11 +295,10 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
.await
|
||||
.map_err(move |e| {
|
||||
crit!(
|
||||
log,
|
||||
"Error during attestation routine";
|
||||
"error" => format!("{:?}", e),
|
||||
"committee_index" => committee_index,
|
||||
"slot" => slot.as_u64(),
|
||||
error = format!("{:?}", e),
|
||||
committee_index,
|
||||
slot = slot.as_u64(),
|
||||
"Error during attestation routine"
|
||||
)
|
||||
})?;
|
||||
}
|
||||
@@ -336,8 +324,6 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
committee_index: CommitteeIndex,
|
||||
validator_duties: &[DutyAndProof],
|
||||
) -> Result<Option<AttestationData>, String> {
|
||||
let log = self.context.log();
|
||||
|
||||
if validator_duties.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -373,13 +359,12 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
// Ensure that the attestation matches the duties.
|
||||
if !duty.match_attestation_data::<E>(attestation_data, &self.context.eth2_config.spec) {
|
||||
crit!(
|
||||
log,
|
||||
"Inconsistent validator duties during signing";
|
||||
"validator" => ?duty.pubkey,
|
||||
"duty_slot" => duty.slot,
|
||||
"attestation_slot" => attestation_data.slot,
|
||||
"duty_index" => duty.committee_index,
|
||||
"attestation_index" => attestation_data.index,
|
||||
validator = ?duty.pubkey,
|
||||
duty_slot = %duty.slot,
|
||||
attestation_slot = %attestation_data.slot,
|
||||
duty_index = duty.committee_index,
|
||||
attestation_index = attestation_data.index,
|
||||
"Inconsistent validator duties during signing"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -396,11 +381,10 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
Ok(attestation) => attestation,
|
||||
Err(err) => {
|
||||
crit!(
|
||||
log,
|
||||
"Invalid validator duties during signing";
|
||||
"validator" => ?duty.pubkey,
|
||||
"duty" => ?duty,
|
||||
"err" => ?err,
|
||||
validator = ?duty.pubkey,
|
||||
?duty,
|
||||
?err,
|
||||
"Invalid validator duties during signing"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -421,24 +405,22 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
// A pubkey can be missing when a validator was recently
|
||||
// removed via the API.
|
||||
warn!(
|
||||
log,
|
||||
"Missing pubkey for attestation";
|
||||
"info" => "a validator may have recently been removed from this VC",
|
||||
"pubkey" => ?pubkey,
|
||||
"validator" => ?duty.pubkey,
|
||||
"committee_index" => committee_index,
|
||||
"slot" => slot.as_u64(),
|
||||
info = "a validator may have recently been removed from this VC",
|
||||
pubkey = ?pubkey,
|
||||
validator = ?duty.pubkey,
|
||||
committee_index = committee_index,
|
||||
slot = slot.as_u64(),
|
||||
"Missing pubkey for attestation"
|
||||
);
|
||||
None
|
||||
}
|
||||
Err(e) => {
|
||||
crit!(
|
||||
log,
|
||||
"Failed to sign attestation";
|
||||
"error" => ?e,
|
||||
"validator" => ?duty.pubkey,
|
||||
"committee_index" => committee_index,
|
||||
"slot" => slot.as_u64(),
|
||||
error = ?e,
|
||||
validator = ?duty.pubkey,
|
||||
committee_index,
|
||||
slot = slot.as_u64(),
|
||||
"Failed to sign attestation"
|
||||
);
|
||||
None
|
||||
}
|
||||
@@ -453,7 +435,7 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
.unzip();
|
||||
|
||||
if attestations.is_empty() {
|
||||
warn!(log, "No attestations were published");
|
||||
warn!("No attestations were published");
|
||||
return Ok(None);
|
||||
}
|
||||
let fork_name = self
|
||||
@@ -481,12 +463,11 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
// This shouldn't happen unless BN and VC are out of sync with
|
||||
// respect to the Electra fork.
|
||||
error!(
|
||||
log,
|
||||
"Unable to convert to SingleAttestation";
|
||||
"error" => ?e,
|
||||
"committee_index" => attestation_data.index,
|
||||
"slot" => slot.as_u64(),
|
||||
"type" => "unaggregated",
|
||||
error = ?e,
|
||||
committee_index = attestation_data.index,
|
||||
slot = slot.as_u64(),
|
||||
"type" = "unaggregated",
|
||||
"Unable to convert to SingleAttestation"
|
||||
);
|
||||
None
|
||||
}
|
||||
@@ -509,22 +490,20 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
.await
|
||||
{
|
||||
Ok(()) => info!(
|
||||
log,
|
||||
"Successfully published attestations";
|
||||
"count" => attestations.len(),
|
||||
"validator_indices" => ?validator_indices,
|
||||
"head_block" => ?attestation_data.beacon_block_root,
|
||||
"committee_index" => attestation_data.index,
|
||||
"slot" => attestation_data.slot.as_u64(),
|
||||
"type" => "unaggregated",
|
||||
count = attestations.len(),
|
||||
validator_indices = ?validator_indices,
|
||||
head_block = ?attestation_data.beacon_block_root,
|
||||
committee_index = attestation_data.index,
|
||||
slot = attestation_data.slot.as_u64(),
|
||||
"type" = "unaggregated",
|
||||
"Successfully published attestations"
|
||||
),
|
||||
Err(e) => error!(
|
||||
log,
|
||||
"Unable to publish attestations";
|
||||
"error" => %e,
|
||||
"committee_index" => attestation_data.index,
|
||||
"slot" => slot.as_u64(),
|
||||
"type" => "unaggregated",
|
||||
error = %e,
|
||||
committee_index = attestation_data.index,
|
||||
slot = slot.as_u64(),
|
||||
"type" = "unaggregated",
|
||||
"Unable to publish attestations"
|
||||
),
|
||||
}
|
||||
|
||||
@@ -550,8 +529,6 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
committee_index: CommitteeIndex,
|
||||
validator_duties: &[DutyAndProof],
|
||||
) -> Result<(), String> {
|
||||
let log = self.context.log();
|
||||
|
||||
if !validator_duties
|
||||
.iter()
|
||||
.any(|duty_and_proof| duty_and_proof.selection_proof.is_some())
|
||||
@@ -609,7 +586,7 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
let selection_proof = duty_and_proof.selection_proof.as_ref()?;
|
||||
|
||||
if !duty.match_attestation_data::<E>(attestation_data, &self.context.eth2_config.spec) {
|
||||
crit!(log, "Inconsistent validator duties during signing");
|
||||
crit!("Inconsistent validator duties during signing");
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -627,19 +604,14 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
|
||||
// A pubkey can be missing when a validator was recently
|
||||
// removed via the API.
|
||||
debug!(
|
||||
log,
|
||||
"Missing pubkey for aggregate";
|
||||
"pubkey" => ?pubkey,
|
||||
);
|
||||
debug!(?pubkey, "Missing pubkey for aggregate");
|
||||
None
|
||||
}
|
||||
Err(e) => {
|
||||
crit!(
|
||||
log,
|
||||
"Failed to sign aggregate";
|
||||
"error" => ?e,
|
||||
"pubkey" => ?duty.pubkey,
|
||||
error = ?e,
|
||||
pubkey = ?duty.pubkey,
|
||||
"Failed to sign aggregate"
|
||||
);
|
||||
None
|
||||
}
|
||||
@@ -683,14 +655,13 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
for signed_aggregate_and_proof in signed_aggregate_and_proofs {
|
||||
let attestation = signed_aggregate_and_proof.message().aggregate();
|
||||
info!(
|
||||
log,
|
||||
"Successfully published attestation";
|
||||
"aggregator" => signed_aggregate_and_proof.message().aggregator_index(),
|
||||
"signatures" => attestation.num_set_aggregation_bits(),
|
||||
"head_block" => format!("{:?}", attestation.data().beacon_block_root),
|
||||
"committee_index" => attestation.committee_index(),
|
||||
"slot" => attestation.data().slot.as_u64(),
|
||||
"type" => "aggregated",
|
||||
aggregator = signed_aggregate_and_proof.message().aggregator_index(),
|
||||
signatures = attestation.num_set_aggregation_bits(),
|
||||
head_block = format!("{:?}", attestation.data().beacon_block_root),
|
||||
committee_index = attestation.committee_index(),
|
||||
slot = attestation.data().slot.as_u64(),
|
||||
"type" = "aggregated",
|
||||
"Successfully published attestation"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -698,13 +669,12 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
|
||||
for signed_aggregate_and_proof in signed_aggregate_and_proofs {
|
||||
let attestation = &signed_aggregate_and_proof.message().aggregate();
|
||||
crit!(
|
||||
log,
|
||||
"Failed to publish attestation";
|
||||
"error" => %e,
|
||||
"aggregator" => signed_aggregate_and_proof.message().aggregator_index(),
|
||||
"committee_index" => attestation.committee_index(),
|
||||
"slot" => attestation.data().slot.as_u64(),
|
||||
"type" => "aggregated",
|
||||
error = %e,
|
||||
aggregator = signed_aggregate_and_proof.message().aggregator_index(),
|
||||
committee_index = attestation.committee_index(),
|
||||
slot = attestation.data().slot.as_u64(),
|
||||
"type" = "aggregated",
|
||||
"Failed to publish attestation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user