gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-17 12:58:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge capella

Browse Source
This commit is contained in:
realbigsean
2023-01-12 12:51:09 -05:00
parent c705be202b d0368b0459
commit 06f71e8cce
142 changed files with 3412 additions and 1965 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
beacon_node/beacon_chain/src/builder.rs
View File

@@ -583,11 +583,13 @@ where
mut self,
auto_register: bool,
validators: Vec<PublicKeyBytes>,
individual_metrics_threshold: usize,
log: Logger,
) -> Self {
self.validator_monitor = Some(ValidatorMonitor::new(
validators,
auto_register,
individual_metrics_threshold,
log.clone(),
));
self
@@ -1009,6 +1011,7 @@ fn descriptive_db_error(item: &str, error: &StoreError) -> String {
#[cfg(test)]
mod test {
use super::*;
use crate::validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD;
use eth2_hashing::hash;
use genesis::{
generate_deterministic_keypairs, interop_genesis_state, DEFAULT_ETH1_BLOCK_HASH,
@@ -1065,7 +1068,12 @@ mod test {
.testing_slot_clock(Duration::from_secs(1))
.expect("should configure testing slot clock")
.shutdown_sender(shutdown_tx)
.monitor_validators(true, vec![], log.clone())
.monitor_validators(
true,
vec![],
DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD,
log.clone(),
)
.build()
.expect("should build");

5
beacon_node/beacon_chain/src/chain_config.rs
View File

@@ -65,6 +65,8 @@ pub struct ChainConfig {
/// Low values are useful for execution engines which don't improve their payload after the
/// first call, and high values are useful for ensuring the EL is given ample notice.
pub prepare_payload_lookahead: Duration,
/// Use EL-free optimistic sync for the finalized part of the chain.
pub optimistic_finalized_sync: bool,
}
impl Default for ChainConfig {
@@ -89,6 +91,9 @@ impl Default for ChainConfig {
count_unrealized_full: CountUnrealizedFull::default(),
checkpoint_sync_url_timeout: 60,
prepare_payload_lookahead: Duration::from_secs(4),
// TODO(capella): disabled until withdrawal verification is implemented
// See: https://github.com/sigp/lighthouse/issues/3870
optimistic_finalized_sync: false,
}
}
}

63
beacon_node/beacon_chain/src/execution_payload.rs
View File

@@ -15,7 +15,7 @@ use crate::{
use execution_layer::{BlockProposalContents, BuilderParams, PayloadAttributes, PayloadStatus};
use fork_choice::{InvalidationOperation, PayloadVerificationStatus};
use proto_array::{Block as ProtoBlock, ExecutionStatus};
use slog::debug;
use slog::{debug, warn};
use slot_clock::SlotClock;
use state_processing::per_block_processing::{
compute_timestamp_at_slot, get_expected_withdrawals, is_execution_enabled,
@@ -60,26 +60,51 @@ impl<T: BeaconChainTypes> PayloadNotifier<T> {
state: &BeaconState<T::EthSpec>,
notify_execution_layer: NotifyExecutionLayer,
) -> Result<Self, BlockError<T::EthSpec>> {
let payload_verification_status = match notify_execution_layer {
NotifyExecutionLayer::No => Some(PayloadVerificationStatus::Optimistic),
NotifyExecutionLayer::Yes => {
if is_execution_enabled(state, block.message().body()) {
// Perform the initial stages of payload verification.
//
// We will duplicate these checks again during `per_block_processing`, however these checks
// are cheap and doing them here ensures we protect the execution engine from junk.
partially_verify_execution_payload::<T::EthSpec, FullPayload<T::EthSpec>>(
state,
block.slot(),
block.message().execution_payload()?,
&chain.spec,
)
.map_err(BlockError::PerBlockProcessingError)?;
None
} else {
Some(PayloadVerificationStatus::Irrelevant)
let payload_verification_status = if is_execution_enabled(state, block.message().body()) {
// Perform the initial stages of payload verification.
//
// We will duplicate these checks again during `per_block_processing`, however these
// checks are cheap and doing them here ensures we have verified them before marking
// the block as optimistically imported. This is particularly relevant in the case
// where we do not send the block to the EL at all.
let block_message = block.message();
let payload = block_message.execution_payload()?;
partially_verify_execution_payload::<_, FullPayload<_>>(
state,
block.slot(),
payload,
&chain.spec,
)
.map_err(BlockError::PerBlockProcessingError)?;
match notify_execution_layer {
NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => {
// Verify the block hash here in Lighthouse and immediately mark the block as
// optimistically imported. This saves a lot of roundtrips to the EL.
let execution_layer = chain
.execution_layer
.as_ref()
.ok_or(ExecutionPayloadError::NoExecutionConnection)?;
if let Err(e) =
execution_layer.verify_payload_block_hash(payload.execution_payload_ref())
{
warn!(
chain.log,
"Falling back to slow block hash verification";
"block_number" => payload.block_number(),
"info" => "you can silence this warning with --disable-optimistic-finalized-sync",
"error" => ?e,
);
None
} else {
Some(PayloadVerificationStatus::Optimistic)
}
}
_ => None,
}
} else {
Some(PayloadVerificationStatus::Irrelevant)
};
Ok(Self {

13
beacon_node/beacon_chain/src/test_utils.rs
View File

@@ -2,6 +2,7 @@ pub use crate::persisted_beacon_chain::PersistedBeaconChain;
pub use crate::{
beacon_chain::{BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, FORK_CHOICE_DB_KEY, OP_POOL_DB_KEY},
migrate::MigratorConfig,
validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD,
BeaconChainError, NotifyExecutionLayer, ProduceBlockVerification,
};
use crate::{
@@ -367,6 +368,7 @@ where
.collect::<Result<_, _>>()
.unwrap();
let spec = MainnetEthSpec::default_spec();
let config = execution_layer::Config {
execution_endpoints: urls,
secret_files: vec![],
@@ -377,6 +379,7 @@ where
config,
self.runtime.task_executor.clone(),
self.log.clone(),
&spec,
)
.unwrap();
@@ -415,13 +418,11 @@ where
});
let mock = MockExecutionLayer::new(
self.runtime.task_executor.clone(),
spec.terminal_total_difficulty,
DEFAULT_TERMINAL_BLOCK,
spec.terminal_block_hash,
spec.terminal_block_hash_activation_epoch,
shanghai_time,
eip4844_time,
Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
spec,
None,
);
self.execution_layer = Some(mock.el.clone());
@@ -443,13 +444,11 @@ where
});
let mock_el = MockExecutionLayer::new(
self.runtime.task_executor.clone(),
spec.terminal_total_difficulty,
DEFAULT_TERMINAL_BLOCK,
spec.terminal_block_hash,
spec.terminal_block_hash_activation_epoch,
shanghai_time,
eip4844_time,
Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
spec.clone(),
Some(builder_url.clone()),
)
.move_to_terminal_block();
@@ -514,7 +513,7 @@ where
log.clone(),
5,
)))
.monitor_validators(true, vec![], log)
.monitor_validators(true, vec![], DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, log)
.trusted_setup(trusted_setup);
builder = if let Some(mutator) = self.initial_mutator {

770
beacon_node/beacon_chain/src/validator_monitor.rs
View File

@@ -21,10 +21,21 @@ use types::{
SignedContributionAndProof, Slot, SyncCommitteeMessage, VoluntaryExit,
};
/// Used for Prometheus labels.
///
/// We've used `total` for this value to align with Nimbus, as per:
/// https://github.com/sigp/lighthouse/pull/3728#issuecomment-1375173063
const TOTAL_LABEL: &str = "total";
/// The validator monitor collects per-epoch data about each monitored validator. Historical data
/// will be kept around for `HISTORIC_EPOCHS` before it is pruned.
pub const HISTORIC_EPOCHS: usize = 4;
/// Once the validator monitor reaches this number of validators it will stop
/// tracking their metrics/logging individually in an effort to reduce
/// Prometheus cardinality and log volume.
pub const DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD: usize = 64;
#[derive(Debug)]
pub enum Error {
InvalidPubkey(String),
@@ -258,16 +269,27 @@ pub struct ValidatorMonitor<T> {
indices: HashMap<u64, PublicKeyBytes>,
/// If true, allow the automatic registration of validators.
auto_register: bool,
/// Once the number of monitored validators goes above this threshold, we
/// will stop tracking metrics/logs on a per-validator basis. This prevents
/// large validator counts causing infeasibly high cardinailty for
/// Prometheus and high log volumes.
individual_tracking_threshold: usize,
log: Logger,
_phantom: PhantomData<T>,
}
impl<T: EthSpec> ValidatorMonitor<T> {
pub fn new(pubkeys: Vec<PublicKeyBytes>, auto_register: bool, log: Logger) -> Self {
pub fn new(
pubkeys: Vec<PublicKeyBytes>,
auto_register: bool,
individual_tracking_threshold: usize,
log: Logger,
) -> Self {
let mut s = Self {
validators: <_>::default(),
indices: <_>::default(),
auto_register,
individual_tracking_threshold,
log,
_phantom: PhantomData,
};
@@ -277,6 +299,13 @@ impl<T: EthSpec> ValidatorMonitor<T> {
s
}
/// Returns `true` when the validator count is sufficiently low enough to
/// emit metrics and logs on a per-validator basis (rather than just an
/// aggregated basis).
fn individual_tracking(&self) -> bool {
self.validators.len() <= self.individual_tracking_threshold
}
/// Add some validators to `self` for additional monitoring.
fn add_validator_pubkey(&mut self, pubkey: PublicKeyBytes) {
let index_opt = self
@@ -317,6 +346,12 @@ impl<T: EthSpec> ValidatorMonitor<T> {
for monitored_validator in self.validators.values() {
if let Some(i) = monitored_validator.index {
monitored_validator.touch_epoch_summary(current_epoch);
// Only log the per-validator metrics if it's enabled.
if !self.individual_tracking() {
continue;
}
let i = i as usize;
let id = &monitored_validator.id;
@@ -379,6 +414,24 @@ impl<T: EthSpec> ValidatorMonitor<T> {
}
}
/// Run `func` with the `TOTAL_LABEL` and optionally the
/// `individual_id`.
///
/// This function is used for registering metrics that can be applied to
/// both all validators and an indivdual validator. For example, the count
/// of missed head votes can be aggregated across all validators in a single
/// metric and also tracked on a per-validator basis.
///
/// We allow disabling tracking metrics on an individual validator basis
/// since it can result in untenable cardinality with high validator counts.
fn aggregatable_metric<F: Fn(&str)>(&self, individual_id: &str, func: F) {
func(TOTAL_LABEL);
if self.individual_tracking() {
func(individual_id);
}
}
pub fn process_validator_statuses(
&self,
epoch: Epoch,
@@ -431,72 +484,92 @@ impl<T: EthSpec> ValidatorMonitor<T> {
// For Base states, this will be *any* attestation whatsoever. For Altair states,
// this will be any attestation that matched a "timely" flag.
if previous_epoch_matched_any {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_HIT,
&[id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_HIT,
&[label],
)
});
attestation_success.push(id);
debug!(
self.log,
"Previous epoch attestation success";
"matched_source" => previous_epoch_matched_source,
"matched_target" => previous_epoch_matched_target,
"matched_head" => previous_epoch_matched_head,
"epoch" => prev_epoch,
"validator" => id,
)
if self.individual_tracking() {
debug!(
self.log,
"Previous epoch attestation success";
"matched_source" => previous_epoch_matched_source,
"matched_target" => previous_epoch_matched_target,
"matched_head" => previous_epoch_matched_head,
"epoch" => prev_epoch,
"validator" => id,
)
}
} else {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_MISS,
&[id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_MISS,
&[label],
);
});
attestation_miss.push(id);
debug!(
self.log,
"Previous epoch attestation missing";
"epoch" => prev_epoch,
"validator" => id,
)
if self.individual_tracking() {
debug!(
self.log,
"Previous epoch attestation missing";
"epoch" => prev_epoch,
"validator" => id,
)
}
}
// Indicates if any on-chain attestation hit the head.
if previous_epoch_matched_head {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_HIT,
&[id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_HIT,
&[label],
);
});
} else {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_MISS,
&[id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_MISS,
&[label],
);
});
head_miss.push(id);
debug!(
self.log,
"Attestation failed to match head";
"epoch" => prev_epoch,
"validator" => id,
);
if self.individual_tracking() {
debug!(
self.log,
"Attestation failed to match head";
"epoch" => prev_epoch,
"validator" => id,
);
}
}
// Indicates if any on-chain attestation hit the target.
if previous_epoch_matched_target {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_HIT,
&[id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_HIT,
&[label],
);
});
} else {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_MISS,
&[id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_MISS,
&[label],
);
});
target_miss.push(id);
debug!(
self.log,
"Attestation failed to match target";
"epoch" => prev_epoch,
"validator" => id,
);
if self.individual_tracking() {
debug!(
self.log,
"Attestation failed to match target";
"epoch" => prev_epoch,
"validator" => id,
);
}
}
// Get the minimum value among the validator monitor observed inclusion distance
@@ -511,21 +584,25 @@ impl<T: EthSpec> ValidatorMonitor<T> {
if let Some(inclusion_delay) = min_inclusion_distance {
if inclusion_delay > spec.min_attestation_inclusion_delay {
suboptimal_inclusion.push(id);
debug!(
self.log,
"Potential sub-optimal inclusion delay";
"optimal" => spec.min_attestation_inclusion_delay,
"delay" => inclusion_delay,
"epoch" => prev_epoch,
"validator" => id,
);
if self.individual_tracking() {
debug!(
self.log,
"Potential sub-optimal inclusion delay";
"optimal" => spec.min_attestation_inclusion_delay,
"delay" => inclusion_delay,
"epoch" => prev_epoch,
"validator" => id,
);
}
}
metrics::set_int_gauge(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_INCLUSION_DISTANCE,
&[id],
inclusion_delay as i64,
);
if self.individual_tracking() {
metrics::set_int_gauge(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_INCLUSION_DISTANCE,
&[id],
inclusion_delay as i64,
);
}
}
// Indicates the number of sync committee signatures that made it into
@@ -536,13 +613,19 @@ impl<T: EthSpec> ValidatorMonitor<T> {
let current_epoch = epoch - 1;
if let Some(sync_committee) = summary.sync_committee() {
if sync_committee.contains(pubkey) {
metrics::set_int_gauge(
&metrics::VALIDATOR_MONITOR_VALIDATOR_IN_CURRENT_SYNC_COMMITTEE,
&[id],
1,
);
if self.individual_tracking() {
metrics::set_int_gauge(
&metrics::VALIDATOR_MONITOR_VALIDATOR_IN_CURRENT_SYNC_COMMITTEE,
&[id],
1,
);
}
let epoch_summary = monitored_validator.summaries.read();
if let Some(summary) = epoch_summary.get(&current_epoch) {
// This log is not gated by
// `self.individual_tracking()` since the number of
// logs that can be generated is capped by the size
// of the sync committee.
info!(
self.log,
"Current epoch sync signatures";
@@ -552,7 +635,7 @@ impl<T: EthSpec> ValidatorMonitor<T> {
"validator" => id,
);
}
} else {
} else if self.individual_tracking() {
metrics::set_int_gauge(
&metrics::VALIDATOR_MONITOR_VALIDATOR_IN_CURRENT_SYNC_COMMITTEE,
&[id],
@@ -693,12 +776,17 @@ impl<T: EthSpec> ValidatorMonitor<T> {
let id = &validator.id;
let delay = get_block_delay_ms(seen_timestamp, block, slot_clock);
metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_BEACON_BLOCK_TOTAL, &[src, id]);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_BEACON_BLOCK_DELAY_SECONDS,
&[src, id],
delay,
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_BEACON_BLOCK_TOTAL,
&[src, label],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_BEACON_BLOCK_DELAY_SECONDS,
&[src, label],
delay,
);
});
info!(
self.log,
@@ -764,27 +852,31 @@ impl<T: EthSpec> ValidatorMonitor<T> {
if let Some(validator) = self.get_validator(*i) {
let id = &validator.id;
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_TOTAL,
&[src, id],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_DELAY_SECONDS,
&[src, id],
delay,
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_TOTAL,
&[src, label],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_DELAY_SECONDS,
&[src, label],
delay,
);
});
info!(
self.log,
"Unaggregated attestation";
"head" => ?data.beacon_block_root,
"index" => %data.index,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %data.slot,
"src" => src,
"validator" => %id,
);
if self.individual_tracking() {
info!(
self.log,
"Unaggregated attestation";
"head" => ?data.beacon_block_root,
"index" => %data.index,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %data.slot,
"src" => src,
"validator" => %id,
);
}
validator.with_epoch_summary(epoch, |summary| {
summary.register_unaggregated_attestation(delay)
@@ -848,27 +940,31 @@ impl<T: EthSpec> ValidatorMonitor<T> {
if let Some(validator) = self.get_validator(aggregator_index) {
let id = &validator.id;
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_TOTAL,
&[src, id],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_DELAY_SECONDS,
&[src, id],
delay,
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_TOTAL,
&[src, label],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_DELAY_SECONDS,
&[src, label],
delay,
);
});
info!(
self.log,
"Aggregated attestation";
"head" => ?data.beacon_block_root,
"index" => %data.index,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %data.slot,
"src" => src,
"validator" => %id,
);
if self.individual_tracking() {
info!(
self.log,
"Aggregated attestation";
"head" => ?data.beacon_block_root,
"index" => %data.index,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %data.slot,
"src" => src,
"validator" => %id,
);
}
validator.with_epoch_summary(epoch, |summary| {
summary.register_aggregated_attestation(delay)
@@ -879,27 +975,31 @@ impl<T: EthSpec> ValidatorMonitor<T> {
if let Some(validator) = self.get_validator(*i) {
let id = &validator.id;
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_TOTAL,
&[src, id],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_DELAY_SECONDS,
&[src, id],
delay,
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_TOTAL,
&[src, label],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_DELAY_SECONDS,
&[src, label],
delay,
);
});
info!(
self.log,
"Attestation included in aggregate";
"head" => ?data.beacon_block_root,
"index" => %data.index,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %data.slot,
"src" => src,
"validator" => %id,
);
if self.individual_tracking() {
info!(
self.log,
"Attestation included in aggregate";
"head" => ?data.beacon_block_root,
"index" => %data.index,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %data.slot,
"src" => src,
"validator" => %id,
);
}
validator.with_epoch_summary(epoch, |summary| {
summary.register_aggregate_attestation_inclusion()
@@ -933,26 +1033,31 @@ impl<T: EthSpec> ValidatorMonitor<T> {
if let Some(validator) = self.get_validator(*i) {
let id = &validator.id;
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_TOTAL,
&["block", id],
);
metrics::set_int_gauge(
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_DELAY_SLOTS,
&["block", id],
delay.as_u64() as i64,
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_TOTAL,
&["block", label],
);
});
info!(
self.log,
"Attestation included in block";
"head" => ?data.beacon_block_root,
"index" => %data.index,
"inclusion_lag" => format!("{} slot(s)", delay),
"epoch" => %epoch,
"slot" => %data.slot,
"validator" => %id,
);
if self.individual_tracking() {
metrics::set_int_gauge(
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_DELAY_SLOTS,
&["block", id],
delay.as_u64() as i64,
);
info!(
self.log,
"Attestation included in block";
"head" => ?data.beacon_block_root,
"index" => %data.index,
"inclusion_lag" => format!("{} slot(s)", delay),
"epoch" => %epoch,
"slot" => %data.slot,
"validator" => %id,
);
}
validator.with_epoch_summary(epoch, |summary| {
summary.register_attestation_block_inclusion(inclusion_distance)
@@ -1010,26 +1115,30 @@ impl<T: EthSpec> ValidatorMonitor<T> {
slot_clock,
);
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGES_TOTAL,
&[src, id],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGES_DELAY_SECONDS,
&[src, id],
delay,
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGES_TOTAL,
&[src, label],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGES_DELAY_SECONDS,
&[src, label],
delay,
);
});
info!(
self.log,
"Sync committee message";
"head" => %sync_committee_message.beacon_block_root,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %sync_committee_message.slot,
"src" => src,
"validator" => %id,
);
if self.individual_tracking() {
info!(
self.log,
"Sync committee message";
"head" => %sync_committee_message.beacon_block_root,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %sync_committee_message.slot,
"src" => src,
"validator" => %id,
);
}
validator.with_epoch_summary(epoch, |summary| {
summary.register_sync_committee_message(delay)
@@ -1094,26 +1203,30 @@ impl<T: EthSpec> ValidatorMonitor<T> {
if let Some(validator) = self.get_validator(aggregator_index) {
let id = &validator.id;
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_SYNC_CONTRIBUTIONS_TOTAL,
&[src, id],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_SYNC_CONTRIBUTIONS_DELAY_SECONDS,
&[src, id],
delay,
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_SYNC_CONTRIBUTIONS_TOTAL,
&[src, label],
);
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_SYNC_CONTRIBUTIONS_DELAY_SECONDS,
&[src, label],
delay,
);
});
info!(
self.log,
"Sync contribution";
"head" => %beacon_block_root,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %slot,
"src" => src,
"validator" => %id,
);
if self.individual_tracking() {
info!(
self.log,
"Sync contribution";
"head" => %beacon_block_root,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %slot,
"src" => src,
"validator" => %id,
);
}
validator.with_epoch_summary(epoch, |summary| {
summary.register_sync_committee_contribution(delay)
@@ -1124,21 +1237,25 @@ impl<T: EthSpec> ValidatorMonitor<T> {
if let Some(validator) = self.validators.get(validator_pubkey) {
let id = &validator.id;
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGE_IN_CONTRIBUTION_TOTAL,
&[src, id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGE_IN_CONTRIBUTION_TOTAL,
&[src, label],
);
});
info!(
self.log,
"Sync signature included in contribution";
"head" => %beacon_block_root,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %slot,
"src" => src,
"validator" => %id,
);
if self.individual_tracking() {
info!(
self.log,
"Sync signature included in contribution";
"head" => %beacon_block_root,
"delay_ms" => %delay.as_millis(),
"epoch" => %epoch,
"slot" => %slot,
"src" => src,
"validator" => %id,
);
}
validator.with_epoch_summary(epoch, |summary| {
summary.register_sync_signature_contribution_inclusion()
@@ -1160,19 +1277,23 @@ impl<T: EthSpec> ValidatorMonitor<T> {
if let Some(validator) = self.validators.get(validator_pubkey) {
let id = &validator.id;
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGE_IN_BLOCK_TOTAL,
&["block", id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGE_IN_BLOCK_TOTAL,
&["block", label],
);
});
info!(
self.log,
"Sync signature included in block";
"head" => %beacon_block_root,
"epoch" => %epoch,
"slot" => %slot,
"validator" => %id,
);
if self.individual_tracking() {
info!(
self.log,
"Sync signature included in block";
"head" => %beacon_block_root,
"epoch" => %epoch,
"slot" => %slot,
"validator" => %id,
);
}
validator.with_epoch_summary(epoch, |summary| {
summary.register_sync_signature_block_inclusions();
@@ -1201,8 +1322,12 @@ impl<T: EthSpec> ValidatorMonitor<T> {
let id = &validator.id;
let epoch = exit.epoch;
metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_EXIT_TOTAL, &[src, id]);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_EXIT_TOTAL, &[src, label]);
});
// Not gated behind `self.individual_tracking()` since it's an
// infrequent and interesting message.
info!(
self.log,
"Voluntary exit";
@@ -1240,11 +1365,15 @@ impl<T: EthSpec> ValidatorMonitor<T> {
if let Some(validator) = self.get_validator(proposer) {
let id = &validator.id;
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PROPOSER_SLASHING_TOTAL,
&[src, id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_PROPOSER_SLASHING_TOTAL,
&[src, label],
);
});
// Not gated behind `self.individual_tracking()` since it's an
// infrequent and interesting message.
crit!(
self.log,
"Proposer slashing";
@@ -1293,11 +1422,15 @@ impl<T: EthSpec> ValidatorMonitor<T> {
let id = &validator.id;
let epoch = data.slot.epoch(T::slots_per_epoch());
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_ATTESTER_SLASHING_TOTAL,
&[src, id],
);
self.aggregatable_metric(id, |label| {
metrics::inc_counter_vec(
&metrics::VALIDATOR_MONITOR_ATTESTER_SLASHING_TOTAL,
&[src, label],
);
});
// Not gated behind `self.individual_tracking()` since it's an
// infrequent and interesting message.
crit!(
self.log,
"Attester slashing";
@@ -1347,69 +1480,80 @@ impl<T: EthSpec> ValidatorMonitor<T> {
/*
* Attestations
*/
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_TOTAL,
&[id],
summary.attestations as i64,
);
if let Some(delay) = summary.attestation_min_delay {
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_MIN_DELAY_SECONDS,
&[id],
delay,
);
self.aggregatable_metric(id, |tag| {
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_MIN_DELAY_SECONDS,
&[tag],
delay,
);
});
}
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_AGGREGATE_INCLUSIONS,
&[id],
summary.attestation_aggregate_inclusions as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_INCLUSIONS,
&[id],
summary.attestation_block_inclusions as i64,
);
if let Some(distance) = summary.attestation_min_block_inclusion_distance {
if self.individual_tracking() {
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_MIN_INCLUSION_DISTANCE,
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_TOTAL,
&[id],
distance.as_u64() as i64,
summary.attestations as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_AGGREGATE_INCLUSIONS,
&[id],
summary.attestation_aggregate_inclusions as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_INCLUSIONS,
&[id],
summary.attestation_block_inclusions as i64,
);
if let Some(distance) = summary.attestation_min_block_inclusion_distance {
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_MIN_INCLUSION_DISTANCE,
&[id],
distance.as_u64() as i64,
);
}
}
/*
* Sync committee messages
*/
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_COMMITTEE_MESSAGES_TOTAL,
&[id],
summary.sync_committee_messages as i64,
);
if let Some(delay) = summary.sync_committee_message_min_delay {
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_COMMITTEE_MESSAGES_MIN_DELAY_SECONDS,
self.aggregatable_metric(id, |tag| {
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_COMMITTEE_MESSAGES_MIN_DELAY_SECONDS,
&[tag],
delay,
);
});
}
if self.individual_tracking() {
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_COMMITTEE_MESSAGES_TOTAL,
&[id],
delay,
summary.sync_committee_messages as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTION_INCLUSIONS,
&[id],
summary.sync_signature_contribution_inclusions as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_SIGNATURE_BLOCK_INCLUSIONS,
&[id],
summary.sync_signature_block_inclusions as i64,
);
}
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTION_INCLUSIONS,
&[id],
summary.sync_signature_contribution_inclusions as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_SIGNATURE_BLOCK_INCLUSIONS,
&[id],
summary.sync_signature_block_inclusions as i64,
);
/*
* Sync contributions
*/
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTIONS_TOTAL,
&[id],
summary.sync_contributions as i64,
);
if self.individual_tracking() {
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTIONS_TOTAL,
&[id],
summary.sync_contributions as i64,
);
}
if let Some(delay) = summary.sync_contribution_min_delay {
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTION_MIN_DELAY_SECONDS,
@@ -1421,51 +1565,61 @@ impl<T: EthSpec> ValidatorMonitor<T> {
/*
* Blocks
*/
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_TOTAL,
&[id],
summary.blocks as i64,
);
if let Some(delay) = summary.block_min_delay {
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_MIN_DELAY_SECONDS,
if self.individual_tracking() {
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_TOTAL,
&[id],
delay,
summary.blocks as i64,
);
}
if let Some(delay) = summary.block_min_delay {
self.aggregatable_metric(id, |tag| {
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_MIN_DELAY_SECONDS,
&[tag],
delay,
);
});
}
/*
* Aggregates
*/
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_TOTAL,
&[id],
summary.aggregates as i64,
);
if let Some(delay) = summary.aggregate_min_delay {
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_MIN_DELAY_SECONDS,
if self.individual_tracking() {
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_TOTAL,
&[id],
delay,
summary.aggregates as i64,
);
}
if let Some(delay) = summary.aggregate_min_delay {
self.aggregatable_metric(id, |tag| {
metrics::observe_timer_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_MIN_DELAY_SECONDS,
&[tag],
delay,
);
});
}
/*
* Other
*/
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_EXITS_TOTAL,
&[id],
summary.exits as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_PROPOSER_SLASHINGS_TOTAL,
&[id],
summary.proposer_slashings as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTER_SLASHINGS_TOTAL,
&[id],
summary.attester_slashings as i64,
);
if self.individual_tracking() {
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_EXITS_TOTAL,
&[id],
summary.exits as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_PROPOSER_SLASHINGS_TOTAL,
&[id],
summary.proposer_slashings as i64,
);
metrics::set_gauge_vec(
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTER_SLASHINGS_TOTAL,
&[id],
summary.attester_slashings as i64,
);
}
}
}
}