mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-10 20:22:02 +00:00
3dc1eb5eb6
## Proposed Changes A user on Discord (`@ChewsMacRibs`) reported that the validator monitor was logging `WARN Attested to an incorrect head` for their validator while it was awaiting activation. This PR modifies the monitor so that it ignores inactive validators, by the logic that they are either awaiting activation, or have already exited. Either way, there's no way for an inactive validator to have their attestations included on chain, so no need for the monitor to report on them. ## Additional Info To reproduce the bug requires registering validator keys manually with `--validator-monitor-pubkeys`. I don't think the bug will present itself with `--validator-monitor-auto`.
1063 lines
38 KiB
Rust
1063 lines
38 KiB
Rust
//! Provides detailed logging and metrics for a set of registered validators.
|
|
//!
|
|
//! This component should not affect consensus.
|
|
|
|
use crate::metrics;
|
|
use parking_lot::RwLock;
|
|
use slog::{crit, error, info, warn, Logger};
|
|
use slot_clock::SlotClock;
|
|
use state_processing::per_epoch_processing::ValidatorStatus;
|
|
use std::collections::{HashMap, HashSet};
|
|
use std::convert::TryFrom;
|
|
use std::io;
|
|
use std::marker::PhantomData;
|
|
use std::str::Utf8Error;
|
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|
use types::{
|
|
AttestationData, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Epoch, EthSpec,
|
|
Hash256, IndexedAttestation, ProposerSlashing, PublicKeyBytes, SignedAggregateAndProof, Slot,
|
|
VoluntaryExit,
|
|
};
|
|
|
|
/// The validator monitor collects per-epoch data about each monitored validator. Historical data
|
|
/// will be kept around for `HISTORIC_EPOCHS` before it is pruned.
|
|
pub const HISTORIC_EPOCHS: usize = 4;
|
|
|
|
#[derive(Debug)]
|
|
pub enum Error {
|
|
InvalidPubkey(String),
|
|
FileError(io::Error),
|
|
InvalidUtf8(Utf8Error),
|
|
}
|
|
|
|
/// Contains data pertaining to one validator for one epoch.
|
|
#[derive(Default)]
|
|
struct EpochSummary {
|
|
/*
|
|
* Attestations with a target in the current epoch.
|
|
*/
|
|
/// The number of attestations seen.
|
|
pub attestations: usize,
|
|
/// The delay between when the attestation should have been produced and when it was observed.
|
|
pub attestation_min_delay: Option<Duration>,
|
|
/// The number of times a validators attestation was seen in an aggregate.
|
|
pub attestation_aggregate_incusions: usize,
|
|
/// The number of times a validators attestation was seen in a block.
|
|
pub attestation_block_inclusions: usize,
|
|
/// The minimum observed inclusion distance for an attestation for this epoch..
|
|
pub attestation_min_block_inclusion_distance: Option<Slot>,
|
|
/*
|
|
* Blocks with a slot in the current epoch.
|
|
*/
|
|
/// The number of blocks observed.
|
|
pub blocks: usize,
|
|
/// The delay between when the block should have been produced and when it was observed.
|
|
pub block_min_delay: Option<Duration>,
|
|
/*
|
|
* Aggregates with a target in the current epoch
|
|
*/
|
|
/// The number of signed aggregate and proofs observed.
|
|
pub aggregates: usize,
|
|
/// The delay between when the aggregate should have been produced and when it was observed.
|
|
pub aggregate_min_delay: Option<Duration>,
|
|
/*
|
|
* Others pertaining to this epoch.
|
|
*/
|
|
/// The number of voluntary exists observed.
|
|
pub exits: usize,
|
|
/// The number of proposer slashings observed.
|
|
pub proposer_slashings: usize,
|
|
/// The number of attester slashings observed.
|
|
pub attester_slashings: usize,
|
|
}
|
|
|
|
impl EpochSummary {
|
|
/// Update `current` if:
|
|
///
|
|
/// - It is `None`.
|
|
/// - `new` is greater than its current value.
|
|
fn update_if_lt<T: Ord>(current: &mut Option<T>, new: T) {
|
|
if let Some(ref mut current) = current {
|
|
if new < *current {
|
|
*current = new
|
|
}
|
|
} else {
|
|
*current = Some(new)
|
|
}
|
|
}
|
|
|
|
pub fn register_unaggregated_attestation(&mut self, delay: Duration) {
|
|
self.attestations += 1;
|
|
Self::update_if_lt(&mut self.attestation_min_delay, delay);
|
|
}
|
|
|
|
pub fn register_aggregated_attestation(&mut self, delay: Duration) {
|
|
self.aggregates += 1;
|
|
Self::update_if_lt(&mut self.aggregate_min_delay, delay);
|
|
}
|
|
|
|
pub fn register_aggregate_attestation_inclusion(&mut self) {
|
|
self.attestation_aggregate_incusions += 1;
|
|
}
|
|
|
|
pub fn register_attestation_block_inclusion(&mut self, delay: Slot) {
|
|
self.attestation_block_inclusions += 1;
|
|
Self::update_if_lt(&mut self.attestation_min_block_inclusion_distance, delay);
|
|
}
|
|
|
|
pub fn register_exit(&mut self) {
|
|
self.exits += 1;
|
|
}
|
|
|
|
pub fn register_proposer_slashing(&mut self) {
|
|
self.proposer_slashings += 1;
|
|
}
|
|
|
|
pub fn register_attester_slashing(&mut self) {
|
|
self.attester_slashings += 1;
|
|
}
|
|
}
|
|
|
|
type SummaryMap = HashMap<Epoch, EpochSummary>;
|
|
|
|
/// A validator that is being monitored by the `ValidatorMonitor`.
|
|
struct MonitoredValidator {
|
|
/// A human-readable identifier for the validator.
|
|
pub id: String,
|
|
/// The validator voting pubkey.
|
|
pub pubkey: PublicKeyBytes,
|
|
/// The validator index in the state.
|
|
pub index: Option<u64>,
|
|
/// A history of the validator over time.
|
|
pub summaries: RwLock<SummaryMap>,
|
|
}
|
|
|
|
impl MonitoredValidator {
|
|
fn new(pubkey: PublicKeyBytes, index: Option<u64>) -> Self {
|
|
Self {
|
|
id: index
|
|
.map(|i| i.to_string())
|
|
.unwrap_or_else(|| pubkey.to_string()),
|
|
pubkey,
|
|
index,
|
|
summaries: <_>::default(),
|
|
}
|
|
}
|
|
|
|
fn set_index(&mut self, validator_index: u64) {
|
|
if self.index.is_none() {
|
|
self.index = Some(validator_index);
|
|
self.id = validator_index.to_string();
|
|
}
|
|
}
|
|
|
|
/// Maps `func` across the `self.summaries`.
|
|
///
|
|
/// ## Warning
|
|
///
|
|
/// It is possible to deadlock this function by trying to obtain a lock on
|
|
/// `self.summary` inside `func`.
|
|
///
|
|
/// ## Notes
|
|
///
|
|
/// - If `epoch` doesn't exist in `self.summaries`, it is created.
|
|
/// - `self.summaries` may be pruned after `func` is run.
|
|
fn with_epoch_summary<F>(&self, epoch: Epoch, func: F)
|
|
where
|
|
F: Fn(&mut EpochSummary),
|
|
{
|
|
let mut summaries = self.summaries.write();
|
|
|
|
func(summaries.entry(epoch).or_default());
|
|
|
|
// Prune
|
|
while summaries.len() > HISTORIC_EPOCHS {
|
|
if let Some(key) = summaries.iter().map(|(epoch, _)| *epoch).min() {
|
|
summaries.remove(&key);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Holds a collection of `MonitoredValidator` and is notified about a variety of events on the P2P
|
|
/// network, HTTP API and `BeaconChain`.
|
|
///
|
|
/// If any of the events pertain to a `MonitoredValidator`, additional logging and metrics will be
|
|
/// performed.
|
|
///
|
|
/// The intention of this struct is to provide users with more logging and Prometheus metrics around
|
|
/// validators that they are interested in.
|
|
pub struct ValidatorMonitor<T> {
|
|
/// The validators that require additional monitoring.
|
|
validators: HashMap<PublicKeyBytes, MonitoredValidator>,
|
|
/// A map of validator index (state.validators) to a validator public key.
|
|
indices: HashMap<u64, PublicKeyBytes>,
|
|
/// If true, allow the automatic registration of validators.
|
|
auto_register: bool,
|
|
log: Logger,
|
|
_phantom: PhantomData<T>,
|
|
}
|
|
|
|
impl<T: EthSpec> ValidatorMonitor<T> {
|
|
pub fn new(pubkeys: Vec<PublicKeyBytes>, auto_register: bool, log: Logger) -> Self {
|
|
let mut s = Self {
|
|
validators: <_>::default(),
|
|
indices: <_>::default(),
|
|
auto_register,
|
|
log,
|
|
_phantom: PhantomData,
|
|
};
|
|
for pubkey in pubkeys {
|
|
s.add_validator_pubkey(pubkey)
|
|
}
|
|
s
|
|
}
|
|
|
|
/// Add some validators to `self` for additional monitoring.
|
|
fn add_validator_pubkey(&mut self, pubkey: PublicKeyBytes) {
|
|
let index_opt = self
|
|
.indices
|
|
.iter()
|
|
.find(|(_, candidate_pk)| **candidate_pk == pubkey)
|
|
.map(|(index, _)| *index);
|
|
|
|
let log = self.log.clone();
|
|
self.validators.entry(pubkey).or_insert_with(|| {
|
|
info!(
|
|
log,
|
|
"Started monitoring validator";
|
|
"pubkey" => %pubkey,
|
|
);
|
|
MonitoredValidator::new(pubkey, index_opt)
|
|
});
|
|
}
|
|
|
|
/// Reads information from the given `state`. The `state` *must* be valid (i.e, able to be
|
|
/// imported).
|
|
pub fn process_valid_state(&mut self, current_epoch: Epoch, state: &BeaconState<T>) {
|
|
// Add any new validator indices.
|
|
state
|
|
.validators
|
|
.iter()
|
|
.enumerate()
|
|
.skip(self.indices.len())
|
|
.for_each(|(i, validator)| {
|
|
let i = i as u64;
|
|
if let Some(validator) = self.validators.get_mut(&validator.pubkey) {
|
|
validator.set_index(i)
|
|
}
|
|
self.indices.insert(i, validator.pubkey);
|
|
});
|
|
|
|
// Update metrics for individual validators.
|
|
for monitored_validator in self.validators.values() {
|
|
if let Some(i) = monitored_validator.index {
|
|
let i = i as usize;
|
|
let id = &monitored_validator.id;
|
|
|
|
if let Some(balance) = state.balances.get(i) {
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_MONITOR_BALANCE_GWEI,
|
|
&[id],
|
|
*balance as i64,
|
|
);
|
|
}
|
|
|
|
if let Some(validator) = state.validators.get(i) {
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_MONITOR_EFFECTIVE_BALANCE_GWEI,
|
|
&[id],
|
|
u64_to_i64(validator.effective_balance),
|
|
);
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_MONITOR_SLASHED,
|
|
&[id],
|
|
if validator.slashed { 1 } else { 0 },
|
|
);
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_MONITOR_ACTIVE,
|
|
&[id],
|
|
if validator.is_active_at(current_epoch) {
|
|
1
|
|
} else {
|
|
0
|
|
},
|
|
);
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_MONITOR_EXITED,
|
|
&[id],
|
|
if validator.is_exited_at(current_epoch) {
|
|
1
|
|
} else {
|
|
0
|
|
},
|
|
);
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_MONITOR_WITHDRAWABLE,
|
|
&[id],
|
|
if validator.is_withdrawable_at(current_epoch) {
|
|
1
|
|
} else {
|
|
0
|
|
},
|
|
);
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_ACTIVATION_ELIGIBILITY_EPOCH,
|
|
&[id],
|
|
u64_to_i64(validator.activation_eligibility_epoch),
|
|
);
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_ACTIVATION_EPOCH,
|
|
&[id],
|
|
u64_to_i64(validator.activation_epoch),
|
|
);
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_EXIT_EPOCH,
|
|
&[id],
|
|
u64_to_i64(validator.exit_epoch),
|
|
);
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_WITHDRAWABLE_EPOCH,
|
|
&[id],
|
|
u64_to_i64(validator.withdrawable_epoch),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn process_validator_statuses(&self, epoch: Epoch, summaries: &[ValidatorStatus]) {
|
|
for monitored_validator in self.validators.values() {
|
|
// We subtract two from the state of the epoch that generated these summaries.
|
|
//
|
|
// - One to account for it being the previous epoch.
|
|
// - One to account for the state advancing an epoch whilst generating the validator
|
|
// statuses.
|
|
let prev_epoch = epoch - 2;
|
|
if let Some(i) = monitored_validator.index {
|
|
let i = i as usize;
|
|
let id = &monitored_validator.id;
|
|
|
|
if let Some(summary) = summaries.get(i) {
|
|
if summary.is_previous_epoch_attester {
|
|
let lag = summary
|
|
.inclusion_info
|
|
.map(|i| format!("{} slot(s)", i.delay.saturating_sub(1).to_string()))
|
|
.unwrap_or_else(|| "??".to_string());
|
|
|
|
info!(
|
|
self.log,
|
|
"Previous epoch attestation success";
|
|
"inclusion_lag" => lag,
|
|
"matched_target" => summary.is_previous_epoch_target_attester,
|
|
"matched_head" => summary.is_previous_epoch_head_attester,
|
|
"epoch" => prev_epoch,
|
|
"validator" => id,
|
|
);
|
|
} else if summary.is_active_in_previous_epoch
|
|
&& !summary.is_previous_epoch_attester
|
|
{
|
|
error!(
|
|
self.log,
|
|
"Previous epoch attestation missing";
|
|
"epoch" => prev_epoch,
|
|
"validator" => id,
|
|
)
|
|
} else if !summary.is_active_in_previous_epoch {
|
|
// Monitored validator is not active, due to awaiting activation
|
|
// or being exited/withdrawn. Do not attempt to report on its
|
|
// attestations.
|
|
continue;
|
|
}
|
|
|
|
if summary.is_previous_epoch_attester {
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_HIT,
|
|
&[id],
|
|
);
|
|
} else {
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_MISS,
|
|
&[id],
|
|
);
|
|
}
|
|
if summary.is_previous_epoch_head_attester {
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_HIT,
|
|
&[id],
|
|
);
|
|
} else {
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_MISS,
|
|
&[id],
|
|
);
|
|
warn!(
|
|
self.log,
|
|
"Attested to an incorrect head";
|
|
"epoch" => prev_epoch,
|
|
"validator" => id,
|
|
);
|
|
}
|
|
if summary.is_previous_epoch_target_attester {
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_HIT,
|
|
&[id],
|
|
);
|
|
} else {
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_MISS,
|
|
&[id],
|
|
);
|
|
warn!(
|
|
self.log,
|
|
"Attested to an incorrect target";
|
|
"epoch" => prev_epoch,
|
|
"validator" => id,
|
|
);
|
|
}
|
|
if let Some(inclusion_info) = summary.inclusion_info {
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_INCLUSION_DISTANCE,
|
|
&[id],
|
|
inclusion_info.delay as i64,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn get_validator_id(&self, validator_index: u64) -> Option<&str> {
|
|
self.indices
|
|
.get(&validator_index)
|
|
.and_then(|pubkey| self.validators.get(pubkey))
|
|
.map(|validator| validator.id.as_str())
|
|
}
|
|
|
|
fn get_validator(&self, validator_index: u64) -> Option<&MonitoredValidator> {
|
|
self.indices
|
|
.get(&validator_index)
|
|
.and_then(|pubkey| self.validators.get(pubkey))
|
|
}
|
|
|
|
/// Returns the number of validators monitored by `self`.
|
|
pub fn num_validators(&self) -> usize {
|
|
self.validators.len()
|
|
}
|
|
|
|
/// If `self.auto_register == true`, add the `validator_index` to `self.monitored_validators`.
|
|
/// Otherwise, do nothing.
|
|
pub fn auto_register_local_validator(&mut self, validator_index: u64) {
|
|
if !self.auto_register {
|
|
return;
|
|
}
|
|
|
|
if let Some(pubkey) = self.indices.get(&validator_index) {
|
|
if !self.validators.contains_key(pubkey) {
|
|
info!(
|
|
self.log,
|
|
"Started monitoring validator";
|
|
"pubkey" => %pubkey,
|
|
"validator" => %validator_index,
|
|
);
|
|
|
|
self.validators.insert(
|
|
*pubkey,
|
|
MonitoredValidator::new(*pubkey, Some(validator_index)),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Process a block received on gossip.
|
|
pub fn register_gossip_block<S: SlotClock>(
|
|
&self,
|
|
seen_timestamp: Duration,
|
|
block: &BeaconBlock<T>,
|
|
block_root: Hash256,
|
|
slot_clock: &S,
|
|
) {
|
|
self.register_beacon_block("gossip", seen_timestamp, block, block_root, slot_clock)
|
|
}
|
|
|
|
/// Process a block received on the HTTP API from a local validator.
|
|
pub fn register_api_block<S: SlotClock>(
|
|
&self,
|
|
seen_timestamp: Duration,
|
|
block: &BeaconBlock<T>,
|
|
block_root: Hash256,
|
|
slot_clock: &S,
|
|
) {
|
|
self.register_beacon_block("api", seen_timestamp, block, block_root, slot_clock)
|
|
}
|
|
|
|
fn register_beacon_block<S: SlotClock>(
|
|
&self,
|
|
src: &str,
|
|
seen_timestamp: Duration,
|
|
block: &BeaconBlock<T>,
|
|
block_root: Hash256,
|
|
slot_clock: &S,
|
|
) {
|
|
if let Some(id) = self.get_validator_id(block.proposer_index) {
|
|
let delay = get_block_delay_ms(seen_timestamp, block, slot_clock);
|
|
|
|
metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_BEACON_BLOCK_TOTAL, &[src, id]);
|
|
metrics::observe_timer_vec(
|
|
&metrics::VALIDATOR_MONITOR_BEACON_BLOCK_DELAY_SECONDS,
|
|
&[src, id],
|
|
delay,
|
|
);
|
|
|
|
info!(
|
|
self.log,
|
|
"Block from API";
|
|
"root" => ?block_root,
|
|
"delay" => %delay.as_millis(),
|
|
"slot" => %block.slot,
|
|
"src" => src,
|
|
"validator" => %id,
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Returns the duration between when the attestation `data` could be produced (1/3rd through
|
|
/// the slot) and `seen_timestamp`.
|
|
fn get_unaggregated_attestation_delay_ms<S: SlotClock>(
|
|
seen_timestamp: Duration,
|
|
data: &AttestationData,
|
|
slot_clock: &S,
|
|
) -> Duration {
|
|
slot_clock
|
|
.start_of(data.slot)
|
|
.and_then(|slot_start| seen_timestamp.checked_sub(slot_start))
|
|
.and_then(|gross_delay| {
|
|
gross_delay.checked_sub(slot_clock.unagg_attestation_production_delay())
|
|
})
|
|
.unwrap_or_else(|| Duration::from_secs(0))
|
|
}
|
|
|
|
/// Register an attestation seen on the gossip network.
|
|
pub fn register_gossip_unaggregated_attestation<S: SlotClock>(
|
|
&self,
|
|
seen_timestamp: Duration,
|
|
indexed_attestation: &IndexedAttestation<T>,
|
|
slot_clock: &S,
|
|
) {
|
|
self.register_unaggregated_attestation(
|
|
"gossip",
|
|
seen_timestamp,
|
|
indexed_attestation,
|
|
slot_clock,
|
|
)
|
|
}
|
|
|
|
/// Register an attestation seen on the HTTP API.
|
|
pub fn register_api_unaggregated_attestation<S: SlotClock>(
|
|
&self,
|
|
seen_timestamp: Duration,
|
|
indexed_attestation: &IndexedAttestation<T>,
|
|
slot_clock: &S,
|
|
) {
|
|
self.register_unaggregated_attestation(
|
|
"api",
|
|
seen_timestamp,
|
|
indexed_attestation,
|
|
slot_clock,
|
|
)
|
|
}
|
|
|
|
fn register_unaggregated_attestation<S: SlotClock>(
|
|
&self,
|
|
src: &str,
|
|
seen_timestamp: Duration,
|
|
indexed_attestation: &IndexedAttestation<T>,
|
|
slot_clock: &S,
|
|
) {
|
|
let data = &indexed_attestation.data;
|
|
let epoch = data.slot.epoch(T::slots_per_epoch());
|
|
let delay = Self::get_unaggregated_attestation_delay_ms(seen_timestamp, data, slot_clock);
|
|
|
|
indexed_attestation.attesting_indices.iter().for_each(|i| {
|
|
if let Some(validator) = self.get_validator(*i) {
|
|
let id = &validator.id;
|
|
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_TOTAL,
|
|
&[src, id],
|
|
);
|
|
metrics::observe_timer_vec(
|
|
&metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_DELAY_SECONDS,
|
|
&[src, id],
|
|
delay,
|
|
);
|
|
|
|
info!(
|
|
self.log,
|
|
"Unaggregated attestation";
|
|
"head" => ?data.beacon_block_root,
|
|
"index" => %data.index,
|
|
"delay_ms" => %delay.as_millis(),
|
|
"epoch" => %epoch,
|
|
"slot" => %data.slot,
|
|
"src" => src,
|
|
"validator" => %id,
|
|
);
|
|
|
|
validator.with_epoch_summary(epoch, |summary| {
|
|
summary.register_unaggregated_attestation(delay)
|
|
});
|
|
}
|
|
})
|
|
}
|
|
|
|
/// Returns the duration between when a `AggregateAndproof` with `data` could be produced (2/3rd
|
|
/// through the slot) and `seen_timestamp`.
|
|
fn get_aggregated_attestation_delay_ms<S: SlotClock>(
|
|
seen_timestamp: Duration,
|
|
data: &AttestationData,
|
|
slot_clock: &S,
|
|
) -> Duration {
|
|
slot_clock
|
|
.start_of(data.slot)
|
|
.and_then(|slot_start| seen_timestamp.checked_sub(slot_start))
|
|
.and_then(|gross_delay| {
|
|
gross_delay.checked_sub(slot_clock.agg_attestation_production_delay())
|
|
})
|
|
.unwrap_or_else(|| Duration::from_secs(0))
|
|
}
|
|
|
|
/// Register a `signed_aggregate_and_proof` seen on the gossip network.
|
|
pub fn register_gossip_aggregated_attestation<S: SlotClock>(
|
|
&self,
|
|
seen_timestamp: Duration,
|
|
signed_aggregate_and_proof: &SignedAggregateAndProof<T>,
|
|
indexed_attestation: &IndexedAttestation<T>,
|
|
slot_clock: &S,
|
|
) {
|
|
self.register_aggregated_attestation(
|
|
"gossip",
|
|
seen_timestamp,
|
|
signed_aggregate_and_proof,
|
|
indexed_attestation,
|
|
slot_clock,
|
|
)
|
|
}
|
|
|
|
/// Register a `signed_aggregate_and_proof` seen on the HTTP API.
|
|
pub fn register_api_aggregated_attestation<S: SlotClock>(
|
|
&self,
|
|
seen_timestamp: Duration,
|
|
signed_aggregate_and_proof: &SignedAggregateAndProof<T>,
|
|
indexed_attestation: &IndexedAttestation<T>,
|
|
slot_clock: &S,
|
|
) {
|
|
self.register_aggregated_attestation(
|
|
"api",
|
|
seen_timestamp,
|
|
signed_aggregate_and_proof,
|
|
indexed_attestation,
|
|
slot_clock,
|
|
)
|
|
}
|
|
|
|
fn register_aggregated_attestation<S: SlotClock>(
|
|
&self,
|
|
src: &str,
|
|
seen_timestamp: Duration,
|
|
signed_aggregate_and_proof: &SignedAggregateAndProof<T>,
|
|
indexed_attestation: &IndexedAttestation<T>,
|
|
slot_clock: &S,
|
|
) {
|
|
let data = &indexed_attestation.data;
|
|
let epoch = data.slot.epoch(T::slots_per_epoch());
|
|
let delay = Self::get_aggregated_attestation_delay_ms(seen_timestamp, data, slot_clock);
|
|
|
|
let aggregator_index = signed_aggregate_and_proof.message.aggregator_index;
|
|
if let Some(validator) = self.get_validator(aggregator_index) {
|
|
let id = &validator.id;
|
|
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_TOTAL,
|
|
&[src, id],
|
|
);
|
|
metrics::observe_timer_vec(
|
|
&metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_DELAY_SECONDS,
|
|
&[src, id],
|
|
delay,
|
|
);
|
|
|
|
info!(
|
|
self.log,
|
|
"Aggregated attestation";
|
|
"head" => ?data.beacon_block_root,
|
|
"index" => %data.index,
|
|
"delay_ms" => %delay.as_millis(),
|
|
"epoch" => %epoch,
|
|
"slot" => %data.slot,
|
|
"src" => src,
|
|
"validator" => %id,
|
|
);
|
|
|
|
validator.with_epoch_summary(epoch, |summary| {
|
|
summary.register_aggregated_attestation(delay)
|
|
});
|
|
}
|
|
|
|
indexed_attestation.attesting_indices.iter().for_each(|i| {
|
|
if let Some(validator) = self.get_validator(*i) {
|
|
let id = &validator.id;
|
|
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_TOTAL,
|
|
&[src, id],
|
|
);
|
|
metrics::observe_timer_vec(
|
|
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_DELAY_SECONDS,
|
|
&[src, id],
|
|
delay,
|
|
);
|
|
|
|
info!(
|
|
self.log,
|
|
"Attestation included in aggregate";
|
|
"head" => ?data.beacon_block_root,
|
|
"index" => %data.index,
|
|
"delay_ms" => %delay.as_millis(),
|
|
"epoch" => %epoch,
|
|
"slot" => %data.slot,
|
|
"src" => src,
|
|
"validator" => %id,
|
|
);
|
|
|
|
validator.with_epoch_summary(epoch, |summary| {
|
|
summary.register_aggregate_attestation_inclusion()
|
|
});
|
|
}
|
|
})
|
|
}
|
|
|
|
/// Register that the `indexed_attestation` was included in a *valid* `BeaconBlock`.
|
|
pub fn register_attestation_in_block(
|
|
&self,
|
|
indexed_attestation: &IndexedAttestation<T>,
|
|
block: &BeaconBlock<T>,
|
|
spec: &ChainSpec,
|
|
) {
|
|
let data = &indexed_attestation.data;
|
|
let delay = (block.slot - data.slot) - spec.min_attestation_inclusion_delay;
|
|
let epoch = data.slot.epoch(T::slots_per_epoch());
|
|
|
|
indexed_attestation.attesting_indices.iter().for_each(|i| {
|
|
if let Some(validator) = self.get_validator(*i) {
|
|
let id = &validator.id;
|
|
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_TOTAL,
|
|
&["block", id],
|
|
);
|
|
metrics::set_int_gauge(
|
|
&metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_DELAY_SLOTS,
|
|
&["block", id],
|
|
delay.as_u64() as i64,
|
|
);
|
|
|
|
info!(
|
|
self.log,
|
|
"Attestation included in block";
|
|
"head" => ?data.beacon_block_root,
|
|
"index" => %data.index,
|
|
"inclusion_lag" => format!("{} slot(s)", delay),
|
|
"epoch" => %epoch,
|
|
"slot" => %data.slot,
|
|
"validator" => %id,
|
|
);
|
|
|
|
validator.with_epoch_summary(epoch, |summary| {
|
|
summary.register_attestation_block_inclusion(delay)
|
|
});
|
|
}
|
|
})
|
|
}
|
|
|
|
/// Register an exit from the gossip network.
|
|
pub fn register_gossip_voluntary_exit(&self, exit: &VoluntaryExit) {
|
|
self.register_voluntary_exit("gossip", exit)
|
|
}
|
|
|
|
/// Register an exit from the HTTP API.
|
|
pub fn register_api_voluntary_exit(&self, exit: &VoluntaryExit) {
|
|
self.register_voluntary_exit("api", exit)
|
|
}
|
|
|
|
/// Register an exit included in a *valid* beacon block.
|
|
pub fn register_block_voluntary_exit(&self, exit: &VoluntaryExit) {
|
|
self.register_voluntary_exit("block", exit)
|
|
}
|
|
|
|
fn register_voluntary_exit(&self, src: &str, exit: &VoluntaryExit) {
|
|
if let Some(validator) = self.get_validator(exit.validator_index) {
|
|
let id = &validator.id;
|
|
let epoch = exit.epoch;
|
|
|
|
metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_EXIT_TOTAL, &[src, id]);
|
|
|
|
info!(
|
|
self.log,
|
|
"Voluntary exit";
|
|
"epoch" => %epoch,
|
|
"validator" => %id,
|
|
"src" => src,
|
|
);
|
|
|
|
validator.with_epoch_summary(epoch, |summary| summary.register_exit());
|
|
}
|
|
}
|
|
|
|
/// Register a proposer slashing from the gossip network.
|
|
pub fn register_gossip_proposer_slashing(&self, slashing: &ProposerSlashing) {
|
|
self.register_proposer_slashing("gossip", slashing)
|
|
}
|
|
|
|
/// Register a proposer slashing from the HTTP API.
|
|
pub fn register_api_proposer_slashing(&self, slashing: &ProposerSlashing) {
|
|
self.register_proposer_slashing("api", slashing)
|
|
}
|
|
|
|
/// Register a proposer slashing included in a *valid* `BeaconBlock`.
|
|
pub fn register_block_proposer_slashing(&self, slashing: &ProposerSlashing) {
|
|
self.register_proposer_slashing("block", slashing)
|
|
}
|
|
|
|
fn register_proposer_slashing(&self, src: &str, slashing: &ProposerSlashing) {
|
|
let proposer = slashing.signed_header_1.message.proposer_index;
|
|
let slot = slashing.signed_header_1.message.slot;
|
|
let epoch = slot.epoch(T::slots_per_epoch());
|
|
let root_1 = slashing.signed_header_1.message.canonical_root();
|
|
let root_2 = slashing.signed_header_2.message.canonical_root();
|
|
|
|
if let Some(validator) = self.get_validator(proposer) {
|
|
let id = &validator.id;
|
|
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_PROPOSER_SLASHING_TOTAL,
|
|
&[src, id],
|
|
);
|
|
|
|
crit!(
|
|
self.log,
|
|
"Proposer slashing";
|
|
"root_2" => %root_2,
|
|
"root_1" => %root_1,
|
|
"slot" => %slot,
|
|
"validator" => %id,
|
|
"src" => src,
|
|
);
|
|
|
|
validator.with_epoch_summary(epoch, |summary| summary.register_proposer_slashing());
|
|
}
|
|
}
|
|
|
|
/// Register an attester slashing from the gossip network.
|
|
pub fn register_gossip_attester_slashing(&self, slashing: &AttesterSlashing<T>) {
|
|
self.register_attester_slashing("gossip", slashing)
|
|
}
|
|
|
|
/// Register an attester slashing from the HTTP API.
|
|
pub fn register_api_attester_slashing(&self, slashing: &AttesterSlashing<T>) {
|
|
self.register_attester_slashing("api", slashing)
|
|
}
|
|
|
|
/// Register an attester slashing included in a *valid* `BeaconBlock`.
|
|
pub fn register_block_attester_slashing(&self, slashing: &AttesterSlashing<T>) {
|
|
self.register_attester_slashing("block", slashing)
|
|
}
|
|
|
|
fn register_attester_slashing(&self, src: &str, slashing: &AttesterSlashing<T>) {
|
|
let data = &slashing.attestation_1.data;
|
|
let attestation_1_indices: HashSet<u64> = slashing
|
|
.attestation_1
|
|
.attesting_indices
|
|
.iter()
|
|
.copied()
|
|
.collect();
|
|
|
|
slashing
|
|
.attestation_2
|
|
.attesting_indices
|
|
.iter()
|
|
.filter(|index| attestation_1_indices.contains(index))
|
|
.filter_map(|index| self.get_validator(*index))
|
|
.for_each(|validator| {
|
|
let id = &validator.id;
|
|
let epoch = data.slot.epoch(T::slots_per_epoch());
|
|
|
|
metrics::inc_counter_vec(
|
|
&metrics::VALIDATOR_MONITOR_ATTESTER_SLASHING_TOTAL,
|
|
&[src, id],
|
|
);
|
|
|
|
crit!(
|
|
self.log,
|
|
"Attester slashing";
|
|
"epoch" => %epoch,
|
|
"slot" => %data.slot,
|
|
"validator" => %id,
|
|
"src" => src,
|
|
);
|
|
|
|
validator.with_epoch_summary(epoch, |summary| summary.register_attester_slashing());
|
|
})
|
|
}
|
|
|
|
/// Scrape `self` for metrics.
|
|
///
|
|
/// Should be called whenever Prometheus is scraping Lighthouse.
|
|
pub fn scrape_metrics<S: SlotClock>(&self, slot_clock: &S, spec: &ChainSpec) {
|
|
metrics::set_gauge(
|
|
&metrics::VALIDATOR_MONITOR_VALIDATORS_TOTAL,
|
|
self.num_validators() as i64,
|
|
);
|
|
|
|
if let Some(slot) = slot_clock.now() {
|
|
let epoch = slot.epoch(T::slots_per_epoch());
|
|
let slot_in_epoch = slot % T::slots_per_epoch();
|
|
|
|
// Only start to report on the current epoch once we've progressed past the point where
|
|
// all attestation should be included in a block.
|
|
//
|
|
// This allows us to set alarms on Grafana to detect when an attestation has been
|
|
// missed. If we didn't delay beyond the attestation inclusion period then we could
|
|
// expect some occasional false-positives on attestation misses.
|
|
//
|
|
// I have chosen 3 as an arbitrary number where we *probably* shouldn't see that many
|
|
// skip slots on mainnet.
|
|
let previous_epoch = if slot_in_epoch > spec.min_attestation_inclusion_delay + 3 {
|
|
epoch - 1
|
|
} else {
|
|
epoch - 2
|
|
};
|
|
|
|
for (_, validator) in self.validators.iter() {
|
|
let id = &validator.id;
|
|
let summaries = validator.summaries.read();
|
|
|
|
if let Some(summary) = summaries.get(&previous_epoch) {
|
|
/*
|
|
* Attestations
|
|
*/
|
|
metrics::set_gauge_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_TOTAL,
|
|
&[id],
|
|
summary.attestations as i64,
|
|
);
|
|
if let Some(delay) = summary.attestation_min_delay {
|
|
metrics::observe_timer_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_MIN_DELAY_SECONDS,
|
|
&[id],
|
|
delay,
|
|
);
|
|
}
|
|
metrics::set_gauge_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_AGGREGATE_INCLUSIONS,
|
|
&[id],
|
|
summary.attestation_aggregate_incusions as i64,
|
|
);
|
|
metrics::set_gauge_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_INCLUSIONS,
|
|
&[id],
|
|
summary.attestation_block_inclusions as i64,
|
|
);
|
|
if let Some(distance) = summary.attestation_min_block_inclusion_distance {
|
|
metrics::set_gauge_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_MIN_INCLUSION_DISTANCE,
|
|
&[id],
|
|
distance.as_u64() as i64,
|
|
);
|
|
}
|
|
/*
|
|
* Blocks
|
|
*/
|
|
metrics::set_gauge_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_TOTAL,
|
|
&[id],
|
|
summary.blocks as i64,
|
|
);
|
|
if let Some(delay) = summary.block_min_delay {
|
|
metrics::observe_timer_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_MIN_DELAY_SECONDS,
|
|
&[id],
|
|
delay,
|
|
);
|
|
}
|
|
/*
|
|
* Aggregates
|
|
*/
|
|
metrics::set_gauge_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_TOTAL,
|
|
&[id],
|
|
summary.aggregates as i64,
|
|
);
|
|
if let Some(delay) = summary.aggregate_min_delay {
|
|
metrics::observe_timer_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_MIN_DELAY_SECONDS,
|
|
&[id],
|
|
delay,
|
|
);
|
|
}
|
|
/*
|
|
* Other
|
|
*/
|
|
metrics::set_gauge_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_EXITS_TOTAL,
|
|
&[id],
|
|
summary.exits as i64,
|
|
);
|
|
metrics::set_gauge_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_PROPOSER_SLASHINGS_TOTAL,
|
|
&[id],
|
|
summary.proposer_slashings as i64,
|
|
);
|
|
metrics::set_gauge_vec(
|
|
&metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTER_SLASHINGS_TOTAL,
|
|
&[id],
|
|
summary.attester_slashings as i64,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Returns the duration since the unix epoch.
|
|
pub fn timestamp_now() -> Duration {
|
|
SystemTime::now()
|
|
.duration_since(UNIX_EPOCH)
|
|
.unwrap_or_else(|_| Duration::from_secs(0))
|
|
}
|
|
|
|
fn u64_to_i64(n: impl Into<u64>) -> i64 {
|
|
i64::try_from(n.into()).unwrap_or(i64::max_value())
|
|
}
|
|
|
|
/// Returns the delay between the start of `block.slot` and `seen_timestamp`.
|
|
pub fn get_block_delay_ms<T: EthSpec, S: SlotClock>(
|
|
seen_timestamp: Duration,
|
|
block: &BeaconBlock<T>,
|
|
slot_clock: &S,
|
|
) -> Duration {
|
|
get_slot_delay_ms::<S>(seen_timestamp, block.slot, slot_clock)
|
|
}
|
|
|
|
/// Returns the delay between the start of `slot` and `seen_timestamp`.
|
|
pub fn get_slot_delay_ms<S: SlotClock>(
|
|
seen_timestamp: Duration,
|
|
slot: Slot,
|
|
slot_clock: &S,
|
|
) -> Duration {
|
|
slot_clock
|
|
.start_of(slot)
|
|
.and_then(|slot_start| seen_timestamp.checked_sub(slot_start))
|
|
.unwrap_or_else(|| Duration::from_secs(0))
|
|
}
|