Detailed validator monitoring (#2151)

## Issue Addressed

- Resolves #2064

## Proposed Changes

Adds a `ValidatorMonitor` struct which provides additional logging and Grafana metrics for specific validators.

Use `lighthouse bn --validator-monitor` to automatically enable monitoring for any validator that hits the [subnet subscription](https://ethereum.github.io/eth2.0-APIs/#/Validator/prepareBeaconCommitteeSubnet) HTTP API endpoint.

Also, use `lighthouse bn --validator-monitor-pubkeys` to supply a list of validators which will always be monitored.

See the new docs included in this PR for more info.

## TODO

- [x] Track validator balance, `slashed` status, etc.
- [x] ~~Register slashings in current epoch, not offense epoch~~
- [ ] Publish Grafana dashboard, update TODO link in docs
- [x] ~~#2130 is merged into this branch, resolve that~~
This commit is contained in:
Paul Hauner
2021-01-20 19:19:38 +00:00
parent 1eb0915301
commit 2b2a358522
29 changed files with 1646 additions and 37 deletions

View File

@@ -12,8 +12,9 @@ mod state_id;
mod validator_inclusion;
use beacon_chain::{
observed_operations::ObservationOutcome, AttestationError as AttnError, BeaconChain,
BeaconChainError, BeaconChainTypes,
attestation_verification::SignatureVerifiedAttestation,
observed_operations::ObservationOutcome, validator_monitor::timestamp_now,
AttestationError as AttnError, BeaconChain, BeaconChainError, BeaconChainTypes,
};
use beacon_proposer_cache::BeaconProposerCache;
use block_id::BlockId;
@@ -816,6 +817,8 @@ pub fn serve<T: BeaconChainTypes>(
network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>,
log: Logger| {
blocking_json_task(move || {
let seen_timestamp = timestamp_now();
// Send the block, regardless of whether or not it is valid. The API
// specification is very clear that this is the desired behaviour.
publish_pubsub_message(
@@ -831,6 +834,14 @@ pub fn serve<T: BeaconChainTypes>(
"root" => format!("{}", root)
);
// Notify the validator monitor.
chain.validator_monitor.read().register_api_block(
seen_timestamp,
&block.message,
root,
&chain.slot_clock,
);
// Update the head since it's likely this block will become the new
// head.
chain
@@ -921,6 +932,7 @@ pub fn serve<T: BeaconChainTypes>(
network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>,
log: Logger| {
blocking_json_task(move || {
let seen_timestamp = timestamp_now();
let mut failures = Vec::new();
for (index, attestation) in attestations.as_slice().iter().enumerate() {
@@ -945,6 +957,16 @@ pub fn serve<T: BeaconChainTypes>(
}
};
// Notify the validator monitor.
chain
.validator_monitor
.read()
.register_api_unaggregated_attestation(
seen_timestamp,
attestation.indexed_attestation(),
&chain.slot_clock,
);
publish_pubsub_message(
&network_tx,
PubsubMessage::Attestation(Box::new((
@@ -1049,6 +1071,12 @@ pub fn serve<T: BeaconChainTypes>(
))
})?;
// Notify the validator monitor.
chain
.validator_monitor
.read()
.register_api_attester_slashing(&slashing);
if let ObservationOutcome::New(slashing) = outcome {
publish_pubsub_message(
&network_tx,
@@ -1100,6 +1128,12 @@ pub fn serve<T: BeaconChainTypes>(
))
})?;
// Notify the validator monitor.
chain
.validator_monitor
.read()
.register_api_proposer_slashing(&slashing);
if let ObservationOutcome::New(slashing) = outcome {
publish_pubsub_message(
&network_tx,
@@ -1149,6 +1183,12 @@ pub fn serve<T: BeaconChainTypes>(
))
})?;
// Notify the validator monitor.
chain
.validator_monitor
.read()
.register_api_voluntary_exit(&exit.message);
if let ObservationOutcome::New(exit) = outcome {
publish_pubsub_message(
&network_tx,
@@ -1970,6 +2010,7 @@ pub fn serve<T: BeaconChainTypes>(
aggregates: Vec<SignedAggregateAndProof<T::EthSpec>>,
network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>, log: Logger| {
blocking_json_task(move || {
let seen_timestamp = timestamp_now();
let mut verified_aggregates = Vec::with_capacity(aggregates.len());
let mut messages = Vec::with_capacity(aggregates.len());
let mut failures = Vec::new();
@@ -1981,6 +2022,18 @@ pub fn serve<T: BeaconChainTypes>(
messages.push(PubsubMessage::AggregateAndProofAttestation(Box::new(
verified_aggregate.aggregate().clone(),
)));
// Notify the validator monitor.
chain
.validator_monitor
.read()
.register_api_aggregated_attestation(
seen_timestamp,
verified_aggregate.aggregate(),
verified_aggregate.indexed_attestation(),
&chain.slot_clock,
);
verified_aggregates.push((index, verified_aggregate));
}
// If we already know the attestation, don't broadcast it or attempt to
@@ -2050,11 +2103,18 @@ pub fn serve<T: BeaconChainTypes>(
.and(warp::path::end())
.and(warp::body::json())
.and(network_tx_filter)
.and(chain_filter.clone())
.and_then(
|subscriptions: Vec<api_types::BeaconCommitteeSubscription>,
network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>| {
network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>,
chain: Arc<BeaconChain<T>>| {
blocking_json_task(move || {
for subscription in &subscriptions {
chain
.validator_monitor
.write()
.auto_register_local_validator(subscription.validator_index);
let subscription = api_types::ValidatorSubscription {
validator_index: subscription.validator_index,
attestation_committee_index: subscription.committee_index,