gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-09 19:51:47 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
605ef8e8e6cbb9bc974b767bb8a35a445d5a1936
lighthouse/beacon_node/network/src/metrics.rs
Eitan Seri-Levi 33e21634cb Custody backfill sync (#7907) 
#7603


  #### Custody backfill sync service
Similar in many ways to the current backfill service. There may be ways to unify the two services. The difficulty there is that the current backfill service tightly couples blocks and their associated blobs/data columns. Any attempts to unify the two services should be left to a separate PR in my opinion.

#### `SyncNeworkContext`
`SyncNetworkContext` manages custody sync data columns by range requests separetly from other sync RPC requests. I think this is a nice separation considering that custody backfill is its own service.

#### Data column import logic
The import logic verifies KZG committments and that the data columns block root matches the block root in the nodes store before importing columns

#### New channel to send messages to `SyncManager`
Now external services can communicate with the `SyncManager`. In this PR this channel is used to trigger a custody sync. Alternatively we may be able to use the existing `mpsc` channel that the `SyncNetworkContext` uses to communicate with the `SyncManager`. I will spend some time reviewing this.


Co-Authored-By: Eitan Seri-Levi <eserilev@ucsc.edu>

Co-Authored-By: Eitan Seri- Levi <eserilev@gmail.com>

Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com>
2025-10-22 03:51:34 +00:00

789 lines
30 KiB
Rust
Raw Blame History

use beacon_chain::{
AvailabilityProcessingStatus, BlockError, attestation_verification::Error as AttnError,
light_client_finality_update_verification::Error as LightClientFinalityUpdateError,
light_client_optimistic_update_verification::Error as LightClientOptimisticUpdateError,
sync_committee_verification::Error as SyncCommitteeError,
};
use fnv::FnvHashMap;
use lighthouse_network::{
GossipTopic, Gossipsub, NetworkGlobals, peer_manager::peerdb::client::ClientKind,
types::GossipKind,
};
pub use metrics::*;
use std::sync::{Arc, LazyLock};
use strum::AsRefStr;
use strum::IntoEnumIterator;
use types::DataColumnSubnetId;
use types::EthSpec;
#[derive(Debug, AsRefStr)]
pub(crate) enum BlockSource {
Gossip,
Rpc,
}
pub static BEACON_BLOCK_MESH_PEERS_PER_CLIENT: LazyLock<Result<IntGaugeVec>> =
LazyLock::new(|| {
try_create_int_gauge_vec(
"block_mesh_peers_per_client",
"Number of mesh peers for BeaconBlock topic per client",
&["Client"],
)
});
pub static BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT: LazyLock<Result<IntGaugeVec>> =
LazyLock::new(|| {
try_create_int_gauge_vec(
"beacon_aggregate_and_proof_mesh_peers_per_client",
"Number of mesh peers for BeaconAggregateAndProof topic per client",
&["Client"],
)
});
/*
* Attestation subnet subscriptions
*/
pub static SUBNET_SUBSCRIPTION_REQUESTS: LazyLock<Result<IntCounter>> = LazyLock::new(|| {
try_create_int_counter(
"validator_attestation_subnet_subscriptions_total",
"Count of validator attestation subscription requests.",
)
});
pub static SUBNET_SUBSCRIPTION_AGGREGATOR_REQUESTS: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"validator_subnet_subscriptions_aggregator_total",
"Count of validator subscription requests where the subscriber is an aggregator.",
)
});
pub static SYNC_COMMITTEE_SUBSCRIPTION_REQUESTS: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"validator_sync_committee_subnet_subscriptions_total",
"Count of validator sync committee subscription requests.",
)
});
/*
* Beacon processor
*/
pub static BEACON_PROCESSOR_MISSING_COMPONENTS: LazyLock<Result<IntCounterVec>> = LazyLock::new(
|| {
try_create_int_counter_vec(
"beacon_processor_missing_components_total",
"Total number of imported individual block components that resulted in missing components",
&["source", "component"],
)
},
);
pub static BEACON_PROCESSOR_IMPORT_ERRORS_PER_TYPE: LazyLock<Result<IntCounterVec>> =
LazyLock::new(|| {
try_create_int_counter_vec(
"beacon_processor_import_errors_total",
"Total number of block components that were not verified",
&["source", "component", "type"],
)
});
pub static BEACON_PROCESSOR_GET_BLOCK_ROOTS_TIME: LazyLock<Result<HistogramVec>> =
LazyLock::new(|| {
try_create_histogram_vec_with_buckets(
"beacon_processor_get_block_roots_time_seconds",
"Time to complete get_block_roots when serving by_range requests",
decimal_buckets(-3, -1),
&["source"],
)
});
/*
* Gossip processor
*/
// Gossip blocks.
pub static BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_gossip_block_verified_total",
"Total number of gossip blocks verified for propagation.",
)
});
pub static BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_gossip_block_imported_total",
"Total number of gossip blocks imported to fork choice, etc.",
)
});
pub static BEACON_PROCESSOR_GOSSIP_BLOCK_REQUEUED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_gossip_block_requeued_total",
"Total number of gossip blocks that arrived early and were re-queued for later processing.",
)
});
pub static BEACON_PROCESSOR_GOSSIP_BLOCK_EARLY_SECONDS: LazyLock<Result<Histogram>> = LazyLock::new(
|| {
try_create_histogram(
"beacon_processor_gossip_block_early_seconds",
"Whenever a gossip block is received early this metrics is set to how early that block was.",
)
},
);
pub static BEACON_PROCESSOR_GOSSIP_BLOB_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_gossip_blob_verified_total",
"Total number of gossip blob verified for propagation.",
)
});
pub static BEACON_PROCESSOR_GOSSIP_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: LazyLock<
Result<IntCounter>,
> = LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_gossip_data_column_verified_total",
"Total number of gossip data column sidecar verified for propagation.",
)
});
// Gossip Exits.
pub static BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_exit_verified_total",
"Total number of voluntary exits verified for propagation.",
)
});
pub static BEACON_PROCESSOR_EXIT_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_exit_imported_total",
"Total number of voluntary exits imported to the op pool.",
)
});
// Gossip proposer slashings.
pub static BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_proposer_slashing_verified_total",
"Total number of proposer slashings verified for propagation.",
)
});
pub static BEACON_PROCESSOR_PROPOSER_SLASHING_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_proposer_slashing_imported_total",
"Total number of proposer slashings imported to the op pool.",
)
});
// Gossip attester slashings.
pub static BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_attester_slashing_verified_total",
"Total number of attester slashings verified for propagation.",
)
});
pub static BEACON_PROCESSOR_ATTESTER_SLASHING_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_attester_slashing_imported_total",
"Total number of attester slashings imported to the op pool.",
)
});
// Gossip BLS to execution changes.
pub static BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_bls_to_execution_change_verified_total",
"Total number of address changes verified for propagation.",
)
});
pub static BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_bls_to_execution_change_imported_total",
"Total number of address changes imported to the op pool.",
)
});
// Rpc blocks.
pub static BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_rpc_block_imported_total",
"Total number of gossip blocks imported to fork choice, etc.",
)
});
// Custody Sync.
pub static BEACON_PROCESSOR_CUSTODY_BACKFILL_COLUMN_IMPORT_SUCCESS_TOTAL: LazyLock<
Result<IntCounter>,
> = LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_custody_backfill_column_import_success_total",
"Total number of custody backfill sync columns successfully processed.",
)
});
pub static BEACON_PROCESSOR_CUSTODY_BACKFILL_BATCH_FAILED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_custody_backfill_batch_failed_total",
"Total number of custody backfill batches that failed to be processed.",
)
});
// Chain segments.
pub static BEACON_PROCESSOR_CHAIN_SEGMENT_SUCCESS_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_chain_segment_success_total",
"Total number of chain segments successfully processed.",
)
});
pub static BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_SUCCESS_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_backfill_chain_segment_success_total",
"Total number of chain segments successfully processed.",
)
});
pub static BEACON_PROCESSOR_CHAIN_SEGMENT_FAILED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_chain_segment_failed_total",
"Total number of chain segments that failed processing.",
)
});
pub static BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_FAILED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_backfill_chain_segment_failed_total",
"Total number of backfill chain segments that failed processing.",
)
});
// Unaggregated attestations.
pub static BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_unaggregated_attestation_verified_total",
"Total number of unaggregated attestations verified for gossip.",
)
});
pub static BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_unaggregated_attestation_imported_total",
"Total number of unaggregated attestations imported to fork choice, etc.",
)
});
pub static BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_REQUEUED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_unaggregated_attestation_requeued_total",
"Total number of unaggregated attestations that referenced an unknown block and were re-queued.",
)
});
// Aggregated attestations.
pub static BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_aggregated_attestation_verified_total",
"Total number of aggregated attestations verified for gossip.",
)
});
pub static BEACON_PROCESSOR_AGGREGATED_ATTESTATION_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_aggregated_attestation_imported_total",
"Total number of aggregated attestations imported to fork choice, etc.",
)
});
pub static BEACON_PROCESSOR_AGGREGATED_ATTESTATION_REQUEUED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_aggregated_attestation_requeued_total",
"Total number of aggregated attestations that referenced an unknown block and were re-queued.",
)
});
// Sync committee messages.
pub static BEACON_PROCESSOR_SYNC_MESSAGE_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_sync_message_verified_total",
"Total number of sync committee messages verified for gossip.",
)
});
pub static BEACON_PROCESSOR_SYNC_MESSAGE_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_sync_message_imported_total",
"Total number of sync committee messages imported to fork choice, etc.",
)
});
// Sync contribution.
pub static BEACON_PROCESSOR_SYNC_CONTRIBUTION_VERIFIED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_sync_contribution_verified_total",
"Total number of sync committee contributions verified for gossip.",
)
});
pub static BEACON_PROCESSOR_SYNC_CONTRIBUTION_IMPORTED_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_sync_contribution_imported_total",
"Total number of sync committee contributions imported to fork choice, etc.",
)
});
/// Errors and Debugging Stats
pub static GOSSIP_ATTESTATION_ERRORS_PER_TYPE: LazyLock<Result<IntCounterVec>> =
LazyLock::new(|| {
try_create_int_counter_vec(
"gossipsub_attestation_errors_per_type",
"Gossipsub attestation errors per error type",
&["type"],
)
});
pub static GOSSIP_SYNC_COMMITTEE_ERRORS_PER_TYPE: LazyLock<Result<IntCounterVec>> =
LazyLock::new(|| {
try_create_int_counter_vec(
"gossipsub_sync_committee_errors_per_type",
"Gossipsub sync_committee errors per error type",
&["type"],
)
});
pub static GOSSIP_FINALITY_UPDATE_ERRORS_PER_TYPE: LazyLock<Result<IntCounterVec>> =
LazyLock::new(|| {
try_create_int_counter_vec(
"gossipsub_light_client_finality_update_errors_per_type",
"Gossipsub light_client_finality_update errors per error type",
&["type"],
)
});
pub static GOSSIP_OPTIMISTIC_UPDATE_ERRORS_PER_TYPE: LazyLock<Result<IntCounterVec>> =
LazyLock::new(|| {
try_create_int_counter_vec(
"gossipsub_light_client_optimistic_update_errors_per_type",
"Gossipsub light_client_optimistic_update errors per error type",
&["type"],
)
});
/*
* Network queue metrics
*/
pub static NETWORK_RECEIVE_EVENTS: LazyLock<Result<IntCounterVec>> = LazyLock::new(|| {
try_create_int_counter_vec(
"network_receive_events",
"Count of events received by the channel to the network service",
&["type"],
)
});
pub static NETWORK_RECEIVE_TIMES: LazyLock<Result<HistogramVec>> = LazyLock::new(|| {
try_create_histogram_vec(
"network_receive_times",
"Time taken for network to handle an event sent to the network service.",
&["type"],
)
});
/*
* Sync related metrics
*/
pub static PEERS_PER_SYNC_TYPE: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
try_create_int_gauge_vec(
"sync_peers_per_status",
"Number of connected peers per sync status type",
&["sync_status"],
)
});
pub static PEERS_PER_COLUMN_SUBNET: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
try_create_int_gauge_vec(
"sync_peers_per_column_subnet",
"Number of connected peers per column subnet",
&["subnet_id"],
)
});
pub static PEERS_PER_CUSTODY_COLUMN_SUBNET: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
try_create_int_gauge_vec(
"sync_peers_per_custody_column_subnet",
"Number of connected peers per custody column subnet",
&["subnet_id"],
)
});
pub static SYNCING_CHAINS_COUNT: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
try_create_int_gauge_vec(
"sync_range_chains",
"Number of Syncing chains in range, per range type",
&["range_type"],
)
});
pub static SYNCING_CHAINS_REMOVED: LazyLock<Result<IntCounterVec>> = LazyLock::new(|| {
try_create_int_counter_vec(
"sync_range_removed_chains_total",
"Total count of range syncing chains removed per range type",
&["range_type"],
)
});
pub static SYNCING_CHAINS_ADDED: LazyLock<Result<IntCounterVec>> = LazyLock::new(|| {
try_create_int_counter_vec(
"sync_range_added_chains_total",
"Total count of range syncing chains added per range type",
&["range_type"],
)
});
pub static SYNCING_CHAINS_DROPPED_BLOCKS: LazyLock<Result<IntCounterVec>> = LazyLock::new(|| {
try_create_int_counter_vec(
"sync_range_chains_dropped_blocks_total",
"Total count of dropped blocks when removing a syncing chain per range type",
&["range_type"],
)
});
pub static SYNCING_CHAINS_IGNORED_BLOCKS: LazyLock<Result<IntCounterVec>> = LazyLock::new(|| {
try_create_int_counter_vec(
"sync_range_chains_ignored_blocks_total",
"Total count of ignored blocks when processing a syncing chain batch per chain type",
&["chain_type"],
)
});
pub static SYNCING_CHAINS_PROCESSED_BATCHES: LazyLock<Result<IntCounterVec>> =
LazyLock::new(|| {
try_create_int_counter_vec(
"sync_range_chains_processed_batches_total",
"Total count of processed batches in a syncing chain batch per chain type",
&["chain_type"],
)
});
pub static SYNCING_CHAIN_BATCH_AWAITING_PROCESSING: LazyLock<Result<Histogram>> =
LazyLock::new(|| {
try_create_histogram_with_buckets(
"sync_range_chain_batch_awaiting_processing_seconds",
"Time range sync batches spend in AwaitingProcessing state",
Ok(vec![
0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0,
]),
)
});
pub static SYNC_SINGLE_BLOCK_LOOKUPS: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge(
"sync_single_block_lookups",
"Number of single block lookups underway",
)
});
pub static SYNC_LOOKUP_CREATED: LazyLock<Result<IntCounter>> = LazyLock::new(|| {
try_create_int_counter(
"sync_lookups_created_total",
"Total count of sync lookups created",
)
});
pub static SYNC_LOOKUP_DROPPED: LazyLock<Result<IntCounterVec>> = LazyLock::new(|| {
try_create_int_counter_vec(
"sync_lookups_dropped_total",
"Total count of sync lookups dropped by reason",
&["reason"],
)
});
pub static SYNC_LOOKUP_COMPLETED: LazyLock<Result<IntCounter>> = LazyLock::new(|| {
try_create_int_counter(
"sync_lookups_completed_total",
"Total count of sync lookups completed",
)
});
pub static SYNC_LOOKUPS_STUCK: LazyLock<Result<IntCounter>> = LazyLock::new(|| {
try_create_int_counter(
"sync_lookups_stuck_total",
"Total count of sync lookups that are stuck and dropped",
)
});
pub static SYNC_ACTIVE_NETWORK_REQUESTS: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
try_create_int_gauge_vec(
"sync_active_network_requests",
"Current count of active network requests from sync",
&["type"],
)
});
pub static SYNC_UNKNOWN_NETWORK_REQUESTS: LazyLock<Result<IntCounterVec>> = LazyLock::new(|| {
try_create_int_counter_vec(
"sync_unknwon_network_request",
"Total count of network messages received for unknown active requests",
&["type"],
)
});
/*
* Block Delay Metrics
*/
pub static BEACON_BLOCK_DELAY_GOSSIP: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge(
"beacon_block_delay_gossip",
"The first time we see this block from gossip as a delay from the start of the slot",
)
});
pub static BEACON_BLOCK_DELAY_GOSSIP_VERIFICATION: LazyLock<Result<IntGauge>> = LazyLock::new(
|| {
try_create_int_gauge(
"beacon_block_delay_gossip_verification",
"Keeps track of the time delay from the start of the slot to the point we propagate the block",
)
},
);
pub static BEACON_BLOCK_DELAY_FULL_VERIFICATION: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge(
"beacon_block_delay_full_verification",
"The time it takes to verify a beacon block.",
)
});
pub static BEACON_BLOCK_DELAY_GOSSIP_ARRIVED_LATE_TOTAL: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_block_delay_gossip_arrived_late_total",
"Count of times when a gossip block arrived from the network later than the attestation deadline.",
)
});
/*
* Blob Delay Metrics
*/
pub static BEACON_BLOB_DELAY_GOSSIP: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge(
"beacon_blob_delay_gossip_last_delay",
"The first time we see this blob as a delay from the start of the slot",
)
});
pub static BEACON_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: LazyLock<
Result<Histogram>,
> = LazyLock::new(|| {
try_create_histogram_with_buckets(
"beacon_data_column_gossip_propagation_verification_delay_time",
"Duration between when the data column sidecar is received over gossip and when it is verified for propagation.",
// [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5]
decimal_buckets(-3, -1),
)
});
pub static BEACON_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: LazyLock<Result<Histogram>> =
LazyLock::new(|| {
try_create_histogram_with_buckets(
"beacon_data_column_gossip_slot_start_delay_time",
"Duration between when the data column sidecar is received over gossip and the start of the slot it belongs to.",
// Create a custom bucket list for greater granularity in block delay
Ok(vec![
0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 5.0,
6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0,
]), // NOTE: Previous values, which we may want to switch back to.
// [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]
//decimal_buckets(-1,2)
)
});
pub static BEACON_BLOB_DELAY_GOSSIP_VERIFICATION: LazyLock<Result<IntGauge>> = LazyLock::new(
|| {
try_create_int_gauge(
"beacon_blob_delay_gossip_verification",
"Keeps track of the time delay from the start of the slot to the point we propagate the blob",
)
},
);
pub static BEACON_BLOB_DELAY_FULL_VERIFICATION: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge(
"beacon_blob_last_full_verification_delay",
"The time it takes to verify a beacon blob",
)
});
pub static BEACON_BLOB_RPC_SLOT_START_DELAY_TIME: LazyLock<Result<Histogram>> = LazyLock::new(
|| {
try_create_histogram_with_buckets(
"beacon_blob_rpc_slot_start_delay_time",
"Duration between when a blob is received over rpc and the start of the slot it belongs to.",
// Create a custom bucket list for greater granularity in block delay
Ok(vec![
0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 5.0,
6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0,
]), // NOTE: Previous values, which we may want to switch back to.
// [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]
//decimal_buckets(-1,2)
)
},
);
pub static BEACON_BLOB_GOSSIP_ARRIVED_LATE_TOTAL: LazyLock<Result<IntCounter>> = LazyLock::new(
|| {
try_create_int_counter(
"beacon_blob_gossip_arrived_late_total",
"Count of times when a gossip blob arrived from the network later than the attestation deadline.",
)
},
);
/*
* Light client update reprocessing queue metrics.
*/
pub static BEACON_PROCESSOR_REPROCESSING_QUEUE_SENT_OPTIMISTIC_UPDATES: LazyLock<
Result<IntCounter>,
> = LazyLock::new(|| {
try_create_int_counter(
"beacon_processor_reprocessing_queue_sent_optimistic_updates",
"Number of queued light client optimistic updates where as matching block has been imported.",
)
});
pub fn register_finality_update_error(error: &LightClientFinalityUpdateError) {
inc_counter_vec(&GOSSIP_FINALITY_UPDATE_ERRORS_PER_TYPE, &[error.as_ref()]);
}
pub fn register_optimistic_update_error(error: &LightClientOptimisticUpdateError) {
inc_counter_vec(&GOSSIP_OPTIMISTIC_UPDATE_ERRORS_PER_TYPE, &[error.as_ref()]);
}
pub fn register_attestation_error(error: &AttnError) {
inc_counter_vec(&GOSSIP_ATTESTATION_ERRORS_PER_TYPE, &[error.as_ref()]);
}
pub fn register_sync_committee_error(error: &SyncCommitteeError) {
inc_counter_vec(&GOSSIP_SYNC_COMMITTEE_ERRORS_PER_TYPE, &[error.as_ref()]);
}
pub(crate) fn register_process_result_metrics(
result: &std::result::Result<AvailabilityProcessingStatus, BlockError>,
source: BlockSource,
block_component: &'static str,
) {
match result {
Ok(status) => match status {
AvailabilityProcessingStatus::Imported { .. } => match source {
BlockSource::Gossip => {
inc_counter(&BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL);
}
BlockSource::Rpc => {
inc_counter(&BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL);
}
},
AvailabilityProcessingStatus::MissingComponents { .. } => {
inc_counter_vec(
&BEACON_PROCESSOR_MISSING_COMPONENTS,
&[source.as_ref(), block_component],
);
}
},
Err(error) => {
inc_counter_vec(
&BEACON_PROCESSOR_IMPORT_ERRORS_PER_TYPE,
&[source.as_ref(), block_component, error.as_ref()],
);
}
}
}
pub fn update_gossip_metrics<E: EthSpec>(
gossipsub: &Gossipsub,
network_globals: &Arc<NetworkGlobals<E>>,
) {
// Mesh peers per client
// Reset the gauges
for client_kind in ClientKind::iter() {
set_gauge_vec(
&BEACON_BLOCK_MESH_PEERS_PER_CLIENT,
&[client_kind.as_ref()],
0_i64,
);
set_gauge_vec(
&BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT,
&[client_kind.as_ref()],
0_i64,
);
}
for topic_hash in gossipsub.topics() {
if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
match topic.kind() {
GossipKind::Attestation(_subnet_id) => {}
GossipKind::BeaconBlock => {
for peer_id in gossipsub.mesh_peers(topic_hash) {
let client = network_globals
.peers
.read()
.peer_info(peer_id)
.map(|peer_info| peer_info.client().kind.into())
.unwrap_or_else(|| "Unknown");
if let Some(v) =
get_int_gauge(&BEACON_BLOCK_MESH_PEERS_PER_CLIENT, &[client])
{
v.inc()
};
}
}
GossipKind::BeaconAggregateAndProof => {
for peer_id in gossipsub.mesh_peers(topic_hash) {
let client = network_globals
.peers
.read()
.peer_info(peer_id)
.map(|peer_info| peer_info.client().kind.into())
.unwrap_or_else(|| "Unknown");
if let Some(v) = get_int_gauge(
&BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT,
&[client],
) {
v.inc()
};
}
}
GossipKind::SyncCommitteeMessage(_subnet_id) => {}
_kind => {}
}
}
}
}
pub fn update_sync_metrics<E: EthSpec>(network_globals: &Arc<NetworkGlobals<E>>) {
// reset the counts
if PEERS_PER_SYNC_TYPE
.as_ref()
.map(|metric| metric.reset())
.is_err()
{
return;
};
// count per sync status, the number of connected peers
let mut peers_per_sync_type = FnvHashMap::default();
let mut peers_per_column_subnet = FnvHashMap::default();
for (_, info) in network_globals.peers.read().connected_peers() {
let sync_type = info.sync_status().as_str();
*peers_per_sync_type.entry(sync_type).or_default() += 1;
for subnet in info.custody_subnets_iter() {
*peers_per_column_subnet.entry(*subnet).or_default() += 1;
}
}
for (sync_type, peer_count) in peers_per_sync_type {
set_gauge_entry(&PEERS_PER_SYNC_TYPE, &[sync_type], peer_count);
}
let all_column_subnets =
(0..network_globals.spec.data_column_sidecar_subnet_count).map(DataColumnSubnetId::new);
let custody_column_subnets = network_globals.sampling_subnets();
// Iterate all subnet values to set to zero the empty entries in peers_per_column_subnet
for subnet in all_column_subnets {
set_gauge_entry(
&PEERS_PER_COLUMN_SUBNET,
&[&format!("{subnet}")],
peers_per_column_subnet.get(&subnet).copied().unwrap_or(0),
);
}
// Registering this metric is a duplicate for supernodes but helpful for fullnodes. This way
// operators can monitor the health of only the subnets of their interest without complex
// Grafana queries.
for subnet in custody_column_subnets.iter() {
set_gauge_entry(
&PEERS_PER_CUSTODY_COLUMN_SUBNET,
&[&format!("{subnet}")],
peers_per_column_subnet.get(subnet).copied().unwrap_or(0),
);
}
}
Reference in New Issue View Git Blame Copy Permalink