use beacon_chain::{ AvailabilityProcessingStatus, BlockError, attestation_verification::Error as AttnError, light_client_finality_update_verification::Error as LightClientFinalityUpdateError, light_client_optimistic_update_verification::Error as LightClientOptimisticUpdateError, sync_committee_verification::Error as SyncCommitteeError, }; use fnv::FnvHashMap; use lighthouse_network::{ GossipTopic, Gossipsub, NetworkGlobals, peer_manager::peerdb::client::ClientKind, types::GossipKind, }; pub use metrics::*; use std::sync::{Arc, LazyLock}; use strum::AsRefStr; use strum::IntoEnumIterator; use types::DataColumnSubnetId; use types::EthSpec; #[derive(Debug, AsRefStr)] pub(crate) enum BlockSource { Gossip, Rpc, } pub static BEACON_BLOCK_MESH_PEERS_PER_CLIENT: LazyLock> = LazyLock::new(|| { try_create_int_gauge_vec( "block_mesh_peers_per_client", "Number of mesh peers for BeaconBlock topic per client", &["Client"], ) }); pub static BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT: LazyLock> = LazyLock::new(|| { try_create_int_gauge_vec( "beacon_aggregate_and_proof_mesh_peers_per_client", "Number of mesh peers for BeaconAggregateAndProof topic per client", &["Client"], ) }); /* * Attestation subnet subscriptions */ pub static SUBNET_SUBSCRIPTION_REQUESTS: LazyLock> = LazyLock::new(|| { try_create_int_counter( "validator_attestation_subnet_subscriptions_total", "Count of validator attestation subscription requests.", ) }); pub static SUBNET_SUBSCRIPTION_AGGREGATOR_REQUESTS: LazyLock> = LazyLock::new(|| { try_create_int_counter( "validator_subnet_subscriptions_aggregator_total", "Count of validator subscription requests where the subscriber is an aggregator.", ) }); pub static SYNC_COMMITTEE_SUBSCRIPTION_REQUESTS: LazyLock> = LazyLock::new(|| { try_create_int_counter( "validator_sync_committee_subnet_subscriptions_total", "Count of validator sync committee subscription requests.", ) }); /* * Beacon processor */ pub static BEACON_PROCESSOR_MISSING_COMPONENTS: LazyLock> = LazyLock::new( || { try_create_int_counter_vec( "beacon_processor_missing_components_total", "Total number of imported individual block components that resulted in missing components", &["source", "component"], ) }, ); pub static BEACON_PROCESSOR_IMPORT_ERRORS_PER_TYPE: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "beacon_processor_import_errors_total", "Total number of block components that were not verified", &["source", "component", "type"], ) }); pub static BEACON_PROCESSOR_GET_BLOCK_ROOTS_TIME: LazyLock> = LazyLock::new(|| { try_create_histogram_vec_with_buckets( "beacon_processor_get_block_roots_time_seconds", "Time to complete get_block_roots when serving by_range requests", decimal_buckets(-3, -1), &["source"], ) }); /* * Gossip processor */ // Gossip blocks. pub static BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_gossip_block_verified_total", "Total number of gossip blocks verified for propagation.", ) }); pub static BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_gossip_block_imported_total", "Total number of gossip blocks imported to fork choice, etc.", ) }); pub static BEACON_PROCESSOR_GOSSIP_BLOCK_REQUEUED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_gossip_block_requeued_total", "Total number of gossip blocks that arrived early and were re-queued for later processing.", ) }); pub static BEACON_PROCESSOR_GOSSIP_BLOCK_EARLY_SECONDS: LazyLock> = LazyLock::new( || { try_create_histogram( "beacon_processor_gossip_block_early_seconds", "Whenever a gossip block is received early this metrics is set to how early that block was.", ) }, ); pub static BEACON_PROCESSOR_GOSSIP_BLOB_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_gossip_blob_verified_total", "Total number of gossip blob verified for propagation.", ) }); pub static BEACON_PROCESSOR_GOSSIP_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: LazyLock< Result, > = LazyLock::new(|| { try_create_int_counter( "beacon_processor_gossip_data_column_verified_total", "Total number of gossip data column sidecar verified for propagation.", ) }); // Gossip Exits. pub static BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_exit_verified_total", "Total number of voluntary exits verified for propagation.", ) }); pub static BEACON_PROCESSOR_EXIT_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_exit_imported_total", "Total number of voluntary exits imported to the op pool.", ) }); // Gossip proposer slashings. pub static BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_proposer_slashing_verified_total", "Total number of proposer slashings verified for propagation.", ) }); pub static BEACON_PROCESSOR_PROPOSER_SLASHING_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_proposer_slashing_imported_total", "Total number of proposer slashings imported to the op pool.", ) }); // Gossip attester slashings. pub static BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_attester_slashing_verified_total", "Total number of attester slashings verified for propagation.", ) }); pub static BEACON_PROCESSOR_ATTESTER_SLASHING_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_attester_slashing_imported_total", "Total number of attester slashings imported to the op pool.", ) }); // Gossip BLS to execution changes. pub static BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_bls_to_execution_change_verified_total", "Total number of address changes verified for propagation.", ) }); pub static BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_bls_to_execution_change_imported_total", "Total number of address changes imported to the op pool.", ) }); // Rpc blocks. pub static BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_rpc_block_imported_total", "Total number of gossip blocks imported to fork choice, etc.", ) }); // Custody Sync. pub static BEACON_PROCESSOR_CUSTODY_BACKFILL_COLUMN_IMPORT_SUCCESS_TOTAL: LazyLock< Result, > = LazyLock::new(|| { try_create_int_counter( "beacon_processor_custody_backfill_column_import_success_total", "Total number of custody backfill sync columns successfully processed.", ) }); pub static BEACON_PROCESSOR_CUSTODY_BACKFILL_BATCH_FAILED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_custody_backfill_batch_failed_total", "Total number of custody backfill batches that failed to be processed.", ) }); // Chain segments. pub static BEACON_PROCESSOR_CHAIN_SEGMENT_SUCCESS_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_chain_segment_success_total", "Total number of chain segments successfully processed.", ) }); pub static BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_SUCCESS_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_backfill_chain_segment_success_total", "Total number of chain segments successfully processed.", ) }); pub static BEACON_PROCESSOR_CHAIN_SEGMENT_FAILED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_chain_segment_failed_total", "Total number of chain segments that failed processing.", ) }); pub static BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_FAILED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_backfill_chain_segment_failed_total", "Total number of backfill chain segments that failed processing.", ) }); // Unaggregated attestations. pub static BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_unaggregated_attestation_verified_total", "Total number of unaggregated attestations verified for gossip.", ) }); pub static BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_unaggregated_attestation_imported_total", "Total number of unaggregated attestations imported to fork choice, etc.", ) }); pub static BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_REQUEUED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_unaggregated_attestation_requeued_total", "Total number of unaggregated attestations that referenced an unknown block and were re-queued.", ) }); // Aggregated attestations. pub static BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_aggregated_attestation_verified_total", "Total number of aggregated attestations verified for gossip.", ) }); pub static BEACON_PROCESSOR_AGGREGATED_ATTESTATION_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_aggregated_attestation_imported_total", "Total number of aggregated attestations imported to fork choice, etc.", ) }); pub static BEACON_PROCESSOR_AGGREGATED_ATTESTATION_REQUEUED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_aggregated_attestation_requeued_total", "Total number of aggregated attestations that referenced an unknown block and were re-queued.", ) }); // Sync committee messages. pub static BEACON_PROCESSOR_SYNC_MESSAGE_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_sync_message_verified_total", "Total number of sync committee messages verified for gossip.", ) }); pub static BEACON_PROCESSOR_SYNC_MESSAGE_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_sync_message_imported_total", "Total number of sync committee messages imported to fork choice, etc.", ) }); // Sync contribution. pub static BEACON_PROCESSOR_SYNC_CONTRIBUTION_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_sync_contribution_verified_total", "Total number of sync committee contributions verified for gossip.", ) }); pub static BEACON_PROCESSOR_SYNC_CONTRIBUTION_IMPORTED_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_processor_sync_contribution_imported_total", "Total number of sync committee contributions imported to fork choice, etc.", ) }); /// Errors and Debugging Stats pub static GOSSIP_ATTESTATION_ERRORS_PER_TYPE: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "gossipsub_attestation_errors_per_type", "Gossipsub attestation errors per error type", &["type"], ) }); pub static GOSSIP_SYNC_COMMITTEE_ERRORS_PER_TYPE: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "gossipsub_sync_committee_errors_per_type", "Gossipsub sync_committee errors per error type", &["type"], ) }); pub static GOSSIP_FINALITY_UPDATE_ERRORS_PER_TYPE: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "gossipsub_light_client_finality_update_errors_per_type", "Gossipsub light_client_finality_update errors per error type", &["type"], ) }); pub static GOSSIP_OPTIMISTIC_UPDATE_ERRORS_PER_TYPE: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "gossipsub_light_client_optimistic_update_errors_per_type", "Gossipsub light_client_optimistic_update errors per error type", &["type"], ) }); /* * Network queue metrics */ pub static NETWORK_RECEIVE_EVENTS: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "network_receive_events", "Count of events received by the channel to the network service", &["type"], ) }); pub static NETWORK_RECEIVE_TIMES: LazyLock> = LazyLock::new(|| { try_create_histogram_vec( "network_receive_times", "Time taken for network to handle an event sent to the network service.", &["type"], ) }); /* * Sync related metrics */ pub static PEERS_PER_SYNC_TYPE: LazyLock> = LazyLock::new(|| { try_create_int_gauge_vec( "sync_peers_per_status", "Number of connected peers per sync status type", &["sync_status"], ) }); pub static PEERS_PER_COLUMN_SUBNET: LazyLock> = LazyLock::new(|| { try_create_int_gauge_vec( "sync_peers_per_column_subnet", "Number of connected peers per column subnet", &["subnet_id"], ) }); pub static PEERS_PER_CUSTODY_COLUMN_SUBNET: LazyLock> = LazyLock::new(|| { try_create_int_gauge_vec( "sync_peers_per_custody_column_subnet", "Number of connected peers per custody column subnet", &["subnet_id"], ) }); pub static SYNCING_CHAINS_COUNT: LazyLock> = LazyLock::new(|| { try_create_int_gauge_vec( "sync_range_chains", "Number of Syncing chains in range, per range type", &["range_type"], ) }); pub static SYNCING_CHAINS_REMOVED: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "sync_range_removed_chains_total", "Total count of range syncing chains removed per range type", &["range_type"], ) }); pub static SYNCING_CHAINS_ADDED: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "sync_range_added_chains_total", "Total count of range syncing chains added per range type", &["range_type"], ) }); pub static SYNCING_CHAINS_DROPPED_BLOCKS: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "sync_range_chains_dropped_blocks_total", "Total count of dropped blocks when removing a syncing chain per range type", &["range_type"], ) }); pub static SYNCING_CHAINS_IGNORED_BLOCKS: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "sync_range_chains_ignored_blocks_total", "Total count of ignored blocks when processing a syncing chain batch per chain type", &["chain_type"], ) }); pub static SYNCING_CHAINS_PROCESSED_BATCHES: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "sync_range_chains_processed_batches_total", "Total count of processed batches in a syncing chain batch per chain type", &["chain_type"], ) }); pub static SYNCING_CHAIN_BATCH_AWAITING_PROCESSING: LazyLock> = LazyLock::new(|| { try_create_histogram_with_buckets( "sync_range_chain_batch_awaiting_processing_seconds", "Time range sync batches spend in AwaitingProcessing state", Ok(vec![ 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, ]), ) }); pub static SYNC_SINGLE_BLOCK_LOOKUPS: LazyLock> = LazyLock::new(|| { try_create_int_gauge( "sync_single_block_lookups", "Number of single block lookups underway", ) }); pub static SYNC_LOOKUP_CREATED: LazyLock> = LazyLock::new(|| { try_create_int_counter( "sync_lookups_created_total", "Total count of sync lookups created", ) }); pub static SYNC_LOOKUP_DROPPED: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "sync_lookups_dropped_total", "Total count of sync lookups dropped by reason", &["reason"], ) }); pub static SYNC_LOOKUP_COMPLETED: LazyLock> = LazyLock::new(|| { try_create_int_counter( "sync_lookups_completed_total", "Total count of sync lookups completed", ) }); pub static SYNC_LOOKUPS_STUCK: LazyLock> = LazyLock::new(|| { try_create_int_counter( "sync_lookups_stuck_total", "Total count of sync lookups that are stuck and dropped", ) }); pub static SYNC_ACTIVE_NETWORK_REQUESTS: LazyLock> = LazyLock::new(|| { try_create_int_gauge_vec( "sync_active_network_requests", "Current count of active network requests from sync", &["type"], ) }); pub static SYNC_UNKNOWN_NETWORK_REQUESTS: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "sync_unknwon_network_request", "Total count of network messages received for unknown active requests", &["type"], ) }); /* * Block Delay Metrics */ pub static BEACON_BLOCK_DELAY_GOSSIP: LazyLock> = LazyLock::new(|| { try_create_int_gauge( "beacon_block_delay_gossip", "The first time we see this block from gossip as a delay from the start of the slot", ) }); pub static BEACON_BLOCK_DELAY_GOSSIP_VERIFICATION: LazyLock> = LazyLock::new( || { try_create_int_gauge( "beacon_block_delay_gossip_verification", "Keeps track of the time delay from the start of the slot to the point we propagate the block", ) }, ); pub static BEACON_BLOCK_DELAY_FULL_VERIFICATION: LazyLock> = LazyLock::new(|| { try_create_int_gauge( "beacon_block_delay_full_verification", "The time it takes to verify a beacon block.", ) }); pub static BEACON_BLOCK_DELAY_GOSSIP_ARRIVED_LATE_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( "beacon_block_delay_gossip_arrived_late_total", "Count of times when a gossip block arrived from the network later than the attestation deadline.", ) }); /* * Blob Delay Metrics */ pub static BEACON_BLOB_DELAY_GOSSIP: LazyLock> = LazyLock::new(|| { try_create_int_gauge( "beacon_blob_delay_gossip_last_delay", "The first time we see this blob as a delay from the start of the slot", ) }); pub static BEACON_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: LazyLock< Result, > = LazyLock::new(|| { try_create_histogram_with_buckets( "beacon_data_column_gossip_propagation_verification_delay_time", "Duration between when the data column sidecar is received over gossip and when it is verified for propagation.", // [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] decimal_buckets(-3, -1), ) }); pub static BEACON_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: LazyLock> = LazyLock::new(|| { try_create_histogram_with_buckets( "beacon_data_column_gossip_slot_start_delay_time", "Duration between when the data column sidecar is received over gossip and the start of the slot it belongs to.", // Create a custom bucket list for greater granularity in block delay Ok(vec![ 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, ]), // NOTE: Previous values, which we may want to switch back to. // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] //decimal_buckets(-1,2) ) }); pub static BEACON_BLOB_DELAY_GOSSIP_VERIFICATION: LazyLock> = LazyLock::new( || { try_create_int_gauge( "beacon_blob_delay_gossip_verification", "Keeps track of the time delay from the start of the slot to the point we propagate the blob", ) }, ); pub static BEACON_BLOB_DELAY_FULL_VERIFICATION: LazyLock> = LazyLock::new(|| { try_create_int_gauge( "beacon_blob_last_full_verification_delay", "The time it takes to verify a beacon blob", ) }); pub static BEACON_BLOB_RPC_SLOT_START_DELAY_TIME: LazyLock> = LazyLock::new( || { try_create_histogram_with_buckets( "beacon_blob_rpc_slot_start_delay_time", "Duration between when a blob is received over rpc and the start of the slot it belongs to.", // Create a custom bucket list for greater granularity in block delay Ok(vec![ 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, ]), // NOTE: Previous values, which we may want to switch back to. // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] //decimal_buckets(-1,2) ) }, ); pub static BEACON_BLOB_GOSSIP_ARRIVED_LATE_TOTAL: LazyLock> = LazyLock::new( || { try_create_int_counter( "beacon_blob_gossip_arrived_late_total", "Count of times when a gossip blob arrived from the network later than the attestation deadline.", ) }, ); /* * Light client update reprocessing queue metrics. */ pub static BEACON_PROCESSOR_REPROCESSING_QUEUE_SENT_OPTIMISTIC_UPDATES: LazyLock< Result, > = LazyLock::new(|| { try_create_int_counter( "beacon_processor_reprocessing_queue_sent_optimistic_updates", "Number of queued light client optimistic updates where as matching block has been imported.", ) }); pub fn register_finality_update_error(error: &LightClientFinalityUpdateError) { inc_counter_vec(&GOSSIP_FINALITY_UPDATE_ERRORS_PER_TYPE, &[error.as_ref()]); } pub fn register_optimistic_update_error(error: &LightClientOptimisticUpdateError) { inc_counter_vec(&GOSSIP_OPTIMISTIC_UPDATE_ERRORS_PER_TYPE, &[error.as_ref()]); } pub fn register_attestation_error(error: &AttnError) { inc_counter_vec(&GOSSIP_ATTESTATION_ERRORS_PER_TYPE, &[error.as_ref()]); } pub fn register_sync_committee_error(error: &SyncCommitteeError) { inc_counter_vec(&GOSSIP_SYNC_COMMITTEE_ERRORS_PER_TYPE, &[error.as_ref()]); } pub(crate) fn register_process_result_metrics( result: &std::result::Result, source: BlockSource, block_component: &'static str, ) { match result { Ok(status) => match status { AvailabilityProcessingStatus::Imported { .. } => match source { BlockSource::Gossip => { inc_counter(&BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL); } BlockSource::Rpc => { inc_counter(&BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL); } }, AvailabilityProcessingStatus::MissingComponents { .. } => { inc_counter_vec( &BEACON_PROCESSOR_MISSING_COMPONENTS, &[source.as_ref(), block_component], ); } }, Err(error) => { inc_counter_vec( &BEACON_PROCESSOR_IMPORT_ERRORS_PER_TYPE, &[source.as_ref(), block_component, error.as_ref()], ); } } } pub fn update_gossip_metrics( gossipsub: &Gossipsub, network_globals: &Arc>, ) { // Mesh peers per client // Reset the gauges for client_kind in ClientKind::iter() { set_gauge_vec( &BEACON_BLOCK_MESH_PEERS_PER_CLIENT, &[client_kind.as_ref()], 0_i64, ); set_gauge_vec( &BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT, &[client_kind.as_ref()], 0_i64, ); } for topic_hash in gossipsub.topics() { if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) { match topic.kind() { GossipKind::Attestation(_subnet_id) => {} GossipKind::BeaconBlock => { for peer_id in gossipsub.mesh_peers(topic_hash) { let client = network_globals .peers .read() .peer_info(peer_id) .map(|peer_info| peer_info.client().kind.into()) .unwrap_or_else(|| "Unknown"); if let Some(v) = get_int_gauge(&BEACON_BLOCK_MESH_PEERS_PER_CLIENT, &[client]) { v.inc() }; } } GossipKind::BeaconAggregateAndProof => { for peer_id in gossipsub.mesh_peers(topic_hash) { let client = network_globals .peers .read() .peer_info(peer_id) .map(|peer_info| peer_info.client().kind.into()) .unwrap_or_else(|| "Unknown"); if let Some(v) = get_int_gauge( &BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT, &[client], ) { v.inc() }; } } GossipKind::SyncCommitteeMessage(_subnet_id) => {} _kind => {} } } } } pub fn update_sync_metrics(network_globals: &Arc>) { // reset the counts if PEERS_PER_SYNC_TYPE .as_ref() .map(|metric| metric.reset()) .is_err() { return; }; // count per sync status, the number of connected peers let mut peers_per_sync_type = FnvHashMap::default(); let mut peers_per_column_subnet = FnvHashMap::default(); for (_, info) in network_globals.peers.read().connected_peers() { let sync_type = info.sync_status().as_str(); *peers_per_sync_type.entry(sync_type).or_default() += 1; for subnet in info.custody_subnets_iter() { *peers_per_column_subnet.entry(*subnet).or_default() += 1; } } for (sync_type, peer_count) in peers_per_sync_type { set_gauge_entry(&PEERS_PER_SYNC_TYPE, &[sync_type], peer_count); } let all_column_subnets = (0..network_globals.spec.data_column_sidecar_subnet_count).map(DataColumnSubnetId::new); let custody_column_subnets = network_globals.sampling_subnets(); // Iterate all subnet values to set to zero the empty entries in peers_per_column_subnet for subnet in all_column_subnets { set_gauge_entry( &PEERS_PER_COLUMN_SUBNET, &[&format!("{subnet}")], peers_per_column_subnet.get(&subnet).copied().unwrap_or(0), ); } // Registering this metric is a duplicate for supernodes but helpful for fullnodes. This way // operators can monitor the health of only the subnets of their interest without complex // Grafana queries. for subnet in custody_column_subnets.iter() { set_gauge_entry( &PEERS_PER_CUSTODY_COLUMN_SUBNET, &[&format!("{subnet}")], peers_per_column_subnet.get(subnet).copied().unwrap_or(0), ); } }