Upgrade to tokio 0.3 (#1839)

## Description

This PR updates Lighthouse to tokio 0.3. It includes a number of dependency updates and some structural changes as to how we create and spawn tasks.

This also brings with it a number of various improvements:

- Discv5 update
- Libp2p update
- Fix for recompilation issues
- Improved UPnP port mapping handling
- Futures dependency update
- Log downgrade to traces for rejecting peers when we've reached our max



Co-authored-by: blacktemplar <blacktemplar@a1.net>

beacon_node/network/src/metrics.rs

@@ -1,5 +1,12 @@
 use beacon_chain::attestation_verification::Error as AttnError;
+use eth2_libp2p::PubsubMessage;
+use eth2_libp2p::{
+    types::GossipKind, BandwidthSinks, GossipTopic, Gossipsub, NetworkGlobals, TopicHash,
+};
+use fnv::FnvHashMap;
 pub use lighthouse_metrics::*;
+use std::{collections::HashMap, sync::Arc};
+use types::{subnet_id::subnet_id_to_string, EthSpec};
 
 lazy_static! {
 
@@ -404,6 +411,27 @@ lazy_static! {
         "gossipsub_attestation_error_beacon_chain_error",
         "Count of a specific error type (see metric name)"
     );
+
+    pub static ref INBOUND_LIBP2P_BYTES: Result<IntGauge> =
+        try_create_int_gauge("libp2p_inbound_bytes", "The inbound bandwidth over libp2p");
+
+    pub static ref OUTBOUND_LIBP2P_BYTES: Result<IntGauge> = try_create_int_gauge(
+        "libp2p_outbound_bytes",
+        "The outbound bandwidth over libp2p"
+    );
+    pub static ref TOTAL_LIBP2P_BANDWIDTH: Result<IntGauge> = try_create_int_gauge(
+        "libp2p_total_bandwidth",
+        "The total inbound/outbound bandwidth over libp2p"
+    );
+}
+
+pub fn update_bandwidth_metrics(bandwidth: Arc<BandwidthSinks>) {
+    set_gauge(&INBOUND_LIBP2P_BYTES, bandwidth.total_inbound() as i64);
+    set_gauge(&OUTBOUND_LIBP2P_BYTES, bandwidth.total_outbound() as i64);
+    set_gauge(
+        &TOTAL_LIBP2P_BANDWIDTH,
+        (bandwidth.total_inbound() + bandwidth.total_outbound()) as i64,
+    );
 }
 
 lazy_static! {
@@ -486,3 +514,359 @@ pub fn register_attestation_error(error: &AttnError) {
         AttnError::BeaconChainError(_) => inc_counter(&GOSSIP_ATTESTATION_ERROR_BEACON_CHAIN_ERROR),
     }
 }
+
+/// Inspects the `messages` that were being sent to the network and updates Prometheus metrics.
+pub fn expose_publish_metrics<T: EthSpec>(messages: &[PubsubMessage<T>]) {
+    for message in messages {
+        match message {
+            PubsubMessage::BeaconBlock(_) => inc_counter(&GOSSIP_BLOCKS_TX),
+            PubsubMessage::Attestation(subnet_id) => {
+                inc_counter_vec(
+                    &ATTESTATIONS_PUBLISHED_PER_SUBNET_PER_SLOT,
+                    &[&subnet_id.0.as_ref()],
+                );
+                inc_counter(&GOSSIP_UNAGGREGATED_ATTESTATIONS_TX)
+            }
+            PubsubMessage::AggregateAndProofAttestation(_) => {
+                inc_counter(&GOSSIP_AGGREGATED_ATTESTATIONS_TX)
+            }
+            _ => {}
+        }
+    }
+}
+
+/// Inspects a `message` received from the network and updates Prometheus metrics.
+pub fn expose_receive_metrics<T: EthSpec>(message: &PubsubMessage<T>) {
+    match message {
+        PubsubMessage::BeaconBlock(_) => inc_counter(&GOSSIP_BLOCKS_RX),
+        PubsubMessage::Attestation(_) => inc_counter(&GOSSIP_UNAGGREGATED_ATTESTATIONS_RX),
+        PubsubMessage::AggregateAndProofAttestation(_) => {
+            inc_counter(&GOSSIP_AGGREGATED_ATTESTATIONS_RX)
+        }
+        _ => {}
+    }
+}
+
+pub fn update_gossip_metrics<T: EthSpec>(
+    gossipsub: &Gossipsub,
+    network_globals: &Arc<NetworkGlobals<T>>,
+) {
+    // Clear the metrics
+    let _ = PEERS_PER_PROTOCOL.as_ref().map(|gauge| gauge.reset());
+    let _ = PEERS_PER_PROTOCOL.as_ref().map(|gauge| gauge.reset());
+    let _ = MESH_PEERS_PER_MAIN_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+
+    let _ = SCORES_BELOW_ZERO_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = SCORES_BELOW_GOSSIP_THRESHOLD_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = SCORES_BELOW_PUBLISH_THRESHOLD_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = SCORES_BELOW_GREYLIST_THRESHOLD_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = MIN_SCORES_PER_CLIENT.as_ref().map(|gauge| gauge.reset());
+    let _ = MEDIAN_SCORES_PER_CLIENT.as_ref().map(|gauge| gauge.reset());
+    let _ = MEAN_SCORES_PER_CLIENT.as_ref().map(|gauge| gauge.reset());
+    let _ = MAX_SCORES_PER_CLIENT.as_ref().map(|gauge| gauge.reset());
+
+    let _ = BEACON_BLOCK_MESH_PEERS_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+
+    // reset the mesh peers, showing all subnets
+    for subnet_id in 0..T::default_spec().attestation_subnet_count {
+        let _ = get_int_gauge(
+            &MESH_PEERS_PER_SUBNET_TOPIC,
+            &[subnet_id_to_string(subnet_id)],
+        )
+        .map(|v| v.set(0));
+
+        let _ = get_int_gauge(
+            &GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
+            &[subnet_id_to_string(subnet_id)],
+        )
+        .map(|v| v.set(0));
+
+        let _ = get_int_gauge(
+            &GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
+            &[subnet_id_to_string(subnet_id)],
+        )
+        .map(|v| v.set(0));
+    }
+
+    // Subnet topics subscribed to
+    for topic_hash in gossipsub.topics() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            if let GossipKind::Attestation(subnet_id) = topic.kind() {
+                let _ = get_int_gauge(
+                    &GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
+                    &[subnet_id_to_string(subnet_id.into())],
+                )
+                .map(|v| v.set(1));
+            }
+        }
+    }
+
+    // Peers per subscribed subnet
+    let mut peers_per_topic: HashMap<TopicHash, usize> = HashMap::new();
+    for (peer_id, topics) in gossipsub.all_peers() {
+        for topic_hash in topics {
+            *peers_per_topic.entry(topic_hash.clone()).or_default() += 1;
+
+            if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+                match topic.kind() {
+                    GossipKind::Attestation(subnet_id) => {
+                        if let Some(v) = get_int_gauge(
+                            &GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
+                            &[subnet_id_to_string(subnet_id.into())],
+                        ) {
+                            v.inc()
+                        };
+
+                        // average peer scores
+                        if let Some(score) = gossipsub.peer_score(peer_id) {
+                            if let Some(v) = get_gauge(
+                                &AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
+                                &[subnet_id_to_string(subnet_id.into())],
+                            ) {
+                                v.add(score)
+                            };
+                        }
+                    }
+                    kind => {
+                        // main topics
+                        if let Some(score) = gossipsub.peer_score(peer_id) {
+                            if let Some(v) = get_gauge(
+                                &AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC,
+                                &[kind.as_ref()],
+                            ) {
+                                v.add(score)
+                            };
+                        }
+                    }
+                }
+            }
+        }
+    }
+    // adjust to average scores by dividing by number of peers
+    for (topic_hash, peers) in peers_per_topic.iter() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::Attestation(subnet_id) => {
+                    // average peer scores
+                    if let Some(v) = get_gauge(
+                        &AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
+                        &[subnet_id_to_string(subnet_id.into())],
+                    ) {
+                        v.set(v.get() / (*peers as f64))
+                    };
+                }
+                kind => {
+                    // main topics
+                    if let Some(v) =
+                        get_gauge(&AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC, &[kind.as_ref()])
+                    {
+                        v.set(v.get() / (*peers as f64))
+                    };
+                }
+            }
+        }
+    }
+
+    // mesh peers
+    for topic_hash in gossipsub.topics() {
+        let peers = gossipsub.mesh_peers(&topic_hash).count();
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::Attestation(subnet_id) => {
+                    if let Some(v) = get_int_gauge(
+                        &MESH_PEERS_PER_SUBNET_TOPIC,
+                        &[subnet_id_to_string(subnet_id.into())],
+                    ) {
+                        v.set(peers as i64)
+                    };
+                }
+                kind => {
+                    // main topics
+                    if let Some(v) = get_int_gauge(&MESH_PEERS_PER_MAIN_TOPIC, &[kind.as_ref()]) {
+                        v.set(peers as i64)
+                    };
+                }
+            }
+        }
+    }
+
+    // protocol peers
+    let mut peers_per_protocol: HashMap<&'static str, i64> = HashMap::new();
+    for (_peer, protocol) in gossipsub.peer_protocol() {
+        *peers_per_protocol
+            .entry(protocol.as_static_ref())
+            .or_default() += 1;
+    }
+
+    for (protocol, peers) in peers_per_protocol.iter() {
+        if let Some(v) = get_int_gauge(&PEERS_PER_PROTOCOL, &[protocol]) {
+            v.set(*peers)
+        };
+    }
+
+    let mut peer_to_client = HashMap::new();
+    let mut scores_per_client: HashMap<&'static str, Vec<f64>> = HashMap::new();
+    {
+        let peers = network_globals.peers.read();
+        for (peer_id, _) in gossipsub.all_peers() {
+            let client = peers
+                .peer_info(peer_id)
+                .map(|peer_info| peer_info.client.kind.as_static_ref())
+                .unwrap_or_else(|| "Unknown");
+
+            peer_to_client.insert(peer_id, client);
+            let score = gossipsub.peer_score(peer_id).unwrap_or(0.0);
+            scores_per_client.entry(client).or_default().push(score);
+        }
+    }
+
+    // mesh peers per client
+    for topic_hash in gossipsub.topics() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::BeaconBlock => {
+                    for peer in gossipsub.mesh_peers(&topic_hash) {
+                        if let Some(client) = peer_to_client.get(peer) {
+                            if let Some(v) =
+                                get_int_gauge(&BEACON_BLOCK_MESH_PEERS_PER_CLIENT, &[client])
+                            {
+                                v.inc()
+                            };
+                        }
+                    }
+                }
+                GossipKind::BeaconAggregateAndProof => {
+                    for peer in gossipsub.mesh_peers(&topic_hash) {
+                        if let Some(client) = peer_to_client.get(peer) {
+                            if let Some(v) = get_int_gauge(
+                                &BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT,
+                                &[client],
+                            ) {
+                                v.inc()
+                            };
+                        }
+                    }
+                }
+                _ => (),
+            }
+        }
+    }
+
+    for (client, scores) in scores_per_client.into_iter() {
+        let c = &[client];
+        let len = scores.len();
+        if len > 0 {
+            let mut below0 = 0;
+            let mut below_gossip_threshold = 0;
+            let mut below_publish_threshold = 0;
+            let mut below_greylist_threshold = 0;
+            let mut min = f64::INFINITY;
+            let mut sum = 0.0;
+            let mut max = f64::NEG_INFINITY;
+
+            let count = scores.len() as f64;
+
+            for &score in &scores {
+                if score < 0.0 {
+                    below0 += 1;
+                }
+                if score < -4000.0 {
+                    //TODO not hardcode
+                    below_gossip_threshold += 1;
+                }
+                if score < -8000.0 {
+                    //TODO not hardcode
+                    below_publish_threshold += 1;
+                }
+                if score < -16000.0 {
+                    //TODO not hardcode
+                    below_greylist_threshold += 1;
+                }
+                if score < min {
+                    min = score;
+                }
+                if score > max {
+                    max = score;
+                }
+                sum += score;
+            }
+
+            let median = if len == 0 {
+                0.0
+            } else if len % 2 == 0 {
+                (scores[len / 2 - 1] + scores[len / 2]) / 2.0
+            } else {
+                scores[len / 2]
+            };
+
+            set_gauge_entry(&SCORES_BELOW_ZERO_PER_CLIENT, c, below0 as f64 / count);
+            set_gauge_entry(
+                &SCORES_BELOW_GOSSIP_THRESHOLD_PER_CLIENT,
+                c,
+                below_gossip_threshold as f64 / count,
+            );
+            set_gauge_entry(
+                &SCORES_BELOW_PUBLISH_THRESHOLD_PER_CLIENT,
+                c,
+                below_publish_threshold as f64 / count,
+            );
+            set_gauge_entry(
+                &SCORES_BELOW_GREYLIST_THRESHOLD_PER_CLIENT,
+                c,
+                below_greylist_threshold as f64 / count,
+            );
+
+            set_gauge_entry(&MIN_SCORES_PER_CLIENT, c, min);
+            set_gauge_entry(&MEDIAN_SCORES_PER_CLIENT, c, median);
+            set_gauge_entry(&MEAN_SCORES_PER_CLIENT, c, sum / count);
+            set_gauge_entry(&MAX_SCORES_PER_CLIENT, c, max);
+        }
+    }
+}
+
+pub fn update_sync_metrics<T: EthSpec>(network_globals: &Arc<NetworkGlobals<T>>) {
+    // reset the counts
+    if PEERS_PER_SYNC_TYPE
+        .as_ref()
+        .map(|metric| metric.reset())
+        .is_err()
+    {
+        return;
+    };
+
+    // count per sync status, the number of connected peers
+    let mut peers_per_sync_type = FnvHashMap::default();
+    for sync_type in network_globals
+        .peers
+        .read()
+        .connected_peers()
+        .map(|(_peer_id, info)| info.sync_status.as_str())
+    {
+        *peers_per_sync_type.entry(sync_type).or_default() += 1;
+    }
+
+    for (sync_type, peer_count) in peers_per_sync_type {
+        set_gauge_entry(&PEERS_PER_SYNC_TYPE, &[sync_type], peer_count);
+    }
+}