More metrics + RPC tweaks (#2041)

## Issue Addressed

NA

## Proposed Changes
This was mostly done to find the reason why LH was dropping peers from Nimbus. It proved to be useful so I think it's worth it. But there is also some functional stuff here
- Add metrics for rpc errors per client, error type and direction
- Add metrics for downscoring events per source type, client and penalty type
- Add metrics for gossip validation results per client for non-accepted messages
- Make the RPC handler return errors and requests/responses in the order we see them
- Allow a small burst for the Ping rate limit, from 1 every 5 seconds to 2 every 10 seconds
- Send rate limiting errors with a particular code and use that same code to identify them. I picked something different to 128 since that is most likely what other clients are using for their own errors
- Remove some unused code in the `PeerAction` and the rpc handler
- Remove the unused variant `RateLimited`. tTis was never produced directly, since the only way to get the request's protocol is via de handler. The handler upon receiving from LH a response with an error (rate limited in this case) emits this event with the missing info (It was always like this, just pointing out that we do downscore rate limiting errors regardless of the change)

Metrics for Nimbus looked like this:
Downscoring events: `increase(libp2p_peer_actions_per_client{client="Nimbus"}[5m])`
![image](https://user-images.githubusercontent.com/26765164/101210880-862bf280-3676-11eb-94c0-399f0bf5aa2e.png)

RPC Errors: `increase(libp2p_rpc_errors_per_client{client="Nimbus"}[5m])`
![image](https://user-images.githubusercontent.com/26765164/101210997-ba071800-3676-11eb-847a-f32405ede002.png)

Unaccepted gossip message: `increase(gossipsub_unaccepted_messages_per_client{client="Nimbus"}[5m])`
![image](https://user-images.githubusercontent.com/26765164/101211124-f470b500-3676-11eb-9459-132ecff058ec.png)

beacon_node/eth2_libp2p/src/peer_manager/mod.rs

@@ -33,7 +33,7 @@ pub(crate) mod score;
 
 pub use peer_info::{ConnectionDirection, PeerConnectionStatus, PeerConnectionStatus::*, PeerInfo};
 pub use peer_sync_status::{PeerSyncStatus, SyncInfo};
-use score::{PeerAction, ScoreState};
+use score::{PeerAction, ReportSource, ScoreState};
 use std::cmp::Ordering;
 use std::collections::HashMap;
 
@@ -145,7 +145,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// All instant disconnections are fatal and we ban the associated peer.
     ///
     /// This will send a goodbye and disconnect the peer if it is connected or dialing.
-    pub fn goodbye_peer(&mut self, peer_id: &PeerId, reason: GoodbyeReason) {
+    pub fn goodbye_peer(&mut self, peer_id: &PeerId, reason: GoodbyeReason, source: ReportSource) {
         // get the peer info
         if let Some(info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
             debug!(self.log, "Sending goodbye to peer"; "peer_id" => %peer_id, "reason" => %reason, "score" => %info.score());
@@ -155,6 +155,14 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
 
             // Goodbye's are fatal
             info.apply_peer_action_to_score(PeerAction::Fatal);
+            metrics::inc_counter_vec(
+                &metrics::PEER_ACTION_EVENTS_PER_CLIENT,
+                &[
+                    info.client.kind.as_static_ref(),
+                    PeerAction::Fatal.as_static_str(),
+                    source.into(),
+                ],
+            );
         }
 
         // Update the peerdb and peer state accordingly
@@ -173,7 +181,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// Reports a peer for some action.
     ///
     /// If the peer doesn't exist, log a warning and insert defaults.
-    pub fn report_peer(&mut self, peer_id: &PeerId, action: PeerAction) {
+    pub fn report_peer(&mut self, peer_id: &PeerId, action: PeerAction, source: ReportSource) {
         // Helper function to avoid any potential deadlocks.
         let mut to_ban_peers = Vec::with_capacity(1);
         let mut to_unban_peers = Vec::with_capacity(1);
@@ -183,6 +191,15 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             if let Some(info) = peer_db.peer_info_mut(peer_id) {
                 let previous_state = info.score_state();
                 info.apply_peer_action_to_score(action);
+                metrics::inc_counter_vec(
+                    &metrics::PEER_ACTION_EVENTS_PER_CLIENT,
+                    &[
+                        info.client.kind.as_static_ref(),
+                        action.as_static_str(),
+                        source.into(),
+                    ],
+                );
+
                 Self::handle_score_transitions(
                     previous_state,
                     peer_id,
@@ -352,7 +369,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         }
     }
 
-    /// An error has occured in the RPC.
+    /// An error has occurred in the RPC.
     ///
     /// This adjusts a peer's score based on the error.
     pub fn handle_rpc_error(
@@ -366,6 +383,14 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         let score = self.network_globals.peers.read().score(peer_id);
         debug!(self.log, "RPC Error"; "protocol" => %protocol, "err" => %err, "client" => %client,
             "peer_id" => %peer_id, "score" => %score, "direction" => ?direction);
+        metrics::inc_counter_vec(
+            &metrics::TOTAL_RPC_ERRORS_PER_CLIENT,
+            &[
+                client.kind.as_static_ref(),
+                err.as_static_str(),
+                direction.as_static_str(),
+            ],
+        );
 
         // Map this error to a `PeerAction` (if any)
         let peer_action = match err {
@@ -389,7 +414,14 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                 RPCResponseErrorCode::Unknown => PeerAction::HighToleranceError,
                 RPCResponseErrorCode::ServerError => PeerAction::MidToleranceError,
                 RPCResponseErrorCode::InvalidRequest => PeerAction::LowToleranceError,
-                RPCResponseErrorCode::RateLimited => PeerAction::LowToleranceError,
+                RPCResponseErrorCode::RateLimited => match protocol {
+                    Protocol::Ping => PeerAction::MidToleranceError,
+                    Protocol::BlocksByRange => PeerAction::MidToleranceError,
+                    Protocol::BlocksByRoot => PeerAction::MidToleranceError,
+                    Protocol::Goodbye => PeerAction::LowToleranceError,
+                    Protocol::MetaData => PeerAction::LowToleranceError,
+                    Protocol::Status => PeerAction::LowToleranceError,
+                },
             },
             RPCError::SSZDecodeError(_) => PeerAction::Fatal,
             RPCError::UnsupportedProtocol => {
@@ -422,17 +454,9 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                 },
             },
             RPCError::NegotiationTimeout => PeerAction::HighToleranceError,
-            RPCError::RateLimited => match protocol {
-                Protocol::Ping => PeerAction::MidToleranceError,
-                Protocol::BlocksByRange => PeerAction::HighToleranceError,
-                Protocol::BlocksByRoot => PeerAction::HighToleranceError,
-                Protocol::Goodbye => PeerAction::LowToleranceError,
-                Protocol::MetaData => PeerAction::LowToleranceError,
-                Protocol::Status => PeerAction::LowToleranceError,
-            },
         };
 
-        self.report_peer(peer_id, peer_action);
+        self.report_peer(peer_id, peer_action, ReportSource::RPC);
     }
 
     /// A ping request has been received.