Global Network Behaviour Refactor (#2442)

* Network upgrades (#2345)

* Discovery patch (#2382)

* Upgrade libp2p and unstable gossip

* Network protocol upgrades

* Correct dependencies, reduce incoming bucket limit

* Clean up dirty DHT entries before repopulating

* Update cargo lock

* Update lockfile

* Update ENR dep

* Update deps to specific versions

* Update test dependencies

* Update docker rust, and remote signer tests

* More remote signer test fixes

* Temp commit

* Update discovery

* Remove cached enrs after dialing

* Increase the session capacity, for improved efficiency

* Bleeding edge discovery (#2435)

* Update discovery banning logic and tokio

* Update to latest discovery

* Shift to latest discovery

* Fmt

* Initial re-factor of the behaviour

* More progress

* Missed changes

* First draft

* Discovery as a behaviour

* Adding back event waker (not convinced its neccessary, but have made this many changes already)

* Corrections

* Speed up discovery

* Remove double log

* Fmt

* After disconnect inform swarm about ban

* More fmt

* Appease clippy

* Improve ban handling

* Update tests

* Update cargo.lock

* Correct tests

* Downgrade log

beacon_node/eth2_libp2p/src/rpc/handler.rs

@@ -1,8 +1,10 @@
 #![allow(clippy::type_complexity)]
 #![allow(clippy::cognitive_complexity)]
 
-use super::methods::{RPCCodedResponse, RPCResponseErrorCode, RequestId, ResponseTermination};
-use super::protocol::{Protocol, RPCError, RPCProtocol};
+use super::methods::{
+    GoodbyeReason, RPCCodedResponse, RPCResponseErrorCode, RequestId, ResponseTermination,
+};
+use super::protocol::{InboundRequest, Protocol, RPCError, RPCProtocol};
 use super::{RPCReceived, RPCSend};
 use crate::rpc::outbound::{OutboundFramed, OutboundRequest};
 use crate::rpc::protocol::InboundFramed;
@@ -221,13 +223,14 @@ where
         }
     }
 
-    /// Initiates the handler's shutdown process, sending an optional last message to the peer.
-    pub fn shutdown(&mut self, final_msg: Option<(RequestId, OutboundRequest<TSpec>)>) {
+    /// Initiates the handler's shutdown process, sending an optional Goodbye message to the
+    /// peer.
+    fn shutdown(&mut self, goodbye_reason: Option<GoodbyeReason>) {
         if matches!(self.state, HandlerState::Active) {
             if !self.dial_queue.is_empty() {
                 debug!(self.log, "Starting handler shutdown"; "unsent_queued_requests" => self.dial_queue.len());
             }
-            // we now drive to completion communications already dialed/established
+            // We now drive to completion communications already dialed/established
             while let Some((id, req)) = self.dial_queue.pop() {
                 self.events_out.push(Err(HandlerErr::Outbound {
                     error: RPCError::HandlerRejected,
@@ -236,9 +239,10 @@ where
                 }));
             }
 
-            // Queue our final message, if any
-            if let Some((id, req)) = final_msg {
-                self.dial_queue.push((id, req));
+            // Queue our goodbye message.
+            if let Some(reason) = goodbye_reason {
+                self.dial_queue
+                    .push((RequestId::Router, OutboundRequest::Goodbye(reason)));
             }
 
             self.state = HandlerState::ShuttingDown(Box::new(sleep_until(
@@ -345,6 +349,11 @@ where
             );
         }
 
+        // If we received a goodbye, shutdown the connection.
+        if let InboundRequest::Goodbye(_) = req {
+            self.shutdown(None);
+        }
+
         self.events_out.push(Ok(RPCReceived::Request(
             self.current_inbound_substream_id,
             req,
@@ -412,6 +421,7 @@ where
         match rpc_event {
             RPCSend::Request(id, req) => self.send_request(id, req),
             RPCSend::Response(inbound_id, response) => self.send_response(inbound_id, response),
+            RPCSend::Shutdown(reason) => self.shutdown(Some(reason)),
         }
     }
 
@@ -512,6 +522,9 @@ where
             if delay.is_elapsed() {
                 self.state = HandlerState::Deactivated;
                 debug!(self.log, "Handler deactivated");
+                return Poll::Ready(ProtocolsHandlerEvent::Close(RPCError::InternalError(
+                    "Shutdown timeout",
+                )));
             }
         }
 
@@ -864,6 +877,19 @@ where
                 protocol: SubstreamProtocol::new(req.clone(), ()).map_info(|()| (id, req)),
             });
         }
+
+        // Check if we have completed sending a goodbye, disconnect.
+        if let HandlerState::ShuttingDown(_) = self.state {
+            if self.dial_queue.is_empty()
+                && self.outbound_substreams.is_empty()
+                && self.inbound_substreams.is_empty()
+                && self.events_out.is_empty()
+                && self.dial_negotiated == 0
+            {
+                return Poll::Ready(ProtocolsHandlerEvent::Close(RPCError::Disconnected));
+            }
+        }
+
         Poll::Pending
     }
 }

beacon_node/eth2_libp2p/src/rpc/methods.rs

@@ -149,9 +149,9 @@ impl From<u64> for GoodbyeReason {
     }
 }
 
-impl Into<u64> for GoodbyeReason {
-    fn into(self) -> u64 {
-        self as u64
+impl From<GoodbyeReason> for u64 {
+    fn from(reason: GoodbyeReason) -> u64 {
+        reason as u64
     }
 }
 

beacon_node/eth2_libp2p/src/rpc/mod.rs

@@ -52,6 +52,8 @@ pub enum RPCSend<TSpec: EthSpec> {
     /// peer. The second parameter is a single chunk of a response. These go over *inbound*
     /// connections.
     Response(SubstreamId, RPCCodedResponse<TSpec>),
+    /// Lighthouse has requested to terminate the connection with a goodbye message.
+    Shutdown(GoodbyeReason),
 }
 
 /// RPC events received from outside Lighthouse.
@@ -77,6 +79,7 @@ impl<T: EthSpec> std::fmt::Display for RPCSend<T> {
         match self {
             RPCSend::Request(id, req) => write!(f, "RPC Request(id: {:?}, {})", id, req),
             RPCSend::Response(id, res) => write!(f, "RPC Response(id: {:?}, {})", id, res),
+            RPCSend::Shutdown(reason) => write!(f, "Sending Goodbye: {}", reason),
         }
     }
 }
@@ -115,11 +118,7 @@ impl<TSpec: EthSpec> RPC<TSpec> {
                 methods::MAX_REQUEST_BLOCKS,
                 Duration::from_secs(10),
             )
-            .n_every(
-                Protocol::BlocksByRoot,
-                methods::MAX_REQUEST_BLOCKS,
-                Duration::from_secs(10),
-            )
+            .n_every(Protocol::BlocksByRoot, 128, Duration::from_secs(10))
             .build()
             .expect("Configuration parameters are valid");
         RPC {
@@ -160,6 +159,16 @@ impl<TSpec: EthSpec> RPC<TSpec> {
             event: RPCSend::Request(request_id, event),
         });
     }
+
+    /// Lighthouse wishes to disconnect from this peer by sending a Goodbye message. This
+    /// gracefully terminates the RPC behaviour with a goodbye message.
+    pub fn shutdown(&mut self, peer_id: PeerId, reason: GoodbyeReason) {
+        self.events.push(NetworkBehaviourAction::NotifyHandler {
+            peer_id,
+            handler: NotifyHandler::Any,
+            event: RPCSend::Shutdown(reason),
+        });
+    }
 }
 
 impl<TSpec> NetworkBehaviour for RPC<TSpec>

beacon_node/eth2_libp2p/src/rpc/protocol.rs

@@ -452,6 +452,8 @@ pub enum RPCError {
     NegotiationTimeout,
     /// Handler rejected this request.
     HandlerRejected,
+    /// We have intentionally disconnected.
+    Disconnected,
 }
 
 impl From<ssz::DecodeError> for RPCError {
@@ -490,6 +492,7 @@ impl std::fmt::Display for RPCError {
             RPCError::InternalError(ref err) => write!(f, "Internal error: {}", err),
             RPCError::NegotiationTimeout => write!(f, "Negotiation timeout"),
             RPCError::HandlerRejected => write!(f, "Handler rejected the request"),
+            RPCError::Disconnected => write!(f, "Gracefully Disconnected"),
         }
     }
 }
@@ -508,6 +511,7 @@ impl std::error::Error for RPCError {
             RPCError::ErrorResponse(_, _) => None,
             RPCError::NegotiationTimeout => None,
             RPCError::HandlerRejected => None,
+            RPCError::Disconnected => None,
         }
     }
 }