Improve tokio task execution (#1181)

* Add logging on shutdown

* Replace tokio::spawn with handle.spawn

* Upgrade tokio

* Add a task executor

* Beacon chain tasks use task executor

* Validator client tasks use task executor

* Rename runtime_handle to executor

* Add duration histograms; minor fixes

* Cleanup

* Fix logs

* Fix tests

* Remove random file

* Get enr dependency instead of libp2p

* Address some review comments

* Libp2p takes a TaskExecutor

* Ugly fix libp2p tests

* Move TaskExecutor to own file

* Upgrade Dockerfile rust version

* Minor fixes

* Revert "Ugly fix libp2p tests"

This reverts commit 58d4bb690f.

* Pretty fix libp2p tests

* Add spawn_without_exit; change Counter to Gauge

* Tidy

* Move log from RuntimeContext to TaskExecutor

* Fix errors

* Replace histogram with int_gauge for async tasks

* Fix todo

* Fix memory leak in test by exiting all spawned tasks at the end

beacon_node/eth2-libp2p/Cargo.toml

@@ -35,6 +35,7 @@ tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
 # Patched for quick updates
 discv5 = { git = "https://github.com/sigp/discv5", rev = "7b3bd40591b62b8c002ffdb85de008aa9f82e2e5" }
 tiny-keccak = "2.0.2"
+environment = { path = "../../lighthouse/environment" }
 libp2p-tcp = { version = "0.19.1", default-features = false, features = ["tokio"] }
 
 [dependencies.libp2p]
@@ -49,3 +50,4 @@ slog-stdlog = "4.0.0"
 slog-term = "2.5.0"
 slog-async = "2.5.0"
 tempdir = "0.3.7"
+exit-future = "0.2.0"

beacon_node/eth2-libp2p/src/service.rs

@@ -84,6 +84,7 @@ pub struct Service<TSpec: EthSpec> {
 
 impl<TSpec: EthSpec> Service<TSpec> {
     pub fn new(
+        executor: environment::TaskExecutor,
         config: &NetworkConfig,
         enr_fork_id: EnrForkId,
         log: &slog::Logger,
@@ -122,15 +123,15 @@ impl<TSpec: EthSpec> Service<TSpec> {
             let behaviour = Behaviour::new(&local_keypair, config, network_globals.clone(), &log)?;
 
             // use the executor for libp2p
-            struct Executor(tokio::runtime::Handle);
+            struct Executor(environment::TaskExecutor);
             impl libp2p::core::Executor for Executor {
                 fn exec(&self, f: Pin<Box<dyn Future<Output = ()> + Send>>) {
-                    self.0.spawn(f);
+                    self.0.spawn(f, "libp2p");
                 }
             }
             SwarmBuilder::new(transport, behaviour, local_peer_id.clone())
                 .peer_connection_limit(MAX_CONNECTIONS_PER_PEER)
-                .executor(Box::new(Executor(tokio::runtime::Handle::current())))
+                .executor(Box::new(Executor(executor)))
                 .build()
         };
 

beacon_node/eth2-libp2p/tests/common/mod.rs

@@ -12,6 +12,21 @@ use types::{EnrForkId, MinimalEthSpec};
 type E = MinimalEthSpec;
 use tempdir::TempDir;
 
+pub struct Libp2pInstance(LibP2PService<E>, exit_future::Signal);
+
+impl std::ops::Deref for Libp2pInstance {
+    type Target = LibP2PService<E>;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl std::ops::DerefMut for Libp2pInstance {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
 pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger {
     let decorator = slog_term::TermDecorator::new().build();
     let drain = slog_term::FullFormat::new(decorator).build().fuse();
@@ -82,13 +97,20 @@ pub fn build_libp2p_instance(
     boot_nodes: Vec<Enr>,
     secret_key: Option<String>,
     log: slog::Logger,
-) -> LibP2PService<E> {
+) -> Libp2pInstance {
     let port = unused_port("tcp").unwrap();
     let config = build_config(port, boot_nodes, secret_key);
     // launch libp2p service
-    LibP2PService::new(&config, EnrForkId::default(), &log)
-        .expect("should build libp2p instance")
-        .1
+
+    let (signal, exit) = exit_future::signal();
+    let executor =
+        environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone());
+    Libp2pInstance(
+        LibP2PService::new(executor, &config, EnrForkId::default(), &log)
+            .expect("should build libp2p instance")
+            .1,
+        signal,
+    )
 }
 
 #[allow(dead_code)]
@@ -99,8 +121,8 @@ pub fn get_enr(node: &LibP2PService<E>) -> Enr {
 
 // Returns `n` libp2p peers in fully connected topology.
 #[allow(dead_code)]
-pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<LibP2PService<E>> {
-    let mut nodes: Vec<LibP2PService<E>> = (0..n)
+pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+    let mut nodes: Vec<_> = (0..n)
         .map(|_| build_libp2p_instance(vec![], None, log.clone()))
         .collect();
     let multiaddrs: Vec<Multiaddr> = nodes
@@ -124,7 +146,7 @@ pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<LibP2PService<E>> {
 // Constructs a pair of nodes with separate loggers. The sender dials the receiver.
 // This returns a (sender, receiver) pair.
 #[allow(dead_code)]
-pub async fn build_node_pair(log: &slog::Logger) -> (LibP2PService<E>, LibP2PService<E>) {
+pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInstance) {
     let sender_log = log.new(o!("who" => "sender"));
     let receiver_log = log.new(o!("who" => "receiver"));
 
@@ -168,8 +190,8 @@ pub async fn build_node_pair(log: &slog::Logger) -> (LibP2PService<E>, LibP2PSer
 
 // Returns `n` peers in a linear topology
 #[allow(dead_code)]
-pub fn build_linear(log: slog::Logger, n: usize) -> Vec<LibP2PService<E>> {
-    let mut nodes: Vec<LibP2PService<E>> = (0..n)
+pub fn build_linear(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+    let mut nodes: Vec<_> = (0..n)
         .map(|_| build_libp2p_instance(vec![], None, log.clone()))
         .collect();
     let multiaddrs: Vec<Multiaddr> = nodes

beacon_node/eth2-libp2p/tests/noise.rs

@@ -136,7 +136,10 @@ async fn test_secio_noise_fallback() {
 
     let port = common::unused_port("tcp").unwrap();
     let noisy_config = common::build_config(port, vec![], None);
-    let mut noisy_node = Service::new(&noisy_config, EnrForkId::default(), &log)
+    let (_signal, exit) = exit_future::signal();
+    let executor =
+        environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone());
+    let mut noisy_node = Service::new(executor, &noisy_config, EnrForkId::default(), &log)
         .expect("should build a libp2p instance")
         .1;