Upgrade to tokio 0.3 (#1839)

## Description

This PR updates Lighthouse to tokio 0.3. It includes a number of dependency updates and some structural changes as to how we create and spawn tasks.

This also brings with it a number of various improvements:

- Discv5 update
- Libp2p update
- Fix for recompilation issues
- Improved UPnP port mapping handling
- Futures dependency update
- Log downgrade to traces for rejecting peers when we've reached our max



Co-authored-by: blacktemplar <blacktemplar@a1.net>

validator_client/src/http_api/create_validator.rs

@@ -21,7 +21,7 @@ use validator_dir::Builder as ValidatorDirBuilder;
 ///
 /// If `key_derivation_path_offset` is supplied then the EIP-2334 validator index will start at
 /// this point.
-pub fn create_validators<P: AsRef<Path>, T: 'static + SlotClock, E: EthSpec>(
+pub async fn create_validators<P: AsRef<Path>, T: 'static + SlotClock, E: EthSpec>(
     mnemonic_opt: Option<Mnemonic>,
     key_derivation_path_offset: Option<u32>,
     validator_requests: &[api_types::ValidatorRequest],
@@ -129,12 +129,9 @@ pub fn create_validators<P: AsRef<Path>, T: 'static + SlotClock, E: EthSpec>(
         let voting_keystore_path = validator_dir.voting_keystore_path();
         drop(validator_dir);
 
-        tokio::runtime::Handle::current()
-            .block_on(validator_store.add_validator_keystore(
-                voting_keystore_path,
-                voting_password_string,
-                request.enable,
-            ))
+        validator_store
+            .add_validator_keystore(voting_keystore_path, voting_password_string, request.enable)
+            .await
             .map_err(|e| {
                 warp_utils::reject::custom_server_error(format!(
                     "failed to initialize validator: {:?}",

validator_client/src/http_api/mod.rs

@@ -14,7 +14,8 @@ use std::future::Future;
 use std::marker::PhantomData;
 use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
 use std::path::PathBuf;
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
+use tokio::runtime::Runtime;
 use types::{ChainSpec, EthSpec, YamlConfig};
 use validator_dir::Builder as ValidatorDirBuilder;
 use warp::{
@@ -50,6 +51,7 @@ impl From<String> for Error {
 ///
 /// The server will gracefully handle the case where any fields are `None`.
 pub struct Context<T: Clone, E: EthSpec> {
+    pub runtime: Weak<Runtime>,
     pub api_secret: ApiSecret,
     pub validator_store: Option<ValidatorStore<T, E>>,
     pub validator_dir: Option<PathBuf>,
@@ -138,6 +140,9 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
             })
         });
 
+    let inner_runtime = ctx.runtime.clone();
+    let runtime_filter = warp::any().map(move || inner_runtime.clone());
+
     let inner_validator_dir = ctx.validator_dir.clone();
     let validator_dir_filter = warp::any()
         .map(move || inner_validator_dir.clone())
@@ -258,26 +263,34 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(validator_store_filter.clone())
         .and(spec_filter.clone())
         .and(signer.clone())
+        .and(runtime_filter.clone())
         .and_then(
             |body: Vec<api_types::ValidatorRequest>,
              validator_dir: PathBuf,
              validator_store: ValidatorStore<T, E>,
              spec: Arc<ChainSpec>,
-             signer| {
+             signer,
+             runtime: Weak<Runtime>| {
                 blocking_signed_json_task(signer, move || {
-                    let (validators, mnemonic) = create_validators(
-                        None,
-                        None,
-                        &body,
-                        &validator_dir,
-                        &validator_store,
-                        &spec,
-                    )?;
-                    let response = api_types::PostValidatorsResponseData {
-                        mnemonic: mnemonic.into_phrase().into(),
-                        validators,
-                    };
-                    Ok(api_types::GenericResponse::from(response))
+                    if let Some(runtime) = runtime.upgrade() {
+                        let (validators, mnemonic) = runtime.block_on(create_validators(
+                            None,
+                            None,
+                            &body,
+                            &validator_dir,
+                            &validator_store,
+                            &spec,
+                        ))?;
+                        let response = api_types::PostValidatorsResponseData {
+                            mnemonic: mnemonic.into_phrase().into(),
+                            validators,
+                        };
+                        Ok(api_types::GenericResponse::from(response))
+                    } else {
+                        Err(warp_utils::reject::custom_server_error(
+                            "Runtime shutdown".into(),
+                        ))
+                    }
                 })
             },
         );
@@ -292,25 +305,37 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(validator_store_filter.clone())
         .and(spec_filter)
         .and(signer.clone())
+        .and(runtime_filter.clone())
         .and_then(
             |body: api_types::CreateValidatorsMnemonicRequest,
              validator_dir: PathBuf,
              validator_store: ValidatorStore<T, E>,
              spec: Arc<ChainSpec>,
-             signer| {
+             signer,
+             runtime: Weak<Runtime>| {
                 blocking_signed_json_task(signer, move || {
-                    let mnemonic = mnemonic_from_phrase(body.mnemonic.as_str()).map_err(|e| {
-                        warp_utils::reject::custom_bad_request(format!("invalid mnemonic: {:?}", e))
-                    })?;
-                    let (validators, _mnemonic) = create_validators(
-                        Some(mnemonic),
-                        Some(body.key_derivation_path_offset),
-                        &body.validators,
-                        &validator_dir,
-                        &validator_store,
-                        &spec,
-                    )?;
-                    Ok(api_types::GenericResponse::from(validators))
+                    if let Some(runtime) = runtime.upgrade() {
+                        let mnemonic =
+                            mnemonic_from_phrase(body.mnemonic.as_str()).map_err(|e| {
+                                warp_utils::reject::custom_bad_request(format!(
+                                    "invalid mnemonic: {:?}",
+                                    e
+                                ))
+                            })?;
+                        let (validators, _mnemonic) = runtime.block_on(create_validators(
+                            Some(mnemonic),
+                            Some(body.key_derivation_path_offset),
+                            &body.validators,
+                            &validator_dir,
+                            &validator_store,
+                            &spec,
+                        ))?;
+                        Ok(api_types::GenericResponse::from(validators))
+                    } else {
+                        Err(warp_utils::reject::custom_server_error(
+                            "Runtime shutdown".into(),
+                        ))
+                    }
                 })
             },
         );
@@ -324,11 +349,13 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(validator_dir_filter)
         .and(validator_store_filter.clone())
         .and(signer.clone())
+        .and(runtime_filter.clone())
         .and_then(
             |body: api_types::KeystoreValidatorsPostRequest,
              validator_dir: PathBuf,
              validator_store: ValidatorStore<T, E>,
-             signer| {
+             signer,
+             runtime: Weak<Runtime>| {
                 blocking_signed_json_task(signer, move || {
                     // Check to ensure the password is correct.
                     let keypair = body
@@ -357,18 +384,26 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                     drop(validator_dir);
                     let voting_password = body.password.clone();
 
-                    let validator_def = tokio::runtime::Handle::current()
-                        .block_on(validator_store.add_validator_keystore(
-                            voting_keystore_path,
-                            voting_password,
-                            body.enable,
-                        ))
-                        .map_err(|e| {
-                            warp_utils::reject::custom_server_error(format!(
-                                "failed to initialize validator: {:?}",
-                                e
-                            ))
-                        })?;
+                    let validator_def = {
+                        if let Some(runtime) = runtime.upgrade() {
+                            runtime
+                                .block_on(validator_store.add_validator_keystore(
+                                    voting_keystore_path,
+                                    voting_password,
+                                    body.enable,
+                                ))
+                                .map_err(|e| {
+                                    warp_utils::reject::custom_server_error(format!(
+                                        "failed to initialize validator: {:?}",
+                                        e
+                                    ))
+                                })?
+                        } else {
+                            return Err(warp_utils::reject::custom_server_error(
+                                "Runtime shutdown".into(),
+                            ));
+                        }
+                    };
 
                     Ok(api_types::GenericResponse::from(api_types::ValidatorData {
                         enabled: body.enable,
@@ -387,11 +422,13 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::body::json())
         .and(validator_store_filter)
         .and(signer)
+        .and(runtime_filter)
         .and_then(
             |validator_pubkey: PublicKey,
              body: api_types::ValidatorPatchRequest,
              validator_store: ValidatorStore<T, E>,
-             signer| {
+             signer,
+             runtime: Weak<Runtime>| {
                 blocking_signed_json_task(signer, move || {
                     let initialized_validators_rw_lock = validator_store.initialized_validators();
                     let mut initialized_validators = initialized_validators_rw_lock.write();
@@ -403,19 +440,24 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                         ))),
                         Some(enabled) if enabled == body.enabled => Ok(()),
                         Some(_) => {
-                            tokio::runtime::Handle::current()
-                                .block_on(
-                                    initialized_validators
-                                        .set_validator_status(&validator_pubkey, body.enabled),
-                                )
-                                .map_err(|e| {
-                                    warp_utils::reject::custom_server_error(format!(
-                                        "unable to set validator status: {:?}",
-                                        e
-                                    ))
-                                })?;
-
-                            Ok(())
+                            if let Some(runtime) = runtime.upgrade() {
+                                runtime
+                                    .block_on(
+                                        initialized_validators
+                                            .set_validator_status(&validator_pubkey, body.enabled),
+                                    )
+                                    .map_err(|e| {
+                                        warp_utils::reject::custom_server_error(format!(
+                                            "unable to set validator status: {:?}",
+                                            e
+                                        ))
+                                    })?;
+                                Ok(())
+                            } else {
+                                Err(warp_utils::reject::custom_server_error(
+                                    "Runtime shutdown".into(),
+                                ))
+                            }
                         }
                     }
                 })
@@ -471,8 +513,8 @@ pub async fn blocking_signed_json_task<S, F, T>(
 ) -> Result<impl warp::Reply, warp::Rejection>
 where
     S: Fn(&[u8]) -> String,
-    F: Fn() -> Result<T, warp::Rejection>,
-    T: Serialize,
+    F: Fn() -> Result<T, warp::Rejection> + Send + 'static,
+    T: Serialize + Send + 'static,
 {
     warp_utils::task::blocking_task(func)
         .await

validator_client/src/http_api/tests.rs

@@ -23,7 +23,9 @@ use std::marker::PhantomData;
 use std::net::Ipv4Addr;
 use std::sync::Arc;
 use tempfile::{tempdir, TempDir};
+use tokio::runtime::Runtime;
 use tokio::sync::oneshot;
+use tokio_compat_02::FutureExt;
 
 const PASSWORD_BYTES: &[u8] = &[42, 50, 37];
 
@@ -37,8 +39,18 @@ struct ApiTester {
     _validator_dir: TempDir,
 }
 
+// Builds a runtime to be used in the testing configuration.
+fn build_runtime() -> Arc<Runtime> {
+    Arc::new(
+        tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .expect("Should be able to build a testing runtime"),
+    )
+}
+
 impl ApiTester {
-    pub async fn new() -> Self {
+    pub async fn new(runtime: std::sync::Weak<Runtime>) -> Self {
         let log = null_logger().unwrap();
 
         let validator_dir = tempdir().unwrap();
@@ -80,6 +92,7 @@ impl ApiTester {
         let initialized_validators = validator_store.initialized_validators();
 
         let context: Arc<Context<TestingSlotClock, E>> = Arc::new(Context {
+            runtime,
             api_secret,
             validator_dir: Some(validator_dir.path().into()),
             validator_store: Some(validator_store),
@@ -420,110 +433,145 @@ struct KeystoreValidatorScenario {
     correct_password: bool,
 }
 
-#[tokio::test(core_threads = 2)]
-async fn invalid_pubkey() {
-    ApiTester::new()
-        .await
-        .invalidate_api_token()
-        .test_get_lighthouse_version_invalid()
-        .await;
+#[test]
+fn invalid_pubkey() {
+    let runtime = build_runtime();
+    let weak_runtime = Arc::downgrade(&runtime);
+    runtime.block_on(
+        async {
+            ApiTester::new(weak_runtime)
+                .await
+                .invalidate_api_token()
+                .test_get_lighthouse_version_invalid()
+                .await;
+        }
+        .compat(),
+    );
 }
 
-#[tokio::test(core_threads = 2)]
-async fn simple_getters() {
-    ApiTester::new()
-        .await
-        .test_get_lighthouse_version()
-        .await
-        .test_get_lighthouse_health()
-        .await
-        .test_get_lighthouse_spec()
-        .await;
+#[test]
+fn simple_getters() {
+    let runtime = build_runtime();
+    let weak_runtime = Arc::downgrade(&runtime);
+    runtime.block_on(
+        async {
+            ApiTester::new(weak_runtime)
+                .await
+                .test_get_lighthouse_version()
+                .await
+                .test_get_lighthouse_health()
+                .await
+                .test_get_lighthouse_spec()
+                .await;
+        }
+        .compat(),
+    );
 }
 
-#[tokio::test(core_threads = 2)]
-async fn hd_validator_creation() {
-    ApiTester::new()
-        .await
-        .assert_enabled_validators_count(0)
-        .assert_validators_count(0)
-        .create_hd_validators(HdValidatorScenario {
-            count: 2,
-            specify_mnemonic: true,
-            key_derivation_path_offset: 0,
-            disabled: vec![],
-        })
-        .await
-        .assert_enabled_validators_count(2)
-        .assert_validators_count(2)
-        .create_hd_validators(HdValidatorScenario {
-            count: 1,
-            specify_mnemonic: false,
-            key_derivation_path_offset: 0,
-            disabled: vec![0],
-        })
-        .await
-        .assert_enabled_validators_count(2)
-        .assert_validators_count(3)
-        .create_hd_validators(HdValidatorScenario {
-            count: 0,
-            specify_mnemonic: true,
-            key_derivation_path_offset: 4,
-            disabled: vec![],
-        })
-        .await
-        .assert_enabled_validators_count(2)
-        .assert_validators_count(3);
+#[test]
+fn hd_validator_creation() {
+    let runtime = build_runtime();
+    let weak_runtime = Arc::downgrade(&runtime);
+    runtime.block_on(
+        async {
+            ApiTester::new(weak_runtime)
+                .await
+                .assert_enabled_validators_count(0)
+                .assert_validators_count(0)
+                .create_hd_validators(HdValidatorScenario {
+                    count: 2,
+                    specify_mnemonic: true,
+                    key_derivation_path_offset: 0,
+                    disabled: vec![],
+                })
+                .await
+                .assert_enabled_validators_count(2)
+                .assert_validators_count(2)
+                .create_hd_validators(HdValidatorScenario {
+                    count: 1,
+                    specify_mnemonic: false,
+                    key_derivation_path_offset: 0,
+                    disabled: vec![0],
+                })
+                .await
+                .assert_enabled_validators_count(2)
+                .assert_validators_count(3)
+                .create_hd_validators(HdValidatorScenario {
+                    count: 0,
+                    specify_mnemonic: true,
+                    key_derivation_path_offset: 4,
+                    disabled: vec![],
+                })
+                .await
+                .assert_enabled_validators_count(2)
+                .assert_validators_count(3);
+        }
+        .compat(),
+    );
 }
 
-#[tokio::test(core_threads = 2)]
-async fn validator_enabling() {
-    ApiTester::new()
-        .await
-        .create_hd_validators(HdValidatorScenario {
-            count: 2,
-            specify_mnemonic: false,
-            key_derivation_path_offset: 0,
-            disabled: vec![],
-        })
-        .await
-        .assert_enabled_validators_count(2)
-        .assert_validators_count(2)
-        .set_validator_enabled(0, false)
-        .await
-        .assert_enabled_validators_count(1)
-        .assert_validators_count(2)
-        .set_validator_enabled(0, true)
-        .await
-        .assert_enabled_validators_count(2)
-        .assert_validators_count(2);
+#[test]
+fn validator_enabling() {
+    let runtime = build_runtime();
+    let weak_runtime = Arc::downgrade(&runtime);
+    runtime.block_on(
+        async {
+            ApiTester::new(weak_runtime)
+                .await
+                .create_hd_validators(HdValidatorScenario {
+                    count: 2,
+                    specify_mnemonic: false,
+                    key_derivation_path_offset: 0,
+                    disabled: vec![],
+                })
+                .await
+                .assert_enabled_validators_count(2)
+                .assert_validators_count(2)
+                .set_validator_enabled(0, false)
+                .await
+                .assert_enabled_validators_count(1)
+                .assert_validators_count(2)
+                .set_validator_enabled(0, true)
+                .await
+                .assert_enabled_validators_count(2)
+                .assert_validators_count(2);
+        }
+        .compat(),
+    );
 }
 
-#[tokio::test(core_threads = 2)]
-async fn keystore_validator_creation() {
-    ApiTester::new()
-        .await
-        .assert_enabled_validators_count(0)
-        .assert_validators_count(0)
-        .create_keystore_validators(KeystoreValidatorScenario {
-            correct_password: true,
-            enabled: true,
-        })
-        .await
-        .assert_enabled_validators_count(1)
-        .assert_validators_count(1)
-        .create_keystore_validators(KeystoreValidatorScenario {
-            correct_password: false,
-            enabled: true,
-        })
-        .await
-        .assert_enabled_validators_count(1)
-        .assert_validators_count(1)
-        .create_keystore_validators(KeystoreValidatorScenario {
-            correct_password: true,
-            enabled: false,
-        })
-        .await
-        .assert_enabled_validators_count(1)
-        .assert_validators_count(2);
+#[test]
+fn keystore_validator_creation() {
+    let runtime = build_runtime();
+    let weak_runtime = Arc::downgrade(&runtime);
+    runtime.block_on(
+        async {
+            ApiTester::new(weak_runtime)
+                .await
+                .assert_enabled_validators_count(0)
+                .assert_validators_count(0)
+                .create_keystore_validators(KeystoreValidatorScenario {
+                    correct_password: true,
+                    enabled: true,
+                })
+                .await
+                .assert_enabled_validators_count(1)
+                .assert_validators_count(1)
+                .create_keystore_validators(KeystoreValidatorScenario {
+                    correct_password: false,
+                    enabled: true,
+                })
+                .await
+                .assert_enabled_validators_count(1)
+                .assert_validators_count(1)
+                .create_keystore_validators(KeystoreValidatorScenario {
+                    correct_password: true,
+                    enabled: false,
+                })
+                .await
+                .assert_enabled_validators_count(1)
+                .assert_validators_count(2);
+        }
+        .compat(),
+    );
 }