gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-29 20:27:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve tokio task execution (#1181)

Browse Source 
* Add logging on shutdown

* Replace tokio::spawn with handle.spawn

* Upgrade tokio

* Add a task executor

* Beacon chain tasks use task executor

* Validator client tasks use task executor

* Rename runtime_handle to executor

* Add duration histograms; minor fixes

* Cleanup

* Fix logs

* Fix tests

* Remove random file

* Get enr dependency instead of libp2p

* Address some review comments

* Libp2p takes a TaskExecutor

* Ugly fix libp2p tests

* Move TaskExecutor to own file

* Upgrade Dockerfile rust version

* Minor fixes

* Revert "Ugly fix libp2p tests"

This reverts commit 58d4bb690f.

* Pretty fix libp2p tests

* Add spawn_without_exit; change Counter to Gauge

* Tidy

* Move log from RuntimeContext to TaskExecutor

* Fix errors

* Replace histogram with int_gauge for async tasks

* Fix todo

* Fix memory leak in test by exiting all spawned tasks at the end
This commit is contained in:
Pawan Dhananjay
2020-06-04 17:18:05 +05:30
committed by GitHub
parent ce10db15da
commit 042e80570c
53 changed files with 541 additions and 361 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
beacon_node/Cargo.toml
View File

@@ -27,7 +27,7 @@ slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_tr
slog-term = "2.5.0"
slog-async = "2.5.0"
ctrlc = { version = "3.1.4", features = ["termination"] }
tokio = {version = "0.2.20", features = ["time"] }
tokio = { version = "0.2.21", features = ["time"] }
exit-future = "0.2.0"
env_logger = "0.7.1"
dirs = "2.0.2"

5
beacon_node/beacon_chain/Cargo.toml
View File

@@ -33,7 +33,7 @@ eth2_ssz_derive = "0.1.0"
state_processing = { path = "../../consensus/state_processing" }
tree_hash = "0.1.0"
types = { path = "../../consensus/types" }
tokio = "0.2.20"
tokio = "0.2.21"
eth1 = { path = "../eth1" }
websocket_server = { path = "../websocket_server" }
futures = "0.3.5"
@@ -46,7 +46,8 @@ tempfile = "3.1.0"
bitvec = "0.17.4"
bls = { path = "../../crypto/bls" }
safe_arith = { path = "../../consensus/safe_arith" }
environment = { path = "../../lighthouse/environment" }
[dev-dependencies]
lazy_static = "1.4.0"
environment = { path = "../../lighthouse/environment" }

7
beacon_node/beacon_chain/src/eth1_chain.rs
View File

@@ -1,4 +1,5 @@
use crate::metrics;
use environment::TaskExecutor;
use eth1::{Config as Eth1Config, Eth1Block, Service as HttpService};
use eth2_hashing::hash;
use slog::{debug, error, trace, Logger};
@@ -285,10 +286,8 @@ impl<T: EthSpec, S: Store<T>> CachingEth1Backend<T, S> {
}
/// Starts the routine which connects to the external eth1 node and updates the caches.
pub fn start(&self, exit: tokio::sync::oneshot::Receiver<()>) {
// don't need to spawn as a task is being spawned in auto_update
// TODO: check if this is correct
HttpService::auto_update(self.core.clone(), exit);
pub fn start(&self, handle: TaskExecutor) {
HttpService::auto_update(self.core.clone(), handle);
}
/// Instantiates `self` from an existing service.

2
beacon_node/client/Cargo.toml
View File

@@ -28,7 +28,7 @@ error-chain = "0.12.2"
serde_yaml = "0.8.11"
slog = { version = "2.5.2", features = ["max_level_trace"] }
slog-async = "2.5.0"
tokio = "0.2.20"
tokio = "0.2.21"
dirs = "2.0.2"
futures = "0.3.5"
reqwest = "0.10.4"

119
beacon_node/client/src/builder.rs
View File

@@ -21,6 +21,7 @@ use std::net::SocketAddr;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use timer::spawn_timer;
use tokio::sync::mpsc::UnboundedSender;
use types::{test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec};
use websocket_server::{Config as WebSocketConfig, WebSocketSender};
@@ -50,7 +51,6 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
beacon_chain_builder: Option<BeaconChainBuilder<T>>,
beacon_chain: Option<Arc<BeaconChain<T>>>,
eth1_service: Option<Eth1Service>,
exit_channels: Vec<tokio::sync::oneshot::Sender<()>>,
event_handler: Option<T::EventHandler>,
network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
network_send: Option<UnboundedSender<NetworkMessage<T::EthSpec>>>,
@@ -84,7 +84,6 @@ where
beacon_chain_builder: None,
beacon_chain: None,
eth1_service: None,
exit_channels: vec![],
event_handler: None,
network_globals: None,
network_send: None,
@@ -132,7 +131,7 @@ where
.ok_or_else(|| "beacon_chain_start_method requires a chain spec".to_string())?;
let builder = BeaconChainBuilder::new(eth_spec_instance)
.logger(context.log.clone())
.logger(context.log().clone())
.store(store)
.store_migrator(store_migrator)
.data_dir(data_dir)
@@ -150,7 +149,7 @@ where
// Alternatively, if there's a beacon chain in the database then always resume
// using it.
let client_genesis = if client_genesis == ClientGenesis::FromStore && !chain_exists {
info!(context.log, "Defaulting to deposit contract genesis");
info!(context.log(), "Defaulting to deposit contract genesis");
ClientGenesis::DepositContract
} else if chain_exists {
@@ -172,7 +171,7 @@ where
genesis_state_bytes,
} => {
info!(
context.log,
context.log(),
"Starting from known genesis state";
);
@@ -183,14 +182,14 @@ where
}
ClientGenesis::DepositContract => {
info!(
context.log,
context.log(),
"Waiting for eth2 genesis from eth1";
"eth1_endpoint" => &config.eth1.endpoint,
"contract_deploy_block" => config.eth1.deposit_contract_deploy_block,
"deposit_contract" => &config.eth1.deposit_contract_address
);
let genesis_service = Eth1GenesisService::new(config.eth1, context.log.clone());
let genesis_service = Eth1GenesisService::new(config.eth1, context.log().clone());
let genesis_state = genesis_service
.wait_for_genesis_state(
@@ -223,19 +222,18 @@ where
.ok_or_else(|| "network requires a runtime_context")?
.clone();
let (network_globals, network_send, network_exit) =
NetworkService::start(beacon_chain, config, &context.runtime_handle, context.log)
let (network_globals, network_send) =
NetworkService::start(beacon_chain, config, context.executor)
.map_err(|e| format!("Failed to start network: {:?}", e))?;
self.network_globals = Some(network_globals);
self.network_send = Some(network_send);
self.exit_channels.push(network_exit);
Ok(self)
}
/// Immediately starts the timer service.
fn timer(mut self) -> Result<Self, String> {
fn timer(self) -> Result<Self, String> {
let context = self
.runtime_context
.as_ref()
@@ -251,13 +249,9 @@ where
.ok_or_else(|| "node timer requires a chain spec".to_string())?
.milliseconds_per_slot;
let timer_exit = context
.runtime_handle
.enter(|| timer::spawn(beacon_chain, milliseconds_per_slot))
spawn_timer(context.executor, beacon_chain, milliseconds_per_slot)
.map_err(|e| format!("Unable to start node timer: {}", e))?;
self.exit_channels.push(timer_exit);
Ok(self)
}
@@ -290,32 +284,28 @@ where
network_chan: network_send,
};
let log = context.log.clone();
let (exit_channel, listening_addr) = context.runtime_handle.enter(|| {
rest_api::start_server(
&client_config.rest_api,
beacon_chain,
network_info,
client_config
.create_db_path()
.map_err(|_| "unable to read data dir")?,
client_config
.create_freezer_db_path()
.map_err(|_| "unable to read freezer DB dir")?,
eth2_config.clone(),
log,
)
.map_err(|e| format!("Failed to start HTTP API: {:?}", e))
})?;
let listening_addr = rest_api::start_server(
context.executor,
&client_config.rest_api,
beacon_chain,
network_info,
client_config
.create_db_path()
.map_err(|_| "unable to read data dir")?,
client_config
.create_freezer_db_path()
.map_err(|_| "unable to read freezer DB dir")?,
eth2_config.clone(),
)
.map_err(|e| format!("Failed to start HTTP API: {:?}", e))?;
self.exit_channels.push(exit_channel);
self.http_listen_addr = Some(listening_addr);
Ok(self)
}
/// Immediately starts the service that periodically logs information each slot.
pub fn notifier(mut self) -> Result<Self, String> {
pub fn notifier(self) -> Result<Self, String> {
let context = self
.runtime_context
.as_ref()
@@ -335,19 +325,13 @@ where
.ok_or_else(|| "slot_notifier requires a chain spec".to_string())?
.milliseconds_per_slot;
let exit_channel = context
.runtime_handle
.enter(|| {
spawn_notifier(
beacon_chain,
network_globals,
milliseconds_per_slot,
context.log.clone(),
)
})
.map_err(|e| format!("Unable to start slot notifier: {}", e))?;
self.exit_channels.push(exit_channel);
spawn_notifier(
context.executor,
beacon_chain,
network_globals,
milliseconds_per_slot,
)
.map_err(|e| format!("Unable to start slot notifier: {}", e))?;
Ok(self)
}
@@ -365,7 +349,6 @@ where
network_globals: self.network_globals,
http_listen_addr: self.http_listen_addr,
websocket_listen_addr: self.websocket_listen_addr,
_exit_channels: self.exit_channels,
}
}
}
@@ -436,22 +419,14 @@ where
.ok_or_else(|| "websocket_event_handler requires a runtime_context")?
.service_context("ws".into());
let (sender, exit_channel, listening_addr): (
WebSocketSender<TEthSpec>,
Option<_>,
Option<_>,
) = if config.enabled {
let (sender, exit, listening_addr) = context
.runtime_handle
.enter(|| websocket_server::start_server(&config, &context.log))?;
(sender, Some(exit), Some(listening_addr))
let (sender, listening_addr): (WebSocketSender<TEthSpec>, Option<_>) = if config.enabled {
let (sender, listening_addr) =
websocket_server::start_server(context.executor, &config)?;
(sender, Some(listening_addr))
} else {
(WebSocketSender::dummy(), None, None)
(WebSocketSender::dummy(), None)
};
if let Some(channel) = exit_channel {
self.exit_channels.push(channel);
}
self.event_handler = Some(sender);
self.websocket_listen_addr = listening_addr;
@@ -494,7 +469,7 @@ where
.clone()
.ok_or_else(|| "disk_store requires a chain spec".to_string())?;
let store = HotColdDB::open(hot_path, cold_path, config, spec, context.log)
let store = HotColdDB::open(hot_path, cold_path, config, spec, context.log().clone())
.map_err(|e| format!("Unable to open database: {:?}", e))?;
self.store = Some(Arc::new(store));
Ok(self)
@@ -555,7 +530,7 @@ where
let store = self.store.clone().ok_or_else(|| {
"background_migrator requires the store to be initialized".to_string()
})?;
self.store_migrator = Some(BackgroundMigrator::new(store, context.log.clone()));
self.store_migrator = Some(BackgroundMigrator::new(store, context.log().clone()));
Ok(self)
}
}
@@ -617,25 +592,23 @@ where
&persisted,
config.clone(),
store.clone(),
&context.log,
&context.log().clone(),
)
.map(|chain| chain.into_backend())
})
.unwrap_or_else(|| {
Ok(CachingEth1Backend::new(config, context.log.clone(), store))
Ok(CachingEth1Backend::new(
config,
context.log().clone(),
store,
))
})?
};
self.eth1_service = None;
let exit = {
let (tx, rx) = tokio::sync::oneshot::channel();
self.exit_channels.push(tx);
rx
};
// Starts the service that connects to an eth1 node and periodically updates caches.
context.runtime_handle.enter(|| backend.start(exit));
backend.start(context.executor);
self.beacon_chain_builder = Some(beacon_chain_builder.eth1_backend(Some(backend)));

2
beacon_node/client/src/lib.rs
View File

@@ -25,8 +25,6 @@ pub struct Client<T: BeaconChainTypes> {
network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
http_listen_addr: Option<SocketAddr>,
websocket_listen_addr: Option<SocketAddr>,
/// Exit channels will complete/error when dropped, causing each service to exit gracefully.
_exit_channels: Vec<tokio::sync::oneshot::Sender<()>>,
}
impl<T: BeaconChainTypes> Client<T> {

11
beacon_node/client/src/notifier.rs
View File

@@ -23,11 +23,11 @@ const SPEEDO_OBSERVATIONS: usize = 4;
/// Spawns a notifier service which periodically logs information about the node.
pub fn spawn_notifier<T: BeaconChainTypes>(
executor: environment::TaskExecutor,
beacon_chain: Arc<BeaconChain<T>>,
network: Arc<NetworkGlobals<T::EthSpec>>,
milliseconds_per_slot: u64,
log: slog::Logger,
) -> Result<tokio::sync::oneshot::Sender<()>, String> {
) -> Result<(), String> {
let slot_duration = Duration::from_millis(milliseconds_per_slot);
let duration_to_next_slot = beacon_chain
.slot_clock
@@ -41,6 +41,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
let interval_duration = slot_duration;
let speedo = Mutex::new(Speedo::default());
let log = executor.log().clone();
let mut interval = tokio::time::interval_at(start_instant, interval_duration);
let interval_future = async move {
@@ -163,12 +164,10 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
Ok::<(), ()>(())
};
let (exit_signal, exit) = tokio::sync::oneshot::channel();
// run the notifier on the current executor
tokio::spawn(futures::future::select(Box::pin(interval_future), exit));
executor.spawn(interval_future.unwrap_or_else(|_| ()), "notifier");
Ok(exit_signal)
Ok(())
}
/// Returns the peer count, returning something helpful if it's `usize::max_value` (effectively a

4
beacon_node/eth1/Cargo.toml
View File

@@ -6,7 +6,6 @@ edition = "2018"
[dev-dependencies]
eth1_test_rig = { path = "../../testing/eth1_test_rig" }
environment = { path = "../../lighthouse/environment" }
toml = "0.5.6"
web3 = "0.11.0"
sloggers = "1.0.0"
@@ -25,8 +24,9 @@ tree_hash = "0.1.0"
eth2_hashing = "0.1.0"
parking_lot = "0.10.2"
slog = "2.5.2"
tokio = { version = "0.2.20", features = ["full"] }
tokio = { version = "0.2.21", features = ["full"] }
state_processing = { path = "../../consensus/state_processing" }
libflate = "1.0.0"
lighthouse_metrics = { path = "../../common/lighthouse_metrics"}
lazy_static = "1.4.0"
environment = { path = "../../lighthouse/environment" }

6
beacon_node/eth1/src/service.rs
View File

@@ -290,7 +290,7 @@ impl Service {
/// - Err(_) if there is an error.
///
/// Emits logs for debugging and errors.
pub fn auto_update(service: Self, exit: tokio::sync::oneshot::Receiver<()>) {
pub fn auto_update(service: Self, handle: environment::TaskExecutor) {
let update_interval = Duration::from_millis(service.config().auto_update_interval_millis);
let mut interval = interval_at(Instant::now(), update_interval);
@@ -303,9 +303,7 @@ impl Service {
}
};
let future = futures::future::select(Box::pin(update_future), exit);
tokio::task::spawn(future);
handle.spawn(update_future, "eth1");
}
async fn do_update(service: Self, update_interval: Duration) -> Result<(), ()> {

2
beacon_node/eth2-libp2p/Cargo.toml
View File

@@ -35,6 +35,7 @@ tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
# Patched for quick updates
discv5 = { git = "https://github.com/sigp/discv5", rev = "7b3bd40591b62b8c002ffdb85de008aa9f82e2e5" }
tiny-keccak = "2.0.2"
environment = { path = "../../lighthouse/environment" }
libp2p-tcp = { version = "0.19.1", default-features = false, features = ["tokio"] }
[dependencies.libp2p]
@@ -49,3 +50,4 @@ slog-stdlog = "4.0.0"
slog-term = "2.5.0"
slog-async = "2.5.0"
tempdir = "0.3.7"
exit-future = "0.2.0"

7
beacon_node/eth2-libp2p/src/service.rs
View File

@@ -84,6 +84,7 @@ pub struct Service<TSpec: EthSpec> {
impl<TSpec: EthSpec> Service<TSpec> {
pub fn new(
executor: environment::TaskExecutor,
config: &NetworkConfig,
enr_fork_id: EnrForkId,
log: &slog::Logger,
@@ -122,15 +123,15 @@ impl<TSpec: EthSpec> Service<TSpec> {
let behaviour = Behaviour::new(&local_keypair, config, network_globals.clone(), &log)?;
// use the executor for libp2p
struct Executor(tokio::runtime::Handle);
struct Executor(environment::TaskExecutor);
impl libp2p::core::Executor for Executor {
fn exec(&self, f: Pin<Box<dyn Future<Output = ()> + Send>>) {
self.0.spawn(f);
self.0.spawn(f, "libp2p");
}
}
SwarmBuilder::new(transport, behaviour, local_peer_id.clone())
.peer_connection_limit(MAX_CONNECTIONS_PER_PEER)
.executor(Box::new(Executor(tokio::runtime::Handle::current())))
.executor(Box::new(Executor(executor)))
.build()
};

40
beacon_node/eth2-libp2p/tests/common/mod.rs
View File

@@ -12,6 +12,21 @@ use types::{EnrForkId, MinimalEthSpec};
type E = MinimalEthSpec;
use tempdir::TempDir;
pub struct Libp2pInstance(LibP2PService<E>, exit_future::Signal);
impl std::ops::Deref for Libp2pInstance {
type Target = LibP2PService<E>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for Libp2pInstance {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger {
let decorator = slog_term::TermDecorator::new().build();
let drain = slog_term::FullFormat::new(decorator).build().fuse();
@@ -82,13 +97,20 @@ pub fn build_libp2p_instance(
boot_nodes: Vec<Enr>,
secret_key: Option<String>,
log: slog::Logger,
) -> LibP2PService<E> {
) -> Libp2pInstance {
let port = unused_port("tcp").unwrap();
let config = build_config(port, boot_nodes, secret_key);
// launch libp2p service
LibP2PService::new(&config, EnrForkId::default(), &log)
.expect("should build libp2p instance")
.1
let (signal, exit) = exit_future::signal();
let executor =
environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone());
Libp2pInstance(
LibP2PService::new(executor, &config, EnrForkId::default(), &log)
.expect("should build libp2p instance")
.1,
signal,
)
}
#[allow(dead_code)]
@@ -99,8 +121,8 @@ pub fn get_enr(node: &LibP2PService<E>) -> Enr {
// Returns `n` libp2p peers in fully connected topology.
#[allow(dead_code)]
pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<LibP2PService<E>> {
let mut nodes: Vec<LibP2PService<E>> = (0..n)
pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
let mut nodes: Vec<_> = (0..n)
.map(|_| build_libp2p_instance(vec![], None, log.clone()))
.collect();
let multiaddrs: Vec<Multiaddr> = nodes
@@ -124,7 +146,7 @@ pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<LibP2PService<E>> {
// Constructs a pair of nodes with separate loggers. The sender dials the receiver.
// This returns a (sender, receiver) pair.
#[allow(dead_code)]
pub async fn build_node_pair(log: &slog::Logger) -> (LibP2PService<E>, LibP2PService<E>) {
pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInstance) {
let sender_log = log.new(o!("who" => "sender"));
let receiver_log = log.new(o!("who" => "receiver"));
@@ -168,8 +190,8 @@ pub async fn build_node_pair(log: &slog::Logger) -> (LibP2PService<E>, LibP2PSer
// Returns `n` peers in a linear topology
#[allow(dead_code)]
pub fn build_linear(log: slog::Logger, n: usize) -> Vec<LibP2PService<E>> {
let mut nodes: Vec<LibP2PService<E>> = (0..n)
pub fn build_linear(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
let mut nodes: Vec<_> = (0..n)
.map(|_| build_libp2p_instance(vec![], None, log.clone()))
.collect();
let multiaddrs: Vec<Multiaddr> = nodes

5
beacon_node/eth2-libp2p/tests/noise.rs
View File

@@ -136,7 +136,10 @@ async fn test_secio_noise_fallback() {
let port = common::unused_port("tcp").unwrap();
let noisy_config = common::build_config(port, vec![], None);
let mut noisy_node = Service::new(&noisy_config, EnrForkId::default(), &log)
let (_signal, exit) = exit_future::signal();
let executor =
environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone());
let mut noisy_node = Service::new(executor, &noisy_config, EnrForkId::default(), &log)
.expect("should build a libp2p instance")
.1;

2
beacon_node/genesis/Cargo.toml
View File

@@ -18,7 +18,7 @@ merkle_proof = { path = "../../consensus/merkle_proof" }
eth2_ssz = "0.1.2"
eth2_hashing = "0.1.0"
tree_hash = "0.1.0"
tokio = { version = "0.2.20", features = ["full"] }
tokio = { version = "0.2.21", features = ["full"] }
parking_lot = "0.10.2"
slog = "2.5.2"
exit-future = "0.2.0"

2
beacon_node/genesis/tests/tests.rs
View File

@@ -24,7 +24,7 @@ pub fn new_env() -> Environment<MinimalEthSpec> {
#[test]
fn basic() {
let mut env = new_env();
let log = env.core_context().log.clone();
let log = env.core_context().log().clone();
let mut spec = env.eth2_config().spec.clone();
env.runtime().block_on(async {

4
beacon_node/network/Cargo.toml
View File

@@ -10,6 +10,7 @@ genesis = { path = "../genesis" }
lazy_static = "1.4.0"
matches = "0.1.8"
tempfile = "3.1.0"
exit-future = "0.2.0"
[dependencies]
beacon_chain = { path = "../beacon_chain" }
@@ -25,7 +26,7 @@ eth2_ssz = "0.1.2"
tree_hash = "0.1.0"
futures = "0.3.5"
error-chain = "0.12.2"
tokio = { version = "0.2.20", features = ["full"] }
tokio = { version = "0.2.21", features = ["full"] }
parking_lot = "0.10.2"
smallvec = "1.4.0"
# TODO: Remove rand crate for mainnet
@@ -34,3 +35,4 @@ fnv = "1.0.6"
rlp = "0.4.5"
lazy_static = "1.4.0"
lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
environment = { path = "../../lighthouse/environment" }

19
beacon_node/network/src/router/mod.rs
View File

@@ -58,7 +58,7 @@ impl<T: BeaconChainTypes> Router<T> {
beacon_chain: Arc<BeaconChain<T>>,
network_globals: Arc<NetworkGlobals<T::EthSpec>>,
network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
runtime_handle: &tokio::runtime::Handle,
executor: environment::TaskExecutor,
log: slog::Logger,
) -> error::Result<mpsc::UnboundedSender<RouterMessage<T::EthSpec>>> {
let message_handler_log = log.new(o!("service"=> "router"));
@@ -68,7 +68,7 @@ impl<T: BeaconChainTypes> Router<T> {
// Initialise a message instance, which itself spawns the syncing thread.
let processor = Processor::new(
runtime_handle,
executor.clone(),
beacon_chain,
network_globals.clone(),
network_send.clone(),
@@ -84,12 +84,15 @@ impl<T: BeaconChainTypes> Router<T> {
};
// spawn handler task and move the message handler instance into the spawned thread
runtime_handle.spawn(async move {
handler_recv
.for_each(move |msg| future::ready(handler.handle_message(msg)))
.await;
debug!(log, "Network message handler terminated.");
});
executor.spawn(
async move {
debug!(log, "Network message router started");
handler_recv
.for_each(move |msg| future::ready(handler.handle_message(msg)))
.await;
},
"router",
);
Ok(handler_send)
}

11
beacon_node/network/src/router/processor.rs
View File

@@ -14,7 +14,7 @@ use slog::{debug, error, o, trace, warn};
use ssz::Encode;
use std::sync::Arc;
use store::Store;
use tokio::sync::{mpsc, oneshot};
use tokio::sync::mpsc;
use types::{
Attestation, ChainSpec, Epoch, EthSpec, Hash256, SignedAggregateAndProof, SignedBeaconBlock,
Slot,
@@ -33,8 +33,6 @@ pub struct Processor<T: BeaconChainTypes> {
chain: Arc<BeaconChain<T>>,
/// A channel to the syncing thread.
sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
/// A oneshot channel for destroying the sync thread.
_sync_exit: oneshot::Sender<()>,
/// A network context to return and handle RPC requests.
network: HandlerNetworkContext<T::EthSpec>,
/// The `RPCHandler` logger.
@@ -44,7 +42,7 @@ pub struct Processor<T: BeaconChainTypes> {
impl<T: BeaconChainTypes> Processor<T> {
/// Instantiate a `Processor` instance
pub fn new(
runtime_handle: &tokio::runtime::Handle,
executor: environment::TaskExecutor,
beacon_chain: Arc<BeaconChain<T>>,
network_globals: Arc<NetworkGlobals<T::EthSpec>>,
network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
@@ -53,8 +51,8 @@ impl<T: BeaconChainTypes> Processor<T> {
let sync_logger = log.new(o!("service"=> "sync"));
// spawn the sync thread
let (sync_send, _sync_exit) = crate::sync::manager::spawn(
runtime_handle,
let sync_send = crate::sync::manager::spawn(
executor,
beacon_chain.clone(),
network_globals,
network_send.clone(),
@@ -64,7 +62,6 @@ impl<T: BeaconChainTypes> Processor<T> {
Processor {
chain: beacon_chain,
sync_send,
_sync_exit,
network: HandlerNetworkContext::new(network_send, log.clone()),
log: log.clone(),
}

27
beacon_node/network/src/service.rs
View File

@@ -14,8 +14,7 @@ use rest_types::ValidatorSubscription;
use slog::{debug, error, info, o, trace};
use std::sync::Arc;
use std::time::Duration;
use tokio::runtime::Handle;
use tokio::sync::{mpsc, oneshot};
use tokio::sync::mpsc;
use tokio::time::Delay;
use types::EthSpec;
@@ -53,13 +52,12 @@ impl<T: BeaconChainTypes> NetworkService<T> {
pub fn start(
beacon_chain: Arc<BeaconChain<T>>,
config: &NetworkConfig,
runtime_handle: &Handle,
network_log: slog::Logger,
executor: environment::TaskExecutor,
) -> error::Result<(
Arc<NetworkGlobals<T::EthSpec>>,
mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
oneshot::Sender<()>,
)> {
let network_log = executor.log().clone();
// build the network channel
let (network_send, network_recv) = mpsc::unbounded_channel::<NetworkMessage<T::EthSpec>>();
// get a reference to the beacon chain store
@@ -75,7 +73,7 @@ impl<T: BeaconChainTypes> NetworkService<T> {
// launch libp2p service
let (network_globals, mut libp2p) =
runtime_handle.enter(|| LibP2PService::new(config, enr_fork_id, &network_log))?;
LibP2PService::new(executor.clone(), config, enr_fork_id, &network_log)?;
for enr in load_dht::<T::Store, T::EthSpec>(store.clone()) {
libp2p.swarm.add_enr(enr);
@@ -88,7 +86,7 @@ impl<T: BeaconChainTypes> NetworkService<T> {
beacon_chain.clone(),
network_globals.clone(),
network_send.clone(),
runtime_handle,
executor.clone(),
network_log.clone(),
)?;
@@ -111,19 +109,20 @@ impl<T: BeaconChainTypes> NetworkService<T> {
propagation_percentage,
};
let network_exit = runtime_handle.enter(|| spawn_service(network_service))?;
spawn_service(executor, network_service)?;
Ok((network_globals, network_send, network_exit))
Ok((network_globals, network_send))
}
}
fn spawn_service<T: BeaconChainTypes>(
executor: environment::TaskExecutor,
mut service: NetworkService<T>,
) -> error::Result<tokio::sync::oneshot::Sender<()>> {
let (network_exit, mut exit_rx) = tokio::sync::oneshot::channel();
) -> error::Result<()> {
let mut exit_rx = executor.exit();
// spawn on the current executor
tokio::spawn(async move {
executor.spawn_without_exit(async move {
loop {
// build the futures to check simultaneously
tokio::select! {
@@ -361,9 +360,9 @@ fn spawn_service<T: BeaconChainTypes>(
}
}
}
});
}, "network");
Ok(network_exit)
Ok(())
}
/// Returns a `Delay` that triggers shortly after the next change in the beacon chain fork version.

8
beacon_node/network/src/service/tests.rs
View File

@@ -32,7 +32,9 @@ mod tests {
let enrs = vec![enr1, enr2];
let runtime = Runtime::new().unwrap();
let handle = runtime.handle().clone();
let (signal, exit) = exit_future::signal();
let executor = environment::TaskExecutor::new(runtime.handle().clone(), exit, log.clone());
let mut config = NetworkConfig::default();
config.libp2p_port = 21212;
@@ -42,8 +44,8 @@ mod tests {
// Create a new network service which implicitly gets dropped at the
// end of the block.
let _ =
NetworkService::start(beacon_chain.clone(), &config, &handle, log.clone()).unwrap();
let _ = NetworkService::start(beacon_chain.clone(), &config, executor).unwrap();
drop(signal);
});
runtime.shutdown_timeout(tokio::time::Duration::from_millis(300));

18
beacon_node/network/src/sync/manager.rs
View File

@@ -48,7 +48,7 @@ use smallvec::SmallVec;
use std::boxed::Box;
use std::ops::Sub;
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
use tokio::sync::mpsc;
use types::{EthSpec, Hash256, SignedBeaconBlock, Slot};
/// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync
@@ -181,17 +181,12 @@ impl SingleBlockRequest {
/// chain. This allows the chain to be
/// dropped during the syncing process which will gracefully end the `SyncManager`.
pub fn spawn<T: BeaconChainTypes>(
runtime_handle: &tokio::runtime::Handle,
executor: environment::TaskExecutor,
beacon_chain: Arc<BeaconChain<T>>,
network_globals: Arc<NetworkGlobals<T::EthSpec>>,
network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
log: slog::Logger,
) -> (
mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
oneshot::Sender<()>,
) {
// generate the exit channel
let (sync_exit, exit_rx) = tokio::sync::oneshot::channel();
) -> mpsc::UnboundedSender<SyncMessage<T::EthSpec>> {
// generate the message channel
let (sync_send, sync_recv) = mpsc::unbounded_channel::<SyncMessage<T::EthSpec>>();
@@ -215,11 +210,8 @@ pub fn spawn<T: BeaconChainTypes>(
// spawn the sync manager thread
debug!(log, "Sync Manager started");
runtime_handle.spawn(async move {
futures::future::select(Box::pin(sync_manager.main()), exit_rx).await;
info!(log.clone(), "Sync Manager shutdown");
});
(sync_send, sync_exit)
executor.spawn(async move { Box::pin(sync_manager.main()).await }, "sync");
sync_send
}
impl<T: BeaconChainTypes> SyncManager<T> {

3
beacon_node/rest_api/Cargo.toml
View File

@@ -25,7 +25,7 @@ state_processing = { path = "../../consensus/state_processing" }
types = { path = "../../consensus/types" }
http = "0.2.1"
hyper = "0.13.5"
tokio = { version = "0.2", features = ["sync"] }
tokio = { version = "0.2.21", features = ["sync"] }
url = "2.1.1"
lazy_static = "1.4.0"
eth2_config = { path = "../../common/eth2_config" }
@@ -36,6 +36,7 @@ parking_lot = "0.10.2"
futures = "0.3.5"
operation_pool = { path = "../operation_pool" }
rayon = "1.3.0"
environment = { path = "../../lighthouse/environment" }
[dev-dependencies]
assert_matches = "1.3.0"

16
beacon_node/rest_api/src/lib.rs
View File

@@ -35,7 +35,7 @@ use std::net::SocketAddr;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
use tokio::sync::mpsc;
use url_query::UrlQuery;
pub use crate::helpers::parse_pubkey_bytes;
@@ -51,14 +51,15 @@ pub struct NetworkInfo<T: BeaconChainTypes> {
// Allowing more than 7 arguments.
#[allow(clippy::too_many_arguments)]
pub fn start_server<T: BeaconChainTypes>(
executor: environment::TaskExecutor,
config: &Config,
beacon_chain: Arc<BeaconChain<T>>,
network_info: NetworkInfo<T>,
db_path: PathBuf,
freezer_db_path: PathBuf,
eth2_config: Eth2Config,
log: slog::Logger,
) -> Result<(oneshot::Sender<()>, SocketAddr), hyper::Error> {
) -> Result<SocketAddr, hyper::Error> {
let log = executor.log();
let inner_log = log.clone();
let eth2_config = Arc::new(eth2_config);
@@ -98,7 +99,7 @@ pub fn start_server<T: BeaconChainTypes>(
let actual_listen_addr = server.local_addr();
// Build a channel to kill the HTTP server.
let (exit_signal, exit) = oneshot::channel::<()>();
let exit = executor.exit();
let inner_log = log.clone();
let server_exit = async move {
let _ = exit.await;
@@ -116,7 +117,8 @@ pub fn start_server<T: BeaconChainTypes>(
inner_log,
"HTTP server failed to start, Unable to bind"; "address" => format!("{:?}", e)
)
});
})
.unwrap_or_else(|_| ());
info!(
log,
@@ -125,9 +127,9 @@ pub fn start_server<T: BeaconChainTypes>(
"port" => actual_listen_addr.port(),
);
tokio::spawn(server_future);
executor.spawn_without_exit(server_future, "http");
Ok((exit_signal, actual_listen_addr))
Ok(actual_listen_addr)
}
#[derive(Clone)]

4
beacon_node/src/lib.rs
View File

@@ -58,7 +58,7 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
&matches,
&context.eth2_config.spec_constants,
&context.eth2_config().spec,
context.log.clone(),
context.log().clone(),
)?;
Self::new(context, client_config).await
}
@@ -75,7 +75,7 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
let client_config_1 = client_config.clone();
let client_genesis = client_config.genesis.clone();
let store_config = client_config.store.clone();
let log = context.log.clone();
let log = context.log().clone();
let db_path = client_config.create_db_path()?;
let freezer_db_path_res = client_config.create_freezer_db_path();

1
beacon_node/tests/test.rs
View File

@@ -51,4 +51,5 @@ fn http_server_genesis_state() {
api_state, db_state,
"genesis state from api should match that from the DB"
);
env.fire_signal();
}

3
beacon_node/timer/Cargo.toml
View File

@@ -8,7 +8,8 @@ edition = "2018"
beacon_chain = { path = "../beacon_chain" }
types = { path = "../../consensus/types" }
slot_clock = { path = "../../common/slot_clock" }
tokio = { version = "0.2.20", features = ["full"] }
tokio = { version = "0.2.21", features = ["full"] }
slog = "2.5.2"
parking_lot = "0.10.2"
futures = "0.3.5"
environment = { path = "../../lighthouse/environment" }

28
beacon_node/timer/src/lib.rs
View File

@@ -3,23 +3,20 @@
//! This service allows task execution on the beacon node for various functionality.
use beacon_chain::{BeaconChain, BeaconChainTypes};
use futures::future;
use futures::stream::StreamExt;
use slog::info;
use slot_clock::SlotClock;
use std::sync::Arc;
use std::time::Duration;
use tokio::time::{interval_at, Instant};
/// Spawns a timer service which periodically executes tasks for the beacon chain
/// TODO: We might not need a `Handle` to the runtime since this function should be
/// called from the context of a runtime and we can simply spawn using task::spawn.
/// Check for issues without the Handle.
pub fn spawn<T: BeaconChainTypes>(
pub fn spawn_timer<T: BeaconChainTypes>(
executor: environment::TaskExecutor,
beacon_chain: Arc<BeaconChain<T>>,
milliseconds_per_slot: u64,
) -> Result<tokio::sync::oneshot::Sender<()>, &'static str> {
let (exit_signal, exit) = tokio::sync::oneshot::channel();
) -> Result<(), &'static str> {
let log = executor.log();
let start_instant = Instant::now()
+ beacon_chain
.slot_clock
@@ -27,14 +24,15 @@ pub fn spawn<T: BeaconChainTypes>(
.ok_or_else(|| "slot_notifier unable to determine time to next slot")?;
// Warning: `interval_at` panics if `milliseconds_per_slot` = 0.
let timer_future = interval_at(start_instant, Duration::from_millis(milliseconds_per_slot))
.for_each(move |_| {
let mut interval = interval_at(start_instant, Duration::from_millis(milliseconds_per_slot));
let timer_future = async move {
while interval.next().await.is_some() {
beacon_chain.per_slot_task();
future::ready(())
});
}
};
let future = futures::future::select(timer_future, exit);
tokio::spawn(future);
executor.spawn(timer_future, "timer");
info!(log, "Timer service started");
Ok(exit_signal)
Ok(())
}

3
beacon_node/websocket_server/Cargo.toml
View File

@@ -12,6 +12,7 @@ serde = "1.0.110"
serde_derive = "1.0.110"
serde_json = "1.0.52"
slog = "2.5.2"
tokio = { version = "0.2.20", features = ["full"] }
tokio = { version = "0.2.21", features = ["full"] }
types = { path = "../../consensus/types" }
ws = "0.9.1"
environment = { path = "../../lighthouse/environment" }

56
beacon_node/websocket_server/src/lib.rs
View File

@@ -1,4 +1,4 @@
use slog::{debug, error, info, warn, Logger};
use slog::{debug, error, info, warn};
use std::marker::PhantomData;
use std::net::SocketAddr;
use types::EthSpec;
@@ -34,16 +34,10 @@ impl<T: EthSpec> WebSocketSender<T> {
}
pub fn start_server<T: EthSpec>(
executor: environment::TaskExecutor,
config: &Config,
log: &Logger,
) -> Result<
(
WebSocketSender<T>,
tokio::sync::oneshot::Sender<()>,
SocketAddr,
),
String,
> {
) -> Result<(WebSocketSender<T>, SocketAddr), String> {
let log = executor.log();
let server_string = format!("{}:{}", config.listen_address, config.port);
// Create a server that simply ignores any incoming messages.
@@ -67,31 +61,26 @@ pub fn start_server<T: EthSpec>(
let broadcaster = server.broadcaster();
// Produce a signal/channel that can gracefully shutdown the websocket server.
let exit_channel = {
let (exit_channel, exit) = tokio::sync::oneshot::channel();
let log_inner = log.clone();
let broadcaster_inner = server.broadcaster();
let exit_future = async move {
let _ = exit.await;
if let Err(e) = broadcaster_inner.shutdown() {
warn!(
log_inner,
"Websocket server errored on shutdown";
"error" => format!("{:?}", e)
);
} else {
info!(log_inner, "Websocket server shutdown");
}
};
// Place a future on the handle that will shutdown the websocket server when the
// application exits.
tokio::spawn(exit_future);
exit_channel
let exit = executor.exit();
let log_inner = log.clone();
let broadcaster_inner = server.broadcaster();
let exit_future = async move {
let _ = exit.await;
if let Err(e) = broadcaster_inner.shutdown() {
warn!(
log_inner,
"Websocket server errored on shutdown";
"error" => format!("{:?}", e)
);
} else {
info!(log_inner, "Websocket server shutdown");
}
};
// Place a future on the handle that will shutdown the websocket server when the
// application exits.
executor.runtime_handle().spawn(exit_future);
let log_inner = log.clone();
let _ = std::thread::spawn(move || match server.run() {
@@ -122,7 +111,6 @@ pub fn start_server<T: EthSpec>(
sender: Some(broadcaster),
_phantom: PhantomData,
},
exit_channel,
actual_listen_addr,
))
}