gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-08 01:05:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade discovery and restructure task execution (#1693)

Browse Source 
* Initial rebase

* Remove old code

* Correct release tests

* Rebase commit

* Remove eth2-testnet dep on eth2libp2p

* Remove crates lost in rebase

* Remove unused dep
This commit is contained in:
Age Manning
2020-10-05 18:45:54 +11:00
committed by GitHub
parent bcb629564a
commit 240181e840
35 changed files with 273 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lighthouse/environment/Cargo.toml
View File

@@ -10,6 +10,7 @@ slog = { version = "2.5.2", features = ["max_level_trace"] }
sloggers = "1.0.0"
types = { "path" = "../../consensus/types" }
eth2_config = { "path" = "../../common/eth2_config" }
task_executor = { "path" = "../../common/task_executor" }
eth2_testnet_config = { path = "../../common/eth2_testnet_config" }
logging = { path = "../../common/logging" }
slog-term = "2.5.0"
@@ -19,6 +20,3 @@ futures = "0.3.5"
parking_lot = "0.11.0"
slog-json = "2.3.0"
exit-future = "0.2.0"
lazy_static = "1.4.0"
lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
discv5 = { version = "0.1.0-alpha.12", features = ["libp2p"] }

156
lighthouse/environment/src/executor.rs
View File

@@ -1,156 +0,0 @@
use crate::metrics;
use futures::channel::mpsc::Sender;
use futures::prelude::*;
use slog::{debug, trace};
use tokio::runtime::Handle;
/// A wrapper over a runtime handle which can spawn async and blocking tasks.
#[derive(Clone)]
pub struct TaskExecutor {
/// The handle to the runtime on which tasks are spawned
pub handle: Handle,
/// The receiver exit future which on receiving shuts down the task
pub(crate) exit: exit_future::Exit,
/// Sender given to tasks, so that if they encounter a state in which execution cannot
/// continue they can request that everything shuts down.
///
/// The task must provide a reason for shutting down.
pub(crate) signal_tx: Sender<&'static str>,
pub(crate) log: slog::Logger,
}
impl TaskExecutor {
/// Create a new task executor.
///
/// Note: this function is mainly useful in tests. A `TaskExecutor` should be normally obtained from
/// a [`RuntimeContext`](struct.RuntimeContext.html)
pub fn new(
handle: Handle,
exit: exit_future::Exit,
log: slog::Logger,
signal_tx: Sender<&'static str>,
) -> Self {
Self {
handle,
exit,
signal_tx,
log,
}
}
/// Spawn a future on the tokio runtime wrapped in an `exit_future::Exit`. The task is canceled
/// when the corresponding exit_future `Signal` is fired/dropped.
///
/// This function generates prometheus metrics on number of tasks and task duration.
pub fn spawn(&self, task: impl Future<Output = ()> + Send + 'static, name: &'static str) {
let exit = self.exit.clone();
let log = self.log.clone();
if let Some(int_gauge) = metrics::get_int_gauge(&metrics::ASYNC_TASKS_COUNT, &[name]) {
// Task is shutdown before it completes if `exit` receives
let int_gauge_1 = int_gauge.clone();
let future = future::select(Box::pin(task), exit).then(move |either| {
match either {
future::Either::Left(_) => trace!(log, "Async task completed"; "task" => name),
future::Either::Right(_) => {
debug!(log, "Async task shutdown, exit received"; "task" => name)
}
}
int_gauge_1.dec();
futures::future::ready(())
});
int_gauge.inc();
self.handle.spawn(future);
}
}
/// Spawn a future on the tokio runtime. This function does not wrap the task in an `exit_future::Exit`
/// like [spawn](#method.spawn).
/// The caller of this function is responsible for wrapping up the task with an `exit_future::Exit` to
/// ensure that the task gets canceled appropriately.
/// This function generates prometheus metrics on number of tasks and task duration.
///
/// This is useful in cases where the future to be spawned needs to do additional cleanup work when
/// the task is completed/canceled (e.g. writing local variables to disk) or the task is created from
/// some framework which does its own cleanup (e.g. a hyper server).
pub fn spawn_without_exit(
&self,
task: impl Future<Output = ()> + Send + 'static,
name: &'static str,
) {
if let Some(int_gauge) = metrics::get_int_gauge(&metrics::ASYNC_TASKS_COUNT, &[name]) {
let int_gauge_1 = int_gauge.clone();
let future = task.then(move |_| {
int_gauge_1.dec();
futures::future::ready(())
});
int_gauge.inc();
self.handle.spawn(future);
}
}
/// Spawn a blocking task on a dedicated tokio thread pool wrapped in an exit future.
/// This function generates prometheus metrics on number of tasks and task duration.
pub fn spawn_blocking<F>(&self, task: F, name: &'static str)
where
F: FnOnce() + Send + 'static,
{
let exit = self.exit.clone();
let log = self.log.clone();
if let Some(metric) = metrics::get_histogram(&metrics::BLOCKING_TASKS_HISTOGRAM, &[name]) {
if let Some(int_gauge) = metrics::get_int_gauge(&metrics::BLOCKING_TASKS_COUNT, &[name])
{
let int_gauge_1 = int_gauge.clone();
let timer = metric.start_timer();
let join_handle = self.handle.spawn_blocking(task);
let future = future::select(join_handle, exit).then(move |either| {
match either {
future::Either::Left(_) => {
trace!(log, "Blocking task completed"; "task" => name)
}
future::Either::Right(_) => {
debug!(log, "Blocking task shutdown, exit received"; "task" => name)
}
}
timer.observe_duration();
int_gauge_1.dec();
futures::future::ready(())
});
int_gauge.inc();
self.handle.spawn(future);
}
}
}
/// Returns the underlying runtime handle.
pub fn runtime_handle(&self) -> Handle {
self.handle.clone()
}
/// Returns a copy of the `exit_future::Exit`.
pub fn exit(&self) -> exit_future::Exit {
self.exit.clone()
}
/// Get a channel to request shutting down.
pub fn shutdown_sender(&self) -> Sender<&'static str> {
self.signal_tx.clone()
}
/// Returns a reference to the logger.
pub fn log(&self) -> &slog::Logger {
&self.log
}
}
impl discv5::Executor for TaskExecutor {
fn spawn(&self, future: std::pin::Pin<Box<dyn Future<Output = ()> + Send>>) {
self.spawn(future, "discv5")
}
}

35
lighthouse/environment/src/lib.rs
View File

@@ -15,7 +15,6 @@ use futures::channel::{
};
use futures::{future, StreamExt};
pub use executor::TaskExecutor;
use slog::{info, o, Drain, Level, Logger};
use sloggers::{null::NullLoggerBuilder, Build};
use std::cell::RefCell;
@@ -23,10 +22,9 @@ use std::ffi::OsStr;
use std::fs::{rename as FsRename, OpenOptions};
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use task_executor::TaskExecutor;
use tokio::runtime::{Builder as RuntimeBuilder, Runtime};
use types::{EthSpec, InteropEthSpec, MainnetEthSpec, MinimalEthSpec};
mod executor;
mod metrics;
pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml";
const LOG_CHANNEL_SIZE: usize = 2048;
@@ -311,12 +309,7 @@ impl<E: EthSpec> RuntimeContext<E> {
/// The generated service will have the `service_name` in all it's logs.
pub fn service_context(&self, service_name: String) -> Self {
Self {
executor: TaskExecutor {
handle: self.executor.handle.clone(),
signal_tx: self.executor.signal_tx.clone(),
exit: self.executor.exit.clone(),
log: self.executor.log.new(o!("service" => service_name)),
},
executor: self.executor.clone_with_name(service_name),
eth_spec_instance: self.eth_spec_instance.clone(),
eth2_config: self.eth2_config.clone(),
}
@@ -361,12 +354,12 @@ impl<E: EthSpec> Environment<E> {
/// Returns a `Context` where no "service" has been added to the logger output.
pub fn core_context(&mut self) -> RuntimeContext<E> {
RuntimeContext {
executor: TaskExecutor {
exit: self.exit.clone(),
signal_tx: self.signal_tx.clone(),
handle: self.runtime().handle().clone(),
log: self.log.clone(),
},
executor: TaskExecutor::new(
self.runtime().handle().clone(),
self.exit.clone(),
self.log.clone(),
self.signal_tx.clone(),
),
eth_spec_instance: self.eth_spec_instance.clone(),
eth2_config: self.eth2_config.clone(),
}
@@ -375,12 +368,12 @@ impl<E: EthSpec> Environment<E> {
/// Returns a `Context` where the `service_name` is added to the logger output.
pub fn service_context(&mut self, service_name: String) -> RuntimeContext<E> {
RuntimeContext {
executor: TaskExecutor {
exit: self.exit.clone(),
signal_tx: self.signal_tx.clone(),
handle: self.runtime().handle().clone(),
log: self.log.new(o!("service" => service_name)),
},
executor: TaskExecutor::new(
self.runtime().handle().clone(),
self.exit.clone(),
self.log.new(o!("service" => service_name)),
self.signal_tx.clone(),
),
eth_spec_instance: self.eth_spec_instance.clone(),
eth2_config: self.eth2_config.clone(),
}

21
lighthouse/environment/src/metrics.rs
View File

@@ -1,21 +0,0 @@
/// Handles async task metrics
use lazy_static::lazy_static;
pub use lighthouse_metrics::*;
lazy_static! {
pub static ref ASYNC_TASKS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
"async_tasks_count",
"Total number of async tasks spawned using spawn",
&["async_task_count"]
);
pub static ref BLOCKING_TASKS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
"blocking_tasks_count",
"Total number of async tasks spawned using spawn_blocking",
&["blocking_task_count"]
);
pub static ref BLOCKING_TASKS_HISTOGRAM: Result<HistogramVec> = try_create_histogram_vec(
"blocking_tasks_histogram",
"Time taken by blocking tasks",
&["blocking_task_hist"]
);
}