gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-21 23:08:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Logging via the HTTP API (#4074)

Browse Source 
This PR adds the ability to read the Lighthouse logs from the HTTP API for both the BN and the VC. 

This is done in such a way to as minimize any kind of performance hit by adding this feature.

The current design creates a tokio broadcast channel and mixes is into a form of slog drain that combines with our main global logger drain, only if the http api is enabled. 

The drain gets the logs, checks the log level and drops them if they are below INFO. If they are INFO or higher, it sends them via a broadcast channel only if there are users subscribed to the HTTP API channel. If not, it drops the logs. 

If there are more than one subscriber, the channel clones the log records and converts them to json in their independent HTTP API tasks. 

Co-authored-by: Michael Sproul <micsproul@gmail.com>
This commit is contained in:
Age Manning
2023-05-22 05:57:08 +00:00
parent c27f2bf9c6
commit aa1ed787e9
23 changed files with 1040 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
lighthouse/environment/src/lib.rs
View File

@@ -12,6 +12,7 @@ use eth2_network_config::Eth2NetworkConfig;
use futures::channel::mpsc::{channel, Receiver, Sender};
use futures::{future, StreamExt};
use logging::SSELoggingComponents;
use serde_derive::{Deserialize, Serialize};
use slog::{error, info, o, warn, Drain, Duplicate, Level, Logger};
use sloggers::{file::FileLoggerBuilder, types::Format, types::Severity, Build};
@@ -36,6 +37,7 @@ use {futures::channel::oneshot, std::cell::RefCell};
pub use task_executor::test_utils::null_logger;
const LOG_CHANNEL_SIZE: usize = 2048;
const SSE_LOG_CHANNEL_SIZE: usize = 2048;
/// The maximum time in seconds the client will wait for all internal tasks to shutdown.
const MAXIMUM_SHUTDOWN_TIME: u64 = 15;
@@ -57,6 +59,7 @@ pub struct LoggerConfig {
pub max_log_number: usize,
pub compression: bool,
pub is_restricted: bool,
pub sse_logging: bool,
}
impl Default for LoggerConfig {
fn default() -> Self {
@@ -72,14 +75,54 @@ impl Default for LoggerConfig {
max_log_number: 5,
compression: false,
is_restricted: true,
sse_logging: false,
}
}
}
/// An execution context that can be used by a service.
///
/// Distinct from an `Environment` because a `Context` is not able to give a mutable reference to a
/// `Runtime`, instead it only has access to a `Runtime`.
#[derive(Clone)]
pub struct RuntimeContext<E: EthSpec> {
pub executor: TaskExecutor,
pub eth_spec_instance: E,
pub eth2_config: Eth2Config,
pub eth2_network_config: Option<Arc<Eth2NetworkConfig>>,
pub sse_logging_components: Option<SSELoggingComponents>,
}
impl<E: EthSpec> RuntimeContext<E> {
/// Returns a sub-context of this context.
///
/// The generated service will have the `service_name` in all it's logs.
pub fn service_context(&self, service_name: String) -> Self {
Self {
executor: self.executor.clone_with_name(service_name),
eth_spec_instance: self.eth_spec_instance.clone(),
eth2_config: self.eth2_config.clone(),
eth2_network_config: self.eth2_network_config.clone(),
sse_logging_components: self.sse_logging_components.clone(),
}
}
/// Returns the `eth2_config` for this service.
pub fn eth2_config(&self) -> &Eth2Config {
&self.eth2_config
}
/// Returns a reference to the logger for this service.
pub fn log(&self) -> &slog::Logger {
self.executor.log()
}
}
/// Builds an `Environment`.
pub struct EnvironmentBuilder<E: EthSpec> {
runtime: Option<Arc<Runtime>>,
log: Option<Logger>,
sse_logging_components: Option<SSELoggingComponents>,
eth_spec_instance: E,
eth2_config: Eth2Config,
eth2_network_config: Option<Eth2NetworkConfig>,
@@ -91,6 +134,7 @@ impl EnvironmentBuilder<MinimalEthSpec> {
Self {
runtime: None,
log: None,
sse_logging_components: None,
eth_spec_instance: MinimalEthSpec,
eth2_config: Eth2Config::minimal(),
eth2_network_config: None,
@@ -104,6 +148,7 @@ impl EnvironmentBuilder<MainnetEthSpec> {
Self {
runtime: None,
log: None,
sse_logging_components: None,
eth_spec_instance: MainnetEthSpec,
eth2_config: Eth2Config::mainnet(),
eth2_network_config: None,
@@ -117,6 +162,7 @@ impl EnvironmentBuilder<GnosisEthSpec> {
Self {
runtime: None,
log: None,
sse_logging_components: None,
eth_spec_instance: GnosisEthSpec,
eth2_config: Eth2Config::gnosis(),
eth2_network_config: None,
@@ -265,7 +311,7 @@ impl<E: EthSpec> EnvironmentBuilder<E> {
.build()
.map_err(|e| format!("Unable to build file logger: {}", e))?;
let log = Logger::root(Duplicate::new(stdout_logger, file_logger).fuse(), o!());
let mut log = Logger::root(Duplicate::new(stdout_logger, file_logger).fuse(), o!());
info!(
log,
@@ -273,6 +319,14 @@ impl<E: EthSpec> EnvironmentBuilder<E> {
"path" => format!("{:?}", path)
);
// If the http API is enabled, we may need to send logs to be consumed by subscribers.
if config.sse_logging {
let sse_logger = SSELoggingComponents::new(SSE_LOG_CHANNEL_SIZE);
self.sse_logging_components = Some(sse_logger.clone());
log = Logger::root(Duplicate::new(log, sse_logger).fuse(), o!());
}
self.log = Some(log);
Ok(self)
@@ -315,6 +369,7 @@ impl<E: EthSpec> EnvironmentBuilder<E> {
signal: Some(signal),
exit,
log: self.log.ok_or("Cannot build environment without log")?,
sse_logging_components: self.sse_logging_components,
eth_spec_instance: self.eth_spec_instance,
eth2_config: self.eth2_config,
eth2_network_config: self.eth2_network_config.map(Arc::new),
@@ -322,42 +377,6 @@ impl<E: EthSpec> EnvironmentBuilder<E> {
}
}
/// An execution context that can be used by a service.
///
/// Distinct from an `Environment` because a `Context` is not able to give a mutable reference to a
/// `Runtime`, instead it only has access to a `Runtime`.
#[derive(Clone)]
pub struct RuntimeContext<E: EthSpec> {
pub executor: TaskExecutor,
pub eth_spec_instance: E,
pub eth2_config: Eth2Config,
pub eth2_network_config: Option<Arc<Eth2NetworkConfig>>,
}
impl<E: EthSpec> RuntimeContext<E> {
/// Returns a sub-context of this context.
///
/// The generated service will have the `service_name` in all it's logs.
pub fn service_context(&self, service_name: String) -> Self {
Self {
executor: self.executor.clone_with_name(service_name),
eth_spec_instance: self.eth_spec_instance.clone(),
eth2_config: self.eth2_config.clone(),
eth2_network_config: self.eth2_network_config.clone(),
}
}
/// Returns the `eth2_config` for this service.
pub fn eth2_config(&self) -> &Eth2Config {
&self.eth2_config
}
/// Returns a reference to the logger for this service.
pub fn log(&self) -> &slog::Logger {
self.executor.log()
}
}
/// An environment where Lighthouse services can run. Used to start a production beacon node or
/// validator client, or to run tests that involve logging and async task execution.
pub struct Environment<E: EthSpec> {
@@ -369,6 +388,7 @@ pub struct Environment<E: EthSpec> {
signal: Option<exit_future::Signal>,
exit: exit_future::Exit,
log: Logger,
sse_logging_components: Option<SSELoggingComponents>,
eth_spec_instance: E,
pub eth2_config: Eth2Config,
pub eth2_network_config: Option<Arc<Eth2NetworkConfig>>,
@@ -395,6 +415,7 @@ impl<E: EthSpec> Environment<E> {
eth_spec_instance: self.eth_spec_instance.clone(),
eth2_config: self.eth2_config.clone(),
eth2_network_config: self.eth2_network_config.clone(),
sse_logging_components: self.sse_logging_components.clone(),
}
}
@@ -410,6 +431,7 @@ impl<E: EthSpec> Environment<E> {
eth_spec_instance: self.eth_spec_instance.clone(),
eth2_config: self.eth2_config.clone(),
eth2_network_config: self.eth2_network_config.clone(),
sse_logging_components: self.sse_logging_components.clone(),
}
}