mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-14 18:32:42 +00:00
Integrate tracing (#6339)
Tracing Integration
- [reference](5bbf1859e9/projects/project-ideas.md (L297))
- [x] replace slog & log with tracing throughout the codebase
- [x] implement custom crit log
- [x] make relevant changes in the formatter
- [x] replace sloggers
- [x] re-write SSE logging components
cc: @macladson @eserilev
This commit is contained in:
ThreeHrSleep
@@ -1,20 +1,20 @@
|
||||
use metrics::{inc_counter, try_create_int_counter, IntCounter, Result as MetricsResult};
|
||||
use slog::Logger;
|
||||
use slog_term::Decorator;
|
||||
use std::io::{Result, Write};
|
||||
use chrono::Local;
|
||||
use logroller::{Compression, LogRollerBuilder, Rotation, RotationSize};
|
||||
use metrics::{try_create_int_counter, IntCounter, Result as MetricsResult};
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::LazyLock;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing_appender::non_blocking::NonBlocking;
|
||||
use tracing_appender::rolling::{RollingFileAppender, Rotation};
|
||||
use tracing_logging_layer::LoggingLayer;
|
||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
|
||||
use tracing::Subscriber;
|
||||
use tracing_appender::non_blocking::{NonBlocking, WorkerGuard};
|
||||
use tracing_subscriber::layer::Context;
|
||||
use tracing_subscriber::{EnvFilter, Layer};
|
||||
|
||||
pub const MAX_MESSAGE_WIDTH: usize = 40;
|
||||
|
||||
pub mod async_record;
|
||||
pub mod macros;
|
||||
mod sse_logging_components;
|
||||
mod tracing_logging_layer;
|
||||
pub mod tracing_logging_layer;
|
||||
mod tracing_metrics_layer;
|
||||
|
||||
pub use sse_logging_components::SSELoggingComponents;
|
||||
@@ -32,169 +32,6 @@ pub static ERRORS_TOTAL: LazyLock<MetricsResult<IntCounter>> =
|
||||
pub static CRITS_TOTAL: LazyLock<MetricsResult<IntCounter>> =
|
||||
LazyLock::new(|| try_create_int_counter("crit_total", "Count of crits logged"));
|
||||
|
||||
pub struct AlignedTermDecorator<D: Decorator> {
|
||||
wrapped: D,
|
||||
message_width: usize,
|
||||
}
|
||||
|
||||
impl<D: Decorator> AlignedTermDecorator<D> {
|
||||
pub fn new(decorator: D, message_width: usize) -> Self {
|
||||
AlignedTermDecorator {
|
||||
wrapped: decorator,
|
||||
message_width,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<D: Decorator> Decorator for AlignedTermDecorator<D> {
|
||||
fn with_record<F>(
|
||||
&self,
|
||||
record: &slog::Record,
|
||||
_logger_values: &slog::OwnedKVList,
|
||||
f: F,
|
||||
) -> Result<()>
|
||||
where
|
||||
F: FnOnce(&mut dyn slog_term::RecordDecorator) -> std::io::Result<()>,
|
||||
{
|
||||
match record.level() {
|
||||
slog::Level::Info => inc_counter(&INFOS_TOTAL),
|
||||
slog::Level::Warning => inc_counter(&WARNS_TOTAL),
|
||||
slog::Level::Error => inc_counter(&ERRORS_TOTAL),
|
||||
slog::Level::Critical => inc_counter(&CRITS_TOTAL),
|
||||
_ => (),
|
||||
}
|
||||
|
||||
self.wrapped.with_record(record, _logger_values, |deco| {
|
||||
f(&mut AlignedRecordDecorator::new(deco, self.message_width))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct AlignedRecordDecorator<'a> {
|
||||
wrapped: &'a mut dyn slog_term::RecordDecorator,
|
||||
message_count: usize,
|
||||
message_active: bool,
|
||||
ignore_comma: bool,
|
||||
message_width: usize,
|
||||
}
|
||||
|
||||
impl<'a> AlignedRecordDecorator<'a> {
|
||||
fn new(
|
||||
decorator: &'a mut dyn slog_term::RecordDecorator,
|
||||
message_width: usize,
|
||||
) -> AlignedRecordDecorator<'a> {
|
||||
AlignedRecordDecorator {
|
||||
wrapped: decorator,
|
||||
message_count: 0,
|
||||
ignore_comma: false,
|
||||
message_active: false,
|
||||
message_width,
|
||||
}
|
||||
}
|
||||
|
||||
fn filtered_write(&mut self, buf: &[u8]) -> Result<usize> {
|
||||
if self.ignore_comma {
|
||||
//don't write comma
|
||||
self.ignore_comma = false;
|
||||
Ok(buf.len())
|
||||
} else if self.message_active {
|
||||
self.wrapped.write(buf).inspect(|n| self.message_count += n)
|
||||
} else {
|
||||
self.wrapped.write(buf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for AlignedRecordDecorator<'_> {
|
||||
fn write(&mut self, buf: &[u8]) -> Result<usize> {
|
||||
if buf.iter().any(u8::is_ascii_control) {
|
||||
let filtered = buf
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|c| if !is_ascii_control(&c) { c } else { b'_' })
|
||||
.collect::<Vec<u8>>();
|
||||
self.filtered_write(&filtered)
|
||||
} else {
|
||||
self.filtered_write(buf)
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
self.wrapped.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl slog_term::RecordDecorator for AlignedRecordDecorator<'_> {
|
||||
fn reset(&mut self) -> Result<()> {
|
||||
self.message_active = false;
|
||||
self.message_count = 0;
|
||||
self.ignore_comma = false;
|
||||
self.wrapped.reset()
|
||||
}
|
||||
|
||||
fn start_whitespace(&mut self) -> Result<()> {
|
||||
self.wrapped.start_whitespace()
|
||||
}
|
||||
|
||||
fn start_msg(&mut self) -> Result<()> {
|
||||
self.message_active = true;
|
||||
self.ignore_comma = false;
|
||||
self.wrapped.start_msg()
|
||||
}
|
||||
|
||||
fn start_timestamp(&mut self) -> Result<()> {
|
||||
self.wrapped.start_timestamp()
|
||||
}
|
||||
|
||||
fn start_level(&mut self) -> Result<()> {
|
||||
self.wrapped.start_level()
|
||||
}
|
||||
|
||||
fn start_comma(&mut self) -> Result<()> {
|
||||
if self.message_active && self.message_count + 1 < self.message_width {
|
||||
self.ignore_comma = true;
|
||||
}
|
||||
self.wrapped.start_comma()
|
||||
}
|
||||
|
||||
fn start_key(&mut self) -> Result<()> {
|
||||
if self.message_active && self.message_count + 1 < self.message_width {
|
||||
write!(
|
||||
self,
|
||||
"{}",
|
||||
" ".repeat(self.message_width - self.message_count)
|
||||
)?;
|
||||
self.message_active = false;
|
||||
self.message_count = 0;
|
||||
self.ignore_comma = false;
|
||||
}
|
||||
self.wrapped.start_key()
|
||||
}
|
||||
|
||||
fn start_value(&mut self) -> Result<()> {
|
||||
self.wrapped.start_value()
|
||||
}
|
||||
|
||||
fn start_separator(&mut self) -> Result<()> {
|
||||
self.wrapped.start_separator()
|
||||
}
|
||||
}
|
||||
|
||||
/// Function to filter out ascii control codes.
|
||||
///
|
||||
/// This helps to keep log formatting consistent.
|
||||
/// Whitespace and padding control codes are excluded.
|
||||
fn is_ascii_control(character: &u8) -> bool {
|
||||
matches!(
|
||||
character,
|
||||
b'\x00'..=b'\x08' |
|
||||
b'\x0b'..=b'\x0c' |
|
||||
b'\x0e'..=b'\x1f' |
|
||||
b'\x7f' |
|
||||
b'\x81'..=b'\x9f'
|
||||
)
|
||||
}
|
||||
|
||||
/// Provides de-bounce functionality for logging.
|
||||
#[derive(Default)]
|
||||
pub struct TimeLatch(Option<Instant>);
|
||||
@@ -214,75 +51,127 @@ impl TimeLatch {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_tracing_layer(base_tracing_log_path: PathBuf) {
|
||||
let mut tracing_log_path = PathBuf::new();
|
||||
pub struct Libp2pDiscv5TracingLayer {
|
||||
pub libp2p_non_blocking_writer: NonBlocking,
|
||||
pub _libp2p_guard: WorkerGuard,
|
||||
pub discv5_non_blocking_writer: NonBlocking,
|
||||
pub _discv5_guard: WorkerGuard,
|
||||
}
|
||||
|
||||
// Ensure that `tracing_log_path` only contains directories.
|
||||
for p in base_tracing_log_path.iter() {
|
||||
tracing_log_path = tracing_log_path.join(p);
|
||||
if let Ok(metadata) = tracing_log_path.metadata() {
|
||||
if !metadata.is_dir() {
|
||||
tracing_log_path.pop();
|
||||
break;
|
||||
}
|
||||
impl<S> Layer<S> for Libp2pDiscv5TracingLayer
|
||||
where
|
||||
S: Subscriber,
|
||||
{
|
||||
fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<S>) {
|
||||
let meta = event.metadata();
|
||||
let log_level = meta.level();
|
||||
let timestamp = Local::now().format("%Y-%m-%d %H:%M:%S").to_string();
|
||||
|
||||
let target = match meta.target().split_once("::") {
|
||||
Some((crate_name, _)) => crate_name,
|
||||
None => "unknown",
|
||||
};
|
||||
|
||||
let mut writer = match target {
|
||||
"gossipsub" => self.libp2p_non_blocking_writer.clone(),
|
||||
"discv5" => self.discv5_non_blocking_writer.clone(),
|
||||
_ => return,
|
||||
};
|
||||
|
||||
let mut visitor = LogMessageExtractor {
|
||||
message: String::default(),
|
||||
};
|
||||
|
||||
event.record(&mut visitor);
|
||||
let message = format!("{} {} {}\n", timestamp, log_level, visitor.message);
|
||||
|
||||
if let Err(e) = writer.write_all(message.as_bytes()) {
|
||||
eprintln!("Failed to write log: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
let filter_layer = match tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.or_else(|_| tracing_subscriber::EnvFilter::try_new("warn"))
|
||||
{
|
||||
Ok(filter) => filter,
|
||||
Err(e) => {
|
||||
eprintln!("Failed to initialize dependency logging {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let Ok(libp2p_writer) = RollingFileAppender::builder()
|
||||
.rotation(Rotation::DAILY)
|
||||
.max_log_files(2)
|
||||
.filename_prefix("libp2p")
|
||||
.filename_suffix("log")
|
||||
.build(tracing_log_path.clone())
|
||||
else {
|
||||
eprintln!("Failed to initialize libp2p rolling file appender");
|
||||
return;
|
||||
};
|
||||
|
||||
let Ok(discv5_writer) = RollingFileAppender::builder()
|
||||
.rotation(Rotation::DAILY)
|
||||
.max_log_files(2)
|
||||
.filename_prefix("discv5")
|
||||
.filename_suffix("log")
|
||||
.build(tracing_log_path)
|
||||
else {
|
||||
eprintln!("Failed to initialize discv5 rolling file appender");
|
||||
return;
|
||||
};
|
||||
|
||||
let (libp2p_non_blocking_writer, _libp2p_guard) = NonBlocking::new(libp2p_writer);
|
||||
let (discv5_non_blocking_writer, _discv5_guard) = NonBlocking::new(discv5_writer);
|
||||
|
||||
let custom_layer = LoggingLayer {
|
||||
libp2p_non_blocking_writer,
|
||||
_libp2p_guard,
|
||||
discv5_non_blocking_writer,
|
||||
_discv5_guard,
|
||||
};
|
||||
|
||||
if let Err(e) = tracing_subscriber::fmt()
|
||||
.with_env_filter(filter_layer)
|
||||
.with_writer(std::io::sink)
|
||||
.finish()
|
||||
.with(MetricsLayer)
|
||||
.with(custom_layer)
|
||||
.try_init()
|
||||
{
|
||||
eprintln!("Failed to initialize dependency logging {e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a logger suitable for test usage.
|
||||
struct LogMessageExtractor {
|
||||
message: String,
|
||||
}
|
||||
|
||||
impl tracing_core::field::Visit for LogMessageExtractor {
|
||||
fn record_debug(&mut self, _: &tracing_core::Field, value: &dyn std::fmt::Debug) {
|
||||
self.message = format!("{} {:?}", self.message, value);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_libp2p_discv5_tracing_layer(
|
||||
base_tracing_log_path: Option<PathBuf>,
|
||||
max_log_size: u64,
|
||||
compression: bool,
|
||||
max_log_number: usize,
|
||||
) -> Libp2pDiscv5TracingLayer {
|
||||
if let Some(mut tracing_log_path) = base_tracing_log_path {
|
||||
// Ensure that `tracing_log_path` only contains directories.
|
||||
for p in tracing_log_path.clone().iter() {
|
||||
tracing_log_path = tracing_log_path.join(p);
|
||||
if let Ok(metadata) = tracing_log_path.metadata() {
|
||||
if !metadata.is_dir() {
|
||||
tracing_log_path.pop();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut libp2p_writer =
|
||||
LogRollerBuilder::new(tracing_log_path.clone(), PathBuf::from("libp2p.log"))
|
||||
.rotation(Rotation::SizeBased(RotationSize::MB(max_log_size)))
|
||||
.max_keep_files(max_log_number.try_into().unwrap_or_else(|e| {
|
||||
eprintln!("Failed to convert max_log_number to u64: {}", e);
|
||||
10
|
||||
}));
|
||||
|
||||
let mut discv5_writer =
|
||||
LogRollerBuilder::new(tracing_log_path.clone(), PathBuf::from("discv5.log"))
|
||||
.rotation(Rotation::SizeBased(RotationSize::MB(max_log_size)))
|
||||
.max_keep_files(max_log_number.try_into().unwrap_or_else(|e| {
|
||||
eprintln!("Failed to convert max_log_number to u64: {}", e);
|
||||
10
|
||||
}));
|
||||
|
||||
if compression {
|
||||
libp2p_writer = libp2p_writer.compression(Compression::Gzip);
|
||||
discv5_writer = discv5_writer.compression(Compression::Gzip);
|
||||
}
|
||||
|
||||
let Ok(libp2p_writer) = libp2p_writer.build() else {
|
||||
eprintln!("Failed to initialize libp2p rolling file appender");
|
||||
std::process::exit(1);
|
||||
};
|
||||
|
||||
let Ok(discv5_writer) = discv5_writer.build() else {
|
||||
eprintln!("Failed to initialize discv5 rolling file appender");
|
||||
std::process::exit(1);
|
||||
};
|
||||
|
||||
let (libp2p_non_blocking_writer, _libp2p_guard) = NonBlocking::new(libp2p_writer);
|
||||
let (discv5_non_blocking_writer, _discv5_guard) = NonBlocking::new(discv5_writer);
|
||||
|
||||
Libp2pDiscv5TracingLayer {
|
||||
libp2p_non_blocking_writer,
|
||||
_libp2p_guard,
|
||||
discv5_non_blocking_writer,
|
||||
_discv5_guard,
|
||||
}
|
||||
} else {
|
||||
let (libp2p_non_blocking_writer, _libp2p_guard) = NonBlocking::new(std::io::sink());
|
||||
let (discv5_non_blocking_writer, _discv5_guard) = NonBlocking::new(std::io::sink());
|
||||
Libp2pDiscv5TracingLayer {
|
||||
libp2p_non_blocking_writer,
|
||||
_libp2p_guard,
|
||||
discv5_non_blocking_writer,
|
||||
_discv5_guard,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a tracing subscriber suitable for test usage.
|
||||
///
|
||||
/// By default no logs will be printed, but they can be enabled via
|
||||
/// the `test_logger` feature. This feature can be enabled for any
|
||||
@@ -290,17 +179,10 @@ pub fn create_tracing_layer(base_tracing_log_path: PathBuf) {
|
||||
/// ```bash
|
||||
/// cargo test -p beacon_chain --features logging/test_logger
|
||||
/// ```
|
||||
pub fn test_logger() -> Logger {
|
||||
use sloggers::Build;
|
||||
|
||||
pub fn create_test_tracing_subscriber() {
|
||||
if cfg!(feature = "test_logger") {
|
||||
sloggers::terminal::TerminalLoggerBuilder::new()
|
||||
.level(sloggers::types::Severity::Debug)
|
||||
.build()
|
||||
.expect("Should build TerminalLoggerBuilder")
|
||||
} else {
|
||||
sloggers::null::NullLoggerBuilder
|
||||
.build()
|
||||
.expect("Should build NullLoggerBuilder")
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(EnvFilter::try_new("debug").unwrap())
|
||||
.try_init();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user