add tracing metrics layer for dependency logging (#4979)

* add metrics layer

* add metrics

* simplify getting the target

* make clippy happy

* fix typos

* unify deps under workspace

* make import statement shorter, fix typos

* enable warn by default, mark flag as deprecated

* do not exit on error when initializing logging fails

* revert exit on error

* adjust bootnode logging

* use target as is by default

* make libp2p events register correctly

* adjust repilcated cli help

* turn on debug logs by default, remove deprecation warning

* suppress output (#5)

---------

Co-authored-by: Age Manning <Age@AgeManning.com>

common/logging/Cargo.toml

@@ -8,15 +8,18 @@ edition = { workspace = true }
 test_logger = [] # Print log output to stderr when running tests instead of dropping it
 
 [dependencies]
-slog = { workspace = true }
-slog-term = { workspace = true }
-tokio = { workspace = true }
-lighthouse_metrics = { workspace = true }
+chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
 lazy_static = { workspace = true }
-sloggers = { workspace = true }
-slog-async = { workspace = true }
-take_mut = "0.2.2"
+lighthouse_metrics = { workspace = true }
 parking_lot = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
+slog = { workspace = true }
+slog-async = { workspace = true }
+slog-term = { workspace = true }
+sloggers = { workspace = true }
+take_mut = "0.2.2"
+tokio = { workspace = true }
+tracing-core = { workspace = true }
+tracing-log = { workspace = true }
+tracing-subscriber = { workspace = true }

common/logging/src/lib.rs

@@ -13,7 +13,10 @@ pub const MAX_MESSAGE_WIDTH: usize = 40;
 
 pub mod async_record;
 mod sse_logging_components;
+mod tracing_metrics_layer;
+
 pub use sse_logging_components::SSELoggingComponents;
+pub use tracing_metrics_layer::MetricsLayer;
 
 /// The minimum interval between log messages indicating that a queue is full.
 const LOG_DEBOUNCE_INTERVAL: Duration = Duration::from_secs(30);

common/logging/src/tracing_metrics_layer.rs

@@ -0,0 +1,63 @@
+//! Exposes [`MetricsLayer`]: A tracing layer that registers metrics of logging events.
+
+use lighthouse_metrics as metrics;
+use tracing_log::NormalizeEvent;
+
+lazy_static! {
+    /// Count of `INFO` logs registered per enabled dependency.
+    pub static ref DEP_INFOS_TOTAL: metrics::Result<metrics::IntCounterVec> =
+        metrics::try_create_int_counter_vec(
+            "dep_info_total",
+            "Count of infos logged per enabled dependency",
+            &["target"]
+        );
+    /// Count of `WARN` logs registered per enabled dependency.
+    pub static ref DEP_WARNS_TOTAL: metrics::Result<metrics::IntCounterVec> =
+        metrics::try_create_int_counter_vec(
+            "dep_warn_total",
+            "Count of warns logged per enabled dependency",
+            &["target"]
+        );
+    /// Count of `ERROR` logs registered per enabled dependency.
+    pub static ref DEP_ERRORS_TOTAL: metrics::Result<metrics::IntCounterVec> =
+        metrics::try_create_int_counter_vec(
+            "dep_error_total",
+            "Count of errors logged per enabled dependency",
+            &["target"]
+        );
+}
+
+/// Layer that registers Prometheus metrics for `INFO`, `WARN` and `ERROR` logs emitted per dependency.
+/// Dependencies are enabled via the `RUST_LOG` env flag.
+pub struct MetricsLayer;
+
+impl<S: tracing_core::Subscriber> tracing_subscriber::layer::Layer<S> for MetricsLayer {
+    fn on_event(
+        &self,
+        event: &tracing_core::Event<'_>,
+        _ctx: tracing_subscriber::layer::Context<'_, S>,
+    ) {
+        // get the event's normalized metadata
+        // this is necessary to get the correct module path for libp2p events
+        let normalized_meta = event.normalized_metadata();
+        let meta = normalized_meta.as_ref().unwrap_or_else(|| event.metadata());
+
+        if !meta.is_event() {
+            // ignore tracing span events
+            return;
+        }
+
+        let full_target = meta.module_path().unwrap_or_else(|| meta.target());
+        let target = full_target
+            .split_once("::")
+            .map(|(name, _rest)| name)
+            .unwrap_or(full_target);
+        let target = &[target];
+        match *meta.level() {
+            tracing_core::Level::INFO => metrics::inc_counter_vec(&DEP_INFOS_TOTAL, target),
+            tracing_core::Level::WARN => metrics::inc_counter_vec(&DEP_WARNS_TOTAL, target),
+            tracing_core::Level::ERROR => metrics::inc_counter_vec(&DEP_ERRORS_TOTAL, target),
+            _ => {}
+        }
+    }
+}