Implement custom OpenTelemetry sampler to filter uninstrumented traces (#8647)

Co-Authored-By: Jimmy Chen <jchen.tc@gmail.com>

lighthouse/Cargo.toml

@@ -53,7 +53,6 @@ environment = { workspace = true }
 eth2_network_config = { workspace = true }
 ethereum_hashing = { workspace = true }
 futures = { workspace = true }
-lighthouse_tracing = { workspace = true }
 lighthouse_version = { workspace = true }
 logging = { workspace = true }
 metrics = { workspace = true }
@@ -70,6 +69,7 @@ task_executor = { workspace = true }
 tracing = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 tracing-subscriber = { workspace = true }
+tracing_samplers = { workspace = true }
 types = { workspace = true }
 validator_client = { workspace = true }
 validator_manager = { path = "../validator_manager" }

lighthouse/src/main.rs

@@ -29,6 +29,7 @@ use std::process::exit;
 use std::sync::LazyLock;
 use task_executor::ShutdownReason;
 use tracing::{Level, info};
+use tracing_samplers::PrefixBasedSampler;
 use tracing_subscriber::{Layer, filter::EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
 use types::{EthSpec, EthSpecId};
 use validator_client::ProductionValidatorClient;
@@ -675,7 +676,15 @@ fn run<E: EthSpec>(
                     _ => "lighthouse".to_string(),
                 });
 
+            // Use ParentBased sampler with PrefixBasedSampler to only export traces
+            // that start from known instrumented code paths (lh_ prefix). Child spans
+            // automatically inherit their parent's sampling decision.
+            let sampler = opentelemetry_sdk::trace::Sampler::ParentBased(Box::new(
+                PrefixBasedSampler::new("lh_"),
+            ));
+
             let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
+                .with_sampler(sampler)
                 .with_batch_exporter(exporter)
                 .with_resource(
                     opentelemetry_sdk::Resource::builder()