Integrate tracing (#6339)

Tracing Integration - [reference](5bbf1859e9/projects/project-ideas.md (L297)) - [x] replace slog & log with tracing throughout the codebase - [x] implement custom crit log - [x] make relevant changes in the formatter - [x] replace sloggers - [x] re-write SSE logging components cc: @macladson @eserilev
2026-03-09 19:51:47 +00:00 · 2025-03-13 04:01:05 +05:30
parent f23f984f85
commit d60c24ef1c
241 changed files with 9485 additions and 9328 deletions
--- a/beacon_node/http_api/src/publish_blocks.rs
+++ b/beacon_node/http_api/src/publish_blocks.rs
@@ -18,13 +18,13 @@ use futures::TryFutureExt;
 use lighthouse_network::{NetworkGlobals, PubsubMessage};
 use network::NetworkMessage;
 use rand::prelude::SliceRandom;
-use slog::{debug, error, info, warn, Logger};
 use slot_clock::SlotClock;
 use std::marker::PhantomData;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::mpsc::UnboundedSender;
+use tracing::{debug, error, info, warn};
 use tree_hash::TreeHash;
 use types::{
    AbstractExecPayload, BeaconBlockRef, BlobSidecar, BlobsList, BlockImportSource,
@@ -80,7 +80,6 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
    provenanced_block: ProvenancedBlock<T, B>,
    chain: Arc<BeaconChain<T>>,
    network_tx: &UnboundedSender<NetworkMessage<T::EthSpec>>,
-    log: Logger,
    validation_level: BroadcastValidation,
    duplicate_status_code: StatusCode,
    network_globals: Arc<NetworkGlobals<T::EthSpec>>,
@@ -99,12 +98,12 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
        "builder"
    };
    let block = unverified_block.inner_block();
-    debug!(log, "Signed block received in HTTP API"; "slot" => block.slot());
+
+    debug!(slot = %block.slot(), "Signed block received in HTTP API");

    /* actually publish a block */
    let publish_block_p2p = move |block: Arc<SignedBeaconBlock<T::EthSpec>>,
                                  sender,
-                                  log,
                                  seen_timestamp|
          -> Result<(), BlockError> {
        let publish_timestamp = timestamp_now();
@@ -119,10 +118,9 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
        );

        info!(
-            log,
-            "Signed block published to network via HTTP API";
-            "slot" => block.slot(),
-            "publish_delay_ms" => publish_delay.as_millis(),
+            slot = %block.slot(),
+            publish_delay_ms = publish_delay.as_millis(),
+            "Signed block published to network via HTTP API"
        );

        crate::publish_pubsub_message(&sender, PubsubMessage::BeaconBlock(block.clone()))
@@ -136,7 +134,7 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
    let sender_clone = network_tx.clone();

    let build_sidecar_task_handle =
-        spawn_build_data_sidecar_task(chain.clone(), block.clone(), unverified_blobs, log.clone())?;
+        spawn_build_data_sidecar_task(chain.clone(), block.clone(), unverified_blobs)?;

    // Gossip verify the block and blobs/data columns separately.
    let gossip_verified_block_result = unverified_block.into_gossip_verified_block(&chain);
@@ -151,19 +149,13 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
    if BroadcastValidation::Gossip == validation_level && should_publish_block {
        if let Some(block_publishing_delay) = block_publishing_delay_for_testing {
            debug!(
-                log,
-                "Publishing block with artificial delay";
-                "block_publishing_delay" => ?block_publishing_delay
+                ?block_publishing_delay,
+                "Publishing block with artificial delay"
            );
            tokio::time::sleep(block_publishing_delay).await;
        }
-        publish_block_p2p(
-            block.clone(),
-            sender_clone.clone(),
-            log.clone(),
-            seen_timestamp,
-        )
-        .map_err(|_| warp_utils::reject::custom_server_error("unable to publish".into()))?;
+        publish_block_p2p(block.clone(), sender_clone.clone(), seen_timestamp)
+            .map_err(|_| warp_utils::reject::custom_server_error("unable to publish".into()))?;
    }

    let publish_fn_completed = Arc::new(AtomicBool::new(false));
@@ -175,15 +167,13 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
                BroadcastValidation::Consensus => publish_block_p2p(
                    block_to_publish.clone(),
                    sender_clone.clone(),
-                    log.clone(),
                    seen_timestamp,
                )?,
                BroadcastValidation::ConsensusAndEquivocation => {
-                    check_slashable(&chain, block_root, &block_to_publish, &log)?;
+                    check_slashable(&chain, block_root, &block_to_publish)?;
                    publish_block_p2p(
                        block_to_publish.clone(),
                        sender_clone.clone(),
-                        log.clone(),
                        seen_timestamp,
                    )?;
                }
@@ -206,11 +196,7 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
            return if let BroadcastValidation::Gossip = validation_level {
                Err(warp_utils::reject::broadcast_without_import(msg))
            } else {
-                error!(
-                    log,
-                    "Invalid blob provided to HTTP API";
-                    "reason" => &msg
-                );
+                error!(reason = &msg, "Invalid blob provided to HTTP API");
                Err(warp_utils::reject::custom_bad_request(msg))
            };
        }
@@ -227,9 +213,8 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
            let delay = data_column_publishing_delay.saturating_sub(block_publishing_delay);
            if !delay.is_zero() {
                debug!(
-                    log,
-                    "Publishing data columns with artificial delay";
-                    "data_column_publishing_delay" => ?data_column_publishing_delay
+                    ?data_column_publishing_delay,
+                    "Publishing data columns with artificial delay"
                );
                tokio::time::sleep(delay).await;
            }
@@ -254,9 +239,8 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
                Err(warp_utils::reject::broadcast_without_import(msg))
            } else {
                error!(
-                    log,
-                    "Invalid data column during block publication";
-                    "reason" => &msg
+                    reason = &msg,
+                    "Invalid data column during block publication"
                );
                Err(warp_utils::reject::custom_bad_request(msg))
            };
@@ -280,7 +264,6 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
                is_locally_built_block,
                seen_timestamp,
                &chain,
-                &log,
            )
            .await
        }
@@ -293,7 +276,6 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
                    is_locally_built_block,
                    seen_timestamp,
                    &chain,
-                    &log,
                )
                .await
            } else {
@@ -313,10 +295,9 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
        }
        Err(BlockError::DuplicateImportStatusUnknown(root)) => {
            debug!(
-                log,
-                "Block previously seen";
-                "block_root" => ?root,
-                "slot" => block.slot(),
+                block_root = ?root,
+                slot = %block.slot(),
+                "Block previously seen"
            );
            let import_result = Box::pin(chain.process_block(
                block_root,
@@ -333,16 +314,14 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
                is_locally_built_block,
                seen_timestamp,
                &chain,
-                &log,
            )
            .await
        }
        Err(e) => {
            warn!(
-                log,
-                "Not publishing block - not gossip verified";
-                "slot" => slot,
-                "error" => %e
+                %slot,
+                error = %e,
+                "Not publishing block - not gossip verified"
            );
            Err(warp_utils::reject::custom_bad_request(e.to_string()))
        }
@@ -365,7 +344,6 @@ fn spawn_build_data_sidecar_task<T: BeaconChainTypes>(
    chain: Arc<BeaconChain<T>>,
    block: Arc<SignedBeaconBlock<T::EthSpec, FullPayload<T::EthSpec>>>,
    proofs_and_blobs: UnverifiedBlobs<T>,
-    log: Logger,
 ) -> Result<impl Future<Output = BuildDataSidecarTaskResult<T>>, Rejection> {
    chain
        .clone()
@@ -380,12 +358,12 @@ fn spawn_build_data_sidecar_task<T: BeaconChainTypes>(
                if !peer_das_enabled {
                    // Pre-PeerDAS: construct blob sidecars for the network.
                    let gossip_verified_blobs =
-                        build_gossip_verified_blobs(&chain, &block, blobs, kzg_proofs, &log)?;
+                        build_gossip_verified_blobs(&chain, &block, blobs, kzg_proofs)?;
                    Ok((gossip_verified_blobs, vec![]))
                } else {
                    // Post PeerDAS: construct data columns.
                    let gossip_verified_data_columns =
-                        build_gossip_verified_data_columns(&chain, &block, blobs, &log)?;
+                        build_gossip_verified_data_columns(&chain, &block, blobs)?;
                    Ok((vec![], gossip_verified_data_columns))
                }
            },
@@ -404,16 +382,14 @@ fn build_gossip_verified_data_columns<T: BeaconChainTypes>(
    chain: &BeaconChain<T>,
    block: &SignedBeaconBlock<T::EthSpec, FullPayload<T::EthSpec>>,
    blobs: BlobsList<T::EthSpec>,
-    log: &Logger,
 ) -> Result<Vec<Option<GossipVerifiedDataColumn<T>>>, Rejection> {
    let slot = block.slot();
    let data_column_sidecars =
        build_blob_data_column_sidecars(chain, block, blobs).map_err(|e| {
            error!(
-                log,
-                "Invalid data column - not publishing block";
-                "error" => ?e,
-                "slot" => slot
+                error = ?e,
+                %slot,
+                "Invalid data column - not publishing block"
            );
            warp_utils::reject::custom_bad_request(format!("{e:?}"))
        })?;
@@ -434,21 +410,19 @@ fn build_gossip_verified_data_columns<T: BeaconChainTypes>(
                    // or some of the other data columns if the block & data columns are only
                    // partially published by the other publisher.
                    debug!(
-                        log,
-                        "Data column for publication already known";
-                        "column_index" => column_index,
-                        "slot" => slot,
-                        "proposer" => proposer,
+                        column_index,
+                        %slot,
+                        proposer,
+                        "Data column for publication already known"
                    );
                    Ok(None)
                }
                Err(e) => {
                    error!(
-                        log,
-                        "Data column for publication is gossip-invalid";
-                        "column_index" => column_index,
-                        "slot" => slot,
-                        "error" => ?e,
+                        column_index,
+                        %slot,
+                        error = ?e,
+                        "Data column for publication is gossip-invalid"
                    );
                    Err(warp_utils::reject::custom_bad_request(format!("{e:?}")))
                }
@@ -464,7 +438,6 @@ fn build_gossip_verified_blobs<T: BeaconChainTypes>(
    block: &SignedBeaconBlock<T::EthSpec, FullPayload<T::EthSpec>>,
    blobs: BlobsList<T::EthSpec>,
    kzg_proofs: KzgProofs<T::EthSpec>,
-    log: &Logger,
 ) -> Result<Vec<Option<GossipVerifiedBlob<T>>>, Rejection> {
    let slot = block.slot();
    let gossip_verified_blobs = kzg_proofs
@@ -479,11 +452,10 @@ fn build_gossip_verified_blobs<T: BeaconChainTypes>(
                .map(Arc::new)
                .map_err(|e| {
                    error!(
-                        log,
-                        "Invalid blob - not publishing block";
-                        "error" => ?e,
-                        "blob_index" => i,
-                        "slot" => slot,
+                        error = ?e,
+                        blob_index = i,
+                        %slot,
+                        "Invalid blob - not publishing block"
                    );
                    warp_utils::reject::custom_bad_request(format!("{e:?}"))
                })?;
@@ -499,21 +471,19 @@ fn build_gossip_verified_blobs<T: BeaconChainTypes>(
                    // or some of the other blobs if the block & blobs are only partially published
                    // by the other publisher.
                    debug!(
-                        log,
-                        "Blob for publication already known";
-                        "blob_index" => blob_sidecar.index,
-                        "slot" => slot,
-                        "proposer" => proposer,
+                        blob_index = blob_sidecar.index,
+                        %slot,
+                        proposer,
+                        "Blob for publication already known"
                    );
                    Ok(None)
                }
                Err(e) => {
                    error!(
-                        log,
-                        "Blob for publication is gossip-invalid";
-                        "blob_index" => blob_sidecar.index,
-                        "slot" => slot,
-                        "error" => ?e,
+                        blob_index = blob_sidecar.index,
+                        %slot,
+                        error = ?e,
+                        "Blob for publication is gossip-invalid"
                    );
                    Err(warp_utils::reject::custom_bad_request(e.to_string()))
                }
@@ -524,6 +494,15 @@ fn build_gossip_verified_blobs<T: BeaconChainTypes>(
    Ok(gossip_verified_blobs)
 }

+fn publish_blob_sidecars<T: BeaconChainTypes>(
+    sender_clone: &UnboundedSender<NetworkMessage<T::EthSpec>>,
+    blob: &GossipVerifiedBlob<T>,
+) -> Result<(), BlockError> {
+    let pubsub_message = PubsubMessage::BlobSidecar(Box::new((blob.index(), blob.clone_blob())));
+    crate::publish_pubsub_message(sender_clone, pubsub_message)
+        .map_err(|_| BlockError::BeaconChainError(BeaconChainError::UnableToPublish))
+}
+
 fn publish_column_sidecars<T: BeaconChainTypes>(
    sender_clone: &UnboundedSender<NetworkMessage<T::EthSpec>>,
    data_column_sidecars: &[Option<GossipVerifiedDataColumn<T>>],
@@ -554,15 +533,6 @@ fn publish_column_sidecars<T: BeaconChainTypes>(
        .map_err(|_| BlockError::BeaconChainError(BeaconChainError::UnableToPublish))
 }

-fn publish_blob_sidecars<T: BeaconChainTypes>(
-    sender_clone: &UnboundedSender<NetworkMessage<T::EthSpec>>,
-    blob: &GossipVerifiedBlob<T>,
-) -> Result<(), BlockError> {
-    let pubsub_message = PubsubMessage::BlobSidecar(Box::new((blob.index(), blob.clone_blob())));
-    crate::publish_pubsub_message(sender_clone, pubsub_message)
-        .map_err(|_| BlockError::BeaconChainError(BeaconChainError::UnableToPublish))
-}
-
 async fn post_block_import_logging_and_response<T: BeaconChainTypes>(
    result: Result<AvailabilityProcessingStatus, BlockError>,
    validation_level: BroadcastValidation,
@@ -570,7 +540,6 @@ async fn post_block_import_logging_and_response<T: BeaconChainTypes>(
    is_locally_built_block: bool,
    seen_timestamp: Duration,
    chain: &Arc<BeaconChain<T>>,
-    log: &Logger,
 ) -> Result<Response, Rejection> {
    match result {
        // The `DuplicateFullyImported` case here captures the case where the block finishes
@@ -582,12 +551,11 @@ async fn post_block_import_logging_and_response<T: BeaconChainTypes>(
        | Err(BlockError::DuplicateFullyImported(root)) => {
            let delay = get_block_delay_ms(seen_timestamp, block.message(), &chain.slot_clock);
            info!(
-                log,
-                "Valid block from HTTP API";
-                "block_delay" => ?delay,
-                "root" => %root,
-                "proposer_index" => block.message().proposer_index(),
-                "slot" => block.slot(),
+                block_delay = ?delay,
+                root = %root,
+                proposer_index = block.message().proposer_index(),
+                slot = %block.slot(),
+                "Valid block from HTTP API"
            );

            // Notify the validator monitor.
@@ -606,7 +574,7 @@ async fn post_block_import_logging_and_response<T: BeaconChainTypes>(
            // blocks built with builders we consider the broadcast time to be
            // when the blinded block is published to the builder.
            if is_locally_built_block {
-                late_block_logging(chain, seen_timestamp, block.message(), root, "local", log)
+                late_block_logging(chain, seen_timestamp, block.message(), root, "local")
            }
            Ok(warp::reply().into_response())
        }
@@ -615,11 +583,7 @@ async fn post_block_import_logging_and_response<T: BeaconChainTypes>(
            if let BroadcastValidation::Gossip = validation_level {
                Err(warp_utils::reject::broadcast_without_import(msg))
            } else {
-                error!(
-                    log,
-                    "Invalid block provided to HTTP API";
-                    "reason" => &msg
-                );
+                error!(reason = &msg, "Invalid block provided to HTTP API");
                Err(warp_utils::reject::custom_bad_request(msg))
            }
        }
@@ -636,9 +600,8 @@ async fn post_block_import_logging_and_response<T: BeaconChainTypes>(
                Err(warp_utils::reject::broadcast_without_import(format!("{e}")))
            } else {
                error!(
-                    log,
-                    "Invalid block provided to HTTP API";
-                    "reason" => ?e,
+                    reason = ?e,
+                    "Invalid block provided to HTTP API"
                );
                Err(warp_utils::reject::custom_bad_request(format!(
                    "Invalid block: {e}"
@@ -654,20 +617,17 @@ pub async fn publish_blinded_block<T: BeaconChainTypes>(
    blinded_block: Arc<SignedBlindedBeaconBlock<T::EthSpec>>,
    chain: Arc<BeaconChain<T>>,
    network_tx: &UnboundedSender<NetworkMessage<T::EthSpec>>,
-    log: Logger,
    validation_level: BroadcastValidation,
    duplicate_status_code: StatusCode,
    network_globals: Arc<NetworkGlobals<T::EthSpec>>,
 ) -> Result<Response, Rejection> {
    let block_root = blinded_block.canonical_root();
-    let full_block =
-        reconstruct_block(chain.clone(), block_root, blinded_block, log.clone()).await?;
+    let full_block = reconstruct_block(chain.clone(), block_root, blinded_block).await?;
    publish_block::<T, _>(
        Some(block_root),
        full_block,
        chain,
        network_tx,
-        log,
        validation_level,
        duplicate_status_code,
        network_globals,
@@ -682,7 +642,6 @@ pub async fn reconstruct_block<T: BeaconChainTypes>(
    chain: Arc<BeaconChain<T>>,
    block_root: Hash256,
    block: Arc<SignedBlindedBeaconBlock<T::EthSpec>>,
-    log: Logger,
 ) -> Result<ProvenancedBlock<T, Arc<SignedBeaconBlock<T::EthSpec>>>, Rejection> {
    let full_payload_opt = if let Ok(payload_header) = block.message().body().execution_payload() {
        let el = chain.execution_layer.as_ref().ok_or_else(|| {
@@ -706,7 +665,7 @@ pub async fn reconstruct_block<T: BeaconChainTypes>(
        } else if let Some(cached_payload) =
            el.get_payload_by_root(&payload_header.tree_hash_root())
        {
-            info!(log, "Reconstructing a full block using a local payload"; "block_hash" => ?cached_payload.block_hash());
+            info!(block_hash = ?cached_payload.block_hash(), "Reconstructing a full block using a local payload");
            ProvenancedPayload::Local(cached_payload)
        // Otherwise, this means we are attempting a blind block proposal.
        } else {
@@ -721,7 +680,6 @@ pub async fn reconstruct_block<T: BeaconChainTypes>(
                block.message(),
                block_root,
                "builder",
-                &log,
            );

            let full_payload = el
@@ -733,7 +691,7 @@ pub async fn reconstruct_block<T: BeaconChainTypes>(
                        e
                    ))
                })?;
-            info!(log, "Successfully published a block to the builder network"; "block_hash" => ?full_payload.block_hash());
+            info!(block_hash = ?full_payload.block_hash(), "Successfully published a block to the builder network");
            ProvenancedPayload::Builder(full_payload)
        };

@@ -775,7 +733,6 @@ fn late_block_logging<T: BeaconChainTypes, P: AbstractExecPayload<T::EthSpec>>(
    block: BeaconBlockRef<T::EthSpec, P>,
    root: Hash256,
    provenance: &str,
-    log: &Logger,
 ) {
    let delay = get_block_delay_ms(seen_timestamp, block, &chain.slot_clock);

@@ -794,23 +751,21 @@ fn late_block_logging<T: BeaconChainTypes, P: AbstractExecPayload<T::EthSpec>>(
    let delayed_threshold = too_late_threshold / 2;
    if delay >= too_late_threshold {
        error!(
-            log,
-            "Block was broadcast too late";
-            "msg" => "system may be overloaded, block likely to be orphaned",
-            "provenance" => provenance,
-            "delay_ms" => delay.as_millis(),
-            "slot" => block.slot(),
-            "root" => ?root,
+            msg = "system may be overloaded, block likely to be orphaned",
+            provenance,
+            delay_ms = delay.as_millis(),
+            slot = %block.slot(),
+            ?root,
+            "Block was broadcast too late"
        )
    } else if delay >= delayed_threshold {
        error!(
-            log,
-            "Block broadcast was delayed";
-            "msg" => "system may be overloaded, block may be orphaned",
-            "provenance" => provenance,
-            "delay_ms" => delay.as_millis(),
-            "slot" => block.slot(),
-            "root" => ?root,
+            msg = "system may be overloaded, block may be orphaned",
+            provenance,
+            delay_ms = delay.as_millis(),
+            slot = %block.slot(),
+            ?root,
+            "Block broadcast was delayed"
        )
    }
 }
@@ -820,7 +775,6 @@ fn check_slashable<T: BeaconChainTypes>(
    chain_clone: &BeaconChain<T>,
    block_root: Hash256,
    block_clone: &SignedBeaconBlock<T::EthSpec, FullPayload<T::EthSpec>>,
-    log_clone: &Logger,
 ) -> Result<(), BlockError> {
    let slashable_cache = chain_clone.observed_slashable.read();
    if slashable_cache
@@ -832,9 +786,8 @@ fn check_slashable<T: BeaconChainTypes>(
        .map_err(|e| BlockError::BeaconChainError(e.into()))?
    {
        warn!(
-            log_clone,
-            "Not publishing equivocating block";
-            "slot" => block_clone.slot()
+            slot = %block_clone.slot(),
+            "Not publishing equivocating block"
        );
        return Err(BlockError::Slashable);
    }