Instrument tracing spans for block processing and import (#7816)

#7815

- removes all existing spans, so some span fields that appear in logs like `service_name` may be lost.
- instruments a few key code paths in the beacon node, starting from **root spans** named below:

* Gossip block and blobs
* `process_gossip_data_column_sidecar`
* `process_gossip_blob`
* `process_gossip_block`
* Rpc block and blobs
* `process_rpc_block`
* `process_rpc_blobs`
* `process_rpc_custody_columns`
* Rpc blocks (range and backfill)
* `process_chain_segment`
* `PendingComponents` lifecycle
* `pending_components`

To test locally:
* Run Grafana and Tempo with https://github.com/sigp/lighthouse-metrics/pull/57
* Run Lighthouse BN with `--telemetry-collector-url http://localhost:4317`

Some captured traces can be found here: https://hackmd.io/@jimmygchen/r1sLOxPPeg

Removing the old spans seem to have reduced the memory usage quite a lot - i think we were using them on long running tasks and too excessively:
<img width="910" height="495" alt="image" src="https://github.com/user-attachments/assets/5208bbe4-53b2-4ead-bc71-0b782c788669" />
This commit is contained in:
Jimmy Chen
2025-08-08 15:32:22 +10:00
committed by GitHub
parent 6dfab22267
commit 40c2fd5ff4
52 changed files with 633 additions and 1164 deletions

View File

@@ -34,6 +34,7 @@ pub use crate::beacon_state::slashings_cache::SlashingsCache;
pub use eth_spec::*;
pub use iter::BlockRootsIter;
pub use milhouse::{interface::Interface, List, Vector};
use tracing::instrument;
#[macro_use]
mod committee_cache;
@@ -1873,6 +1874,7 @@ impl<E: EthSpec> BeaconState<E> {
}
/// Build the total active balance cache for the current epoch if it is not already built.
#[instrument(skip_all, level = "debug")]
pub fn build_total_active_balance_cache(&mut self, spec: &ChainSpec) -> Result<(), Error> {
if self
.get_total_active_balance_at_epoch(self.current_epoch())
@@ -1931,6 +1933,7 @@ impl<E: EthSpec> BeaconState<E> {
}
/// Build all caches (except the tree hash cache), if they need to be built.
#[instrument(skip_all, level = "debug")]
pub fn build_caches(&mut self, spec: &ChainSpec) -> Result<(), Error> {
self.build_all_committee_caches(spec)?;
self.update_pubkey_cache()?;
@@ -1941,6 +1944,7 @@ impl<E: EthSpec> BeaconState<E> {
}
/// Build all committee caches, if they need to be built.
#[instrument(skip_all, level = "debug")]
pub fn build_all_committee_caches(&mut self, spec: &ChainSpec) -> Result<(), Error> {
self.build_committee_cache(RelativeEpoch::Previous, spec)?;
self.build_committee_cache(RelativeEpoch::Current, spec)?;
@@ -1949,6 +1953,7 @@ impl<E: EthSpec> BeaconState<E> {
}
/// Build the exit cache, if it needs to be built.
#[instrument(skip_all, level = "debug")]
pub fn build_exit_cache(&mut self, spec: &ChainSpec) -> Result<(), Error> {
if self.exit_cache().check_initialized().is_err() {
*self.exit_cache_mut() = ExitCache::new(self.validators(), spec)?;
@@ -1957,6 +1962,7 @@ impl<E: EthSpec> BeaconState<E> {
}
/// Build the slashings cache if it needs to be built.
#[instrument(skip_all, level = "debug")]
pub fn build_slashings_cache(&mut self) -> Result<(), Error> {
let latest_block_slot = self.latest_block_header().slot;
if !self.slashings_cache().is_initialized(latest_block_slot) {
@@ -1994,6 +2000,7 @@ impl<E: EthSpec> BeaconState<E> {
}
/// Build a committee cache, unless it is has already been built.
#[instrument(skip_all, level = "debug")]
pub fn build_committee_cache(
&mut self,
relative_epoch: RelativeEpoch,
@@ -2114,6 +2121,7 @@ impl<E: EthSpec> BeaconState<E> {
///
/// Adds all `pubkeys` from the `validators` which are not already in the cache. Will
/// never re-add a pubkey.
#[instrument(skip_all, level = "debug")]
pub fn update_pubkey_cache(&mut self) -> Result<(), Error> {
let mut pubkey_cache = mem::take(self.pubkey_cache_mut());
let start_index = pubkey_cache.len();
@@ -2194,6 +2202,7 @@ impl<E: EthSpec> BeaconState<E> {
/// Compute the tree hash root of the state using the tree hash cache.
///
/// Initialize the tree hash cache if it isn't already initialized.
#[instrument(skip_all, level = "debug")]
pub fn update_tree_hash_cache<'a>(&'a mut self) -> Result<Hash256, Error> {
self.apply_pending_mutations()?;
map_beacon_state_ref!(&'a _, self.to_ref(), |inner, cons| {