gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-17 21:08:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Instrument tracing spans for block processing and import (#7816)

Browse Source 
#7815

- removes all existing spans, so some span fields that appear in logs like `service_name` may be lost.
- instruments a few key code paths in the beacon node, starting from **root spans** named below:

* Gossip block and blobs
* `process_gossip_data_column_sidecar`
* `process_gossip_blob`
* `process_gossip_block`
* Rpc block and blobs
* `process_rpc_block`
* `process_rpc_blobs`
* `process_rpc_custody_columns`
* Rpc blocks (range and backfill)
* `process_chain_segment`
* `PendingComponents` lifecycle
* `pending_components`

To test locally:
* Run Grafana and Tempo with https://github.com/sigp/lighthouse-metrics/pull/57
* Run Lighthouse BN with `--telemetry-collector-url http://localhost:4317`

Some captured traces can be found here: https://hackmd.io/@jimmygchen/r1sLOxPPeg

Removing the old spans seem to have reduced the memory usage quite a lot - i think we were using them on long running tasks and too excessively:
<img width="910" height="495" alt="image" src="https://github.com/user-attachments/assets/5208bbe4-53b2-4ead-bc71-0b782c788669" />
This commit is contained in:
Jimmy Chen
2025-08-08 15:32:22 +10:00
committed by GitHub
parent 6dfab22267
commit 40c2fd5ff4
52 changed files with 633 additions and 1164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

303
Cargo.lock generated
View File

@@ -1253,7 +1253,7 @@ dependencies = [
"eth2",
"ethereum_ssz",
"lighthouse_version",
"reqwest",
"reqwest 0.11.27",
"sensitive_url",
"serde",
"serde_json",
@@ -1707,7 +1707,7 @@ dependencies = [
"futures-core",
"prost",
"prost-types",
"tonic",
"tonic 0.12.3",
"tracing-core",
]
@@ -1731,7 +1731,7 @@ dependencies = [
"thread_local",
"tokio",
"tokio-stream",
"tonic",
"tonic 0.12.3",
"tracing",
"tracing-core",
"tracing-subscriber",
@@ -2270,7 +2270,7 @@ dependencies = [
"ethabi 16.0.0",
"ethereum_ssz",
"hex",
"reqwest",
"reqwest 0.11.27",
"serde_json",
"sha2 0.9.9",
"tree_hash",
@@ -2835,7 +2835,7 @@ dependencies = [
"pretty_reqwest_error",
"proto_array",
"rand 0.8.5",
"reqwest",
"reqwest 0.11.27",
"reqwest-eventsource",
"sensitive_url",
"serde",
@@ -2913,7 +2913,7 @@ dependencies = [
"ethereum_ssz",
"kzg",
"pretty_reqwest_error",
"reqwest",
"reqwest 0.11.27",
"sensitive_url",
"serde_yaml",
"sha2 0.9.9",
@@ -3146,7 +3146,7 @@ checksum = "a9713f525348e5dde025d09b0a4217429f8074e8ff22c886263cc191e87d8216"
dependencies = [
"ethers-core",
"getrandom 0.2.15",
"reqwest",
"reqwest 0.11.27",
"semver 1.0.26",
"serde",
"serde-aux",
@@ -3171,7 +3171,7 @@ dependencies = [
"futures-locks",
"futures-util",
"instant",
"reqwest",
"reqwest 0.11.27",
"serde",
"serde_json",
"thiserror 1.0.69",
@@ -3201,7 +3201,7 @@ dependencies = [
"once_cell",
"parking_lot 0.11.2",
"pin-project",
"reqwest",
"reqwest 0.11.27",
"serde",
"serde_json",
"thiserror 1.0.69",
@@ -3287,7 +3287,7 @@ dependencies = [
"futures",
"hex",
"logging",
"reqwest",
"reqwest 0.11.27",
"sensitive_url",
"serde_json",
"task_executor",
@@ -3326,7 +3326,7 @@ dependencies = [
"parking_lot 0.12.3",
"pretty_reqwest_error",
"rand 0.8.5",
"reqwest",
"reqwest 0.11.27",
"sensitive_url",
"serde",
"serde_json",
@@ -4326,7 +4326,7 @@ dependencies = [
"logging",
"malloc_utils",
"metrics",
"reqwest",
"reqwest 0.11.27",
"serde",
"slot_clock",
"store",
@@ -4793,7 +4793,7 @@ dependencies = [
"metrics",
"parking_lot 0.12.3",
"rand 0.8.5",
"reqwest",
"reqwest 0.11.27",
"serde",
"serde_json",
"signing_method",
@@ -5637,6 +5637,9 @@ dependencies = [
"logging",
"malloc_utils",
"metrics",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry_sdk",
"sensitive_url",
"serde",
"serde_json",
@@ -5647,6 +5650,7 @@ dependencies = [
"task_executor",
"tempfile",
"tracing",
"tracing-opentelemetry",
"tracing-subscriber",
"types",
"unused_port",
@@ -6175,7 +6179,7 @@ dependencies = [
"lighthouse_version",
"metrics",
"regex",
"reqwest",
"reqwest 0.11.27",
"sensitive_url",
"serde",
"serde_json",
@@ -6698,6 +6702,80 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "opentelemetry"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6"
dependencies = [
"futures-core",
"futures-sink",
"js-sys",
"pin-project-lite",
"thiserror 2.0.12",
"tracing",
]
[[package]]
name = "opentelemetry-http"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f6639e842a97dbea8886e3439710ae463120091e2e064518ba8e716e6ac36d"
dependencies = [
"async-trait",
"bytes",
"http 1.3.0",
"opentelemetry",
"reqwest 0.12.15",
]
[[package]]
name = "opentelemetry-otlp"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbee664a43e07615731afc539ca60c6d9f1a9425e25ca09c57bc36c87c55852b"
dependencies = [
"http 1.3.0",
"opentelemetry",
"opentelemetry-http",
"opentelemetry-proto",
"opentelemetry_sdk",
"prost",
"reqwest 0.12.15",
"thiserror 2.0.12",
"tokio",
"tonic 0.13.1",
"tracing",
]
[[package]]
name = "opentelemetry-proto"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc"
dependencies = [
"opentelemetry",
"opentelemetry_sdk",
"prost",
"tonic 0.13.1",
]
[[package]]
name = "opentelemetry_sdk"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11f644aa9e5e31d11896e024305d7e3c98a88884d9f8919dbf37a9991bc47a4b"
dependencies = [
"futures-channel",
"futures-executor",
"futures-util",
"opentelemetry",
"percent-encoding",
"rand 0.9.0",
"serde_json",
"thiserror 2.0.12",
]
[[package]]
name = "operation_pool"
version = "0.2.0"
@@ -7129,7 +7207,7 @@ dependencies = [
name = "pretty_reqwest_error"
version = "0.1.0"
dependencies = [
"reqwest",
"reqwest 0.11.27",
"sensitive_url",
]
@@ -7760,6 +7838,43 @@ dependencies = [
"winreg",
]
[[package]]
name = "reqwest"
version = "0.12.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"http 1.3.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.6.0",
"hyper-util",
"ipnet",
"js-sys",
"log",
"mime",
"once_cell",
"percent-encoding",
"pin-project-lite",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper 1.0.2",
"tokio",
"tower 0.5.2",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"windows-registry",
]
[[package]]
name = "reqwest-eventsource"
version = "0.5.0"
@@ -7772,7 +7887,7 @@ dependencies = [
"mime",
"nom",
"pin-project-lite",
"reqwest",
"reqwest 0.11.27",
"thiserror 1.0.69",
]
@@ -8605,7 +8720,7 @@ dependencies = [
"ethereum_serde_utils",
"lockfile",
"parking_lot 0.12.3",
"reqwest",
"reqwest 0.11.27",
"serde",
"task_executor",
"types",
@@ -8852,6 +8967,7 @@ dependencies = [
"ssz_types",
"test_random_derive",
"tokio",
"tracing",
"tree_hash",
"types",
]
@@ -9003,6 +9119,9 @@ name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
[[package]]
name = "synstructure"
@@ -9517,6 +9636,32 @@ dependencies = [
"tracing",
]
[[package]]
name = "tonic"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9"
dependencies = [
"async-trait",
"base64 0.22.1",
"bytes",
"http 1.3.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.6.0",
"hyper-timeout",
"hyper-util",
"percent-encoding",
"pin-project",
"prost",
"tokio",
"tokio-stream",
"tower 0.5.2",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower"
version = "0.4.13"
@@ -9545,10 +9690,15 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
dependencies = [
"futures-core",
"futures-util",
"indexmap 2.8.0",
"pin-project-lite",
"slab",
"sync_wrapper 1.0.2",
"tokio",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
@@ -9629,6 +9779,24 @@ dependencies = [
"tracing-core",
]
[[package]]
name = "tracing-opentelemetry"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddcf5959f39507d0d04d6413119c04f33b623f4f951ebcbdddddfad2d0623a9c"
dependencies = [
"js-sys",
"once_cell",
"opentelemetry",
"opentelemetry_sdk",
"smallvec",
"tracing",
"tracing-core",
"tracing-log",
"tracing-subscriber",
"web-time",
]
[[package]]
name = "tracing-serde"
version = "0.2.0"
@@ -9962,7 +10130,7 @@ dependencies = [
"metrics",
"monitoring_api",
"parking_lot 0.12.3",
"reqwest",
"reqwest 0.11.27",
"sensitive_url",
"serde",
"slashing_protection",
@@ -10389,7 +10557,7 @@ dependencies = [
"lighthouse_validator_store",
"logging",
"parking_lot 0.12.3",
"reqwest",
"reqwest 0.11.27",
"serde",
"serde_json",
"serde_yaml",
@@ -10525,7 +10693,7 @@ dependencies = [
"windows-implement",
"windows-interface",
"windows-result 0.2.0",
"windows-strings",
"windows-strings 0.1.0",
"windows-targets 0.52.6",
]
@@ -10557,6 +10725,17 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3"
[[package]]
name = "windows-registry"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3"
dependencies = [
"windows-result 0.3.1",
"windows-strings 0.3.1",
"windows-targets 0.53.2",
]
[[package]]
name = "windows-result"
version = "0.1.2"
@@ -10575,6 +10754,15 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-strings"
version = "0.1.0"
@@ -10585,6 +10773,15 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-strings"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
@@ -10660,13 +10857,29 @@ dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef"
dependencies = [
"windows_aarch64_gnullvm 0.53.0",
"windows_aarch64_msvc 0.53.0",
"windows_i686_gnu 0.53.0",
"windows_i686_gnullvm 0.53.0",
"windows_i686_msvc 0.53.0",
"windows_x86_64_gnu 0.53.0",
"windows_x86_64_gnullvm 0.53.0",
"windows_x86_64_msvc 0.53.0",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.2"
@@ -10685,6 +10898,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
@@ -10703,6 +10922,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
@@ -10721,12 +10946,24 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
@@ -10745,6 +10982,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
@@ -10763,6 +11006,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
@@ -10781,6 +11030,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
@@ -10799,6 +11054,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
[[package]]
name = "winnow"
version = "0.5.40"