gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-03 00:31:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bump SSZ version for larger bitfield SmallVec (#6915)

Browse Source 
NA


  Bumps the `ethereum_ssz` version, along with other crates that share the dep.

Primarily, this give us bitfields which can store 128 bytes on the stack before allocating, rather than 32 bytes (https://github.com/sigp/ethereum_ssz/pull/38). The validator count has increase massively since we set it at 32 bytes, so aggregation bitfields (et al) now require a heap allocation. This new value of 128 should get us to ~2m active validators.
This commit is contained in:
Paul Hauner
2025-03-10 19:18:33 +11:00
committed by GitHub
parent b4e79edf2a
commit 8d1abce26e
13 changed files with 192 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
common/malloc_utils/src/jemalloc.rs
View File

@@ -7,9 +7,11 @@
//!
//! A) `JEMALLOC_SYS_WITH_MALLOC_CONF` at compile-time.
//! B) `_RJEM_MALLOC_CONF` at runtime.
use metrics::{set_gauge, try_create_int_gauge, IntGauge};
use metrics::{
set_gauge, set_gauge_vec, try_create_int_gauge, try_create_int_gauge_vec, IntGauge, IntGaugeVec,
};
use std::sync::LazyLock;
use tikv_jemalloc_ctl::{arenas, epoch, stats, Access, AsName, Error};
use tikv_jemalloc_ctl::{arenas, epoch, raw, stats, Access, AsName, Error};
#[global_allocator]
static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
@@ -33,6 +35,38 @@ pub static BYTES_RESIDENT: LazyLock<metrics::Result<IntGauge>> = LazyLock::new(|
pub static BYTES_RETAINED: LazyLock<metrics::Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge("jemalloc_bytes_retained", "Equivalent to stats.retained")
});
pub static JEMALLOC_ARENAS_SMALL_NMALLOC: LazyLock<metrics::Result<IntGaugeVec>> =
LazyLock::new(|| {
try_create_int_gauge_vec(
"jemalloc_arenas_small_nmalloc",
"Equivalent to stats.arenas.<i>.small.nmalloc",
&["arena"],
)
});
pub static JEMALLOC_ARENAS_SMALL_NDALLOC: LazyLock<metrics::Result<IntGaugeVec>> =
LazyLock::new(|| {
try_create_int_gauge_vec(
"jemalloc_arenas_small_ndalloc",
"Equivalent to stats.arenas.<i>.small.ndalloc",
&["arena"],
)
});
pub static JEMALLOC_ARENAS_LARGE_NMALLOC: LazyLock<metrics::Result<IntGaugeVec>> =
LazyLock::new(|| {
try_create_int_gauge_vec(
"jemalloc_arenas_large_nmalloc",
"Equivalent to stats.arenas.<i>.large.nmalloc",
&["arena"],
)
});
pub static JEMALLOC_ARENAS_LARGE_NDALLOC: LazyLock<metrics::Result<IntGaugeVec>> =
LazyLock::new(|| {
try_create_int_gauge_vec(
"jemalloc_arenas_large_ndalloc",
"Equivalent to stats.arenas.<i>.large.ndalloc",
&["arena"],
)
});
pub fn scrape_jemalloc_metrics() {
scrape_jemalloc_metrics_fallible().unwrap()
@@ -42,7 +76,8 @@ pub fn scrape_jemalloc_metrics_fallible() -> Result<(), Error> {
// Advance the epoch so that the underlying statistics are updated.
epoch::advance()?;
set_gauge(&NUM_ARENAS, arenas::narenas::read()? as i64);
let num_arenas = arenas::narenas::read()?;
set_gauge(&NUM_ARENAS, num_arenas as i64);
set_gauge(&BYTES_ALLOCATED, stats::allocated::read()? as i64);
set_gauge(&BYTES_ACTIVE, stats::active::read()? as i64);
set_gauge(&BYTES_MAPPED, stats::mapped::read()? as i64);
@@ -50,9 +85,40 @@ pub fn scrape_jemalloc_metrics_fallible() -> Result<(), Error> {
set_gauge(&BYTES_RESIDENT, stats::resident::read()? as i64);
set_gauge(&BYTES_RETAINED, stats::retained::read()? as i64);
for arena in 0..num_arenas {
unsafe {
set_stats_gauge(
&JEMALLOC_ARENAS_SMALL_NMALLOC,
arena,
&format!("stats.arenas.{arena}.small.nmalloc\0"),
);
set_stats_gauge(
&JEMALLOC_ARENAS_SMALL_NDALLOC,
arena,
&format!("stats.arenas.{arena}.small.ndalloc\0"),
);
set_stats_gauge(
&JEMALLOC_ARENAS_LARGE_NMALLOC,
arena,
&format!("stats.arenas.{arena}.large.nmalloc\0"),
);
set_stats_gauge(
&JEMALLOC_ARENAS_LARGE_NDALLOC,
arena,
&format!("stats.arenas.{arena}.large.ndalloc\0"),
);
}
}
Ok(())
}
unsafe fn set_stats_gauge(metric: &metrics::Result<IntGaugeVec>, arena: u32, stat: &str) {
if let Ok(val) = raw::read::<usize>(stat.as_bytes()) {
set_gauge_vec(metric, &[&format!("arena_{arena}")], val as i64);
}
}
pub fn page_size() -> Result<usize, Error> {
// Full list of keys: https://jemalloc.net/jemalloc.3.html
"arenas.page\0".name().read()