Fix malloc_utils features (sysmalloc) (#7770)

Follow-up to:

- https://github.com/sigp/lighthouse/pull/7764

The `heaptrack` feature added in my previous PR was ineffective, because the jemalloc feature was turned on by the Linux target-specific dependency.

This PR tweaks the features such that:

- The jemalloc feature is just used to control whether jemalloc is compiled in. It is enabled on Linux by the target-specific dependency (see `lighthouse/Cargo.toml`), and completely disabled on Windows.
- If the `sysmalloc` feature is set on Linux then it overrides jemalloc when selecting an allocator, _even if_ the jemalloc feature is enabled (and the jemalloc dep was compiled).

Makefile

@@ -16,7 +16,7 @@ BUILD_PATH_RISCV64 = "target/$(RISCV64_TAG)/release"
 PINNED_NIGHTLY ?= nightly
 
 # List of features to use when cross-compiling. Can be overridden via the environment.
-CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc,beacon-node-leveldb,beacon-node-redb
+CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,beacon-node-leveldb,beacon-node-redb
 
 # Cargo profile for Cross builds. Default is for local builds, CI uses an override.
 CROSS_PROFILE ?= release

book/src/installation_source.md

@@ -166,8 +166,8 @@ Commonly used features include:
 - `slasher-lmdb`: support for the LMDB slasher backend. Enabled by default.
 - `slasher-mdbx`: support for the MDBX slasher backend.
 - `beacon-node-leveldb`: support for the leveldb backend. Enabled by default.
-- `jemalloc`: use [`jemalloc`][jemalloc] to allocate memory. Enabled by default on Linux and macOS.
+- `sysmalloc`: use the system memory allocator rather than jemalloc. This is always enabled on
-  Not supported on Windows.
+  Windows.
 - `spec-minimal`: support for the minimal preset (useful for testing).
 
 Default features (e.g. `slasher-lmdb`, `beacon-node-leveldb`) may be opted out of using the `--no-default-features`
@@ -178,8 +178,6 @@ E.g.
 CARGO_INSTALL_EXTRA_FLAGS="--no-default-features" make
 ```
 
-[jemalloc]: https://jemalloc.net/
-
 ## Compilation Profiles
 
 You can customise the compiler settings used to compile Lighthouse via

common/malloc_utils/Cargo.toml

@@ -4,10 +4,23 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
 
+# Features are not rich enough to express the complexity of our defaults, so we choose to just
+# use the jemalloc feature to control whether the dependency is compiled, but avoid using it if
+# the `sysmalloc` feature is set.
+#
+# On Windows, setting the jemalloc feature will result in a compile-time error.
 [features]
+default = []
 mallinfo2 = []
-jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"]
+# The jemalloc feature enables the compilation of jemalloc dependencies. Jemalloc is also the
+# default allocator, unless `sysmalloc` is set.
+#
+# It should be turned off on Windows.
+jemalloc = ["tikv-jemalloc-ctl", "tikv-jemallocator"]
 jemalloc-profiling = ["tikv-jemallocator/profiling"]
+# Force the use of system malloc (or glibc) rather than jemalloc.
+# This is a no-op on Windows where jemalloc is always disabled.
+sysmalloc = []
 
 [dependencies]
 libc = "0.2.79"

common/malloc_utils/src/lib.rs

@@ -25,28 +25,35 @@
 //! functions, then try to compile with the `not_glibc_interface` module.
 
 #[cfg(all(
+    any(feature = "sysmalloc", not(feature = "jemalloc")),
     target_os = "linux",
-    not(target_env = "musl"),
+    not(target_env = "musl")
-    not(feature = "jemalloc")
 ))]
 pub mod glibc;
 
-#[cfg(feature = "jemalloc")]
+#[cfg(all(unix, not(feature = "sysmalloc"), feature = "jemalloc"))]
 pub mod jemalloc;
 
 pub use interface::*;
 
+// Glibc malloc is the default on non-musl Linux if the sysmalloc feature is enabled, or jemalloc
+// is disabled.
 #[cfg(all(
+    any(feature = "sysmalloc", not(feature = "jemalloc")),
     target_os = "linux",
-    not(target_env = "musl"),
+    not(target_env = "musl")
-    not(feature = "jemalloc")
 ))]
 mod interface {
     pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator;
     pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics;
+
+    pub fn allocator_name() -> String {
+        "glibc".to_string()
+    }
 }
 
-#[cfg(feature = "jemalloc")]
+// Jemalloc is the default on UNIX (including musl) unless the sysmalloc feature is enabled.
+#[cfg(all(unix, not(feature = "sysmalloc"), feature = "jemalloc"))]
 mod interface {
     #[allow(dead_code)]
     pub fn configure_memory_allocator() -> Result<(), String> {
@@ -54,11 +61,21 @@ mod interface {
     }
 
     pub use crate::jemalloc::scrape_jemalloc_metrics as scrape_allocator_metrics;
+
+    pub fn allocator_name() -> String {
+        match crate::jemalloc::page_size() {
+            Ok(page_size) => format!("jemalloc ({}K)", page_size / 1024),
+            Err(e) => format!("jemalloc (error: {e:?})"),
+        }
+    }
 }
 
-#[cfg(all(
+#[cfg(any(
-    any(not(target_os = "linux"), target_env = "musl"),
+    not(unix),
-    not(feature = "jemalloc")
+    all(
+        any(feature = "sysmalloc", not(feature = "jemalloc")),
+        any(not(target_os = "linux"), target_env = "musl")
+    )
 ))]
 mod interface {
     #[allow(dead_code, clippy::unnecessary_wraps)]
@@ -68,4 +85,8 @@ mod interface {
 
     #[allow(dead_code)]
     pub fn scrape_allocator_metrics() {}
+
+    pub fn allocator_name() -> String {
+        "system".to_string()
+    }
 }

lcli/Cargo.toml

@@ -11,7 +11,6 @@ normal = ["malloc_utils"]
 [features]
 portable = ["bls/supranational-portable"]
 fake_crypto = ['bls/fake_crypto']
-jemalloc = ["malloc_utils/jemalloc"]
 
 [dependencies]
 account_utils = { workspace = true }
@@ -31,7 +30,6 @@ hex = { workspace = true }
 lighthouse_network = { workspace = true }
 lighthouse_version = { workspace = true }
 log = { workspace = true }
-malloc_utils = { workspace = true }
 rayon = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -44,3 +42,9 @@ tracing-subscriber = { workspace = true }
 tree_hash = { workspace = true }
 types = { workspace = true }
 validator_dir = { workspace = true }
+
+[target.'cfg(not(target_os = "windows"))'.dependencies]
+malloc_utils = { workspace = true, features = ["jemalloc"] }
+
+[target.'cfg(target_os = "windows")'.dependencies]
+malloc_utils = { workspace = true, features = [] }

lighthouse/Cargo.toml

@@ -35,11 +35,8 @@ beacon-node-leveldb = ["store/leveldb"]
 beacon-node-redb = ["store/redb"]
 # Supports console subscriber for debugging
 console-subscriber = ["console-subscriber/default"]
-# Turns off jemalloc so that heaptrack may be used to analyse memory usage.
+# Force the use of the system memory allocator rather than jemalloc.
-heaptrack = []
+sysmalloc = ["malloc_utils/sysmalloc"]
-
-# Deprecated. This is now enabled by default on non windows targets (unless heaptrack is enabled).
-jemalloc = []
 
 [dependencies]
 account_manager = { "path" = "../account_manager" }
@@ -58,7 +55,6 @@ ethereum_hashing = { workspace = true }
 futures = { workspace = true }
 lighthouse_version = { workspace = true }
 logging = { workspace = true }
-malloc_utils = { workspace = true }
 metrics = { workspace = true }
 opentelemetry = { workspace = true }
 opentelemetry-otlp = { workspace = true }
@@ -77,12 +73,12 @@ unused_port = { workspace = true }
 validator_client = { workspace = true }
 validator_manager = { path = "../validator_manager" }
 
-[target.'cfg(any(target_os = "windows", features = "heaptrack"))'.dependencies]
+[target.'cfg(not(target_os = "windows"))'.dependencies]
-malloc_utils = { workspace = true }
-
-[target.'cfg(not(any(target_os = "windows", features = "heaptrack")))'.dependencies]
 malloc_utils = { workspace = true, features = ["jemalloc"] }
 
+[target.'cfg(target_os = "windows")'.dependencies]
+malloc_utils = { workspace = true, features = [] }
+
 [dev-dependencies]
 beacon_node_fallback = { workspace = true }
 beacon_processor = { workspace = true }

lighthouse/src/main.rs

@@ -76,15 +76,7 @@ fn bls_hardware_acceleration() -> bool {
 }
 
 fn allocator_name() -> String {
-    #[cfg(any(feature = "heaptrack", target_os = "windows"))]
+    malloc_utils::allocator_name()
-    {
-        "system".to_string()
-    }
-    #[cfg(not(any(feature = "heaptrack", target_os = "windows")))]
-    match malloc_utils::jemalloc::page_size() {
-        Ok(page_size) => format!("jemalloc ({}K)", page_size / 1024),
-        Err(e) => format!("jemalloc (error: {e:?})"),
-    }
 }
 
 fn build_profile_name() -> String {

wordlist.txt

@@ -1,3 +1,4 @@
+allocator
 APIs
 ARMv
 AUR