Switch allocator to jemalloc (#3697)

Squashed commit of the following: commit 974b3359f8 Merge: ac205b7ba 480309fb9 Author: Michael Sproul <michael@sigmaprime.io> Date: Wed Jan 18 10:01:26 2023 +1100 Merge remote-tracking branch 'origin/unstable' into jemalloc commit 480309fb96 Author: aliask <aliask@gmail.com> Date: Tue Jan 17 05:13:49 2023 +0000 Fix some dead links in markdown files (#3885) ## Issue Addressed No issue has been raised for these broken links. ## Proposed Changes Update links with the new URLs for the same document. ## Additional Info ~The link for the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb/) mailing list is also broken, but I can't find the correct link.~ Co-authored-by: Paul Hauner <paul@paulhauner.com> commit b4d9fc03ee Author: GeemoCandama <geemo@tutanota.com> Date: Tue Jan 17 05:13:48 2023 +0000 add logging for starting request and receiving block (#3858) ## Issue Addressed #3853 ## Proposed Changes Added `INFO` level logs for requesting and receiving the unsigned block. ## Additional Info Logging for successfully publishing the signed block is already there. And seemingly there is a log for when "We realize we are going to produce a block" in the `start_update_service`: `info!(log, "Block production service started"); `. Is there anywhere else you'd like to see logging around this event? Co-authored-by: GeemoCandama <104614073+GeemoCandama@users.noreply.github.com> commit 9a970ce3a2 Author: David Theodore <prodigalsonsolutions@gmail.com> Date: Tue Jan 17 05:13:47 2023 +0000 add better err reporting UnableToOpenVotingKeystore (#3781) ## Issue Addressed #3780 ## Proposed Changes Add error reporting that notifies the node operator that the `voting_keystore_path` in their `validator_definitions.yml` file may be incorrect. ## Additional Info There is more info in issue #3780 Co-authored-by: Paul Hauner <paul@paulhauner.com> commit ac205b7bab Merge: 93457d85b bf533c8e4 Author: Michael Sproul <michael@sigmaprime.io> Date: Fri Nov 25 16:32:33 2022 +1100 Merge remote-tracking branch 'origin/unstable' into jemalloc commit 93457d85b7 Author: Michael Sproul <michael@sigmaprime.io> Date: Wed Nov 9 11:53:59 2022 +1100 Fix cargo-udeps commit 6c42aef1b5 Author: Michael Sproul <micsproul@gmail.com> Date: Tue Nov 8 19:12:19 2022 +1100 Fixups commit f14b87bb88 Author: Michael Sproul <michael@sigmaprime.io> Date: Tue Nov 8 16:28:16 2022 +1100 Update docs commit 5005dc3b65 Author: Michael Sproul <michael@sigmaprime.io> Date: Tue Nov 8 16:22:42 2022 +1100 Fix lcli commit a082ba5904 Author: Michael Sproul <michael@sigmaprime.io> Date: Tue Nov 8 16:17:10 2022 +1100 Remove check-consensus commit 81441e9cea Author: Michael Sproul <micsproul@gmail.com> Date: Tue Nov 8 15:28:11 2022 +1100 Disable jemalloc on Windows commit 41eac5d0c1 Author: Michael Sproul <micsproul@gmail.com> Date: Tue Nov 8 13:46:17 2022 +1100 Compatibility with macOS commit 69ecba7876 Author: Michael Sproul <michael@sigmaprime.io> Date: Mon Nov 7 18:48:31 2022 +1100 Add jemalloc support
2026-05-08 09:16:00 +00:00 · 2023-01-18 10:07:21 +11:00
parent a70ee29c08
commit 44a106a8af
17 changed files with 149 additions and 40 deletions
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -306,16 +306,6 @@ jobs:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Typecheck benchmark code without running it
      run: make check-benches
-  check-consensus:
-    name: check-consensus
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
-    steps:
-    - uses: actions/checkout@v3
-    - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Typecheck consensus code in strict mode
-      run: make check-consensus
  clippy:
    name: clippy
    runs-on: ubuntu-latest
@@ -382,14 +372,12 @@ jobs:
    - uses: actions/checkout@v3
    - name: Install Rust (${{ env.PINNED_NIGHTLY }})
      run: rustup toolchain install $PINNED_NIGHTLY
-    # NOTE: cargo-udeps version is pinned until this issue is resolved:
-    # https://github.com/est31/cargo-udeps/issues/135
    - name: Install Protoc
      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
    - name: Install cargo-udeps
-      run: cargo install cargo-udeps --locked --force --version 0.1.30
+      run: cargo install cargo-udeps --locked --force --version $CARGO_UDEPS_VERSION
    - name: Create Cargo config dir
      run: mkdir -p .cargo
    - name: Install custom Cargo config
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3596,10 +3596,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"

 [[package]]
-name = "jemalloc-sys"
-version = "0.3.2"
+name = "jemalloc-ctl"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45"
+checksum = "c1891c671f3db85d8ea8525dd43ab147f9977041911d24a03e5a36187a7bfde9"
+dependencies = [
+ "jemalloc-sys",
+ "libc",
+ "paste",
+]
+
+[[package]]
+name = "jemalloc-sys"
+version = "0.5.2+5.3.0-patched"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "134163979b6eed9564c98637b710b40979939ba351f59952708234ea11b5f3f8"
 dependencies = [
 "cc",
 "fs_extra",
@@ -3608,9 +3619,9 @@ dependencies = [

 [[package]]
 name = "jemallocator"
-version = "0.3.2"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69"
+checksum = "16c2514137880c52b0b4822b563fadd38257c1f380858addb74a400889696ea6"
 dependencies = [
 "jemalloc-sys",
 "libc",
@@ -3733,6 +3744,7 @@ dependencies = [
 "lighthouse_network",
 "lighthouse_version",
 "log",
+ "malloc_utils",
 "sensitive_url",
 "serde",
 "serde_json",
@@ -4541,7 +4553,7 @@ dependencies = [
 name = "malloc_utils"
 version = "0.1.0"
 dependencies = [
- "jemalloc-sys",
+ "jemalloc-ctl",
 "jemallocator",
 "lazy_static",
 "libc",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -81,6 +81,7 @@ members = [
    "validator_client",
    "validator_client/slashing_protection",
 ]
+resolver = "2"

 [patch]
 [patch.crates-io]
--- a/14
+++ b/14
@@ -14,8 +14,16 @@ BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
 PINNED_NIGHTLY ?= nightly
 CLIPPY_PINNED_NIGHTLY=nightly-2022-05-19

+# List of features to use when building natively. Can be overriden via the environment.
+# No jemalloc on Windows
+ifeq ($(OS),Windows_NT)
+    FEATURES?=
+else
+    FEATURES?=jemalloc
+endif
+
 # List of features to use when cross-compiling. Can be overridden via the environment.
-CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx
+CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,jemalloc

 # Cargo profile for Cross builds. Default is for local builds, CI uses an override.
 CROSS_PROFILE ?= release
@@ -101,10 +109,6 @@ cargo-fmt:
 check-benches:
 	cargo check --workspace --benches

-# Typechecks consensus code *without* allowing deprecated legacy arithmetic or metrics.
-check-consensus:
-	cargo check -p state_processing --no-default-features
-
 # Runs only the ef-test vectors.
 run-ef-tests:
 	rm -rf $(EF_TESTS)/.accessed_file_log.txt
--- a/README.md
+++ b/README.md
@@ -66,7 +66,7 @@ of the Lighthouse book.
 The best place for discussion is the [Lighthouse Discord
 server](https://discord.gg/cyAszAh).

-Sign up to the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb/) mailing list for email
+Sign up to the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb) mailing list for email
 notifications about releases, network status and other important information.

 Encrypt sensitive messages using our [PGP
--- a/book/src/installation-source.md
+++ b/book/src/installation-source.md
@@ -64,6 +64,7 @@ choco install protoc
 These dependencies are for compiling Lighthouse natively on Windows. Lighthouse can also run
 successfully under the [Windows Subsystem for Linux (WSL)][WSL]. If using Ubuntu under WSL, you
 should follow the instructions for Ubuntu listed in the [Dependencies (Ubuntu)](#ubuntu) section.
+
 [WSL]: https://docs.microsoft.com/en-us/windows/wsl/about

 ## Build Lighthouse
@@ -128,8 +129,12 @@ Commonly used features include:
 * `gnosis`: support for the Gnosis Beacon Chain.
 * `portable`: support for legacy hardware.
 * `modern`: support for exclusively modern hardware.
-* `slasher-mdbx`: support for the MDBX slasher backend (enabled by default).
+* `slasher-mdbx`: support for the MDBX slasher backend. Enabled by default.
 * `slasher-lmdb`: support for the LMDB slasher backend.
+* `jemalloc`: use [`jemalloc`][jemalloc] to allocate memory. Enabled by default on Linux and macOS.
+  Not supported on Windows.
+
+[jemalloc]: https://jemalloc.net/

 ## Compilation Profiles

--- a/book/src/merge-migration.md
+++ b/book/src/merge-migration.md
@@ -58,7 +58,7 @@ supported.
 Each execution engine has its own flags for configuring the engine API and JWT. Please consult
 the relevant page for your execution engine for the required flags:

- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/interface/consensus-clients)
+- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/getting-started/consensus-clients)
 - [Nethermind: Running Nethermind Post Merge](https://docs.nethermind.io/nethermind/first-steps-with-nethermind/running-nethermind-post-merge)
 - [Besu: Prepare For The Merge](https://besu.hyperledger.org/en/stable/HowTo/Upgrade/Prepare-for-The-Merge/)
 - [Erigon: Beacon Chain (Consensus Layer)](https://github.com/ledgerwatch/erigon#beacon-chain-consensus-layer)
@@ -203,5 +203,5 @@ guidance for specific setups.
 - [Ethereum.org: The Merge](https://ethereum.org/en/upgrades/merge/)
 - [Ethereum Staking Launchpad: Merge Readiness](https://launchpad.ethereum.org/en/merge-readiness).
 - [CoinCashew: Ethereum Merge Upgrade Checklist](https://www.coincashew.com/coins/overview-eth/ethereum-merge-upgrade-checklist-for-home-stakers-and-validators)
- [EthDocker: Merge Preparation](https://eth-docker.net/docs/About/MergePrep/)
+- [EthDocker: Merge Preparation](https://eth-docker.net/About/MergePrep/)
 - [Remy Roy: How to join the Goerli/Prater merge testnet](https://github.com/remyroy/ethstaker/blob/main/merge-goerli-prater.md)
--- a/book/src/run_a_node.md
+++ b/book/src/run_a_node.md
@@ -26,7 +26,7 @@ has authority to control the execution engine.
 Each execution engine has its own flags for configuring the engine API and JWT.
 Please consult the relevant page of your execution engine for the required flags:

- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/interface/consensus-clients)
+- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/getting-started/consensus-clients)
 - [Nethermind: Running Nethermind & CL](https://docs.nethermind.io/nethermind/first-steps-with-nethermind/running-nethermind-post-merge)
 - [Besu: Connect to Mainnet](https://besu.hyperledger.org/en/stable/public-networks/get-started/connect/mainnet/)
 - [Erigon: Beacon Chain (Consensus Layer)](https://github.com/ledgerwatch/erigon#beacon-chain-consensus-layer)
--- a/bors.toml
+++ b/bors.toml
@@ -10,7 +10,6 @@ status = [
    "merge-transition-ubuntu",
    "no-eth1-simulator-ubuntu",
    "check-benchmarks",
-    "check-consensus",
    "clippy",
    "arbitrary-check",
    "cargo-audit",
--- a/common/malloc_utils/Cargo.toml
+++ b/common/malloc_utils/Cargo.toml
@@ -4,18 +4,21 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2021"

-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 lighthouse_metrics = { path = "../lighthouse_metrics" }
 lazy_static = "1.4.0"
 libc = "0.2.79"
 parking_lot = "0.12.0"
-jemallocator = { version = "0.3.0", optional = true, features = ["background_threads"] }
-jemalloc-sys = { version = "0.3.0", optional = true }
+jemalloc-ctl = { version = "0.5.0", optional = true }
+
+# Jemalloc's background_threads feature requires Linux (pthreads).
+[target.'cfg(target_os = "linux")'.dependencies]
+jemallocator = { version = "0.5.0", optional = true, features = ["stats", "background_threads"] }
+
+[target.'cfg(not(target_os = "linux"))'.dependencies]
+jemallocator = { version = "0.5.0", optional = true, features = ["stats"] }

 [features]
 mallinfo2 = []
-jemalloc = ["jemallocator", "jemalloc-sys"]
-jemalloc-stats = ["jemallocator/stats"]
+jemalloc = ["jemallocator", "jemalloc-ctl"]
 jemalloc-profiling = ["jemallocator/profiling"]
--- a/common/malloc_utils/src/jemalloc.rs
+++ b/common/malloc_utils/src/jemalloc.rs
@@ -7,6 +7,46 @@
 //!
 //! A) `JEMALLOC_SYS_WITH_MALLOC_CONF` at compile-time.
 //! B) `_RJEM_MALLOC_CONF` at runtime.
+use jemalloc_ctl::{arenas, epoch, stats, Error};
+use lazy_static::lazy_static;
+use lighthouse_metrics::{set_gauge, try_create_int_gauge, IntGauge};

 #[global_allocator]
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
+
+// Metrics for jemalloc.
+lazy_static! {
+    pub static ref NUM_ARENAS: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_num_arenas", "The number of arenas in use");
+    pub static ref BYTES_ALLOCATED: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_allocated", "Equivalent to stats.allocated");
+    pub static ref BYTES_ACTIVE: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_active", "Equivalent to stats.active");
+    pub static ref BYTES_MAPPED: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_mapped", "Equivalent to stats.mapped");
+    pub static ref BYTES_METADATA: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_metadata", "Equivalent to stats.metadata");
+    pub static ref BYTES_RESIDENT: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_resident", "Equivalent to stats.resident");
+    pub static ref BYTES_RETAINED: lighthouse_metrics::Result<IntGauge> =
+        try_create_int_gauge("jemalloc_bytes_retained", "Equivalent to stats.retained");
+}
+
+pub fn scrape_jemalloc_metrics() {
+    scrape_jemalloc_metrics_fallible().unwrap()
+}
+
+pub fn scrape_jemalloc_metrics_fallible() -> Result<(), Error> {
+    // Advance the epoch so that the underlying statistics are updated.
+    epoch::advance()?;
+
+    set_gauge(&NUM_ARENAS, arenas::narenas::read()? as i64);
+    set_gauge(&BYTES_ALLOCATED, stats::allocated::read()? as i64);
+    set_gauge(&BYTES_ACTIVE, stats::active::read()? as i64);
+    set_gauge(&BYTES_MAPPED, stats::mapped::read()? as i64);
+    set_gauge(&BYTES_METADATA, stats::metadata::read()? as i64);
+    set_gauge(&BYTES_RESIDENT, stats::resident::read()? as i64);
+    set_gauge(&BYTES_RETAINED, stats::retained::read()? as i64);
+
+    Ok(())
+}
--- a/common/malloc_utils/src/lib.rs
+++ b/common/malloc_utils/src/lib.rs
@@ -26,7 +26,8 @@

 #[cfg(all(
    target_os = "linux",
-    not(any(target_env = "musl", feature = "jemalloc"))
+    not(target_env = "musl"),
+    not(feature = "jemalloc")
 ))]
 mod glibc;

@@ -37,14 +38,28 @@ pub use interface::*;

 #[cfg(all(
    target_os = "linux",
-    not(any(target_env = "musl", feature = "jemalloc"))
+    not(target_env = "musl"),
+    not(feature = "jemalloc")
 ))]
 mod interface {
    pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator;
    pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics;
 }

-#[cfg(any(not(target_os = "linux"), target_env = "musl", feature = "jemalloc"))]
+#[cfg(feature = "jemalloc")]
+mod interface {
+    #[allow(dead_code)]
+    pub fn configure_memory_allocator() -> Result<(), String> {
+        Ok(())
+    }
+
+    pub use crate::jemalloc::scrape_jemalloc_metrics as scrape_allocator_metrics;
+}
+
+#[cfg(all(
+    any(not(target_os = "linux"), target_env = "musl"),
+    not(feature = "jemalloc")
+))]
 mod interface {
    #[allow(dead_code, clippy::unnecessary_wraps)]
    pub fn configure_memory_allocator() -> Result<(), String> {
--- a/lcli/Cargo.toml
+++ b/lcli/Cargo.toml
@@ -8,6 +8,7 @@ edition = "2021"
 [features]
 portable = ["bls/supranational-portable"]
 fake_crypto = ['bls/fake_crypto']
+jemalloc = ["malloc_utils/jemalloc"]

 [dependencies]
 bls = { path = "../crypto/bls" }
@@ -40,3 +41,7 @@ eth2 = { path = "../common/eth2" }
 snap = "1.0.1"
 beacon_chain = { path = "../beacon_node/beacon_chain" }
 store = { path = "../beacon_node/store" }
+malloc_utils = { path = "../common/malloc_utils" }
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["malloc_utils"]
--- a/lighthouse/Cargo.toml
+++ b/lighthouse/Cargo.toml
@@ -24,6 +24,8 @@ gnosis = []
 slasher-mdbx = ["slasher/mdbx"]
 # Support slasher LMDB backend.
 slasher-lmdb = ["slasher/lmdb"]
+# Use jemalloc.
+jemalloc = ["malloc_utils/jemalloc"]

 [dependencies]
 beacon_node = { "path" = "../beacon_node" }
--- a/lighthouse/src/main.rs
+++ b/lighthouse/src/main.rs
@@ -31,6 +31,14 @@ fn bls_library_name() -> &'static str {
    }
 }

+fn allocator_name() -> &'static str {
+    if cfg!(feature = "jemalloc") {
+        "jemalloc"
+    } else {
+        "system"
+    }
+}
+
 fn main() {
    // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided.
    if std::env::var("RUST_BACKTRACE").is_err() {
@@ -51,10 +59,12 @@ fn main() {
                "{}\n\
                 BLS library: {}\n\
                 SHA256 hardware acceleration: {}\n\
+                 Allocator: {}\n\
                 Specs: mainnet (true), minimal ({}), gnosis ({})",
                 VERSION.replace("Lighthouse/", ""),
                 bls_library_name(),
                 have_sha_extensions(),
+                 allocator_name(),
                 cfg!(feature = "spec-minimal"),
                 cfg!(feature = "gnosis"),
            ).as_str()
--- a/validator_client/src/block_service.rs
+++ b/validator_client/src/block_service.rs
@@ -333,6 +333,11 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
        let proposer_index = self.validator_store.validator_index(&validator_pubkey);
        let validator_pubkey_ref = &validator_pubkey;

+        info!(
+            log,
+            "Requesting unsigned block";
+            "slot" => slot.as_u64(),
+        );
        // Request block from first responsive beacon node.
        let block = self
            .beacon_nodes
@@ -383,6 +388,11 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
                        }
                    };

+                    info!(
+                        log,
+                        "Received unsigned block";
+                        "slot" => slot.as_u64(),
+                    );
                    if proposer_index != Some(block.proposer_index()) {
                        return Err(BlockError::Recoverable(
                            "Proposer index does not match block proposer. Beacon chain re-orged"
@@ -401,6 +411,11 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
            .await
            .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))?;

+        info!(
+            log,
+            "Publishing signed block";
+            "slot" => slot.as_u64(),
+        );
        // Publish block with first available beacon node.
        self.beacon_nodes
            .first_success(
--- a/validator_client/src/lib.rs
+++ b/validator_client/src/lib.rs
@@ -31,6 +31,7 @@ use crate::beacon_node_fallback::{
 };
 use crate::doppelganger_service::DoppelgangerService;
 use crate::graffiti_file::GraffitiFile;
+use crate::initialized_validators::Error::UnableToOpenVotingKeystore;
 use account_utils::validator_definitions::ValidatorDefinitions;
 use attestation_service::{AttestationService, AttestationServiceBuilder};
 use block_service::{BlockService, BlockServiceBuilder};
@@ -184,7 +185,16 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
            log.clone(),
        )
        .await
-        .map_err(|e| format!("Unable to initialize validators: {:?}", e))?;
+        .map_err(|e| {
+            match e {
+                UnableToOpenVotingKeystore(err) => {
+                    format!("Unable to initialize validators: {:?}. If you have recently moved the location of your data directory \
+                    make sure to update the location of voting_keystore_path in your validator_definitions.yml", err)
+                },
+                err => {
+                    format!("Unable to initialize validators: {:?}", err)}
+                }
+            })?;

        let voting_pubkeys: Vec<_> = validators.iter_voting_pubkeys().collect();