From d9be76afe7647c75dfa150417b3a6938ee77c399 Mon Sep 17 00:00:00 2001 From: jking-aus <72330194+jking-aus@users.noreply.github.com> Date: Tue, 5 May 2026 01:39:33 +0200 Subject: [PATCH 01/18] fix: payload_attestation_data when no block received for slot (#9225) Addresses issue #9220 The `payload_attestation_data` endpoint returns 400 when no block has been received for the requested slot. This causes the VC to log at CRIT level for what is expected behaviour per spec: validators should simply not submit a payload attestation when no block has been seen. - Return 404 (Not Found) instead of 400 from `payload_attestation_data` when no block exists for the slot. This is consistent with other beacon api endpoints. - Downgrade the VC log from `crit` to `debug` when a 503 is received, since this is an expected no-op per spec. - Add `BlockNotFound` rejection type to `warp_utils`. - Add a test asserting the 404 response for an empty slot. Co-Authored-By: Josh King Co-Authored-By: Eitan Seri-Levi Co-Authored-By: Jimmy Chen --- beacon_node/http_api/src/validator/mod.rs | 8 +++- beacon_node/http_api/tests/tests.rs | 38 ++++++++++++++++++- common/eth2/src/lib.rs | 22 +++++++---- common/warp_utils/src/reject.rs | 14 +++++++ .../src/payload_attestation_service.rs | 16 ++++++-- 5 files changed, 83 insertions(+), 15 deletions(-) diff --git a/beacon_node/http_api/src/validator/mod.rs b/beacon_node/http_api/src/validator/mod.rs index 044f2089ce..77df94bc36 100644 --- a/beacon_node/http_api/src/validator/mod.rs +++ b/beacon_node/http_api/src/validator/mod.rs @@ -333,8 +333,12 @@ pub fn get_validator_payload_attestation_data( let payload_attestation_data = chain .produce_payload_attestation_data(slot) .map_err(|e| match e { - BeaconChainError::InvalidSlot(_) - | BeaconChainError::NoBlockForSlot(_) => { + BeaconChainError::NoBlockForSlot(_) => { + warp_utils::reject::block_not_found(format!( + "No block received for slot {slot}" + )) + } + BeaconChainError::InvalidSlot(_) => { warp_utils::reject::custom_bad_request(format!( "Unable to produce payload attestation data: {e:?}" )) diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index d6c621f996..0d6735ff61 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -4807,7 +4807,8 @@ impl ApiTester { .client .get_validator_payload_attestation_data(slot) .await - .unwrap(); + .unwrap() + .expect("expected payload attestation data for slot with block"); assert_eq!(response.version(), Some(fork_name)); @@ -4823,7 +4824,8 @@ impl ApiTester { .client .get_validator_payload_attestation_data_ssz(slot) .await - .unwrap(); + .unwrap() + .expect("expected SSZ payload attestation data for slot with block"); assert_eq!(ssz_result, expected); @@ -4894,6 +4896,7 @@ impl ApiTester { .get_validator_payload_attestation_data(slot) .await .unwrap() + .expect("expected payload attestation data for slot with block") .into_data(); assert_eq!(pa_data.beacon_block_root, block_root); @@ -4926,6 +4929,26 @@ impl ApiTester { self } + pub async fn test_get_validator_payload_attestation_data_no_block(self) -> Self { + // Advance the slot clock without producing a block + self.harness.advance_slot(); + let slot = self.chain.slot().unwrap(); + + // Should return None when no block exists for the slot + let result = self + .client + .get_validator_payload_attestation_data(slot) + .await + .unwrap(); + + assert!( + result.is_none(), + "expected None for empty slot, got: {result:?}" + ); + + self + } + #[allow(clippy::await_holding_lock)] // This is a test, so it should be fine. pub async fn test_get_validator_aggregate_attestation_v1(self) -> Self { let attestation = self @@ -8597,6 +8620,17 @@ async fn get_validator_payload_attestation_data_pre_gloas() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_payload_attestation_data_no_block() { + if !fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } + ApiTester::new_with_hard_forks() + .await + .test_get_validator_payload_attestation_data_no_block() + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn payload_attestation_present_after_envelope_publish() { ApiTester::new_with_hard_forks() diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index c314825413..becbe550a6 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -3030,10 +3030,11 @@ impl BeaconNodeHttpClient { } /// `GET validator/payload_attestation_data/{slot}` + /// Returns `None` if no block has been received for the requested slot (404). pub async fn get_validator_payload_attestation_data( &self, slot: Slot, - ) -> Result, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -3042,16 +3043,23 @@ impl BeaconNodeHttpClient { .push("payload_attestation_data") .push(&slot.to_string()); - self.get_with_timeout(path, self.timeouts.payload_attestation) + let opt_response = self + .get_response(path, |b| b.timeout(self.timeouts.payload_attestation)) .await - .map(BeaconResponse::ForkVersioned) + .optional()?; + + match opt_response { + Some(response) => Ok(Some(BeaconResponse::ForkVersioned(response.json().await?))), + None => Ok(None), + } } /// `GET validator/payload_attestation_data/{slot}` in SSZ format + /// Returns `None` if no block has been received for the requested slot (404). pub async fn get_validator_payload_attestation_data_ssz( &self, slot: Slot, - ) -> Result { + ) -> Result, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -3064,9 +3072,9 @@ impl BeaconNodeHttpClient { .get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.payload_attestation) .await?; - let response_bytes = opt_response.ok_or(Error::StatusCode(StatusCode::NOT_FOUND))?; - - PayloadAttestationData::from_ssz_bytes(&response_bytes).map_err(Error::InvalidSsz) + opt_response + .map(|bytes| PayloadAttestationData::from_ssz_bytes(&bytes).map_err(Error::InvalidSsz)) + .transpose() } /// `GET v1/validator/aggregate_attestation?slot,attestation_data_root` diff --git a/common/warp_utils/src/reject.rs b/common/warp_utils/src/reject.rs index c478870950..b88fd79b23 100644 --- a/common/warp_utils/src/reject.rs +++ b/common/warp_utils/src/reject.rs @@ -110,6 +110,17 @@ pub fn not_synced(msg: String) -> warp::reject::Rejection { warp::reject::custom(NotSynced(msg)) } +/// A 404 Not Found response for when no block has been received for the +/// requested slot. +#[derive(Debug)] +pub struct BlockNotFound(pub String); + +impl Reject for BlockNotFound {} + +pub fn block_not_found(msg: String) -> warp::reject::Rejection { + warp::reject::custom(BlockNotFound(msg)) +} + #[derive(Debug)] pub struct InvalidAuthorization(pub String); @@ -199,6 +210,9 @@ pub async fn handle_rejection(err: warp::Rejection) -> Result() { code = StatusCode::SERVICE_UNAVAILABLE; message = format!("SERVICE_UNAVAILABLE: beacon node is syncing: {}", e.0); + } else if let Some(e) = err.find::() { + code = StatusCode::NOT_FOUND; + message = format!("NOT_FOUND: {}", e.0); } else if let Some(e) = err.find::() { code = StatusCode::FORBIDDEN; message = format!("FORBIDDEN: Invalid auth token: {}", e.0); diff --git a/validator_client/validator_services/src/payload_attestation_service.rs b/validator_client/validator_services/src/payload_attestation_service.rs index 24949edc1f..f41893941f 100644 --- a/validator_client/validator_services/src/payload_attestation_service.rs +++ b/validator_client/validator_services/src/payload_attestation_service.rs @@ -139,14 +139,22 @@ impl PayloadAttestationServ beacon_node .get_validator_payload_attestation_data(slot) .await - .map_err(|e| format!("Failed to get payload attestation data: {e:?}")) - .map(|resp| resp.into_data()) + .map(|opt| opt.map(|resp| resp.into_data())) }) .await { - Ok(data) => data, + Ok(Some(data)) => data, + Ok(None) => { + // Per the consensus spec, validators should not submit a + // payload attestation when no block has been seen for the slot. + debug!( + %slot, + "No block received for slot, skipping payload attestation" + ); + return; + } Err(e) => { - crit!( + error!( error = %e, %slot, "Failed to produce payload attestation data" From 4b314d8e79e4f508192562565500c6b47aeb2766 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 5 May 2026 07:35:06 +0530 Subject: [PATCH 02/18] Remove libssl dependency for cargo udeps (#9263) N/A libssl download seems to be failing on [CI](https://github.com/sigp/lighthouse/actions/runs/25346412432/job/74316275231?pr=9126). This was originally added to unblock CI in https://github.com/sigp/lighthouse/pull/6777, but we may not need this anymore. Co-Authored-By: Pawan Dhananjay --- .github/workflows/test-suite.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index c632042351..9e646af9a7 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -414,10 +414,6 @@ jobs: cache: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Fetch libssl1.1 - run: wget https://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb - - name: Install libssl1.1 - run: sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb - name: Create Cargo config dir run: mkdir -p .cargo - name: Install custom Cargo config From 3351db1ba892b97f7ad851d366ac2a1921ed527f Mon Sep 17 00:00:00 2001 From: Mac L Date: Tue, 5 May 2026 10:35:57 +0400 Subject: [PATCH 03/18] Remove `TestRandom` (#9006) We have a legacy `TestRandom` trait which generates random types for testing and fuzzing. This function overlaps with `arbitrary` which is used very commonly in the ecosystem. Remove `TestRandom` and generate random type instances using `Arbitrary`. Co-Authored-By: Mac L Co-Authored-By: Michael Sproul --- .github/forbidden-files.txt | 2 + .github/workflows/test-suite.yml | 4 +- Cargo.lock | 30 ++-- Cargo.toml | 2 +- beacon_node/beacon_chain/Cargo.toml | 6 + .../src/data_availability_checker.rs | 30 ++-- .../overflow_lru_cache.rs | 10 +- .../src/naive_aggregation_pool.rs | 8 +- .../beacon_chain/src/observed_aggregates.rs | 6 +- beacon_node/beacon_chain/src/test_utils.rs | 42 +++--- beacon_node/beacon_chain/tests/events.rs | 8 +- beacon_node/beacon_chain/tests/store_tests.rs | 3 +- beacon_node/builder_client/Cargo.toml | 2 + beacon_node/builder_client/src/lib.rs | 6 +- beacon_node/network/Cargo.toml | 3 + .../src/sync/block_sidecar_coupling.rs | 38 +++-- beacon_node/network/src/sync/tests/lookups.rs | 18 +-- beacon_node/network/src/sync/tests/mod.rs | 4 +- common/eth2/Cargo.toml | 4 +- common/eth2/src/types.rs | 105 +++++++------ common/test_random_derive/Cargo.toml | 13 -- common/test_random_derive/src/lib.rs | 59 -------- consensus/state_processing/Cargo.toml | 2 +- .../state_processing/src/verify_operation.rs | 22 +-- consensus/types/Cargo.toml | 4 +- .../src/attestation/aggregate_and_proof.rs | 3 - .../types/src/attestation/attestation.rs | 5 +- .../types/src/attestation/attestation_data.rs | 15 +- consensus/types/src/attestation/checkpoint.rs | 3 - .../src/attestation/indexed_attestation.rs | 14 +- .../indexed_payload_attestation.rs | 4 +- .../src/attestation/participation_flags.rs | 8 +- .../src/attestation/payload_attestation.rs | 4 +- .../attestation/payload_attestation_data.rs | 6 +- .../payload_attestation_message.rs | 4 +- .../src/attestation/pending_attestation.rs | 5 +- .../attestation/signed_aggregate_and_proof.rs | 3 - consensus/types/src/block/beacon_block.rs | 98 +++--------- .../types/src/block/beacon_block_body.rs | 3 - .../types/src/block/beacon_block_header.rs | 6 +- .../types/src/block/signed_beacon_block.rs | 3 - .../src/block/signed_beacon_block_header.rs | 6 +- consensus/types/src/builder/builder.rs | 6 +- consensus/types/src/builder/builder_bid.rs | 10 +- .../src/builder/builder_pending_payment.rs | 15 +- .../src/builder/builder_pending_withdrawal.rs | 15 +- .../types/src/builder/proposer_preferences.rs | 8 +- .../consolidation/consolidation_request.rs | 6 +- .../consolidation/pending_consolidation.rs | 7 +- consensus/types/src/core/enr_fork_id.rs | 7 +- .../types/src/core/execution_block_hash.rs | 12 +- consensus/types/src/core/graffiti.rs | 9 -- consensus/types/src/core/signing_data.rs | 5 +- consensus/types/src/core/slot_epoch.rs | 6 +- consensus/types/src/core/slot_epoch_macros.rs | 6 - consensus/types/src/data/blob_sidecar.rs | 4 +- .../types/src/data/data_column_sidecar.rs | 3 - .../src/data/partial_data_column_sidecar.rs | 4 +- consensus/types/src/deposit/deposit.rs | 7 +- consensus/types/src/deposit/deposit_data.rs | 6 +- .../types/src/deposit/deposit_message.rs | 4 +- .../types/src/deposit/deposit_request.rs | 7 +- .../src/deposit/deposit_tree_snapshot.rs | 8 +- .../types/src/deposit/pending_deposit.rs | 6 +- .../src/execution/bls_to_execution_change.rs | 6 +- consensus/types/src/execution/eth1_data.rs | 16 +- .../types/src/execution/execution_payload.rs | 3 - .../src/execution/execution_payload_bid.rs | 6 +- .../execution/execution_payload_envelope.rs | 9 +- .../src/execution/execution_payload_header.rs | 3 - .../types/src/execution/execution_requests.rs | 6 +- consensus/types/src/execution/payload.rs | 17 +-- .../signed_bls_to_execution_change.rs | 7 +- .../execution/signed_execution_payload_bid.rs | 4 +- .../signed_execution_payload_envelope.rs | 9 +- .../types/src/exit/signed_voluntary_exit.rs | 7 +- consensus/types/src/exit/voluntary_exit.rs | 6 +- consensus/types/src/fork/fork.rs | 15 +- consensus/types/src/fork/fork_data.rs | 6 +- .../light_client/light_client_bootstrap.rs | 14 +- .../light_client_finality_update.rs | 14 +- .../src/light_client/light_client_header.rs | 14 +- .../light_client_optimistic_update.rs | 14 +- .../src/light_client/light_client_update.rs | 14 +- .../types/src/slashing/attester_slashing.rs | 14 -- .../types/src/slashing/proposer_slashing.rs | 7 +- consensus/types/src/state/beacon_state.rs | 70 +++++---- consensus/types/src/state/historical_batch.rs | 8 +- .../types/src/state/historical_summary.rs | 3 - .../sync_committee/contribution_and_proof.rs | 4 +- .../signed_contribution_and_proof.rs | 4 +- .../src/sync_committee/sync_aggregate.rs | 4 +- .../sync_aggregator_selection_data.rs | 6 +- .../src/sync_committee/sync_committee.rs | 5 +- .../sync_committee_contribution.rs | 6 +- .../sync_committee/sync_committee_message.rs | 4 +- .../generate_random_block_and_blobs.rs | 29 ++-- consensus/types/src/test_utils/macros.rs | 8 +- consensus/types/src/test_utils/mod.rs | 29 +++- .../src/test_utils/test_random/address.rs | 9 -- .../test_random/aggregate_signature.rs | 12 -- .../src/test_utils/test_random/bitfield.rs | 43 ------ .../src/test_utils/test_random/hash256.rs | 9 -- .../test_utils/test_random/kzg_commitment.rs | 9 -- .../src/test_utils/test_random/kzg_proof.rs | 11 -- .../types/src/test_utils/test_random/mod.rs | 15 -- .../src/test_utils/test_random/public_key.rs | 9 -- .../test_random/public_key_bytes.rs | 17 --- .../src/test_utils/test_random/secret_key.rs | 11 -- .../src/test_utils/test_random/signature.rs | 12 -- .../test_utils/test_random/signature_bytes.rs | 16 -- .../src/test_utils/test_random/test_random.rs | 140 ------------------ .../src/test_utils/test_random/uint256.rs | 9 -- consensus/types/src/validator/validator.rs | 6 +- .../withdrawal/pending_partial_withdrawal.rs | 7 +- consensus/types/src/withdrawal/withdrawal.rs | 6 +- .../src/withdrawal/withdrawal_request.rs | 7 +- consensus/types/tests/state.rs | 11 +- crypto/bls/src/macros.rs | 15 +- .../doppelganger_service/Cargo.toml | 2 + .../doppelganger_service/src/lib.rs | 10 +- 121 files changed, 418 insertions(+), 1141 deletions(-) delete mode 100644 common/test_random_derive/Cargo.toml delete mode 100644 common/test_random_derive/src/lib.rs delete mode 100644 consensus/types/src/test_utils/test_random/address.rs delete mode 100644 consensus/types/src/test_utils/test_random/aggregate_signature.rs delete mode 100644 consensus/types/src/test_utils/test_random/bitfield.rs delete mode 100644 consensus/types/src/test_utils/test_random/hash256.rs delete mode 100644 consensus/types/src/test_utils/test_random/kzg_commitment.rs delete mode 100644 consensus/types/src/test_utils/test_random/kzg_proof.rs delete mode 100644 consensus/types/src/test_utils/test_random/mod.rs delete mode 100644 consensus/types/src/test_utils/test_random/public_key.rs delete mode 100644 consensus/types/src/test_utils/test_random/public_key_bytes.rs delete mode 100644 consensus/types/src/test_utils/test_random/secret_key.rs delete mode 100644 consensus/types/src/test_utils/test_random/signature.rs delete mode 100644 consensus/types/src/test_utils/test_random/signature_bytes.rs delete mode 100644 consensus/types/src/test_utils/test_random/test_random.rs delete mode 100644 consensus/types/src/test_utils/test_random/uint256.rs diff --git a/.github/forbidden-files.txt b/.github/forbidden-files.txt index 8649fbb574..1c5e9acab9 100644 --- a/.github/forbidden-files.txt +++ b/.github/forbidden-files.txt @@ -12,4 +12,6 @@ beacon_node/http_api/src/block_rewards.rs common/eth2/src/lighthouse/attestation_performance.rs common/eth2/src/lighthouse/block_packing_efficiency.rs common/eth2/src/lighthouse/block_rewards.rs +common/test_random_derive/ consensus/types/src/execution/state_payload_status.rs +consensus/types/src/test_utils/test_random/ diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 9e646af9a7..1d66bd30e7 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -85,8 +85,8 @@ jobs: while IFS= read -r file || [ -n "$file" ]; do # Skip comments and empty lines [[ "$file" =~ ^#.*$ || -z "$file" ]] && continue - if [ -f "$file" ]; then - echo "::error::Forbidden file exists: $file" + if [ -e "$file" ]; then + echo "::error::Forbidden file or directory exists: $file" status=1 fi done < .github/forbidden-files.txt diff --git a/Cargo.lock b/Cargo.lock index aefd51a950..078f699f3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1224,6 +1224,8 @@ name = "beacon_chain" version = "0.2.0" dependencies = [ "alloy-primitives", + "arbitrary", + "beacon_chain", "bitvec", "bls", "criterion", @@ -1258,6 +1260,7 @@ dependencies = [ "parking_lot", "proto_array", "rand 0.9.2", + "rand_xorshift 0.4.0", "rayon", "safe_arith", "sensitive_url", @@ -1610,6 +1613,7 @@ dependencies = [ name = "builder_client" version = "0.1.0" dependencies = [ + "arbitrary", "bls", "context_deserialize", "eth2", @@ -1621,6 +1625,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "types", ] [[package]] @@ -2740,6 +2745,7 @@ dependencies = [ name = "doppelganger_service" version = "0.1.0" dependencies = [ + "arbitrary", "beacon_node_fallback", "bls", "environment", @@ -3116,6 +3122,7 @@ dependencies = [ name = "eth2" version = "0.1.0" dependencies = [ + "arbitrary", "bls", "context_deserialize", "educe", @@ -3132,7 +3139,6 @@ dependencies = [ "multiaddr", "pretty_reqwest_error", "proto_array", - "rand 0.9.2", "reqwest", "reqwest-eventsource", "sensitive_url", @@ -3140,7 +3146,6 @@ dependencies = [ "serde_json", "ssz_types", "superstruct", - "test_random_derive", "tokio", "types", "zeroize", @@ -3277,9 +3282,9 @@ dependencies = [ [[package]] name = "ethereum_ssz" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2128a84f7a3850d54ee343334e3392cca61f9f6aa9441eec481b9394b43c238b" +checksum = "368a4a4e4273b0135111fe9464e35465067766a8f664615b5a86338b73864407" dependencies = [ "alloy-primitives", "arbitrary", @@ -3294,9 +3299,9 @@ dependencies = [ [[package]] name = "ethereum_ssz_derive" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd596f91cff004fc8d02be44c21c0f9b93140a04b66027ae052f5f8e05b48eba" +checksum = "f2cd82c68120c89361e1a457245cf212f7d9f541bffaffed530c8f2d54a160b2" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -6053,6 +6058,7 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "anyhow", + "arbitrary", "async-channel 1.9.0", "beacon_chain", "beacon_processor", @@ -8428,7 +8434,7 @@ dependencies = [ "safe_arith", "smallvec", "ssz_types", - "test_random_derive", + "state_processing", "tokio", "tracing", "tree_hash", @@ -8713,14 +8719,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" -[[package]] -name = "test_random_derive" -version = "0.2.0" -dependencies = [ - "quote", - "syn 2.0.117", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -9361,12 +9359,12 @@ dependencies = [ "superstruct", "swap_or_not_shuffle", "tempfile", - "test_random_derive", "tokio", "tracing", "tree_hash", "tree_hash_derive", "typenum", + "types", "yaml_serde", ] diff --git a/Cargo.toml b/Cargo.toml index 1f58c322f1..71398530fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,6 @@ members = [ "common/system_health", "common/target_check", "common/task_executor", - "common/test_random_derive", "common/tracing_samplers", "common/validator_dir", "common/warp_utils", @@ -200,6 +199,7 @@ proto_array = { path = "consensus/proto_array" } quote = "1" r2d2 = "0.8" rand = "0.9.0" +rand_xorshift = "0.4.0" rayon = "1.7" regex = "1" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "stream", "rustls-tls"] } diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index a06db8934b..47ef4d7a03 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -16,9 +16,11 @@ participation_metrics = [] fork_from_env = [] portable = ["bls/supranational-portable"] test_backfill = [] +arbitrary = ["dep:arbitrary", "types/arbitrary"] [dependencies] alloy-primitives = { workspace = true } +arbitrary = { workspace = true, optional = true } bitvec = { workspace = true } bls = { workspace = true } educe = { workspace = true } @@ -74,11 +76,15 @@ types = { workspace = true } zstd = { workspace = true } [dev-dependencies] +arbitrary = { workspace = true } +beacon_chain = { path = ".", features = ["arbitrary"] } criterion = { workspace = true } maplit = { workspace = true } mockall = { workspace = true } mockall_double = { workspace = true } +rand_xorshift = { workspace = true } serde_json = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } [[bench]] name = "benches" diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 9d8b76aaed..f0fa9c7794 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -1041,8 +1041,6 @@ mod test { EphemeralHarnessType, NumBlobs, generate_data_column_indices_rand_order, generate_rand_block_and_data_columns, get_kzg, }; - use rand::SeedableRng; - use rand::prelude::StdRng; use slot_clock::{SlotClock, TestingSlotClock}; use std::collections::HashSet; use std::sync::Arc; @@ -1061,7 +1059,7 @@ mod test { fn should_exclude_rpc_columns_not_required_for_sampling() { // SETUP let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec())); - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let da_checker = new_da_checker(spec.clone()); let custody_context = &da_checker.custody_context; @@ -1093,9 +1091,10 @@ mod test { let (_, data_columns) = generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, - ); + ) + .unwrap(); let block_root = Hash256::random(); // Get 10 columns using the "latest" CGC (head) that block lookup would use. // The CGC change becomes effective after CUSTODY_CHANGE_DA_EFFECTIVE_DELAY_SECONDS, @@ -1147,7 +1146,7 @@ mod test { fn should_exclude_gossip_columns_not_required_for_sampling() { // SETUP let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec())); - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let da_checker = new_da_checker(spec.clone()); let custody_context = &da_checker.custody_context; @@ -1180,9 +1179,10 @@ mod test { let (_, data_columns) = generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, - ); + ) + .unwrap(); let block_root = Hash256::random(); // Get 10 columns using the "latest" CGC that gossip subscriptions would use. // The CGC change becomes effective after CUSTODY_CHANGE_DA_EFFECTIVE_DELAY_SECONDS, @@ -1230,7 +1230,7 @@ mod test { #[test] fn verify_kzg_for_range_sync_blocks_should_not_truncate_data_columns_fulu() { let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec())); - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let da_checker = new_da_checker(spec.clone()); // GIVEN multiple RPC blocks with data columns totalling more than 128 @@ -1239,9 +1239,10 @@ mod test { let (block, data_columns) = generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, - ); + ) + .unwrap(); let custody_columns = if index == 0 { // 128 valid data columns in the first block @@ -1293,7 +1294,7 @@ mod test { fn should_exclude_reconstructed_columns_not_required_for_sampling() { // SETUP let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec())); - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let da_checker = new_da_checker(spec.clone()); let custody_context = &da_checker.custody_context; @@ -1314,9 +1315,10 @@ mod test { let (block, data_columns) = generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, - ); + ) + .unwrap(); let block_root = Hash256::random(); // Add the block to the DA checker da_checker diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 8f1d4464e1..7d1bba2de9 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -1077,13 +1077,11 @@ mod pending_components_tests { use crate::PayloadVerificationOutcome; use crate::block_verification_types::BlockImportData; use crate::test_utils::{NumBlobs, generate_rand_block_and_blobs, test_spec}; + use arbitrary::Arbitrary; use fixed_bytes::FixedBytesExtended; use fork_choice::PayloadVerificationStatus; use kzg::KzgCommitment; - use rand::SeedableRng; - use rand::rngs::StdRng; use state_processing::ConsensusContext; - use types::test_utils::TestRandom; use types::{BeaconState, ForkName, MainnetEthSpec, SignedBeaconBlock, Slot}; type E = MainnetEthSpec; @@ -1096,10 +1094,10 @@ mod pending_components_tests { ); pub fn pre_setup() -> Setup { - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let spec = test_spec::(); let (block, blobs_vec) = - generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Random, &mut rng); + generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Random, &mut u).unwrap(); let max_len = spec.max_blobs_per_block(block.epoch()) as usize; let mut blobs: RuntimeFixedVector>>> = RuntimeFixedVector::default(max_len); @@ -1115,7 +1113,7 @@ mod pending_components_tests { for (index, blob) in blobs.iter().enumerate() { if let Some(invalid_blob) = blob { let mut blob_copy = invalid_blob.as_ref().clone(); - blob_copy.kzg_commitment = KzgCommitment::random_for_test(&mut rng); + blob_copy.kzg_commitment = KzgCommitment::arbitrary(&mut u).unwrap(); *invalid_blobs.get_mut(index).unwrap() = Some(Arc::new(blob_copy)); } } diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index 72080b92da..4d192cb5b9 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -582,20 +582,20 @@ mod tests { use tree_hash::TreeHash; use types::{ Attestation, AttestationBase, AttestationElectra, Fork, Hash256, SyncCommitteeMessage, - test_utils::{generate_deterministic_keypair, test_random_instance}, + test_utils::{generate_deterministic_keypair, test_arbitrary_instance}, }; type E = types::MainnetEthSpec; fn get_attestation_base(slot: Slot) -> Attestation { - let mut a: AttestationBase = test_random_instance(); + let mut a: AttestationBase = test_arbitrary_instance(); a.data.slot = slot; a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); Attestation::Base(a) } fn get_attestation_electra(slot: Slot) -> Attestation { - let mut a: AttestationElectra = test_random_instance(); + let mut a: AttestationElectra = test_arbitrary_instance(); a.data.slot = slot; a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); a.committee_bits = BitVector::new(); @@ -606,7 +606,7 @@ mod tests { } fn get_sync_contribution(slot: Slot) -> SyncCommitteeContribution { - let mut a: SyncCommitteeContribution = test_random_instance(); + let mut a: SyncCommitteeContribution = test_arbitrary_instance(); a.slot = slot; a.aggregation_bits = BitVector::new(); a diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index 7ecd581e85..8d4be693ac 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -474,12 +474,12 @@ where mod tests { use super::*; use fixed_bytes::FixedBytesExtended; - use types::{AttestationBase, Hash256, test_utils::test_random_instance}; + use types::{AttestationBase, Hash256, test_utils::test_arbitrary_instance}; type E = types::MainnetEthSpec; fn get_attestation(slot: Slot, beacon_block_root: u64) -> Attestation { - let a: AttestationBase = test_random_instance(); + let a: AttestationBase = test_arbitrary_instance(); let mut a = Attestation::Base(a); a.data_mut().slot = slot; a.data_mut().beacon_block_root = Hash256::from_low_u64_be(beacon_block_root); @@ -487,7 +487,7 @@ mod tests { } fn get_sync_contribution(slot: Slot, beacon_block_root: u64) -> SyncCommitteeContribution { - let mut a: SyncCommitteeContribution = test_random_instance(); + let mut a: SyncCommitteeContribution = test_arbitrary_instance(); a.slot = slot; a.beacon_block_root = Hash256::from_low_u64_be(beacon_block_root); a diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 8f437998c7..ca55811a70 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -20,6 +20,8 @@ pub use crate::{ sync_committee_verification::Error as SyncCommitteeError, validator_monitor::{ValidatorMonitor, ValidatorMonitorConfig}, }; +#[cfg(feature = "arbitrary")] +use arbitrary::Arbitrary; use bls::get_withdrawal_credentials; use bls::{ AggregateSignature, Keypair, PublicKey, PublicKeyBytes, SecretKey, Signature, SignatureBytes, @@ -73,7 +75,6 @@ use typenum::U4294967296; use types::attestation::IndexedAttestationBase; use types::data::CustodyIndex; use types::execution::BlockProductionVersion; -use types::test_utils::TestRandom; pub use types::test_utils::generate_deterministic_keypairs; use types::*; @@ -96,7 +97,9 @@ pub const TEST_DATA_COLUMN_SIDECARS_GLOAS_SSZ: &[u8] = pub const DEFAULT_TARGET_AGGREGATORS: u64 = u64::MAX; // Minimum and maximum number of blobs to generate in each slot when using the `NumBlobs::Random` option (default). +#[cfg(feature = "arbitrary")] const DEFAULT_MIN_BLOBS: usize = 1; +#[cfg(feature = "arbitrary")] const DEFAULT_MAX_BLOBS: usize = 2; static KZG: LazyLock> = LazyLock::new(|| { @@ -3741,10 +3744,11 @@ pub enum NumBlobs { None, } +#[cfg(feature = "arbitrary")] macro_rules! add_blob_transactions { - ($message:expr, $payload_type:ty, $num_blobs:expr, $rng:expr, $fork_name:expr) => {{ + ($message:expr, $payload_type:ty, $num_blobs:expr, $u:expr, $fork_name:expr) => {{ let num_blobs = match $num_blobs { - NumBlobs::Random => $rng.random_range(DEFAULT_MIN_BLOBS..=DEFAULT_MAX_BLOBS), + NumBlobs::Random => $u.int_in_range(DEFAULT_MIN_BLOBS..=DEFAULT_MAX_BLOBS)?, NumBlobs::Number(n) => n, NumBlobs::None => 0, }; @@ -3761,28 +3765,30 @@ macro_rules! add_blob_transactions { }}; } +#[cfg(feature = "arbitrary")] +#[allow(clippy::type_complexity)] pub fn generate_rand_block_and_blobs( fork_name: ForkName, num_blobs: NumBlobs, - rng: &mut impl Rng, -) -> (SignedBeaconBlock>, Vec>) { - let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(rng)); + u: &mut arbitrary::Unstructured, +) -> arbitrary::Result<(SignedBeaconBlock>, Vec>)> { + let inner = map_fork_name!(fork_name, BeaconBlock, <_>::arbitrary(&mut *u)?); - let mut block = SignedBeaconBlock::from_block(inner, Signature::random_for_test(rng)); + let mut block = SignedBeaconBlock::from_block(inner, Signature::arbitrary(&mut *u)?); let mut blob_sidecars = vec![]; let bundle = match block { SignedBeaconBlock::Deneb(SignedBeaconBlockDeneb { ref mut message, .. - }) => add_blob_transactions!(message, FullPayloadDeneb, num_blobs, rng, fork_name), + }) => add_blob_transactions!(message, FullPayloadDeneb, num_blobs, u, fork_name), SignedBeaconBlock::Electra(SignedBeaconBlockElectra { ref mut message, .. - }) => add_blob_transactions!(message, FullPayloadElectra, num_blobs, rng, fork_name), + }) => add_blob_transactions!(message, FullPayloadElectra, num_blobs, u, fork_name), SignedBeaconBlock::Fulu(SignedBeaconBlockFulu { ref mut message, .. - }) => add_blob_transactions!(message, FullPayloadFulu, num_blobs, rng, fork_name), + }) => add_blob_transactions!(message, FullPayloadFulu, num_blobs, u, fork_name), // TODO(EIP-7732) Add `SignedBeaconBlock::Gloas` variant - _ => return (block, blob_sidecars), + _ => return Ok((block, blob_sidecars)), }; let eth2::types::BlobsBundle { @@ -3807,21 +3813,23 @@ pub fn generate_rand_block_and_blobs( .unwrap(), }); } - (block, blob_sidecars) + Ok((block, blob_sidecars)) } +#[cfg(feature = "arbitrary")] +#[allow(clippy::type_complexity)] pub fn generate_rand_block_and_data_columns( fork_name: ForkName, num_blobs: NumBlobs, - rng: &mut impl Rng, + u: &mut arbitrary::Unstructured, spec: &ChainSpec, -) -> ( +) -> arbitrary::Result<( SignedBeaconBlock>, DataColumnSidecarList, -) { - let (block, _blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, rng); +)> { + let (block, _blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, u)?; let data_columns = generate_data_column_sidecars_from_block(&block, spec); - (block, data_columns) + Ok((block, data_columns)) } /// Generate data column sidecars from pre-computed cells and proofs. diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index e943514c4e..cd0e700109 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -1,3 +1,4 @@ +use arbitrary::Arbitrary; use beacon_chain::blob_verification::GossipVerifiedBlob; use beacon_chain::data_column_verification::GossipVerifiedDataColumn; use beacon_chain::test_utils::{ @@ -8,7 +9,6 @@ use rand::SeedableRng; use rand::rngs::StdRng; use std::sync::Arc; use types::data::FixedBlobSidecarList; -use types::test_utils::TestRandom; use types::{ BlobSidecar, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSidecarGloas, Domain, EthSpec, MinimalEthSpec, PayloadAttestationData, PayloadAttestationMessage, SignedRoot, Slot, @@ -74,19 +74,19 @@ async fn data_column_sidecar_event_on_process_gossip_data_column() { let mut data_column_event_receiver = event_handler.subscribe_data_column_sidecar(); // build and process a gossip verified data column - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let sidecar = { let slot = Slot::new(10); let fork_name = harness.spec.fork_name_at_slot::(slot); // DA checker only accepts sampling columns, so we need to create one with a sampling index. if fork_name.gloas_enabled() { - let mut random_sidecar = DataColumnSidecarGloas::random_for_test(&mut rng); + let mut random_sidecar = DataColumnSidecarGloas::arbitrary(&mut u).unwrap(); let epoch = slot.epoch(E::slots_per_epoch()); random_sidecar.slot = slot; random_sidecar.index = harness.chain.sampling_columns_for_epoch(epoch)[0]; DataColumnSidecar::Gloas(random_sidecar) } else { - let mut random_sidecar = DataColumnSidecarFulu::random_for_test(&mut rng); + let mut random_sidecar = DataColumnSidecarFulu::arbitrary(&mut u).unwrap(); let epoch = slot.epoch(E::slots_per_epoch()); random_sidecar.signed_block_header.message.slot = slot; random_sidecar.index = harness.chain.sampling_columns_for_epoch(epoch)[0]; diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 86adf50995..1576092c81 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -31,7 +31,9 @@ use fork_choice::PayloadStatus; use logging::create_test_tracing_subscriber; use maplit::hashset; use rand::Rng; +use rand::SeedableRng; use rand::rngs::StdRng; +use rand_xorshift::XorShiftRng; use slot_clock::{SlotClock, TestingSlotClock}; use ssz_types::VariableList; use state_processing::{BlockReplayer, state_advance::complete_state_advance}; @@ -50,7 +52,6 @@ use store::{ }; use tempfile::{TempDir, tempdir}; use tracing::info; -use types::test_utils::{SeedableRng, XorShiftRng}; use types::*; // Should ideally be divisible by 3. diff --git a/beacon_node/builder_client/Cargo.toml b/beacon_node/builder_client/Cargo.toml index 09bf3f48b4..a329379160 100644 --- a/beacon_node/builder_client/Cargo.toml +++ b/beacon_node/builder_client/Cargo.toml @@ -16,5 +16,7 @@ serde = { workspace = true } serde_json = { workspace = true } [dev-dependencies] +arbitrary = { workspace = true } mockito = { workspace = true } tokio = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index 7dc0cbfc6d..bd064ca8bf 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -540,10 +540,10 @@ impl BuilderHttpClient { #[cfg(test)] mod tests { use super::*; + use arbitrary::Arbitrary; use bls::Signature; use eth2::types::MainnetEthSpec; use eth2::types::builder::{BuilderBid, BuilderBidFulu}; - use eth2::types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use mockito::{Matcher, Server, ServerGuard}; type E = MainnetEthSpec; @@ -689,12 +689,12 @@ mod tests { } fn fulu_signed_builder_bid() -> ForkVersionedResponse> { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); ForkVersionedResponse { version: ForkName::Fulu, metadata: EmptyMetadata {}, data: SignedBuilderBid { - message: BuilderBid::Fulu(BuilderBidFulu::random_for_test(rng)), + message: BuilderBid::Fulu(BuilderBidFulu::arbitrary(&mut u).unwrap()), signature: Signature::empty(), }, } diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index 319ea2b149..607f231a66 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -49,6 +49,8 @@ typenum = { workspace = true } types = { workspace = true } [dev-dependencies] +arbitrary = { workspace = true } +beacon_chain = { workspace = true, features = ["arbitrary"] } bls = { workspace = true } eth2 = { workspace = true } eth2_network_config = { workspace = true } @@ -62,3 +64,4 @@ rand_08 = { package = "rand", version = "0.8.5" } rand_chacha = "0.9.0" rand_chacha_03 = { package = "rand_chacha", version = "0.3.1" } serde_json = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index 98cf3e0a1f..f5c0fdb4e5 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -501,10 +501,9 @@ mod tests { DataColumnsByRangeRequestId, DataColumnsByRangeRequester, Id, RangeRequestId, }, }; - use rand::SeedableRng; use std::{collections::HashMap, sync::Arc}; use tracing::Span; - use types::{Epoch, ForkName, MinimalEthSpec as E, SignedBeaconBlock, test_utils::XorShiftRng}; + use types::{Epoch, ForkName, MinimalEthSpec as E, SignedBeaconBlock}; fn components_id() -> ComponentsByRangeRequestId { ComponentsByRangeRequestId { @@ -549,10 +548,11 @@ mod tests { #[test] fn no_blobs_into_responses() { - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..4) .map(|_| { - generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng) + generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut u) + .unwrap() .0 .into() }) @@ -574,11 +574,12 @@ mod tests { #[test] fn empty_blobs_into_responses() { - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..4) .map(|_| { // Always generate some blobs. - generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Number(3), &mut rng) + generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Number(3), &mut u) + .unwrap() .0 .into() }) @@ -619,15 +620,16 @@ mod tests { .custody_context() .sampling_columns_for_epoch(Epoch::new(0), &spec) .to_vec(); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..4) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); @@ -729,15 +731,16 @@ mod tests { Span::none(), ); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..4) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); @@ -787,15 +790,16 @@ mod tests { .custody_context() .sampling_columns_for_epoch(Epoch::new(0), &spec) .to_vec(); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..2) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); @@ -884,15 +888,16 @@ mod tests { .custody_context() .sampling_columns_for_epoch(Epoch::new(0), &spec) .to_vec(); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..2) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); @@ -999,15 +1004,16 @@ mod tests { .custody_context() .sampling_columns_for_epoch(Epoch::new(0), &spec) .to_vec(); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..1) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index a26996ec5e..d27c92c21a 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -38,7 +38,6 @@ use tracing::info; use types::{ BlobSidecar, BlockImportSource, ColumnIndex, DataColumnSidecar, EthSpec, ForkContext, ForkName, Hash256, MinimalEthSpec as E, SignedBeaconBlock, Slot, - test_utils::{SeedableRng, XorShiftRng}, }; const D: Duration = Duration::new(0, 0); @@ -279,7 +278,6 @@ impl TestRig { // deterministic seed let rng_08 = ::from_seed([0u8; 32]); - let rng = ChaCha20Rng::from_seed([0u8; 32]); init_tracing(); @@ -291,7 +289,7 @@ impl TestRig { sync_rx, sync_rx_queue: vec![], rng_08, - rng, + unstructured: types::test_utils::test_unstructured(), network_globals: beacon_processor.network_globals.clone(), sync_manager: SyncManager::new( chain, @@ -1492,8 +1490,7 @@ impl TestRig { num_blobs: NumBlobs, ) -> (SignedBeaconBlock, Vec>) { let fork_name = self.fork_name; - let rng = &mut self.rng; - generate_rand_block_and_blobs::(fork_name, num_blobs, rng) + generate_rand_block_and_blobs::(fork_name, num_blobs, &mut self.unstructured).unwrap() } pub fn send_sync_message(&mut self, sync_message: SyncMessage) { @@ -1829,16 +1826,17 @@ impl TestRig { } #[test] -fn stable_rng() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let (block, _) = generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng); +fn stable_arbitrary() { + let mut u = types::test_utils::test_unstructured(); + let (block, _) = + generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut u).unwrap(); assert_eq!( block.canonical_root(), Hash256::from_slice( - &hex::decode("adfd2e9e7a7976e8ccaed6eaf0257ed36a5b476732fee63ff44966602fd099ec") + &hex::decode("7348573d99ca404b502e2be790593203a1d899f9cf04f42ec9c5b4975803e3c5") .unwrap() ), - "rng produces a consistent value" + "arbitrary produces a consistent value" ); } diff --git a/beacon_node/network/src/sync/tests/mod.rs b/beacon_node/network/src/sync/tests/mod.rs index 8ffe24dda5..dd8c3ae432 100644 --- a/beacon_node/network/src/sync/tests/mod.rs +++ b/beacon_node/network/src/sync/tests/mod.rs @@ -11,7 +11,6 @@ use beacon_processor::WorkEvent; use lighthouse_network::rpc::RequestType; use lighthouse_network::service::api_types::{AppRequestId, Id}; use lighthouse_network::{NetworkGlobals, PeerId}; -use rand_chacha::ChaCha20Rng; use slot_clock::ManualSlotClock; use std::collections::{HashMap, HashSet}; use std::fs::OpenOptions; @@ -72,9 +71,8 @@ struct TestRig { network_globals: Arc>, /// Beacon chain harness harness: BeaconChainHarness>, - /// `rng` for generating test blocks and blobs. rng_08: rand_chacha_03::ChaCha20Rng, - rng: ChaCha20Rng, + unstructured: arbitrary::Unstructured<'static>, fork_name: ForkName, /// Blocks that will be used in the test but may not be known to `harness` yet. network_blocks_by_root: HashMap>, diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index 974508492a..5e015f2713 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -38,6 +38,6 @@ types = { workspace = true } zeroize = { workspace = true, optional = true } [dev-dependencies] -rand = { workspace = true } -test_random_derive = { path = "../../common/test_random_derive" } +arbitrary = { workspace = true } tokio = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index e1a1166ba7..dfa0fbd87d 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -26,11 +26,6 @@ use std::sync::Arc; use std::time::Duration; use superstruct::superstruct; -#[cfg(test)] -use test_random_derive::TestRandom; -#[cfg(test)] -use types::test_utils::TestRandom; - // TODO(mac): Temporary module and re-export hack to expose old `consensus/types` via `eth2/types`. pub use crate::beacon_response::*; pub mod beacon_response { @@ -2364,7 +2359,7 @@ pub enum ContentType { Ssz, } -#[cfg_attr(test, derive(TestRandom))] +#[cfg_attr(test, derive(arbitrary::Arbitrary))] #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode, Decode)] #[serde(bound = "E: EthSpec")] pub struct BlobsBundle { @@ -2470,7 +2465,7 @@ pub struct BlobWrapper { mod test { use std::fmt::Debug; - use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use arbitrary::Arbitrary; use super::*; @@ -2498,13 +2493,16 @@ mod test { assert_eq!(request, deserialized_request); }; - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); for fork_name in ForkName::list_all() { - let signed_beacon_block = - map_fork_name!(fork_name, SignedBeaconBlock, <_>::random_for_test(rng)); + let signed_beacon_block = map_fork_name!( + fork_name, + SignedBeaconBlock, + <_>::arbitrary(&mut u).unwrap() + ); let request = if fork_name.deneb_enabled() && !fork_name.gloas_enabled() { - let kzg_proofs = KzgProofs::::random_for_test(rng); - let blobs = BlobsList::::random_for_test(rng); + let kzg_proofs = KzgProofs::::arbitrary(&mut u).unwrap(); + let blobs = BlobsList::::arbitrary(&mut u).unwrap(); let block_contents = SignedBlockContents { signed_block: Arc::new(signed_beacon_block), kzg_proofs, @@ -2532,12 +2530,15 @@ mod test { }; let mut fork_name = ForkName::Deneb; - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); loop { - let signed_beacon_block = - map_fork_name!(fork_name, SignedBeaconBlock, <_>::random_for_test(rng)); - let kzg_proofs = KzgProofs::::random_for_test(rng); - let blobs = BlobsList::::random_for_test(rng); + let signed_beacon_block = map_fork_name!( + fork_name, + SignedBeaconBlock, + <_>::arbitrary(&mut u).unwrap() + ); + let kzg_proofs = KzgProofs::::arbitrary(&mut u).unwrap(); + let blobs = BlobsList::::arbitrary(&mut u).unwrap(); let block_contents = SignedBlockContents { signed_block: Arc::new(signed_beacon_block), kzg_proofs, @@ -2555,25 +2556,27 @@ mod test { #[test] fn test_execution_payload_execution_payload_deserialize_by_fork() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let payloads = [ ExecutionPayload::Bellatrix( - ExecutionPayloadBellatrix::::random_for_test(rng), + ExecutionPayloadBellatrix::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Capella( + ExecutionPayloadCapella::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Deneb( + ExecutionPayloadDeneb::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Electra( + ExecutionPayloadElectra::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Fulu( + ExecutionPayloadFulu::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Gloas( + ExecutionPayloadGloas::::arbitrary(&mut u).unwrap(), ), - ExecutionPayload::Capella(ExecutionPayloadCapella::::random_for_test( - rng, - )), - ExecutionPayload::Deneb(ExecutionPayloadDeneb::::random_for_test( - rng, - )), - ExecutionPayload::Electra(ExecutionPayloadElectra::::random_for_test( - rng, - )), - ExecutionPayload::Fulu(ExecutionPayloadFulu::::random_for_test(rng)), - ExecutionPayload::Gloas(ExecutionPayloadGloas::::random_for_test( - rng, - )), ]; let merged_forks = &ForkName::list_all()[2..]; assert_eq!( @@ -2592,48 +2595,44 @@ mod test { #[test] fn test_execution_payload_and_blobs_deserialize_by_fork() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let payloads = [ { - let execution_payload = - ExecutionPayload::Deneb( - ExecutionPayloadDeneb::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); + let execution_payload = ExecutionPayload::Deneb( + ExecutionPayloadDeneb::::arbitrary(&mut u).unwrap(), + ); + let blobs_bundle = BlobsBundle::::arbitrary(&mut u).unwrap(); ExecutionPayloadAndBlobs { execution_payload, blobs_bundle, } }, { - let execution_payload = - ExecutionPayload::Electra( - ExecutionPayloadElectra::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); + let execution_payload = ExecutionPayload::Electra( + ExecutionPayloadElectra::::arbitrary(&mut u).unwrap(), + ); + let blobs_bundle = BlobsBundle::::arbitrary(&mut u).unwrap(); ExecutionPayloadAndBlobs { execution_payload, blobs_bundle, } }, { - let execution_payload = - ExecutionPayload::Fulu( - ExecutionPayloadFulu::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); + let execution_payload = ExecutionPayload::Fulu( + ExecutionPayloadFulu::::arbitrary(&mut u).unwrap(), + ); + let blobs_bundle = BlobsBundle::::arbitrary(&mut u).unwrap(); ExecutionPayloadAndBlobs { execution_payload, blobs_bundle, } }, { - let execution_payload = - ExecutionPayload::Gloas( - ExecutionPayloadGloas::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); + let execution_payload = ExecutionPayload::Gloas( + ExecutionPayloadGloas::::arbitrary(&mut u).unwrap(), + ); + let blobs_bundle = BlobsBundle::::arbitrary(&mut u).unwrap(); ExecutionPayloadAndBlobs { execution_payload, blobs_bundle, diff --git a/common/test_random_derive/Cargo.toml b/common/test_random_derive/Cargo.toml deleted file mode 100644 index b38d5ef63a..0000000000 --- a/common/test_random_derive/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "test_random_derive" -version = "0.2.0" -authors = ["thojest "] -edition = { workspace = true } -description = "Procedural derive macros for implementation of TestRandom trait" - -[lib] -proc-macro = true - -[dependencies] -quote = { workspace = true } -syn = { workspace = true } diff --git a/common/test_random_derive/src/lib.rs b/common/test_random_derive/src/lib.rs deleted file mode 100644 index bf57d79aaa..0000000000 --- a/common/test_random_derive/src/lib.rs +++ /dev/null @@ -1,59 +0,0 @@ -use proc_macro::TokenStream; -use quote::quote; -use syn::{DeriveInput, parse_macro_input}; - -/// Returns true if some field has an attribute declaring it should be generated from default (not -/// randomized). -/// -/// The field attribute is: `#[test_random(default)]` -fn should_use_default(field: &syn::Field) -> bool { - field.attrs.iter().any(|attr| { - attr.path().is_ident("test_random") - && matches!(&attr.meta, syn::Meta::List(list) if list.tokens.to_string().replace(' ', "") == "default") - }) -} - -#[proc_macro_derive(TestRandom, attributes(test_random))] -pub fn test_random_derive(input: TokenStream) -> TokenStream { - let derived_input = parse_macro_input!(input as DeriveInput); - let name = &derived_input.ident; - let (impl_generics, ty_generics, where_clause) = &derived_input.generics.split_for_impl(); - - let syn::Data::Struct(struct_data) = &derived_input.data else { - panic!("test_random_derive only supports structs."); - }; - - // Build quotes for fields that should be generated and those that should be built from - // `Default`. - let mut quotes = vec![]; - for field in &struct_data.fields { - match &field.ident { - Some(ident) => { - if should_use_default(field) { - quotes.push(quote! { - #ident: <_>::default(), - }); - } else { - quotes.push(quote! { - #ident: <_>::random_for_test(rng), - }); - } - } - _ => panic!("test_random_derive only supports named struct fields."), - }; - } - - let output = quote! { - impl #impl_generics TestRandom for #name #ty_generics #where_clause { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - Self { - #( - #quotes - )* - } - } - } - }; - - output.into() -} diff --git a/consensus/state_processing/Cargo.toml b/consensus/state_processing/Cargo.toml index ae0af03231..72d0e17d99 100644 --- a/consensus/state_processing/Cargo.toml +++ b/consensus/state_processing/Cargo.toml @@ -37,7 +37,6 @@ rayon = { workspace = true } safe_arith = { workspace = true } smallvec = { workspace = true } ssz_types = { workspace = true } -test_random_derive = { path = "../../common/test_random_derive" } tracing = { workspace = true } tree_hash = { workspace = true } typenum = { workspace = true } @@ -45,4 +44,5 @@ types = { workspace = true } [dev-dependencies] beacon_chain = { workspace = true } +state_processing = { path = ".", features = ["arbitrary"] } tokio = { workspace = true } diff --git a/consensus/state_processing/src/verify_operation.rs b/consensus/state_processing/src/verify_operation.rs index 1e9c3d5fe3..8e67c3da43 100644 --- a/consensus/state_processing/src/verify_operation.rs +++ b/consensus/state_processing/src/verify_operation.rs @@ -14,11 +14,10 @@ use smallvec::{SmallVec, smallvec}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::marker::PhantomData; -use test_random_derive::TestRandom; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingOnDisk, AttesterSlashingRefOnDisk, BeaconState, ChainSpec, Epoch, EthSpec, Fork, ForkVersion, ProposerSlashing, - SignedBlsToExecutionChange, SignedVoluntaryExit, test_utils::TestRandom, + SignedBlsToExecutionChange, SignedVoluntaryExit, }; const MAX_FORKS_VERIFIED_AGAINST: usize = 2; @@ -138,7 +137,7 @@ struct SigVerifiedOpDecode { /// /// We need to store multiple `ForkVersion`s because attester slashings contain two indexed /// attestations which may be signed using different versions. -#[derive(Debug, PartialEq, Eq, Clone, Hash, Encode, Decode, TestRandom)] +#[derive(Debug, PartialEq, Eq, Clone, Hash, Encode, Decode)] #[cfg_attr(feature = "arbitrary", derive(Arbitrary))] pub struct VerifiedAgainst { fork_versions: SmallVec<[ForkVersion; MAX_FORKS_VERIFIED_AGAINST]>, @@ -423,20 +422,21 @@ impl TransformPersist for SignedBlsToExecutionChange { #[cfg(all(test, not(debug_assertions)))] mod test { use super::*; - use types::{ - MainnetEthSpec, - test_utils::{SeedableRng, TestRandom, XorShiftRng}, - }; + use types::MainnetEthSpec; type E = MainnetEthSpec; - fn roundtrip_test() { + fn roundtrip_test<'a, T>() + where + T: arbitrary::Arbitrary<'a> + TransformPersist + PartialEq + std::fmt::Debug, + { let runs = 10; - let mut rng = XorShiftRng::seed_from_u64(0xff0af5a356af1123); + let mut u = types::test_utils::test_unstructured(); for _ in 0..runs { - let op = T::random_for_test(&mut rng); - let verified_against = VerifiedAgainst::random_for_test(&mut rng); + let op = T::arbitrary(&mut u).expect("arbitrary op"); + let verified_against = + VerifiedAgainst::arbitrary(&mut u).expect("arbitrary verified_against"); let verified_op = SigVerifiedOp { op, diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 4aae4b7f39..9ee827c7b9 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -45,7 +45,7 @@ metastruct = "0.1.0" milhouse = { workspace = true } parking_lot = { workspace = true } rand = { workspace = true } -rand_xorshift = "0.4.0" +rand_xorshift = { workspace = true } rayon = { workspace = true } regex = { workspace = true } rpds = { workspace = true } @@ -58,7 +58,6 @@ ssz_types = { workspace = true } superstruct = { workspace = true } swap_or_not_shuffle = { workspace = true } tempfile = { workspace = true } -test_random_derive = { path = "../../common/test_random_derive" } tracing = { workspace = true } tree_hash = { workspace = true } tree_hash_derive = { workspace = true } @@ -71,6 +70,7 @@ criterion = { workspace = true } paste = { workspace = true } state_processing = { workspace = true } tokio = { workspace = true } +types = { path = ".", features = ["arbitrary"] } [lints.clippy] module_inception = "allow" diff --git a/consensus/types/src/attestation/aggregate_and_proof.rs b/consensus/types/src/attestation/aggregate_and_proof.rs index 4c6e775e56..76e33faf88 100644 --- a/consensus/types/src/attestation/aggregate_and_proof.rs +++ b/consensus/types/src/attestation/aggregate_and_proof.rs @@ -3,7 +3,6 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -12,7 +11,6 @@ use crate::{ }, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; #[superstruct( @@ -26,7 +24,6 @@ use crate::{ Deserialize, Encode, Decode, - TestRandom, TreeHash, ), context_deserialize(ForkName), diff --git a/consensus/types/src/attestation/attestation.rs b/consensus/types/src/attestation/attestation.rs index 28059efee6..4cfb7a4d24 100644 --- a/consensus/types/src/attestation/attestation.rs +++ b/consensus/types/src/attestation/attestation.rs @@ -10,7 +10,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{BitList, BitVector}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -20,7 +19,6 @@ use crate::{ }, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot, Slot, SlotData}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; #[derive(Debug, PartialEq, Clone)] @@ -49,7 +47,6 @@ impl From for Error { Deserialize, Decode, Encode, - TestRandom, Educe, TreeHash, ), @@ -614,7 +611,7 @@ impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for Vec> */ #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive(Debug, Clone, Serialize, Deserialize, Decode, Encode, TestRandom, TreeHash, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, Decode, Encode, TreeHash, PartialEq)] #[context_deserialize(ForkName)] pub struct SingleAttestation { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/attestation/attestation_data.rs b/consensus/types/src/attestation/attestation_data.rs index f3fceb9b70..2d88bce2b9 100644 --- a/consensus/types/src/attestation/attestation_data.rs +++ b/consensus/types/src/attestation/attestation_data.rs @@ -1,14 +1,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ attestation::Checkpoint, core::{Hash256, SignedRoot, Slot, SlotData}, fork::ForkName, - test_utils::TestRandom, }; /// The data upon which an attestation is based. @@ -16,18 +14,7 @@ use crate::{ /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[derive( - Debug, - Clone, - PartialEq, - Eq, - Serialize, - Deserialize, - Hash, - Encode, - Decode, - TreeHash, - TestRandom, - Default, + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Encode, Decode, TreeHash, Default, )] #[context_deserialize(ForkName)] pub struct AttestationData { diff --git a/consensus/types/src/attestation/checkpoint.rs b/consensus/types/src/attestation/checkpoint.rs index f5a95f0ad9..09f8f06e6e 100644 --- a/consensus/types/src/attestation/checkpoint.rs +++ b/consensus/types/src/attestation/checkpoint.rs @@ -1,13 +1,11 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Epoch, Hash256}, fork::ForkName, - test_utils::TestRandom, }; /// Casper FFG checkpoint, used in attestations. @@ -27,7 +25,6 @@ use crate::{ Encode, Decode, TreeHash, - TestRandom, )] #[context_deserialize(ForkName)] pub struct Checkpoint { diff --git a/consensus/types/src/attestation/indexed_attestation.rs b/consensus/types/src/attestation/indexed_attestation.rs index 272b015d90..ae15f474f3 100644 --- a/consensus/types/src/attestation/indexed_attestation.rs +++ b/consensus/types/src/attestation/indexed_attestation.rs @@ -11,10 +11,9 @@ use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName, test_utils::TestRandom}; +use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName}; /// Details an attestation that can be slashable. /// @@ -31,7 +30,6 @@ use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName, test_ut Deserialize, Decode, Encode, - TestRandom, Educe, TreeHash, ), @@ -212,10 +210,8 @@ impl Hash for IndexedAttestation { #[cfg(test)] mod tests { use super::*; - use crate::{ - core::{Epoch, MainnetEthSpec}, - test_utils::{SeedableRng, XorShiftRng}, - }; + use crate::core::{Epoch, MainnetEthSpec}; + use arbitrary::Arbitrary; #[test] pub fn test_is_double_vote_true() { @@ -278,9 +274,9 @@ mod tests { target_epoch: u64, source_epoch: u64, ) -> IndexedAttestation { - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let mut indexed_vote = - IndexedAttestation::Base(IndexedAttestationBase::random_for_test(&mut rng)); + IndexedAttestation::Base(IndexedAttestationBase::arbitrary(&mut u).unwrap()); indexed_vote.data_mut().source.epoch = Epoch::new(source_epoch); indexed_vote.data_mut().target.epoch = Epoch::new(target_epoch); diff --git a/consensus/types/src/attestation/indexed_payload_attestation.rs b/consensus/types/src/attestation/indexed_payload_attestation.rs index bb2087e330..67fdf77bdf 100644 --- a/consensus/types/src/attestation/indexed_payload_attestation.rs +++ b/consensus/types/src/attestation/indexed_payload_attestation.rs @@ -1,14 +1,12 @@ -use crate::test_utils::TestRandom; use crate::{EthSpec, ForkName, PayloadAttestationData}; use bls::AggregateSignature; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(TestRandom, TreeHash, Debug, Clone, PartialEq, Encode, Decode, Serialize, Deserialize)] +#[derive(TreeHash, Debug, Clone, PartialEq, Encode, Decode, Serialize, Deserialize)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[serde(bound = "E: EthSpec", deny_unknown_fields)] #[cfg_attr(feature = "arbitrary", arbitrary(bound = "E: EthSpec"))] diff --git a/consensus/types/src/attestation/participation_flags.rs b/consensus/types/src/attestation/participation_flags.rs index 66831abfac..a88ea0d3f7 100644 --- a/consensus/types/src/attestation/participation_flags.rs +++ b/consensus/types/src/attestation/participation_flags.rs @@ -1,15 +1,11 @@ use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use ssz::{Decode, DecodeError, Encode}; -use test_random_derive::TestRandom; use tree_hash::{PackedEncoding, TreeHash, TreeHashType}; -use crate::{ - core::{Hash256, consts::altair::NUM_FLAG_INDICES}, - test_utils::TestRandom, -}; +use crate::core::{Hash256, consts::altair::NUM_FLAG_INDICES}; -#[derive(Debug, Default, Clone, Copy, PartialEq, Deserialize, Serialize, TestRandom)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Deserialize, Serialize)] #[serde(transparent)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub struct ParticipationFlags { diff --git a/consensus/types/src/attestation/payload_attestation.rs b/consensus/types/src/attestation/payload_attestation.rs index 115a5ec4d6..d5e76f941b 100644 --- a/consensus/types/src/attestation/payload_attestation.rs +++ b/consensus/types/src/attestation/payload_attestation.rs @@ -1,5 +1,4 @@ use crate::attestation::payload_attestation_data::PayloadAttestationData; -use crate::test_utils::TestRandom; use crate::{EthSpec, ForkName}; use bls::AggregateSignature; use context_deserialize::context_deserialize; @@ -7,10 +6,9 @@ use educe::Educe; use serde::{Deserialize, Serialize}; use ssz::BitVector; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(TestRandom, TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] +#[derive(TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[serde(bound = "E: EthSpec", deny_unknown_fields)] #[cfg_attr(feature = "arbitrary", arbitrary(bound = "E: EthSpec"))] diff --git a/consensus/types/src/attestation/payload_attestation_data.rs b/consensus/types/src/attestation/payload_attestation_data.rs index 58d36fd01d..198d380c14 100644 --- a/consensus/types/src/attestation/payload_attestation_data.rs +++ b/consensus/types/src/attestation/payload_attestation_data.rs @@ -1,14 +1,10 @@ -use crate::test_utils::TestRandom; use crate::{ForkName, Hash256, SignedRoot, Slot}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive( - TestRandom, TreeHash, Debug, Clone, PartialEq, Eq, Encode, Decode, Serialize, Deserialize, Hash, -)] +#[derive(TreeHash, Debug, Clone, PartialEq, Eq, Encode, Decode, Serialize, Deserialize, Hash)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[context_deserialize(ForkName)] pub struct PayloadAttestationData { diff --git a/consensus/types/src/attestation/payload_attestation_message.rs b/consensus/types/src/attestation/payload_attestation_message.rs index 82e2137b09..7be022efd3 100644 --- a/consensus/types/src/attestation/payload_attestation_message.rs +++ b/consensus/types/src/attestation/payload_attestation_message.rs @@ -1,14 +1,12 @@ use crate::ForkName; use crate::attestation::payload_attestation_data::PayloadAttestationData; -use crate::test_utils::TestRandom; use bls::Signature; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(TestRandom, TreeHash, Debug, Clone, PartialEq, Encode, Decode, Serialize, Deserialize)] +#[derive(TreeHash, Debug, Clone, PartialEq, Encode, Decode, Serialize, Deserialize)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[context_deserialize(ForkName)] pub struct PayloadAttestationMessage { diff --git a/consensus/types/src/attestation/pending_attestation.rs b/consensus/types/src/attestation/pending_attestation.rs index 84353ac118..79a77b47cb 100644 --- a/consensus/types/src/attestation/pending_attestation.rs +++ b/consensus/types/src/attestation/pending_attestation.rs @@ -2,10 +2,9 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::BitList; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName, test_utils::TestRandom}; +use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName}; /// An attestation that has been included in the state but not yet fully processed. /// @@ -15,7 +14,7 @@ use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName, test_ut derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct PendingAttestation { pub aggregation_bits: BitList, diff --git a/consensus/types/src/attestation/signed_aggregate_and_proof.rs b/consensus/types/src/attestation/signed_aggregate_and_proof.rs index 48c3f4c567..f9db76e9d2 100644 --- a/consensus/types/src/attestation/signed_aggregate_and_proof.rs +++ b/consensus/types/src/attestation/signed_aggregate_and_proof.rs @@ -3,7 +3,6 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -13,7 +12,6 @@ use crate::{ }, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; /// A Validators signed aggregate proof to publish on the `beacon_aggregate_and_proof` @@ -31,7 +29,6 @@ use crate::{ Deserialize, Encode, Decode, - TestRandom, TreeHash, ), context_deserialize(ForkName), diff --git a/consensus/types/src/block/beacon_block.rs b/consensus/types/src/block/beacon_block.rs index 3360728eaa..639a89d7e6 100644 --- a/consensus/types/src/block/beacon_block.rs +++ b/consensus/types/src/block/beacon_block.rs @@ -9,7 +9,6 @@ use ssz::{Decode, DecodeError}; use ssz_derive::{Decode, Encode}; use ssz_types::{BitList, BitVector, FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use typenum::Unsigned; @@ -34,7 +33,6 @@ use crate::{ slashing::{AttesterSlashingBase, ProposerSlashing}, state::BeaconStateError, sync_committee::SyncAggregate, - test_utils::TestRandom, }; /// A block of the `BeaconChain`. @@ -49,7 +47,6 @@ use crate::{ Encode, Decode, TreeHash, - TestRandom, Educe, ), educe(PartialEq, Hash(bound(E: EthSpec, Payload: AbstractExecPayload))), @@ -935,10 +932,8 @@ impl fmt::Display for BlockImportSource { #[cfg(test)] mod tests { use super::*; - use crate::{ - core::MainnetEthSpec, - test_utils::{SeedableRng, XorShiftRng, test_ssz_tree_hash_pair_with}, - }; + use crate::{core::MainnetEthSpec, test_utils::test_ssz_tree_hash_pair_with}; + use arbitrary::Arbitrary; use ssz::Encode; type BeaconBlock = super::BeaconBlock; @@ -947,16 +942,10 @@ mod tests { #[test] fn roundtrip_base_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Base.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockBase { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyBase::random_for_test(rng), - }; + let inner_block = BeaconBlockBase::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Base(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -966,16 +955,10 @@ mod tests { #[test] fn roundtrip_altair_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Altair.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockAltair { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyAltair::random_for_test(rng), - }; + let inner_block = BeaconBlockAltair::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Altair(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -985,16 +968,10 @@ mod tests { #[test] fn roundtrip_capella_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Capella.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockCapella { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyCapella::random_for_test(rng), - }; + let inner_block = BeaconBlockCapella::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Capella(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1004,16 +981,10 @@ mod tests { #[test] fn roundtrip_deneb_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Deneb.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockDeneb { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyDeneb::random_for_test(rng), - }; + let inner_block = BeaconBlockDeneb::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Deneb(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1023,17 +994,10 @@ mod tests { #[test] fn roundtrip_electra_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Electra.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockElectra { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyElectra::random_for_test(rng), - }; - + let inner_block = BeaconBlockElectra::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Electra(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1043,17 +1007,10 @@ mod tests { #[test] fn roundtrip_fulu_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Fulu.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockFulu { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyFulu::random_for_test(rng), - }; - + let inner_block = BeaconBlockFulu::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Fulu(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1063,17 +1020,10 @@ mod tests { #[test] fn roundtrip_gloas_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Gloas.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockGloas { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyGloas::random_for_test(rng), - }; - + let inner_block = BeaconBlockGloas::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Gloas(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1086,7 +1036,7 @@ mod tests { type E = MainnetEthSpec; let mut spec = E::default_spec(); - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let altair_fork_epoch = spec.altair_fork_epoch.unwrap(); @@ -1116,7 +1066,7 @@ mod tests { { let good_base_block = BeaconBlock::Base(BeaconBlockBase { slot: base_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have a base block with a slot higher than the fork epoch. let bad_base_block = { @@ -1138,7 +1088,7 @@ mod tests { { let good_altair_block = BeaconBlock::Altair(BeaconBlockAltair { slot: altair_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have an Altair block with a epoch lower than the fork epoch. let bad_altair_block = { @@ -1160,7 +1110,7 @@ mod tests { { let good_block = BeaconBlock::Capella(BeaconBlockCapella { slot: capella_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have an Capella block with a epoch lower than the fork epoch. let bad_block = { @@ -1182,7 +1132,7 @@ mod tests { { let good_block = BeaconBlock::Deneb(BeaconBlockDeneb { slot: deneb_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have a Deneb block with a epoch lower than the fork epoch. let bad_block = { @@ -1204,7 +1154,7 @@ mod tests { { let good_block = BeaconBlock::Electra(BeaconBlockElectra { slot: electra_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have an Electra block with a epoch lower than the fork epoch. let bad_block = { @@ -1226,7 +1176,7 @@ mod tests { { let good_block = BeaconBlock::Fulu(BeaconBlockFulu { slot: fulu_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); assert_eq!( @@ -1240,7 +1190,7 @@ mod tests { { let good_block = BeaconBlock::Gloas(BeaconBlockGloas { slot: gloas_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have a Fulu block with a epoch lower than the fork epoch. let _bad_block = { diff --git a/consensus/types/src/block/beacon_block_body.rs b/consensus/types/src/block/beacon_block_body.rs index 25695dbdda..071c9e76d4 100644 --- a/consensus/types/src/block/beacon_block_body.rs +++ b/consensus/types/src/block/beacon_block_body.rs @@ -9,7 +9,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -38,7 +37,6 @@ use crate::{ }, state::BeaconStateError, sync_committee::SyncAggregate, - test_utils::TestRandom, }; /// The number of leaves (including padding) on the `BeaconBlockBody` Merkle tree. @@ -65,7 +63,6 @@ pub const BLOB_KZG_COMMITMENTS_INDEX: usize = 11; Encode, Decode, TreeHash, - TestRandom, Educe, ), educe(PartialEq, Hash(bound(E: EthSpec, Payload: AbstractExecPayload))), diff --git a/consensus/types/src/block/beacon_block_header.rs b/consensus/types/src/block/beacon_block_header.rs index 06e1023d91..3d5b02d6b6 100644 --- a/consensus/types/src/block/beacon_block_header.rs +++ b/consensus/types/src/block/beacon_block_header.rs @@ -2,7 +2,6 @@ use bls::SecretKey; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -10,16 +9,13 @@ use crate::{ block::SignedBeaconBlockHeader, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot, Slot}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; /// A header of a `BeaconBlock`. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct BeaconBlockHeader { pub slot: Slot, diff --git a/consensus/types/src/block/signed_beacon_block.rs b/consensus/types/src/block/signed_beacon_block.rs index dd6f52426a..76bb9a09db 100644 --- a/consensus/types/src/block/signed_beacon_block.rs +++ b/consensus/types/src/block/signed_beacon_block.rs @@ -8,7 +8,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tracing::instrument; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -33,7 +32,6 @@ use crate::{ fork::{Fork, ForkName, ForkVersionDecode, InconsistentFork, map_fork_name}, kzg_ext::format_kzg_commitments, state::BeaconStateError, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] @@ -77,7 +75,6 @@ impl From for Hash256 { Decode, TreeHash, Educe, - TestRandom ), educe(PartialEq, Hash(bound(E: EthSpec))), serde(bound = "E: EthSpec, Payload: AbstractExecPayload"), diff --git a/consensus/types/src/block/signed_beacon_block_header.rs b/consensus/types/src/block/signed_beacon_block_header.rs index 2fcd8a705f..6e81850a3f 100644 --- a/consensus/types/src/block/signed_beacon_block_header.rs +++ b/consensus/types/src/block/signed_beacon_block_header.rs @@ -2,23 +2,19 @@ use bls::{PublicKey, Signature}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ block::BeaconBlockHeader, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; /// A signed header of a `BeaconBlock`. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SignedBeaconBlockHeader { pub message: BeaconBlockHeader, diff --git a/consensus/types/src/builder/builder.rs b/consensus/types/src/builder/builder.rs index 2bd50f42cc..18961c5969 100644 --- a/consensus/types/src/builder/builder.rs +++ b/consensus/types/src/builder/builder.rs @@ -1,18 +1,14 @@ -use crate::test_utils::TestRandom; use crate::{Address, Epoch, ForkName}; use bls::PublicKeyBytes; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; pub type BuilderIndex = u64; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash, -)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct Builder { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/builder/builder_bid.rs b/consensus/types/src/builder/builder_bid.rs index e706b01283..df7893b909 100644 --- a/consensus/types/src/builder/builder_bid.rs +++ b/consensus/types/src/builder/builder_bid.rs @@ -5,7 +5,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz::Decode; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -17,7 +16,6 @@ use crate::{ }, fork::{ForkName, ForkVersionDecode}, kzg_ext::KzgCommitments, - test_utils::TestRandom, }; #[superstruct( @@ -32,9 +30,13 @@ use crate::{ TreeHash, Decode, Clone, - TestRandom ), - serde(bound = "E: EthSpec", deny_unknown_fields) + serde(bound = "E: EthSpec", deny_unknown_fields), + cfg_attr( + feature = "arbitrary", + derive(arbitrary::Arbitrary), + arbitrary(bound = "E: EthSpec"), + ), ), map_ref_into(ExecutionPayloadHeaderRef), map_ref_mut_into(ExecutionPayloadHeaderRefMut) diff --git a/consensus/types/src/builder/builder_pending_payment.rs b/consensus/types/src/builder/builder_pending_payment.rs index 0f1b68ad97..61c76dfc15 100644 --- a/consensus/types/src/builder/builder_pending_payment.rs +++ b/consensus/types/src/builder/builder_pending_payment.rs @@ -1,24 +1,11 @@ -use crate::test_utils::TestRandom; use crate::{BuilderPendingWithdrawal, ForkName}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[derive( - Debug, - PartialEq, - Eq, - Hash, - Clone, - Default, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, + Debug, PartialEq, Eq, Hash, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, )] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/builder/builder_pending_withdrawal.rs b/consensus/types/src/builder/builder_pending_withdrawal.rs index dbbb029a5d..4b1003a28b 100644 --- a/consensus/types/src/builder/builder_pending_withdrawal.rs +++ b/consensus/types/src/builder/builder_pending_withdrawal.rs @@ -1,24 +1,11 @@ -use crate::test_utils::TestRandom; use crate::{Address, ForkName}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[derive( - Debug, - PartialEq, - Eq, - Hash, - Clone, - Default, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, + Debug, PartialEq, Eq, Hash, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, )] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/builder/proposer_preferences.rs b/consensus/types/src/builder/proposer_preferences.rs index 38f1b36be3..e3773e333d 100644 --- a/consensus/types/src/builder/proposer_preferences.rs +++ b/consensus/types/src/builder/proposer_preferences.rs @@ -1,16 +1,12 @@ -use crate::test_utils::TestRandom; use crate::{Address, ForkName, Hash256, SignedRoot, Slot}; use bls::Signature; use context_deserialize::context_deserialize; use educe::Educe; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive( - Default, Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe, TestRandom, -)] +#[derive(Default, Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[educe(PartialEq, Hash)] #[context_deserialize(ForkName)] @@ -25,7 +21,7 @@ pub struct ProposerPreferences { impl SignedRoot for ProposerPreferences {} -#[derive(TestRandom, TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] +#[derive(TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[educe(PartialEq, Hash)] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/consolidation/consolidation_request.rs b/consensus/types/src/consolidation/consolidation_request.rs index 3f09517a90..b24d0bee66 100644 --- a/consensus/types/src/consolidation/consolidation_request.rs +++ b/consensus/types/src/consolidation/consolidation_request.rs @@ -3,19 +3,15 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Address, SignedRoot}, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct ConsolidationRequest { pub source_address: Address, diff --git a/consensus/types/src/consolidation/pending_consolidation.rs b/consensus/types/src/consolidation/pending_consolidation.rs index fcd76e43b6..df71316f07 100644 --- a/consensus/types/src/consolidation/pending_consolidation.rs +++ b/consensus/types/src/consolidation/pending_consolidation.rs @@ -1,15 +1,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{fork::ForkName, test_utils::TestRandom}; +use crate::fork::ForkName; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct PendingConsolidation { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/core/enr_fork_id.rs b/consensus/types/src/core/enr_fork_id.rs index c3b400cd13..f4ad072175 100644 --- a/consensus/types/src/core/enr_fork_id.rs +++ b/consensus/types/src/core/enr_fork_id.rs @@ -1,18 +1,15 @@ use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Epoch, test_utils::TestRandom}; +use crate::core::Epoch; /// Specifies a fork which allows nodes to identify each other on the network. This fork is used in /// a nodes local ENR. /// /// Spec v0.11 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct EnrForkId { /// Fork digest of the current fork computed from [`ChainSpec::compute_fork_digest`]. #[serde(with = "serde_utils::bytes_4_hex")] diff --git a/consensus/types/src/core/execution_block_hash.rs b/consensus/types/src/core/execution_block_hash.rs index 71e63727ee..41e00115c6 100644 --- a/consensus/types/src/core/execution_block_hash.rs +++ b/consensus/types/src/core/execution_block_hash.rs @@ -1,14 +1,10 @@ use std::fmt; use fixed_bytes::FixedBytesExtended; -use rand::RngCore; use serde::{Deserialize, Serialize}; use ssz::{Decode, DecodeError, Encode}; -use crate::{ - core::{Hash256, Hash256Ext}, - test_utils::TestRandom, -}; +use crate::core::{Hash256, Hash256Ext}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[derive(Default, Clone, Copy, Serialize, Deserialize, Eq, PartialEq, Hash)] @@ -95,12 +91,6 @@ impl tree_hash::TreeHash for ExecutionBlockHash { } } -impl TestRandom for ExecutionBlockHash { - fn random_for_test(rng: &mut impl RngCore) -> Self { - Self(Hash256::random_for_test(rng)) - } -} - impl std::str::FromStr for ExecutionBlockHash { type Err = String; diff --git a/consensus/types/src/core/graffiti.rs b/consensus/types/src/core/graffiti.rs index d0e0e1b1a8..02b805a2a8 100644 --- a/consensus/types/src/core/graffiti.rs +++ b/consensus/types/src/core/graffiti.rs @@ -1,13 +1,10 @@ use std::{fmt, str::FromStr}; -use rand::RngCore; use regex::bytes::Regex; use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error}; use ssz::{Decode, DecodeError, Encode}; use tree_hash::{PackedEncoding, TreeHash}; -use crate::{core::Hash256, test_utils::TestRandom}; - pub const GRAFFITI_BYTES_LEN: usize = 32; /// The 32-byte `graffiti` field on a beacon block. @@ -180,9 +177,3 @@ impl TreeHash for Graffiti { self.0.tree_hash_root() } } - -impl TestRandom for Graffiti { - fn random_for_test(rng: &mut impl RngCore) -> Self { - Self::from(Hash256::random_for_test(rng).0) - } -} diff --git a/consensus/types/src/core/signing_data.rs b/consensus/types/src/core/signing_data.rs index 907f03fac7..e698b4fdbe 100644 --- a/consensus/types/src/core/signing_data.rs +++ b/consensus/types/src/core/signing_data.rs @@ -1,14 +1,13 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; -use crate::{core::Hash256, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Hash256, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SigningData { pub object_root: Hash256, diff --git a/consensus/types/src/core/slot_epoch.rs b/consensus/types/src/core/slot_epoch.rs index 837391546c..177161a2ab 100644 --- a/consensus/types/src/core/slot_epoch.rs +++ b/consensus/types/src/core/slot_epoch.rs @@ -12,15 +12,11 @@ use std::{fmt, hash::Hash}; -use rand::RngCore; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use ssz::{Decode, DecodeError, Encode}; -use crate::{ - core::{ChainSpec, SignedRoot}, - test_utils::TestRandom, -}; +use crate::core::{ChainSpec, SignedRoot}; #[cfg(feature = "saturating-arith")] use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssign}; diff --git a/consensus/types/src/core/slot_epoch_macros.rs b/consensus/types/src/core/slot_epoch_macros.rs index 1b0c3bcfc1..09e0f1d120 100644 --- a/consensus/types/src/core/slot_epoch_macros.rs +++ b/consensus/types/src/core/slot_epoch_macros.rs @@ -293,12 +293,6 @@ macro_rules! impl_ssz { } impl SignedRoot for $type {} - - impl TestRandom for $type { - fn random_for_test(rng: &mut impl RngCore) -> Self { - $type::from(u64::random_for_test(rng)) - } - } }; } diff --git a/consensus/types/src/data/blob_sidecar.rs b/consensus/types/src/data/blob_sidecar.rs index 70b95615e5..4020278d64 100644 --- a/consensus/types/src/data/blob_sidecar.rs +++ b/consensus/types/src/data/blob_sidecar.rs @@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, RuntimeFixedVector, RuntimeVariableList, VariableList}; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -25,7 +24,6 @@ use crate::{ fork::ForkName, kzg_ext::KzgProofs, state::BeaconStateError, - test_utils::TestRandom, }; /// Container of the data that identifies an individual blob. @@ -55,7 +53,7 @@ impl Ord for BlobIdentifier { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Educe)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, Educe)] #[context_deserialize(ForkName)] #[serde(bound = "E: EthSpec")] #[educe(PartialEq, Eq, Hash(bound(E: EthSpec)))] diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index 109c9472a5..170aa99666 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -12,7 +12,6 @@ use ssz_derive::{Decode, Encode}; use ssz_types::Error as SszError; use ssz_types::{FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -26,7 +25,6 @@ use crate::{ fork::ForkName, kzg_ext::{KzgCommitments, KzgError}, state::BeaconStateError, - test_utils::TestRandom, }; pub type ColumnIndex = u64; @@ -53,7 +51,6 @@ pub type DataColumnSidecarList = Vec>>; Deserialize, Decode, Encode, - TestRandom, Educe, TreeHash, ), diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index df65be1ae3..c0e713b4b8 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -5,7 +5,6 @@ use crate::{ execution::AbstractExecPayload, kzg_ext::KzgCommitments, state::BeaconStateError, - test_utils::TestRandom, }; use educe::Educe; use kzg::KzgProof; @@ -14,7 +13,6 @@ use ssz::BitList; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, ListEncodedOption, VariableList}; use std::fmt::Display; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -134,7 +132,7 @@ impl PartialDataColumnSidecar { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, Encode, Decode, TreeHash, TestRandom, Educe)] +#[derive(Debug, Clone, Encode, Decode, TreeHash, Educe)] #[educe(PartialEq, Eq, Hash(bound = "E: EthSpec"))] pub struct PartialDataColumnHeader { pub kzg_commitments: KzgCommitments, diff --git a/consensus/types/src/deposit/deposit.rs b/consensus/types/src/deposit/deposit.rs index 0b08bd6509..22dbdfbb71 100644 --- a/consensus/types/src/deposit/deposit.rs +++ b/consensus/types/src/deposit/deposit.rs @@ -2,11 +2,10 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use typenum::U33; -use crate::{core::Hash256, deposit::DepositData, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Hash256, deposit::DepositData, fork::ForkName}; pub const DEPOSIT_TREE_DEPTH: usize = 32; @@ -14,9 +13,7 @@ pub const DEPOSIT_TREE_DEPTH: usize = 32; /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct Deposit { pub proof: FixedVector, diff --git a/consensus/types/src/deposit/deposit_data.rs b/consensus/types/src/deposit/deposit_data.rs index 51697f5d1a..bd39643ebd 100644 --- a/consensus/types/src/deposit/deposit_data.rs +++ b/consensus/types/src/deposit/deposit_data.rs @@ -2,23 +2,19 @@ use bls::{PublicKeyBytes, SecretKey, SignatureBytes}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, Hash256, SignedRoot}, deposit::DepositMessage, fork::ForkName, - test_utils::TestRandom, }; /// The data supplied by the user to the deposit contract. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct DepositData { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/deposit/deposit_message.rs b/consensus/types/src/deposit/deposit_message.rs index 4495a5c023..9cb282e2d9 100644 --- a/consensus/types/src/deposit/deposit_message.rs +++ b/consensus/types/src/deposit/deposit_message.rs @@ -2,20 +2,18 @@ use bls::PublicKeyBytes; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Hash256, SignedRoot}, fork::ForkName, - test_utils::TestRandom, }; /// The data supplied by the user to the deposit contract. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct DepositMessage { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/deposit/deposit_request.rs b/consensus/types/src/deposit/deposit_request.rs index 8d3c6e88ba..b17450a851 100644 --- a/consensus/types/src/deposit/deposit_request.rs +++ b/consensus/types/src/deposit/deposit_request.rs @@ -3,15 +3,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Hash256, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Hash256, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct DepositRequest { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/deposit/deposit_tree_snapshot.rs b/consensus/types/src/deposit/deposit_tree_snapshot.rs index 24f41397a0..979f266d1b 100644 --- a/consensus/types/src/deposit/deposit_tree_snapshot.rs +++ b/consensus/types/src/deposit/deposit_tree_snapshot.rs @@ -3,11 +3,10 @@ use fixed_bytes::FixedBytesExtended; use int_to_bytes::int_to_bytes32; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; -use crate::{core::Hash256, deposit::DEPOSIT_TREE_DEPTH, test_utils::TestRandom}; +use crate::{core::Hash256, deposit::DEPOSIT_TREE_DEPTH}; -#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq, TestRandom)] +#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq)] pub struct FinalizedExecutionBlock { pub deposit_root: Hash256, pub deposit_count: u64, @@ -26,7 +25,8 @@ impl From<&DepositTreeSnapshot> for FinalizedExecutionBlock { } } -#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq, TestRandom)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq)] pub struct DepositTreeSnapshot { pub finalized: Vec, pub deposit_root: Hash256, diff --git a/consensus/types/src/deposit/pending_deposit.rs b/consensus/types/src/deposit/pending_deposit.rs index 4c039af39c..ed0f866ecc 100644 --- a/consensus/types/src/deposit/pending_deposit.rs +++ b/consensus/types/src/deposit/pending_deposit.rs @@ -2,19 +2,15 @@ use bls::{PublicKeyBytes, SignatureBytes}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Hash256, Slot}, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct PendingDeposit { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/execution/bls_to_execution_change.rs b/consensus/types/src/execution/bls_to_execution_change.rs index de14f1b4c5..48a089bc63 100644 --- a/consensus/types/src/execution/bls_to_execution_change.rs +++ b/consensus/types/src/execution/bls_to_execution_change.rs @@ -2,20 +2,16 @@ use bls::{PublicKeyBytes, SecretKey}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Address, ChainSpec, Domain, Hash256, SignedRoot}, execution::SignedBlsToExecutionChange, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct BlsToExecutionChange { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/execution/eth1_data.rs b/consensus/types/src/execution/eth1_data.rs index 89a4e634a6..f2a00ca87b 100644 --- a/consensus/types/src/execution/eth1_data.rs +++ b/consensus/types/src/execution/eth1_data.rs @@ -1,28 +1,16 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Hash256, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Hash256, fork::ForkName}; /// Contains data obtained from the Eth1 chain. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[derive( - Debug, - PartialEq, - Clone, - Default, - Eq, - Hash, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, + Debug, PartialEq, Clone, Default, Eq, Hash, Serialize, Deserialize, Encode, Decode, TreeHash, )] #[context_deserialize(ForkName)] pub struct Eth1Data { diff --git a/consensus/types/src/execution/execution_payload.rs b/consensus/types/src/execution/execution_payload.rs index c84a46874d..c444c03157 100644 --- a/consensus/types/src/execution/execution_payload.rs +++ b/consensus/types/src/execution/execution_payload.rs @@ -6,14 +6,12 @@ use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Address, EthSpec, ExecutionBlockHash, Hash256, Slot}, fork::{ForkName, ForkVersionDecode}, state::BeaconStateError, - test_utils::TestRandom, withdrawal::Withdrawals, }; @@ -35,7 +33,6 @@ pub type Transactions = VariableList< Encode, Decode, TreeHash, - TestRandom, Educe, ), context_deserialize(ForkName), diff --git a/consensus/types/src/execution/execution_payload_bid.rs b/consensus/types/src/execution/execution_payload_bid.rs index b2438681c1..87097bbd3b 100644 --- a/consensus/types/src/execution/execution_payload_bid.rs +++ b/consensus/types/src/execution/execution_payload_bid.rs @@ -1,16 +1,12 @@ use crate::kzg_ext::KzgCommitments; -use crate::test_utils::TestRandom; use crate::{Address, EthSpec, ExecutionBlockHash, ForkName, Hash256, SignedRoot, Slot}; use context_deserialize::context_deserialize; use educe::Educe; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive( - Default, Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe, TestRandom, -)] +#[derive(Default, Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe)] #[cfg_attr( feature = "arbitrary", derive(arbitrary::Arbitrary), diff --git a/consensus/types/src/execution/execution_payload_envelope.rs b/consensus/types/src/execution/execution_payload_envelope.rs index a6d123bd21..87a0ea7a63 100644 --- a/consensus/types/src/execution/execution_payload_envelope.rs +++ b/consensus/types/src/execution/execution_payload_envelope.rs @@ -1,5 +1,4 @@ use crate::execution::{ExecutionPayloadGloas, ExecutionRequests}; -use crate::test_utils::TestRandom; use crate::{EthSpec, ForkName, Hash256, SignedRoot, Slot}; use context_deserialize::context_deserialize; use educe::Educe; @@ -7,10 +6,14 @@ use fixed_bytes::FixedBytesExtended; use serde::{Deserialize, Serialize}; use ssz::{BYTES_PER_LENGTH_OFFSET, Encode as SszEncode}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(Debug, Clone, Serialize, Encode, Decode, Deserialize, TestRandom, TreeHash, Educe)] +#[cfg_attr( + feature = "arbitrary", + derive(arbitrary::Arbitrary), + arbitrary(bound = "E: EthSpec") +)] +#[derive(Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe)] #[educe(PartialEq, Hash(bound(E: EthSpec)))] #[context_deserialize(ForkName)] #[serde(bound = "E: EthSpec")] diff --git a/consensus/types/src/execution/execution_payload_header.rs b/consensus/types/src/execution/execution_payload_header.rs index 0b8556634a..54cc182448 100644 --- a/consensus/types/src/execution/execution_payload_header.rs +++ b/consensus/types/src/execution/execution_payload_header.rs @@ -6,7 +6,6 @@ use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -19,7 +18,6 @@ use crate::{ fork::ForkName, map_execution_payload_ref_into_execution_payload_header, state::BeaconStateError, - test_utils::TestRandom, }; #[superstruct( @@ -34,7 +32,6 @@ use crate::{ Encode, Decode, TreeHash, - TestRandom, Educe, ), educe(PartialEq, Hash(bound(E: EthSpec))), diff --git a/consensus/types/src/execution/execution_requests.rs b/consensus/types/src/execution/execution_requests.rs index 92d717778e..218b7edc17 100644 --- a/consensus/types/src/execution/execution_requests.rs +++ b/consensus/types/src/execution/execution_requests.rs @@ -6,7 +6,6 @@ use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -14,7 +13,6 @@ use crate::{ core::{EthSpec, Hash256}, deposit::DepositRequest, fork::ForkName, - test_utils::TestRandom, withdrawal::WithdrawalRequest, }; @@ -30,9 +28,7 @@ pub type ConsolidationRequests = derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive( - Debug, Educe, Default, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, Educe, Default, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[educe(PartialEq, Eq, Hash(bound(E: EthSpec)))] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/execution/payload.rs b/consensus/types/src/execution/payload.rs index c51369034c..0b3ba23e12 100644 --- a/consensus/types/src/execution/payload.rs +++ b/consensus/types/src/execution/payload.rs @@ -6,7 +6,6 @@ use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; use std::{borrow::Cow, fmt::Debug, hash::Hash}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -22,7 +21,6 @@ use crate::{ fork::ForkName, map_execution_payload_into_blinded_payload, map_execution_payload_into_full_payload, state::BeaconStateError, - test_utils::TestRandom, }; #[derive(Debug, PartialEq)] @@ -71,7 +69,6 @@ pub trait OwnedExecPayload: + DeserializeOwned + Encode + Decode - + TestRandom + for<'a> arbitrary::Arbitrary<'a> + 'static { @@ -84,7 +81,6 @@ impl OwnedExecPayload for P where + DeserializeOwned + Encode + Decode - + TestRandom + for<'a> arbitrary::Arbitrary<'a> + 'static { @@ -93,19 +89,12 @@ impl OwnedExecPayload for P where /// `ExecPayload` functionality the requires ownership. #[cfg(not(feature = "arbitrary"))] pub trait OwnedExecPayload: - ExecPayload + Default + Serialize + DeserializeOwned + Encode + Decode + TestRandom + 'static + ExecPayload + Default + Serialize + DeserializeOwned + Encode + Decode + 'static { } #[cfg(not(feature = "arbitrary"))] impl OwnedExecPayload for P where - P: ExecPayload - + Default - + Serialize - + DeserializeOwned - + Encode - + Decode - + TestRandom - + 'static + P: ExecPayload + Default + Serialize + DeserializeOwned + Encode + Decode + 'static { } @@ -166,7 +155,6 @@ pub trait AbstractExecPayload: Deserialize, Encode, Decode, - TestRandom, TreeHash, Educe, ), @@ -533,7 +521,6 @@ impl TryFrom> for FullPayload { Deserialize, Encode, Decode, - TestRandom, TreeHash, Educe, ), diff --git a/consensus/types/src/execution/signed_bls_to_execution_change.rs b/consensus/types/src/execution/signed_bls_to_execution_change.rs index 535960fb3f..0ed7de5350 100644 --- a/consensus/types/src/execution/signed_bls_to_execution_change.rs +++ b/consensus/types/src/execution/signed_bls_to_execution_change.rs @@ -2,15 +2,12 @@ use bls::Signature; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{execution::BlsToExecutionChange, fork::ForkName, test_utils::TestRandom}; +use crate::{execution::BlsToExecutionChange, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SignedBlsToExecutionChange { pub message: BlsToExecutionChange, diff --git a/consensus/types/src/execution/signed_execution_payload_bid.rs b/consensus/types/src/execution/signed_execution_payload_bid.rs index 48da445332..3d4f45a267 100644 --- a/consensus/types/src/execution/signed_execution_payload_bid.rs +++ b/consensus/types/src/execution/signed_execution_payload_bid.rs @@ -1,15 +1,13 @@ use crate::execution::ExecutionPayloadBid; -use crate::test_utils::TestRandom; use crate::{EthSpec, ForkName}; use bls::Signature; use context_deserialize::context_deserialize; use educe::Educe; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(TestRandom, TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] +#[derive(TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] #[cfg_attr( feature = "arbitrary", derive(arbitrary::Arbitrary), diff --git a/consensus/types/src/execution/signed_execution_payload_envelope.rs b/consensus/types/src/execution/signed_execution_payload_envelope.rs index 522c8b3f54..316a580476 100644 --- a/consensus/types/src/execution/signed_execution_payload_envelope.rs +++ b/consensus/types/src/execution/signed_execution_payload_envelope.rs @@ -1,4 +1,3 @@ -use crate::test_utils::TestRandom; use crate::{ BeaconState, BeaconStateError, ChainSpec, Domain, Epoch, EthSpec, ExecutionBlockHash, ExecutionPayloadEnvelope, Fork, ForkName, Hash256, SignedRoot, Slot, @@ -10,10 +9,14 @@ use educe::Educe; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(Debug, Clone, Serialize, Encode, Decode, Deserialize, TestRandom, TreeHash, Educe)] +#[cfg_attr( + feature = "arbitrary", + derive(arbitrary::Arbitrary), + arbitrary(bound = "E: EthSpec") +)] +#[derive(Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe)] #[educe(PartialEq, Hash(bound(E: EthSpec)))] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/exit/signed_voluntary_exit.rs b/consensus/types/src/exit/signed_voluntary_exit.rs index b49401a721..072541e766 100644 --- a/consensus/types/src/exit/signed_voluntary_exit.rs +++ b/consensus/types/src/exit/signed_voluntary_exit.rs @@ -2,18 +2,15 @@ use bls::Signature; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{exit::VoluntaryExit, fork::ForkName, test_utils::TestRandom}; +use crate::{exit::VoluntaryExit, fork::ForkName}; /// An exit voluntarily submitted a validator who wishes to withdraw. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SignedVoluntaryExit { pub message: VoluntaryExit, diff --git a/consensus/types/src/exit/voluntary_exit.rs b/consensus/types/src/exit/voluntary_exit.rs index 30c6a97c4d..fac0a4ad0b 100644 --- a/consensus/types/src/exit/voluntary_exit.rs +++ b/consensus/types/src/exit/voluntary_exit.rs @@ -2,23 +2,19 @@ use bls::SecretKey; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, Domain, Epoch, Hash256, SignedRoot}, exit::SignedVoluntaryExit, fork::ForkName, - test_utils::TestRandom, }; /// An exit voluntarily submitted a validator who wishes to withdraw. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct VoluntaryExit { /// Earliest epoch when voluntary exit can be processed. diff --git a/consensus/types/src/fork/fork.rs b/consensus/types/src/fork/fork.rs index 371b11e05c..675d61cc52 100644 --- a/consensus/types/src/fork/fork.rs +++ b/consensus/types/src/fork/fork.rs @@ -1,27 +1,16 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Epoch, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Epoch, fork::ForkName}; /// Specifies a fork of the `BeaconChain`, to prevent replay attacks. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[derive( - Debug, - Clone, - Copy, - PartialEq, - Default, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, + Debug, Clone, Copy, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, )] #[context_deserialize(ForkName)] pub struct Fork { diff --git a/consensus/types/src/fork/fork_data.rs b/consensus/types/src/fork/fork_data.rs index 1b9c8bad9f..5f98132f62 100644 --- a/consensus/types/src/fork/fork_data.rs +++ b/consensus/types/src/fork/fork_data.rs @@ -1,22 +1,18 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Hash256, SignedRoot}, fork::ForkName, - test_utils::TestRandom, }; /// Specifies a fork of the `BeaconChain`, to prevent replay attacks. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct ForkData { #[serde(with = "serde_utils::bytes_4_hex")] diff --git a/consensus/types/src/light_client/light_client_bootstrap.rs b/consensus/types/src/light_client/light_client_bootstrap.rs index fbcc0ef2b0..18ff246df7 100644 --- a/consensus/types/src/light_client/light_client_bootstrap.rs +++ b/consensus/types/src/light_client/light_client_bootstrap.rs @@ -7,7 +7,6 @@ use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -21,7 +20,6 @@ use crate::{ }, state::BeaconState, sync_committee::SyncCommittee, - test_utils::TestRandom, }; /// A LightClientBootstrap is the initializer we send over to light_client nodes @@ -29,17 +27,7 @@ use crate::{ #[superstruct( variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( - derive( - Debug, - Clone, - Serialize, - Deserialize, - Educe, - Decode, - Encode, - TestRandom, - TreeHash, - ), + derive(Debug, Clone, Serialize, Deserialize, Educe, Decode, Encode, TreeHash,), educe(PartialEq), serde(bound = "E: EthSpec", deny_unknown_fields), cfg_attr( diff --git a/consensus/types/src/light_client/light_client_finality_update.rs b/consensus/types/src/light_client/light_client_finality_update.rs index b503785b85..42afbdfc4b 100644 --- a/consensus/types/src/light_client/light_client_finality_update.rs +++ b/consensus/types/src/light_client/light_client_finality_update.rs @@ -6,7 +6,6 @@ use ssz_derive::Decode; use ssz_derive::Encode; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -19,23 +18,12 @@ use crate::{ LightClientHeaderElectra, LightClientHeaderFulu, }, sync_committee::SyncAggregate, - test_utils::TestRandom, }; #[superstruct( variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( - derive( - Debug, - Clone, - Serialize, - Deserialize, - Educe, - Decode, - Encode, - TestRandom, - TreeHash, - ), + derive(Debug, Clone, Serialize, Deserialize, Educe, Decode, Encode, TreeHash,), educe(PartialEq), serde(bound = "E: EthSpec", deny_unknown_fields), cfg_attr( diff --git a/consensus/types/src/light_client/light_client_header.rs b/consensus/types/src/light_client/light_client_header.rs index fdf9f234ef..df6d884ba8 100644 --- a/consensus/types/src/light_client/light_client_header.rs +++ b/consensus/types/src/light_client/light_client_header.rs @@ -7,7 +7,6 @@ use ssz::Decode; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -19,23 +18,12 @@ use crate::{ }, fork::ForkName, light_client::{ExecutionPayloadProofLen, LightClientError, consts::EXECUTION_PAYLOAD_INDEX}, - test_utils::TestRandom, }; #[superstruct( variants(Altair, Capella, Deneb, Electra, Fulu,), variant_attributes( - derive( - Debug, - Clone, - Serialize, - Deserialize, - Educe, - Decode, - Encode, - TestRandom, - TreeHash, - ), + derive(Debug, Clone, Serialize, Deserialize, Educe, Decode, Encode, TreeHash,), educe(PartialEq), serde(bound = "E: EthSpec", deny_unknown_fields), cfg_attr( diff --git a/consensus/types/src/light_client/light_client_optimistic_update.rs b/consensus/types/src/light_client/light_client_optimistic_update.rs index 139c4b6a08..f762c4ad61 100644 --- a/consensus/types/src/light_client/light_client_optimistic_update.rs +++ b/consensus/types/src/light_client/light_client_optimistic_update.rs @@ -4,7 +4,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::Hash256; use tree_hash_derive::TreeHash; @@ -17,7 +16,6 @@ use crate::{ LightClientHeaderDeneb, LightClientHeaderElectra, LightClientHeaderFulu, }, sync_committee::SyncAggregate, - test_utils::TestRandom, }; /// A LightClientOptimisticUpdate is the update we send on each slot, @@ -25,17 +23,7 @@ use crate::{ #[superstruct( variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( - derive( - Debug, - Clone, - Serialize, - Deserialize, - Educe, - Decode, - Encode, - TestRandom, - TreeHash, - ), + derive(Debug, Clone, Serialize, Deserialize, Educe, Decode, Encode, TreeHash,), educe(PartialEq), serde(bound = "E: EthSpec", deny_unknown_fields), cfg_attr( diff --git a/consensus/types/src/light_client/light_client_update.rs b/consensus/types/src/light_client/light_client_update.rs index cd33f6ae54..0e7e285651 100644 --- a/consensus/types/src/light_client/light_client_update.rs +++ b/consensus/types/src/light_client/light_client_update.rs @@ -10,7 +10,6 @@ use ssz_derive::Decode; use ssz_derive::Encode; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use typenum::{U4, U5, U6, U7}; @@ -23,7 +22,6 @@ use crate::{ LightClientHeaderDeneb, LightClientHeaderElectra, LightClientHeaderFulu, }, sync_committee::{SyncAggregate, SyncCommittee}, - test_utils::TestRandom, }; pub type FinalizedRootProofLen = U6; @@ -47,17 +45,7 @@ type NextSyncCommitteeBranchElectra = FixedVector AttesterSlashing { } } -impl TestRandom for AttesterSlashing { - fn random_for_test(rng: &mut impl RngCore) -> Self { - if rng.random_bool(0.5) { - AttesterSlashing::Base(AttesterSlashingBase::random_for_test(rng)) - } else { - AttesterSlashing::Electra(AttesterSlashingElectra::random_for_test(rng)) - } - } -} - impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for Vec> { fn context_deserialize(deserializer: D, context: ForkName) -> Result where diff --git a/consensus/types/src/slashing/proposer_slashing.rs b/consensus/types/src/slashing/proposer_slashing.rs index 697bd1a9aa..b5ffbc562c 100644 --- a/consensus/types/src/slashing/proposer_slashing.rs +++ b/consensus/types/src/slashing/proposer_slashing.rs @@ -1,18 +1,15 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{block::SignedBeaconBlockHeader, fork::ForkName, test_utils::TestRandom}; +use crate::{block::SignedBeaconBlockHeader, fork::ForkName}; /// Two conflicting proposals from the same proposer (validator). /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct ProposerSlashing { pub signed_header_1: SignedBeaconBlockHeader, diff --git a/consensus/types/src/state/beacon_state.rs b/consensus/types/src/state/beacon_state.rs index e821ca922b..4d2c7533ca 100644 --- a/consensus/types/src/state/beacon_state.rs +++ b/consensus/types/src/state/beacon_state.rs @@ -17,7 +17,6 @@ use ssz_types::{BitVector, FixedVector}; use std::collections::BTreeMap; use superstruct::superstruct; use swap_or_not_shuffle::compute_shuffled_index; -use test_random_derive::TestRandom; use tracing::instrument; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -50,7 +49,6 @@ use crate::{ get_active_validator_indices, }, sync_committee::{SyncCommittee, SyncDuty}, - test_utils::TestRandom, validator::Validator, withdrawal::PendingPartialWithdrawal, }; @@ -289,7 +287,6 @@ impl From for Hash256 { Encode, Decode, TreeHash, - TestRandom, CompareFields, ), serde(bound = "E: EthSpec", deny_unknown_fields), @@ -455,21 +452,21 @@ where // History #[metastruct(exclude_from(tree_lists))] pub latest_block_header: BeaconBlockHeader, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[compare_fields(as_iter)] pub block_roots: Vector, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[compare_fields(as_iter)] pub state_roots: Vector, // Frozen in Capella, replaced by historical_summaries - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[compare_fields(as_iter)] pub historical_roots: List, // Ethereum 1.0 chain data #[metastruct(exclude_from(tree_lists))] pub eth1_data: Eth1Data, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub eth1_data_votes: List, #[superstruct(getter(copy))] #[metastruct(exclude_from(tree_lists))] @@ -478,42 +475,42 @@ where // Registry #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub validators: Validators, #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub balances: List, // Randomness - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub randao_mixes: Vector, // Slashings - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[serde(with = "ssz_types::serde_utils::quoted_u64_fixed_vec")] pub slashings: Vector, // Attestations (genesis fork only) #[superstruct(only(Base))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub previous_epoch_attestations: List, E::MaxPendingAttestations>, #[superstruct(only(Base))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub current_epoch_attestations: List, E::MaxPendingAttestations>, // Participation (Altair and later) #[compare_fields(as_iter)] #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[compare_fields(as_iter)] pub previous_epoch_participation: List, #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub current_epoch_participation: List, // Finality - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude_from(tree_lists))] pub justification_bits: BitVector, #[superstruct(getter(copy))] @@ -529,7 +526,7 @@ where // Inactivity #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub inactivity_scores: List, // Light-client sync committees @@ -571,7 +568,7 @@ where )] #[metastruct(exclude_from(tree_lists))] pub latest_execution_payload_header: ExecutionPayloadHeaderFulu, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas))] #[metastruct(exclude_from(tree_lists))] pub latest_block_hash: ExecutionBlockHash, @@ -585,7 +582,7 @@ where pub next_withdrawal_validator_index: u64, // Deep history valid from Capella onwards. #[superstruct(only(Capella, Deneb, Electra, Fulu, Gloas))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub historical_summaries: List, // Electra @@ -612,28 +609,28 @@ where #[metastruct(exclude_from(tree_lists))] pub earliest_consolidation_epoch: Epoch, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Electra, Fulu, Gloas))] pub pending_deposits: List, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Electra, Fulu, Gloas))] pub pending_partial_withdrawals: List, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Electra, Fulu, Gloas))] pub pending_consolidations: List, // Fulu #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Fulu, Gloas))] #[serde(with = "ssz_types::serde_utils::quoted_u64_fixed_vec")] pub proposer_lookahead: Vector, // Gloas #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas))] pub builders: List, @@ -642,33 +639,34 @@ where #[superstruct(only(Gloas), partial_getter(copy))] pub next_withdrawal_builder_index: BuilderIndex, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas))] #[metastruct(exclude_from(tree_lists))] pub execution_payload_availability: BitVector, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas))] pub builder_pending_payments: Vector, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas))] pub builder_pending_withdrawals: List, + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas))] #[metastruct(exclude_from(tree_lists))] pub latest_execution_payload_bid: ExecutionPayloadBid, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas))] pub payload_expected_withdrawals: List, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas))] pub ptc_window: Vector, E::PtcWindowLength>, @@ -676,44 +674,44 @@ where #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub total_active_balance: Option<(Epoch, u64)>, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub committee_caches: [Arc; CACHED_EPOCHS], #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub progressive_balances_cache: ProgressiveBalancesCache, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub pubkey_cache: PubkeyCache, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub exit_cache: ExitCache, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub slashings_cache: SlashingsCache, /// Epoch cache of values that are useful for block processing that are static over an epoch. #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub epoch_cache: EpochCache, } diff --git a/consensus/types/src/state/historical_batch.rs b/consensus/types/src/state/historical_batch.rs index 0167d64f62..6e6e31eceb 100644 --- a/consensus/types/src/state/historical_batch.rs +++ b/consensus/types/src/state/historical_batch.rs @@ -2,13 +2,11 @@ use context_deserialize::context_deserialize; use milhouse::Vector; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{EthSpec, Hash256}, fork::ForkName, - test_utils::TestRandom, }; /// Historical block and state roots. @@ -19,12 +17,12 @@ use crate::{ derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct HistoricalBatch { - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub block_roots: Vector, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub state_roots: Vector, } diff --git a/consensus/types/src/state/historical_summary.rs b/consensus/types/src/state/historical_summary.rs index f520e46483..80c65316c9 100644 --- a/consensus/types/src/state/historical_summary.rs +++ b/consensus/types/src/state/historical_summary.rs @@ -2,7 +2,6 @@ use compare_fields::CompareFields; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -10,7 +9,6 @@ use crate::{ core::{EthSpec, Hash256}, fork::ForkName, state::BeaconState, - test_utils::TestRandom, }; /// `HistoricalSummary` matches the components of the phase0 `HistoricalBatch` @@ -28,7 +26,6 @@ use crate::{ Encode, Decode, TreeHash, - TestRandom, CompareFields, Clone, Copy, diff --git a/consensus/types/src/sync_committee/contribution_and_proof.rs b/consensus/types/src/sync_committee/contribution_and_proof.rs index 2a344b89de..2b0a1c63f0 100644 --- a/consensus/types/src/sync_committee/contribution_and_proof.rs +++ b/consensus/types/src/sync_committee/contribution_and_proof.rs @@ -2,14 +2,12 @@ use bls::{SecretKey, Signature}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, sync_committee::{SyncCommitteeContribution, SyncSelectionProof}, - test_utils::TestRandom, }; /// A Validators aggregate sync committee contribution and selection proof. @@ -18,7 +16,7 @@ use crate::{ derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] pub struct ContributionAndProof { diff --git a/consensus/types/src/sync_committee/signed_contribution_and_proof.rs b/consensus/types/src/sync_committee/signed_contribution_and_proof.rs index 0027003b9f..c788b01b13 100644 --- a/consensus/types/src/sync_committee/signed_contribution_and_proof.rs +++ b/consensus/types/src/sync_committee/signed_contribution_and_proof.rs @@ -2,14 +2,12 @@ use bls::{SecretKey, Signature}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, sync_committee::{ContributionAndProof, SyncCommitteeContribution, SyncSelectionProof}, - test_utils::TestRandom, }; /// A Validators signed contribution proof to publish on the `sync_committee_contribution_and_proof` @@ -19,7 +17,7 @@ use crate::{ derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] pub struct SignedContributionAndProof { diff --git a/consensus/types/src/sync_committee/sync_aggregate.rs b/consensus/types/src/sync_committee/sync_aggregate.rs index e5848aa22c..263faf1286 100644 --- a/consensus/types/src/sync_committee/sync_aggregate.rs +++ b/consensus/types/src/sync_committee/sync_aggregate.rs @@ -5,14 +5,12 @@ use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::BitVector; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{EthSpec, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT}, fork::ForkName, sync_committee::SyncCommitteeContribution, - test_utils::TestRandom, }; #[derive(Debug, PartialEq)] @@ -32,7 +30,7 @@ impl From for Error { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Educe)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, Educe)] #[educe(PartialEq, Hash(bound(E: EthSpec)))] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/sync_committee/sync_aggregator_selection_data.rs b/consensus/types/src/sync_committee/sync_aggregator_selection_data.rs index e905ca036b..c828e874e0 100644 --- a/consensus/types/src/sync_committee/sync_aggregator_selection_data.rs +++ b/consensus/types/src/sync_committee/sync_aggregator_selection_data.rs @@ -1,19 +1,15 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{SignedRoot, Slot}, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Clone, Serialize, Deserialize, Hash, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Hash, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SyncAggregatorSelectionData { pub slot: Slot, diff --git a/consensus/types/src/sync_committee/sync_committee.rs b/consensus/types/src/sync_committee/sync_committee.rs index 5448411800..413258f77d 100644 --- a/consensus/types/src/sync_committee/sync_committee.rs +++ b/consensus/types/src/sync_committee/sync_committee.rs @@ -6,10 +6,9 @@ use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::EthSpec, fork::ForkName, sync_committee::SyncSubnetId, test_utils::TestRandom}; +use crate::{core::EthSpec, fork::ForkName, sync_committee::SyncSubnetId}; #[derive(Debug, PartialEq)] pub enum Error { @@ -32,7 +31,7 @@ impl From for Error { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] pub struct SyncCommittee { diff --git a/consensus/types/src/sync_committee/sync_committee_contribution.rs b/consensus/types/src/sync_committee/sync_committee_contribution.rs index 09376fbe5c..c646d0b7e3 100644 --- a/consensus/types/src/sync_committee/sync_committee_contribution.rs +++ b/consensus/types/src/sync_committee/sync_committee_contribution.rs @@ -3,14 +3,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::BitVector; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{EthSpec, Hash256, SignedRoot, Slot, SlotData}, fork::ForkName, sync_committee::SyncCommitteeMessage, - test_utils::TestRandom, }; #[derive(Debug, PartialEq)] @@ -26,7 +24,7 @@ pub enum Error { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] pub struct SyncCommitteeContribution { @@ -79,7 +77,7 @@ impl SyncCommitteeContribution { impl SignedRoot for Hash256 {} /// This is not in the spec, but useful for determining uniqueness of sync committee contributions -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct SyncContributionData { pub slot: Slot, pub beacon_block_root: Hash256, diff --git a/consensus/types/src/sync_committee/sync_committee_message.rs b/consensus/types/src/sync_committee/sync_committee_message.rs index ed42555c43..87291c59c4 100644 --- a/consensus/types/src/sync_committee/sync_committee_message.rs +++ b/consensus/types/src/sync_committee/sync_committee_message.rs @@ -2,18 +2,16 @@ use bls::{SecretKey, Signature}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot, Slot, SlotData}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; /// The data upon which a `SyncCommitteeContribution` is based. #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SyncCommitteeMessage { pub slot: Slot, diff --git a/consensus/types/src/test_utils/generate_random_block_and_blobs.rs b/consensus/types/src/test_utils/generate_random_block_and_blobs.rs index 2a38b5be1f..c511fd72e7 100644 --- a/consensus/types/src/test_utils/generate_random_block_and_blobs.rs +++ b/consensus/types/src/test_utils/generate_random_block_and_blobs.rs @@ -1,6 +1,5 @@ -use bls::Signature; +use arbitrary::Arbitrary; use kzg::{KzgCommitment, KzgProof}; -use rand::Rng; use crate::{ block::{BeaconBlock, SignedBeaconBlock}, @@ -9,22 +8,22 @@ use crate::{ execution::FullPayload, fork::{ForkName, map_fork_name}, kzg_ext::{KzgCommitments, KzgProofs}, - test_utils::TestRandom, }; type BlobsBundle = (KzgCommitments, KzgProofs, BlobsList); +#[allow(clippy::type_complexity)] pub fn generate_rand_block_and_blobs( fork_name: ForkName, num_blobs: usize, - rng: &mut impl Rng, -) -> (SignedBeaconBlock>, Vec>) { - let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(rng)); - let mut block = SignedBeaconBlock::from_block(inner, Signature::random_for_test(rng)); + u: &mut arbitrary::Unstructured, +) -> arbitrary::Result<(SignedBeaconBlock>, Vec>)> { + let inner = map_fork_name!(fork_name, BeaconBlock, <_>::arbitrary(u)?); + let mut block = SignedBeaconBlock::from_block(inner, bls::Signature::arbitrary(u)?); let mut blob_sidecars = vec![]; if block.fork_name_unchecked() < ForkName::Deneb { - return (block, blob_sidecars); + return Ok((block, blob_sidecars)); } let (commitments, proofs, blobs) = generate_blobs::(num_blobs).unwrap(); @@ -50,7 +49,7 @@ pub fn generate_rand_block_and_blobs( .unwrap(), }); } - (block, blob_sidecars) + Ok((block, blob_sidecars)) } pub fn generate_blobs(n_blobs: usize) -> Result, String> { @@ -74,13 +73,13 @@ pub fn generate_blobs(n_blobs: usize) -> Result, Stri #[cfg(test)] mod test { use super::*; - use rand::rng; use ssz_types::FixedVector; #[test] fn test_verify_blob_inclusion_proof() { + let mut u = crate::test_utils::test_unstructured(); let (_block, blobs) = - generate_rand_block_and_blobs::(ForkName::Deneb, 2, &mut rng()); + generate_rand_block_and_blobs::(ForkName::Deneb, 2, &mut u).unwrap(); for blob in blobs { assert!(blob.verify_blob_sidecar_inclusion_proof()); } @@ -88,8 +87,9 @@ mod test { #[test] fn test_verify_blob_inclusion_proof_from_existing_proof() { + let mut u = crate::test_utils::test_unstructured(); let (block, mut blob_sidecars) = - generate_rand_block_and_blobs::(ForkName::Deneb, 1, &mut rng()); + generate_rand_block_and_blobs::(ForkName::Deneb, 1, &mut u).unwrap(); let BlobSidecar { index, blob, @@ -105,11 +105,12 @@ mod test { #[test] fn test_verify_blob_inclusion_proof_invalid() { + let mut u = crate::test_utils::test_unstructured(); let (_block, blobs) = - generate_rand_block_and_blobs::(ForkName::Deneb, 1, &mut rng()); + generate_rand_block_and_blobs::(ForkName::Deneb, 1, &mut u).unwrap(); for mut blob in blobs { - blob.kzg_commitment_inclusion_proof = FixedVector::random_for_test(&mut rng()); + blob.kzg_commitment_inclusion_proof = FixedVector::arbitrary(&mut u).unwrap(); assert!(!blob.verify_blob_sidecar_inclusion_proof()); } } diff --git a/consensus/types/src/test_utils/macros.rs b/consensus/types/src/test_utils/macros.rs index 662527f5a4..09afd27ae3 100644 --- a/consensus/types/src/test_utils/macros.rs +++ b/consensus/types/src/test_utils/macros.rs @@ -14,10 +14,8 @@ macro_rules! ssz_tests { #[test] pub fn test_ssz_round_trip() { use ssz::{Decode, ssz_encode}; - use $crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = <$type>::random_for_test(&mut rng); + let original: $type = $crate::test_utils::test_arbitrary_instance(); let bytes = ssz_encode(&original); let decoded = <$type>::from_ssz_bytes(&bytes).unwrap(); @@ -33,10 +31,8 @@ macro_rules! tree_hash_tests { #[test] pub fn test_tree_hash_root() { use tree_hash::TreeHash; - use $crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = <$type>::random_for_test(&mut rng); + let original: $type = $crate::test_utils::test_arbitrary_instance(); // Tree hashing should not panic. original.tree_hash_root(); diff --git a/consensus/types/src/test_utils/mod.rs b/consensus/types/src/test_utils/mod.rs index c4409b4392..5cf728be66 100644 --- a/consensus/types/src/test_utils/mod.rs +++ b/consensus/types/src/test_utils/mod.rs @@ -5,15 +5,36 @@ mod macros; mod generate_deterministic_keypairs; #[cfg(test)] mod generate_random_block_and_blobs; -mod test_random; pub use generate_deterministic_keypairs::generate_deterministic_keypair; pub use generate_deterministic_keypairs::generate_deterministic_keypairs; pub use generate_deterministic_keypairs::load_keypairs_from_yaml; -pub use test_random::{TestRandom, test_random_instance}; -pub use rand::{RngCore, SeedableRng}; -pub use rand_xorshift::XorShiftRng; +/// Deterministic 256 KiB seed. +#[cfg(feature = "arbitrary")] +static SEED: std::sync::LazyLock> = std::sync::LazyLock::new(|| { + use rand::RngCore; + use rand::SeedableRng; + let mut bytes = vec![0u8; 256 * 1024]; + rand_xorshift::XorShiftRng::from_seed([0x42; 16]).fill_bytes(&mut bytes); + bytes +}); + +/// Generates an arbitrary instance of `T` from a deterministic seed. +/// Suitable for one-shot test instance creation. +#[cfg(feature = "arbitrary")] +pub fn test_arbitrary_instance<'a, T: arbitrary::Arbitrary<'a>>() -> T { + let mut u = arbitrary::Unstructured::new(&SEED); + T::arbitrary(&mut u).expect("sufficient bytes for arbitrary generation") +} + +/// Returns an `Unstructured` from a deterministic seed. +/// Use this when you need to pass an `Unstructured` to helpers like +/// `generate_rand_block_and_blobs`. +#[cfg(feature = "arbitrary")] +pub fn test_unstructured() -> arbitrary::Unstructured<'static> { + arbitrary::Unstructured::new(&SEED) +} use ssz::{Decode, Encode, ssz_encode}; use std::fmt::Debug; diff --git a/consensus/types/src/test_utils/test_random/address.rs b/consensus/types/src/test_utils/test_random/address.rs deleted file mode 100644 index 2f601cb91e..0000000000 --- a/consensus/types/src/test_utils/test_random/address.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::{core::Address, test_utils::TestRandom}; - -impl TestRandom for Address { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut key_bytes = vec![0; 20]; - rng.fill_bytes(&mut key_bytes); - Address::from_slice(&key_bytes[..]) - } -} diff --git a/consensus/types/src/test_utils/test_random/aggregate_signature.rs b/consensus/types/src/test_utils/test_random/aggregate_signature.rs deleted file mode 100644 index f9f3dd9567..0000000000 --- a/consensus/types/src/test_utils/test_random/aggregate_signature.rs +++ /dev/null @@ -1,12 +0,0 @@ -use bls::{AggregateSignature, Signature}; - -use crate::test_utils::TestRandom; - -impl TestRandom for AggregateSignature { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let signature = Signature::random_for_test(rng); - let mut aggregate_signature = AggregateSignature::infinity(); - aggregate_signature.add_assign(&signature); - aggregate_signature - } -} diff --git a/consensus/types/src/test_utils/test_random/bitfield.rs b/consensus/types/src/test_utils/test_random/bitfield.rs deleted file mode 100644 index 762f41eb34..0000000000 --- a/consensus/types/src/test_utils/test_random/bitfield.rs +++ /dev/null @@ -1,43 +0,0 @@ -use smallvec::smallvec; -use ssz_types::{BitList, BitVector}; -use typenum::Unsigned; - -use crate::test_utils::TestRandom; - -impl TestRandom for BitList { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let initial_len = std::cmp::max(1, N::to_usize().div_ceil(8)); - let mut raw_bytes = smallvec![0; initial_len]; - rng.fill_bytes(&mut raw_bytes); - - let non_zero_bytes = raw_bytes - .iter() - .enumerate() - .rev() - .find_map(|(i, byte)| (*byte > 0).then_some(i + 1)) - .unwrap_or(0); - - if non_zero_bytes < initial_len { - raw_bytes.truncate(non_zero_bytes); - } - - Self::from_bytes(raw_bytes).expect("we generate a valid BitList") - } -} - -impl TestRandom for BitVector { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut raw_bytes = smallvec![0; std::cmp::max(1, N::to_usize().div_ceil(8))]; - rng.fill_bytes(&mut raw_bytes); - // If N isn't divisible by 8 - // zero out bits greater than N - if let Some(last_byte) = raw_bytes.last_mut() { - let mut mask = 0; - for i in 0..N::to_usize() % 8 { - mask |= 1 << i; - } - *last_byte &= mask; - } - Self::from_bytes(raw_bytes).expect("we generate a valid BitVector") - } -} diff --git a/consensus/types/src/test_utils/test_random/hash256.rs b/consensus/types/src/test_utils/test_random/hash256.rs deleted file mode 100644 index 4d7570fb55..0000000000 --- a/consensus/types/src/test_utils/test_random/hash256.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::{core::Hash256, test_utils::TestRandom}; - -impl TestRandom for Hash256 { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut key_bytes = vec![0; 32]; - rng.fill_bytes(&mut key_bytes); - Hash256::from_slice(&key_bytes[..]) - } -} diff --git a/consensus/types/src/test_utils/test_random/kzg_commitment.rs b/consensus/types/src/test_utils/test_random/kzg_commitment.rs deleted file mode 100644 index 31e316a198..0000000000 --- a/consensus/types/src/test_utils/test_random/kzg_commitment.rs +++ /dev/null @@ -1,9 +0,0 @@ -use kzg::KzgCommitment; - -use crate::test_utils::TestRandom; - -impl TestRandom for KzgCommitment { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - KzgCommitment(<[u8; 48] as TestRandom>::random_for_test(rng)) - } -} diff --git a/consensus/types/src/test_utils/test_random/kzg_proof.rs b/consensus/types/src/test_utils/test_random/kzg_proof.rs deleted file mode 100644 index 4465d5ab39..0000000000 --- a/consensus/types/src/test_utils/test_random/kzg_proof.rs +++ /dev/null @@ -1,11 +0,0 @@ -use kzg::{BYTES_PER_COMMITMENT, KzgProof}; - -use crate::test_utils::TestRandom; - -impl TestRandom for KzgProof { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut bytes = [0; BYTES_PER_COMMITMENT]; - rng.fill_bytes(&mut bytes); - Self(bytes) - } -} diff --git a/consensus/types/src/test_utils/test_random/mod.rs b/consensus/types/src/test_utils/test_random/mod.rs deleted file mode 100644 index 41812593fa..0000000000 --- a/consensus/types/src/test_utils/test_random/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -mod address; -mod aggregate_signature; -mod bitfield; -mod hash256; -mod kzg_commitment; -mod kzg_proof; -mod public_key; -mod public_key_bytes; -mod secret_key; -mod signature; -mod signature_bytes; -mod test_random; -mod uint256; - -pub use test_random::{TestRandom, test_random_instance}; diff --git a/consensus/types/src/test_utils/test_random/public_key.rs b/consensus/types/src/test_utils/test_random/public_key.rs deleted file mode 100644 index 9d287c23d7..0000000000 --- a/consensus/types/src/test_utils/test_random/public_key.rs +++ /dev/null @@ -1,9 +0,0 @@ -use bls::{PublicKey, SecretKey}; - -use crate::test_utils::TestRandom; - -impl TestRandom for PublicKey { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - SecretKey::random_for_test(rng).public_key() - } -} diff --git a/consensus/types/src/test_utils/test_random/public_key_bytes.rs b/consensus/types/src/test_utils/test_random/public_key_bytes.rs deleted file mode 100644 index 587c3baf8f..0000000000 --- a/consensus/types/src/test_utils/test_random/public_key_bytes.rs +++ /dev/null @@ -1,17 +0,0 @@ -use bls::{PUBLIC_KEY_BYTES_LEN, PublicKey, PublicKeyBytes}; - -use crate::test_utils::TestRandom; - -impl TestRandom for PublicKeyBytes { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - //50-50 chance for signature to be "valid" or invalid - if bool::random_for_test(rng) { - //valid signature - PublicKeyBytes::from(PublicKey::random_for_test(rng)) - } else { - //invalid signature, just random bytes - PublicKeyBytes::deserialize(&<[u8; PUBLIC_KEY_BYTES_LEN]>::random_for_test(rng)) - .unwrap() - } - } -} diff --git a/consensus/types/src/test_utils/test_random/secret_key.rs b/consensus/types/src/test_utils/test_random/secret_key.rs deleted file mode 100644 index a8295d968a..0000000000 --- a/consensus/types/src/test_utils/test_random/secret_key.rs +++ /dev/null @@ -1,11 +0,0 @@ -use bls::SecretKey; - -use crate::test_utils::TestRandom; - -impl TestRandom for SecretKey { - fn random_for_test(_rng: &mut impl rand::RngCore) -> Self { - // TODO: Not deterministic generation. Using `SecretKey::deserialize` results in - // `BlstError(BLST_BAD_ENCODING)`, need to debug with blst source on what encoding expects. - SecretKey::random() - } -} diff --git a/consensus/types/src/test_utils/test_random/signature.rs b/consensus/types/src/test_utils/test_random/signature.rs deleted file mode 100644 index 006aba9650..0000000000 --- a/consensus/types/src/test_utils/test_random/signature.rs +++ /dev/null @@ -1,12 +0,0 @@ -use bls::Signature; - -use crate::test_utils::TestRandom; - -impl TestRandom for Signature { - fn random_for_test(_rng: &mut impl rand::RngCore) -> Self { - // TODO: `SecretKey::random_for_test` does not return a deterministic signature. Since this - // signature will not pass verification we could just return the generator point or the - // generator point multiplied by a random scalar if we want disctint signatures. - Signature::infinity().expect("infinity signature is valid") - } -} diff --git a/consensus/types/src/test_utils/test_random/signature_bytes.rs b/consensus/types/src/test_utils/test_random/signature_bytes.rs deleted file mode 100644 index 6992e57467..0000000000 --- a/consensus/types/src/test_utils/test_random/signature_bytes.rs +++ /dev/null @@ -1,16 +0,0 @@ -use bls::{SIGNATURE_BYTES_LEN, Signature, SignatureBytes}; - -use crate::test_utils::TestRandom; - -impl TestRandom for SignatureBytes { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - //50-50 chance for signature to be "valid" or invalid - if bool::random_for_test(rng) { - //valid signature - SignatureBytes::from(Signature::random_for_test(rng)) - } else { - //invalid signature, just random bytes - SignatureBytes::deserialize(&<[u8; SIGNATURE_BYTES_LEN]>::random_for_test(rng)).unwrap() - } - } -} diff --git a/consensus/types/src/test_utils/test_random/test_random.rs b/consensus/types/src/test_utils/test_random/test_random.rs deleted file mode 100644 index 101fbec51b..0000000000 --- a/consensus/types/src/test_utils/test_random/test_random.rs +++ /dev/null @@ -1,140 +0,0 @@ -use std::{marker::PhantomData, sync::Arc}; - -use rand::{RngCore, SeedableRng}; -use rand_xorshift::XorShiftRng; -use smallvec::{SmallVec, smallvec}; -use ssz_types::VariableList; -use typenum::Unsigned; - -pub fn test_random_instance() -> T { - let mut rng = XorShiftRng::from_seed([0x42; 16]); - T::random_for_test(&mut rng) -} - -pub trait TestRandom { - fn random_for_test(rng: &mut impl RngCore) -> Self; -} - -impl TestRandom for PhantomData { - fn random_for_test(_rng: &mut impl RngCore) -> Self { - PhantomData - } -} - -impl TestRandom for bool { - fn random_for_test(rng: &mut impl RngCore) -> Self { - (rng.next_u32() % 2) == 1 - } -} - -impl TestRandom for u64 { - fn random_for_test(rng: &mut impl RngCore) -> Self { - rng.next_u64() - } -} - -impl TestRandom for u32 { - fn random_for_test(rng: &mut impl RngCore) -> Self { - rng.next_u32() - } -} - -impl TestRandom for u8 { - fn random_for_test(rng: &mut impl RngCore) -> Self { - rng.next_u32().to_be_bytes()[0] - } -} - -impl TestRandom for usize { - fn random_for_test(rng: &mut impl RngCore) -> Self { - rng.next_u32() as usize - } -} - -impl TestRandom for Vec -where - U: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - let mut output = vec![]; - - for _ in 0..(usize::random_for_test(rng) % 4) { - output.push(::random_for_test(rng)); - } - - output - } -} - -impl TestRandom for Arc -where - U: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - Arc::new(U::random_for_test(rng)) - } -} - -impl TestRandom for ssz_types::FixedVector -where - T: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - Self::new( - (0..N::to_usize()) - .map(|_| T::random_for_test(rng)) - .collect(), - ) - .expect("N items provided") - } -} - -impl TestRandom for VariableList -where - T: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - let mut output = vec![]; - - if N::to_usize() != 0 { - for _ in 0..(usize::random_for_test(rng) % std::cmp::min(4, N::to_usize())) { - output.push(::random_for_test(rng)); - } - } - - output.try_into().unwrap() - } -} - -impl TestRandom for SmallVec<[U; N]> -where - U: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - let mut output = smallvec![]; - - for _ in 0..(usize::random_for_test(rng) % 4) { - output.push(::random_for_test(rng)); - } - - output - } -} - -macro_rules! impl_test_random_for_u8_array { - ($len: expr) => { - impl TestRandom for [u8; $len] { - fn random_for_test(rng: &mut impl RngCore) -> Self { - let mut bytes = [0; $len]; - rng.fill_bytes(&mut bytes); - bytes - } - } - }; -} - -impl_test_random_for_u8_array!(3); -impl_test_random_for_u8_array!(4); -impl_test_random_for_u8_array!(32); -impl_test_random_for_u8_array!(48); -impl_test_random_for_u8_array!(96); diff --git a/consensus/types/src/test_utils/test_random/uint256.rs b/consensus/types/src/test_utils/test_random/uint256.rs deleted file mode 100644 index eccf476595..0000000000 --- a/consensus/types/src/test_utils/test_random/uint256.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::{core::Uint256, test_utils::TestRandom}; - -impl TestRandom for Uint256 { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut key_bytes = [0; 32]; - rng.fill_bytes(&mut key_bytes); - Self::from_le_slice(&key_bytes[..]) - } -} diff --git a/consensus/types/src/validator/validator.rs b/consensus/types/src/validator/validator.rs index 5c5bfc761f..a56093c0b5 100644 --- a/consensus/types/src/validator/validator.rs +++ b/consensus/types/src/validator/validator.rs @@ -3,7 +3,6 @@ use context_deserialize::context_deserialize; use fixed_bytes::FixedBytesExtended; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -11,16 +10,13 @@ use crate::{ core::{Address, ChainSpec, Epoch, EthSpec, Hash256}, fork::ForkName, state::BeaconState, - test_utils::TestRandom, }; /// Information about a `BeaconChain` validator. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash, -)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct Validator { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/withdrawal/pending_partial_withdrawal.rs b/consensus/types/src/withdrawal/pending_partial_withdrawal.rs index cd866369a4..0b3842808d 100644 --- a/consensus/types/src/withdrawal/pending_partial_withdrawal.rs +++ b/consensus/types/src/withdrawal/pending_partial_withdrawal.rs @@ -1,15 +1,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Epoch, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Epoch, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct PendingPartialWithdrawal { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/withdrawal/withdrawal.rs b/consensus/types/src/withdrawal/withdrawal.rs index d75bd4f501..da69227626 100644 --- a/consensus/types/src/withdrawal/withdrawal.rs +++ b/consensus/types/src/withdrawal/withdrawal.rs @@ -2,19 +2,15 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Address, EthSpec}, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct Withdrawal { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/withdrawal/withdrawal_request.rs b/consensus/types/src/withdrawal/withdrawal_request.rs index 98a40016f9..a89fe9b825 100644 --- a/consensus/types/src/withdrawal/withdrawal_request.rs +++ b/consensus/types/src/withdrawal/withdrawal_request.rs @@ -3,15 +3,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Address, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Address, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct WithdrawalRequest { #[serde(with = "serde_utils::address_hex")] diff --git a/consensus/types/tests/state.rs b/consensus/types/tests/state.rs index 5e223092cf..2168da9afc 100644 --- a/consensus/types/tests/state.rs +++ b/consensus/types/tests/state.rs @@ -2,15 +2,14 @@ use std::ops::Mul; use std::sync::LazyLock; +use arbitrary::Arbitrary; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use bls::Keypair; use fixed_bytes::FixedBytesExtended; use milhouse::Vector; -use rand::SeedableRng; -use rand_xorshift::XorShiftRng; use ssz::Encode; use swap_or_not_shuffle::compute_shuffled_index; -use types::test_utils::{TestRandom, generate_deterministic_keypairs}; +use types::test_utils::generate_deterministic_keypairs; use types::*; pub const MAX_VALIDATOR_COUNT: usize = 129; @@ -315,7 +314,7 @@ fn decode_base_and_altair() { type E = MainnetEthSpec; let spec = E::default_spec(); - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let fork_epoch = spec.altair_fork_epoch.unwrap(); @@ -328,7 +327,7 @@ fn decode_base_and_altair() { { let good_base_state: BeaconState = BeaconState::Base(BeaconStateBase { slot: base_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have a base state with a slot higher than the fork slot. let bad_base_state = { @@ -351,7 +350,7 @@ fn decode_base_and_altair() { let good_altair_state: BeaconState = BeaconState::Altair(BeaconStateAltair { slot: altair_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have an Altair state with a slot lower than the fork slot. let bad_altair_state = { diff --git a/crypto/bls/src/macros.rs b/crypto/bls/src/macros.rs index 58b1ec7d6c..4f2be22dc3 100644 --- a/crypto/bls/src/macros.rs +++ b/crypto/bls/src/macros.rs @@ -165,13 +165,26 @@ macro_rules! impl_debug { /// Contains the functions required for an `Arbitrary` implementation. /// /// Does not include the `Impl` section since it gets very complicated when it comes to generics. +/// +/// For `GenericPublicKeyBytes` and `GenericSignatureBytes`, this implementation works correctly +/// without falling back to zeros. +/// +/// For `GenericPublicKey`, `GenericSignature` and `GenericAggregateSignature`, this implementation +/// will almost always fail and fallback to zeros. This matches the behavior of the previous +/// `TestRandom` impls. +/// +/// TODO: For proper fuzzing, this implementation needs more consideration on how to +/// arbitrarily construct valid types. #[cfg(feature = "arbitrary")] macro_rules! impl_arbitrary { ($byte_size: expr) => { fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { let mut bytes = [0u8; $byte_size]; u.fill_buffer(&mut bytes)?; - Self::deserialize(&bytes).map_err(|_| arbitrary::Error::IncorrectFormat) + Ok(Self::deserialize(&bytes).unwrap_or_else(|_| { + // All-zeros is the "empty" encoding accepted by every BLS type. + Self::deserialize(&[0u8; $byte_size]).expect("all-zeros is a valid encoding") + })) } }; } diff --git a/validator_client/doppelganger_service/Cargo.toml b/validator_client/doppelganger_service/Cargo.toml index 66b27eb39d..a0c579e11f 100644 --- a/validator_client/doppelganger_service/Cargo.toml +++ b/validator_client/doppelganger_service/Cargo.toml @@ -19,5 +19,7 @@ types = { workspace = true } validator_store = { workspace = true } [dev-dependencies] +arbitrary = { workspace = true } futures = { workspace = true } logging = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } diff --git a/validator_client/doppelganger_service/src/lib.rs b/validator_client/doppelganger_service/src/lib.rs index 600ae82c54..0842638bfa 100644 --- a/validator_client/doppelganger_service/src/lib.rs +++ b/validator_client/doppelganger_service/src/lib.rs @@ -598,14 +598,12 @@ impl DoppelgangerService { #[cfg(test)] mod test { use super::*; + use arbitrary::Arbitrary; use futures::executor::block_on; use slot_clock::TestingSlotClock; use std::future; use std::time::Duration; - use types::{ - MainnetEthSpec, - test_utils::{SeedableRng, TestRandom, XorShiftRng}, - }; + use types::MainnetEthSpec; use validator_store::DoppelgangerStatus; const DEFAULT_VALIDATORS: usize = 8; @@ -641,12 +639,12 @@ mod test { impl TestBuilder { fn build(self) -> TestScenario { - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let slot_clock = TestingSlotClock::new(Slot::new(0), GENESIS_TIME, SLOT_DURATION); TestScenario { validators: (0..self.validator_count) - .map(|_| PublicKeyBytes::random_for_test(&mut rng)) + .map(|_| PublicKeyBytes::arbitrary(&mut u).unwrap()) .collect(), doppelganger: DoppelgangerService::default(), slot_clock, From 31e5f308c3acb86dabfe7da5979d56601a2c3b8d Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 6 May 2026 05:25:46 +0300 Subject: [PATCH 04/18] Generalise reconstruct_historic_states for ranged replay (#9222) Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> --- beacon_node/store/src/reconstruct.rs | 143 ++++++++++++++++----------- 1 file changed, 85 insertions(+), 58 deletions(-) diff --git a/beacon_node/store/src/reconstruct.rs b/beacon_node/store/src/reconstruct.rs index 7aca692ef9..04a519af02 100644 --- a/beacon_node/store/src/reconstruct.rs +++ b/beacon_node/store/src/reconstruct.rs @@ -1,7 +1,8 @@ //! Implementation of historic state reconstruction (given complete block history). +use crate::forwards_iter::FrozenForwardsIterator; use crate::hot_cold_store::{HotColdDB, HotColdDBError}; use crate::metrics; -use crate::{Error, ItemStore}; +use crate::{DBColumn, Error, ItemStore}; use itertools::{Itertools, process_results}; use state_processing::{ BlockSignatureStrategy, ConsensusContext, VerifyBlockRoot, per_block_processing, @@ -9,7 +10,7 @@ use state_processing::{ }; use std::sync::Arc; use tracing::{debug, info}; -use types::EthSpec; +use types::{EthSpec, Slot}; impl HotColdDB where @@ -35,13 +36,6 @@ where }); } - debug!( - start_slot = %anchor.state_lower_limit, - "Starting state reconstruction batch" - ); - - let _t = metrics::start_timer(&metrics::STORE_BEACON_RECONSTRUCTION_TIME); - // Iterate blocks from the state lower limit to the upper limit. let split = self.get_split_info(); let lower_limit_slot = anchor.state_lower_limit; @@ -56,20 +50,86 @@ where // If `num_blocks` is not specified iterate all blocks. Add 1 so that we end on an epoch // boundary when `num_blocks` is a multiple of an epoch boundary. We want to be *inclusive* // of the state at slot `lower_limit_slot + num_blocks`. - let block_root_iter = self - .forwards_block_roots_iterator_until(lower_limit_slot, upper_limit_slot - 1, || { - Err(Error::StateShouldNotBeRequired(upper_limit_slot - 1)) - })? - .take(num_blocks.map_or(usize::MAX, |n| n + 1)); + let to_slot = num_blocks + .map(|n| std::cmp::min(lower_limit_slot + n as u64 + 1, upper_limit_slot)) + .unwrap_or(upper_limit_slot); + + let on_commit = |slot: Slot| -> Result<(), Error> { + info!( + %slot, + remaining = %(upper_limit_slot - 1 - slot), + "State reconstruction in progress" + ); + + // Update anchor. + let old_anchor = anchor.clone(); + let reconstruction_complete = slot + 1 == upper_limit_slot; + + if reconstruction_complete { + // The two limits have met in the middle! We're done! + let new_anchor = old_anchor.as_archive_anchor(); + self.compare_and_set_anchor_info_with_write(old_anchor, new_anchor)?; + } else { + // The lower limit has been raised, store it. + anchor.state_lower_limit = slot; + self.compare_and_set_anchor_info_with_write(old_anchor, anchor.clone())?; + } + + Ok(()) + }; + + self.reconstruct_historic_states_on_range(lower_limit_slot, to_slot, on_commit)?; + + // Check that the split point wasn't mutated during the state reconstruction process. + // It shouldn't have been, due to the serialization of requests through the store migrator, + // so this is just a paranoid check. + let latest_split = self.get_split_info(); + if split != latest_split { + return Err(Error::SplitPointModified(latest_split.slot, split.slot)); + } + + Ok(()) + } + + /// Reconstruct historic states for the slot range `(with_state_at_slot, to_slot)`. + /// + /// Loads the state at `with_state_at_slot` and replays blocks up to and including slot + /// `to_slot - 1`, writing all intermediate states to the freezer DB. + /// + /// The `BeaconBlockRoots` column must be populated for the range before this is called. + /// + /// `on_commit(slot)` is invoked after each atomic commit (whenever the hierarchy says to + /// commit, plus once at the final slot) so callers can update anchor metadata or log + /// progress. + pub fn reconstruct_historic_states_on_range( + self: &Arc, + with_state_at_slot: Slot, + to_slot: Slot, + mut on_commit: impl FnMut(Slot) -> Result<(), Error>, + ) -> Result<(), Error> { + debug!( + from_slot = %(with_state_at_slot + 1), + %to_slot, + "Starting state reconstruction batch" + ); + + let _t = metrics::start_timer(&metrics::STORE_BEACON_RECONSTRUCTION_TIME); + + // Iterate from `with_state_at_slot` so `tuple_windows` gives us the predecessor block + // root at each step for skip detection. + let block_root_iter = FrozenForwardsIterator::new( + self, + DBColumn::BeaconBlockRoots, + with_state_at_slot, + to_slot, + )?; // The state to be advanced. - let mut state = self.load_cold_state_by_slot(lower_limit_slot)?; - + let mut state = self.load_cold_state_by_slot(with_state_at_slot)?; state.build_caches(&self.spec)?; process_results(block_root_iter, |iter| -> Result<(), Error> { let mut io_batch = vec![]; - let mut prev_state_root = None; for ((prev_block_root, _), (block_root, slot)) in iter.tuple_windows() { @@ -114,32 +174,16 @@ where // Stage state for storage in freezer DB. self.store_cold_state(&state_root, &state, &mut io_batch)?; - let batch_complete = - num_blocks.is_some_and(|n_blocks| slot == lower_limit_slot + n_blocks as u64); - let reconstruction_complete = slot + 1 == upper_limit_slot; + let batch_complete = slot + 1 == to_slot; // Commit the I/O batch if: // // - The diff/snapshot for this slot is required for future slots, or - // - The reconstruction batch is complete (we are about to return), or - // - Reconstruction is complete. - if self.hierarchy.should_commit_immediately(slot)? - || batch_complete - || reconstruction_complete - { - info!( - %slot, - remaining = %(upper_limit_slot - 1 - slot), - "State reconstruction in progress" - ); - + // - The reconstruction batch is complete (we are about to return). + if self.hierarchy.should_commit_immediately(slot)? || batch_complete { self.cold_db.do_atomically(std::mem::take(&mut io_batch))?; - // Update anchor. - let old_anchor = anchor.clone(); - - if reconstruction_complete { - // The two limits have met in the middle! We're done! + if batch_complete { // Perform one last integrity check on the state reached. let computed_state_root = state.update_tree_hash_cache()?; if computed_state_root != state_root { @@ -149,23 +193,15 @@ where computed: computed_state_root, }); } - - let new_anchor = old_anchor.as_archive_anchor(); - self.compare_and_set_anchor_info_with_write(old_anchor, new_anchor)?; - - return Ok(()); - } else { - // The lower limit has been raised, store it. - anchor.state_lower_limit = slot; - - self.compare_and_set_anchor_info_with_write(old_anchor, anchor.clone())?; } + on_commit(slot)?; + // If this is the end of the batch, return Ok. The caller will run another // batch when there is idle capacity. if batch_complete { debug!( - start_slot = %lower_limit_slot, + start_slot = %(with_state_at_slot + 1), end_slot = %slot, "Finished state reconstruction batch" ); @@ -174,19 +210,10 @@ where } } - // Should always reach the `upper_limit_slot` or the end of the batch and return early - // above. + // Should always reach `to_slot` or the end of the batch and return early above. Err(Error::StateReconstructionLogicError) })??; - // Check that the split point wasn't mutated during the state reconstruction process. - // It shouldn't have been, due to the serialization of requests through the store migrator, - // so this is just a paranoid check. - let latest_split = self.get_split_info(); - if split != latest_split { - return Err(Error::SplitPointModified(latest_split.slot, split.slot)); - } - Ok(()) } } From 7148bfcdd1389ea6410193654758f843572e57ac Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 6 May 2026 20:41:01 -0600 Subject: [PATCH 05/18] Implement beacon_blocks_by_head (#9237) Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> --- beacon_node/beacon_processor/src/lib.rs | 10 + .../src/scheduler/work_queue.rs | 5 + .../src/peer_manager/mod.rs | 3 + .../lighthouse_network/src/rpc/codec.rs | 51 +++- .../lighthouse_network/src/rpc/config.rs | 9 + .../lighthouse_network/src/rpc/methods.rs | 37 ++- .../lighthouse_network/src/rpc/protocol.rs | 32 +++ .../src/rpc/rate_limiter.rs | 15 + .../src/service/api_types.rs | 7 + .../lighthouse_network/src/service/mod.rs | 12 + .../src/network_beacon_processor/mod.rs | 24 +- .../network_beacon_processor/rpc_methods.rs | 264 +++++++++++++++++- .../src/network_beacon_processor/tests.rs | 164 ++++++++++- beacon_node/network/src/router.rs | 12 + 14 files changed, 637 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index ea87e9bc71..25944bcf8a 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -431,6 +431,7 @@ pub enum Work { Status(BlockingFn), BlocksByRangeRequest(AsyncFn), BlocksByRootsRequest(AsyncFn), + BlocksByHeadRequest(AsyncFn), PayloadEnvelopesByRangeRequest(AsyncFn), PayloadEnvelopesByRootRequest(AsyncFn), BlobsByRangeRequest(BlockingFn), @@ -491,6 +492,7 @@ pub enum WorkType { Status, BlocksByRangeRequest, BlocksByRootsRequest, + BlocksByHeadRequest, PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, BlobsByRangeRequest, @@ -553,6 +555,7 @@ impl Work { Work::Status(_) => WorkType::Status, Work::BlocksByRangeRequest(_) => WorkType::BlocksByRangeRequest, Work::BlocksByRootsRequest(_) => WorkType::BlocksByRootsRequest, + Work::BlocksByHeadRequest(_) => WorkType::BlocksByHeadRequest, Work::PayloadEnvelopesByRangeRequest(_) => WorkType::PayloadEnvelopesByRangeRequest, Work::PayloadEnvelopesByRootRequest(_) => WorkType::PayloadEnvelopesByRootRequest, Work::BlobsByRangeRequest(_) => WorkType::BlobsByRangeRequest, @@ -1000,6 +1003,8 @@ impl BeaconProcessor { Some(item) } else if let Some(item) = work_queues.block_broots_queue.pop() { Some(item) + } else if let Some(item) = work_queues.block_bhead_queue.pop() { + Some(item) } else if let Some(item) = work_queues.blob_brange_queue.pop() { Some(item) } else if let Some(item) = work_queues.blob_broots_queue.pop() { @@ -1206,6 +1211,9 @@ impl BeaconProcessor { Work::BlocksByRootsRequest { .. } => { work_queues.block_broots_queue.push(work, work_id) } + Work::BlocksByHeadRequest { .. } => { + work_queues.block_bhead_queue.push(work, work_id) + } Work::PayloadEnvelopesByRangeRequest { .. } => work_queues .payload_envelopes_brange_queue .push(work, work_id), @@ -1331,6 +1339,7 @@ impl BeaconProcessor { WorkType::Status => work_queues.status_queue.len(), WorkType::BlocksByRangeRequest => work_queues.block_brange_queue.len(), WorkType::BlocksByRootsRequest => work_queues.block_broots_queue.len(), + WorkType::BlocksByHeadRequest => work_queues.block_bhead_queue.len(), WorkType::PayloadEnvelopesByRangeRequest => { work_queues.payload_envelopes_brange_queue.len() } @@ -1531,6 +1540,7 @@ impl BeaconProcessor { } Work::BlocksByRangeRequest(work) | Work::BlocksByRootsRequest(work) + | Work::BlocksByHeadRequest(work) | Work::PayloadEnvelopesByRangeRequest(work) | Work::PayloadEnvelopesByRootRequest(work) => task_spawner.spawn_async(work), Work::ChainSegmentBackfill(process_fn) => { diff --git a/beacon_node/beacon_processor/src/scheduler/work_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_queue.rs index f7163d538b..eb57b97df2 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_queue.rs @@ -132,6 +132,7 @@ pub struct BeaconProcessorQueueLengths { status_queue: usize, block_brange_queue: usize, block_broots_queue: usize, + block_bhead_queue: usize, blob_broots_queue: usize, blob_brange_queue: usize, dcbroots_queue: usize, @@ -206,6 +207,7 @@ impl BeaconProcessorQueueLengths { status_queue: 1024, block_brange_queue: 1024, block_broots_queue: 1024, + block_bhead_queue: 1024, blob_broots_queue: 1024, blob_brange_queue: 1024, dcbroots_queue: 1024, @@ -263,6 +265,7 @@ pub struct WorkQueues { pub status_queue: FifoQueue>, pub block_brange_queue: FifoQueue>, pub block_broots_queue: FifoQueue>, + pub block_bhead_queue: FifoQueue>, pub payload_envelopes_brange_queue: FifoQueue>, pub payload_envelopes_broots_queue: FifoQueue>, pub blob_broots_queue: FifoQueue>, @@ -334,6 +337,7 @@ impl WorkQueues { let status_queue = FifoQueue::new(queue_lengths.status_queue); let block_brange_queue = FifoQueue::new(queue_lengths.block_brange_queue); let block_broots_queue = FifoQueue::new(queue_lengths.block_broots_queue); + let block_bhead_queue = FifoQueue::new(queue_lengths.block_bhead_queue); let blob_broots_queue = FifoQueue::new(queue_lengths.blob_broots_queue); let blob_brange_queue = FifoQueue::new(queue_lengths.blob_brange_queue); let dcbroots_queue = FifoQueue::new(queue_lengths.dcbroots_queue); @@ -399,6 +403,7 @@ impl WorkQueues { status_queue, block_brange_queue, block_broots_queue, + block_bhead_queue, blob_broots_queue, blob_brange_queue, dcbroots_queue, diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index d7285c5c8e..6b5144fa6f 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -589,6 +589,7 @@ impl PeerManager { Protocol::Ping => PeerAction::MidToleranceError, Protocol::BlocksByRange => PeerAction::MidToleranceError, Protocol::BlocksByRoot => PeerAction::MidToleranceError, + Protocol::BlocksByHead => PeerAction::MidToleranceError, Protocol::BlobsByRange => PeerAction::MidToleranceError, Protocol::PayloadEnvelopesByRange => PeerAction::MidToleranceError, Protocol::PayloadEnvelopesByRoot => PeerAction::MidToleranceError, @@ -617,6 +618,7 @@ impl PeerManager { Protocol::Ping => PeerAction::Fatal, Protocol::BlocksByRange => return, Protocol::BlocksByRoot => return, + Protocol::BlocksByHead => return, Protocol::PayloadEnvelopesByRange => return, Protocol::PayloadEnvelopesByRoot => return, Protocol::BlobsByRange => return, @@ -642,6 +644,7 @@ impl PeerManager { Protocol::Ping => PeerAction::LowToleranceError, Protocol::BlocksByRange => PeerAction::MidToleranceError, Protocol::BlocksByRoot => PeerAction::MidToleranceError, + Protocol::BlocksByHead => PeerAction::MidToleranceError, Protocol::PayloadEnvelopesByRange => PeerAction::MidToleranceError, Protocol::PayloadEnvelopesByRoot => PeerAction::MidToleranceError, Protocol::BlobsByRange => PeerAction::MidToleranceError, diff --git a/beacon_node/lighthouse_network/src/rpc/codec.rs b/beacon_node/lighthouse_network/src/rpc/codec.rs index 75e035ae82..ba95fff5e8 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec.rs @@ -18,7 +18,7 @@ use tokio_util::codec::{Decoder, Encoder}; use types::SignedExecutionPayloadEnvelope; use types::{ BlobSidecar, ChainSpec, DataColumnSidecar, DataColumnsByRootIdentifier, EthSpec, ForkContext, - ForkName, Hash256, LightClientBootstrap, LightClientFinalityUpdate, + ForkName, ForkVersionDecode, Hash256, LightClientBootstrap, LightClientFinalityUpdate, LightClientOptimisticUpdate, LightClientUpdate, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, SignedBeaconBlockFulu, @@ -77,6 +77,7 @@ impl SSZSnappyInboundCodec { }, RpcSuccessResponse::BlocksByRange(res) => res.as_ssz_bytes(), RpcSuccessResponse::BlocksByRoot(res) => res.as_ssz_bytes(), + RpcSuccessResponse::BlocksByHead(res) => res.as_ssz_bytes(), RpcSuccessResponse::PayloadEnvelopesByRange(res) => res.as_ssz_bytes(), RpcSuccessResponse::PayloadEnvelopesByRoot(res) => res.as_ssz_bytes(), RpcSuccessResponse::BlobsByRange(res) => res.as_ssz_bytes(), @@ -359,6 +360,7 @@ impl Encoder> for SSZSnappyOutboundCodec { BlocksByRootRequest::V1(req) => req.block_roots.as_ssz_bytes(), BlocksByRootRequest::V2(req) => req.block_roots.as_ssz_bytes(), }, + RequestType::BlocksByHead(req) => req.as_ssz_bytes(), RequestType::PayloadEnvelopesByRange(req) => req.as_ssz_bytes(), RequestType::PayloadEnvelopesByRoot(req) => req.beacon_block_roots.as_ssz_bytes(), RequestType::BlobsByRange(req) => req.as_ssz_bytes(), @@ -553,6 +555,9 @@ fn handle_rpc_request( )?, }), ))), + SupportedProtocol::BlocksByHeadV1 => Ok(Some(RequestType::BlocksByHead( + BlocksByHeadRequest::from_ssz_bytes(decoded_buffer)?, + ))), SupportedProtocol::PayloadEnvelopesByRangeV1 => { Ok(Some(RequestType::PayloadEnvelopesByRange( PayloadEnvelopesByRangeRequest::from_ssz_bytes(decoded_buffer)?, @@ -943,6 +948,18 @@ fn handle_rpc_response( ), )), }, + SupportedProtocol::BlocksByHeadV1 => match fork_name { + Some(fork_name) => Ok(Some(RpcSuccessResponse::BlocksByHead(Arc::new( + SignedBeaconBlock::from_ssz_bytes_by_fork(decoded_buffer, fork_name)?, + )))), + None => Err(RPCError::ErrorResponse( + RpcErrorResponse::InvalidRequest, + format!( + "No context bytes provided for {:?} response", + versioned_protocol + ), + )), + }, } } @@ -1319,6 +1336,9 @@ mod tests { RequestType::BlocksByRoot(bbroot) => { assert_eq!(decoded, RequestType::BlocksByRoot(bbroot)) } + RequestType::BlocksByHead(bbhead) => { + assert_eq!(decoded, RequestType::BlocksByHead(bbhead)) + } RequestType::BlobsByRange(blbrange) => { assert_eq!(decoded, RequestType::BlobsByRange(blbrange)) } @@ -1867,6 +1887,31 @@ mod tests { ); } + // BlocksByHead is introduced in Fulu but the response is just `SignedBeaconBlock`, + // so the codec must accept blocks of any fork variant — the chain a Fulu peer walks + // back may straddle the Fulu boundary and include pre-Fulu canonical blocks. + #[test] + fn test_blocks_by_head_decodes_all_forks() { + let chain_spec = spec_with_all_forks_enabled(); + for (block, fork) in [ + (empty_base_block(&chain_spec), ForkName::Base), + (altair_block(&chain_spec), ForkName::Altair), + (bellatrix_block_small(&chain_spec), ForkName::Bellatrix), + ] { + let block_arc = Arc::new(block); + assert_eq!( + encode_then_decode_response( + SupportedProtocol::BlocksByHeadV1, + RpcResponse::Success(RpcSuccessResponse::BlocksByHead(block_arc.clone())), + fork, + &chain_spec, + ), + Ok(Some(RpcSuccessResponse::BlocksByHead(block_arc))), + "BlocksByHeadV1 must round-trip a {fork} block" + ); + } + } + // Test RPCResponse encoding/decoding for V2 messages #[test] fn test_context_bytes_v2() { @@ -2063,6 +2108,10 @@ mod tests { RequestType::BlobsByRange(blbrange_request()), RequestType::DataColumnsByRange(dcbrange_request()), RequestType::MetaData(MetadataRequest::new_v2()), + RequestType::BlocksByHead(BlocksByHeadRequest { + beacon_root: Hash256::zero(), + count: 32, + }), ]; for req in requests.iter() { for fork_name in ForkName::list_all() { diff --git a/beacon_node/lighthouse_network/src/rpc/config.rs b/beacon_node/lighthouse_network/src/rpc/config.rs index 9e1c6541ec..59f0b8e9a2 100644 --- a/beacon_node/lighthouse_network/src/rpc/config.rs +++ b/beacon_node/lighthouse_network/src/rpc/config.rs @@ -89,6 +89,7 @@ pub struct RateLimiterConfig { pub(super) goodbye_quota: Quota, pub(super) blocks_by_range_quota: Quota, pub(super) blocks_by_root_quota: Quota, + pub(super) blocks_by_head_quota: Quota, pub(super) payload_envelopes_by_range_quota: Quota, pub(super) payload_envelopes_by_root_quota: Quota, pub(super) blobs_by_range_quota: Quota, @@ -113,6 +114,8 @@ impl RateLimiterConfig { Quota::n_every(NonZeroU64::new(128).unwrap(), 10); pub const DEFAULT_BLOCKS_BY_ROOT_QUOTA: Quota = Quota::n_every(NonZeroU64::new(128).unwrap(), 10); + pub const DEFAULT_BLOCKS_BY_HEAD_QUOTA: Quota = + Quota::n_every(NonZeroU64::new(128).unwrap(), 10); pub const DEFAULT_PAYLOAD_ENVELOPES_BY_RANGE_QUOTA: Quota = Quota::n_every(NonZeroU64::new(128).unwrap(), 10); pub const DEFAULT_PAYLOAD_ENVELOPES_BY_ROOT_QUOTA: Quota = @@ -143,6 +146,7 @@ impl Default for RateLimiterConfig { goodbye_quota: Self::DEFAULT_GOODBYE_QUOTA, blocks_by_range_quota: Self::DEFAULT_BLOCKS_BY_RANGE_QUOTA, blocks_by_root_quota: Self::DEFAULT_BLOCKS_BY_ROOT_QUOTA, + blocks_by_head_quota: Self::DEFAULT_BLOCKS_BY_HEAD_QUOTA, payload_envelopes_by_range_quota: Self::DEFAULT_PAYLOAD_ENVELOPES_BY_RANGE_QUOTA, payload_envelopes_by_root_quota: Self::DEFAULT_PAYLOAD_ENVELOPES_BY_ROOT_QUOTA, blobs_by_range_quota: Self::DEFAULT_BLOBS_BY_RANGE_QUOTA, @@ -177,6 +181,7 @@ impl Debug for RateLimiterConfig { .field("goodbye", fmt_q!(&self.goodbye_quota)) .field("blocks_by_range", fmt_q!(&self.blocks_by_range_quota)) .field("blocks_by_root", fmt_q!(&self.blocks_by_root_quota)) + .field("blocks_by_head", fmt_q!(&self.blocks_by_head_quota)) .field( "payload_envelopes_by_range", fmt_q!(&self.payload_envelopes_by_range_quota), @@ -213,6 +218,7 @@ impl FromStr for RateLimiterConfig { let mut goodbye_quota = None; let mut blocks_by_range_quota = None; let mut blocks_by_root_quota = None; + let mut blocks_by_head_quota = None; let mut payload_envelopes_by_range_quota = None; let mut payload_envelopes_by_root_quota = None; let mut blobs_by_range_quota = None; @@ -232,6 +238,7 @@ impl FromStr for RateLimiterConfig { Protocol::Goodbye => goodbye_quota = goodbye_quota.or(quota), Protocol::BlocksByRange => blocks_by_range_quota = blocks_by_range_quota.or(quota), Protocol::BlocksByRoot => blocks_by_root_quota = blocks_by_root_quota.or(quota), + Protocol::BlocksByHead => blocks_by_head_quota = blocks_by_head_quota.or(quota), Protocol::PayloadEnvelopesByRange => { payload_envelopes_by_range_quota = payload_envelopes_by_range_quota.or(quota) } @@ -274,6 +281,8 @@ impl FromStr for RateLimiterConfig { .unwrap_or(Self::DEFAULT_BLOCKS_BY_RANGE_QUOTA), blocks_by_root_quota: blocks_by_root_quota .unwrap_or(Self::DEFAULT_BLOCKS_BY_ROOT_QUOTA), + blocks_by_head_quota: blocks_by_head_quota + .unwrap_or(Self::DEFAULT_BLOCKS_BY_HEAD_QUOTA), payload_envelopes_by_range_quota: payload_envelopes_by_range_quota .unwrap_or(Self::DEFAULT_PAYLOAD_ENVELOPES_BY_RANGE_QUOTA), payload_envelopes_by_root_quota: payload_envelopes_by_root_quota diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index baabf48683..f3f294d913 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -488,6 +488,18 @@ impl From for OldBlocksByRangeRequest { } } +/// Request a contiguous range of beacon blocks by walking the parent chain of `beacon_root`. +/// +/// New in Fulu (see consensus-specs PR 5181). The responder walks the parent chain of +/// `beacon_root` (inclusive) and emits up to `count` blocks in descending slot order. +#[derive(Encode, Decode, Clone, Debug, PartialEq)] +pub struct BlocksByHeadRequest { + /// The block root to start the parent walk from (inclusive). + pub beacon_root: Hash256, + /// The maximum number of blocks to return. + pub count: u64, +} + /// Request a number of beacon block bodies from a peer. #[superstruct(variants(V1, V2), variant_attributes(derive(Clone, Debug, PartialEq)))] #[derive(Clone, Debug, PartialEq)] @@ -622,6 +634,9 @@ pub enum RpcSuccessResponse { /// A response to a get BLOCKS_BY_ROOT request. BlocksByRoot(Arc>), + /// A response to a get BEACON_BLOCKS_BY_HEAD request. + BlocksByHead(Arc>), + /// A response to a get EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE request. A None response signifies /// the end of the batch. PayloadEnvelopesByRange(Arc>), @@ -669,6 +684,9 @@ pub enum ResponseTermination { /// Blocks by root stream termination. BlocksByRoot, + /// Blocks by head stream termination. + BlocksByHead, + /// Execution payload envelopes by range stream termination. PayloadEnvelopesByRange, @@ -696,6 +714,7 @@ impl ResponseTermination { match self { ResponseTermination::BlocksByRange => Protocol::BlocksByRange, ResponseTermination::BlocksByRoot => Protocol::BlocksByRoot, + ResponseTermination::BlocksByHead => Protocol::BlocksByHead, ResponseTermination::PayloadEnvelopesByRange => Protocol::PayloadEnvelopesByRange, ResponseTermination::PayloadEnvelopesByRoot => Protocol::PayloadEnvelopesByRoot, ResponseTermination::BlobsByRange => Protocol::BlobsByRange, @@ -793,6 +812,7 @@ impl RpcSuccessResponse { RpcSuccessResponse::Status(_) => Protocol::Status, RpcSuccessResponse::BlocksByRange(_) => Protocol::BlocksByRange, RpcSuccessResponse::BlocksByRoot(_) => Protocol::BlocksByRoot, + RpcSuccessResponse::BlocksByHead(_) => Protocol::BlocksByHead, RpcSuccessResponse::PayloadEnvelopesByRange(_) => Protocol::PayloadEnvelopesByRange, RpcSuccessResponse::PayloadEnvelopesByRoot(_) => Protocol::PayloadEnvelopesByRoot, RpcSuccessResponse::BlobsByRange(_) => Protocol::BlobsByRange, @@ -812,7 +832,9 @@ impl RpcSuccessResponse { pub fn slot(&self) -> Option { match self { - Self::BlocksByRange(r) | Self::BlocksByRoot(r) => Some(r.slot()), + Self::BlocksByRange(r) | Self::BlocksByRoot(r) | Self::BlocksByHead(r) => { + Some(r.slot()) + } Self::PayloadEnvelopesByRoot(r) | Self::PayloadEnvelopesByRange(r) => Some(r.slot()), Self::BlobsByRange(r) | Self::BlobsByRoot(r) => Some(r.slot()), Self::DataColumnsByRange(r) | Self::DataColumnsByRoot(r) => Some(r.slot()), @@ -864,6 +886,9 @@ impl std::fmt::Display for RpcSuccessResponse { RpcSuccessResponse::BlocksByRoot(block) => { write!(f, "BlocksByRoot: Block slot: {}", block.slot()) } + RpcSuccessResponse::BlocksByHead(block) => { + write!(f, "BlocksByHead: Block slot: {}", block.slot()) + } RpcSuccessResponse::PayloadEnvelopesByRange(envelope) => { write!( f, @@ -975,6 +1000,16 @@ impl std::fmt::Display for OldBlocksByRangeRequest { } } +impl std::fmt::Display for BlocksByHeadRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "BlocksByHead: beacon_root: {}, count: {}", + self.beacon_root, self.count + ) + } +} + impl std::fmt::Display for BlobsByRootRequest { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index c949dfe17d..056ffc03b8 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -262,6 +262,9 @@ pub enum Protocol { /// The `BlocksByRoot` protocol name. #[strum(serialize = "beacon_blocks_by_root")] BlocksByRoot, + /// The `BlocksByHead` protocol name. + #[strum(serialize = "beacon_blocks_by_head")] + BlocksByHead, /// The `BlobsByRange` protocol name. #[strum(serialize = "blob_sidecars_by_range")] BlobsByRange, @@ -306,6 +309,7 @@ impl Protocol { Protocol::Goodbye => None, Protocol::BlocksByRange => Some(ResponseTermination::BlocksByRange), Protocol::BlocksByRoot => Some(ResponseTermination::BlocksByRoot), + Protocol::BlocksByHead => Some(ResponseTermination::BlocksByHead), Protocol::PayloadEnvelopesByRange => Some(ResponseTermination::PayloadEnvelopesByRange), Protocol::PayloadEnvelopesByRoot => Some(ResponseTermination::PayloadEnvelopesByRoot), Protocol::BlobsByRange => Some(ResponseTermination::BlobsByRange), @@ -338,6 +342,7 @@ pub enum SupportedProtocol { BlocksByRangeV2, BlocksByRootV1, BlocksByRootV2, + BlocksByHeadV1, PayloadEnvelopesByRangeV1, PayloadEnvelopesByRootV1, BlobsByRangeV1, @@ -366,6 +371,7 @@ impl SupportedProtocol { SupportedProtocol::PayloadEnvelopesByRootV1 => "1", SupportedProtocol::BlocksByRootV1 => "1", SupportedProtocol::BlocksByRootV2 => "2", + SupportedProtocol::BlocksByHeadV1 => "1", SupportedProtocol::BlobsByRangeV1 => "1", SupportedProtocol::BlobsByRootV1 => "1", SupportedProtocol::DataColumnsByRootV1 => "1", @@ -390,6 +396,7 @@ impl SupportedProtocol { SupportedProtocol::BlocksByRangeV2 => Protocol::BlocksByRange, SupportedProtocol::BlocksByRootV1 => Protocol::BlocksByRoot, SupportedProtocol::BlocksByRootV2 => Protocol::BlocksByRoot, + SupportedProtocol::BlocksByHeadV1 => Protocol::BlocksByHead, SupportedProtocol::PayloadEnvelopesByRangeV1 => Protocol::PayloadEnvelopesByRange, SupportedProtocol::PayloadEnvelopesByRootV1 => Protocol::PayloadEnvelopesByRoot, SupportedProtocol::BlobsByRangeV1 => Protocol::BlobsByRange, @@ -458,6 +465,13 @@ impl SupportedProtocol { ), ]); } + // BeaconBlocksByHead is new in Fulu (consensus-specs PR 5181). + if fork_context.fork_exists(ForkName::Fulu) { + supported.push(ProtocolId::new( + SupportedProtocol::BlocksByHeadV1, + Encoding::SSZSnappy, + )); + } supported } } @@ -564,6 +578,10 @@ impl ProtocolId { ::ssz_fixed_len(), ), Protocol::BlocksByRoot => RpcLimits::new(0, spec.max_blocks_by_root_request), + Protocol::BlocksByHead => RpcLimits::new( + ::ssz_fixed_len(), + ::ssz_fixed_len(), + ), Protocol::PayloadEnvelopesByRange => RpcLimits::new( ::ssz_fixed_len(), ::ssz_fixed_len(), @@ -609,6 +627,7 @@ impl ProtocolId { Protocol::Goodbye => RpcLimits::new(0, 0), // Goodbye request has no response Protocol::BlocksByRange => rpc_block_limits_by_fork(fork_context.current_fork_name()), Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork_name()), + Protocol::BlocksByHead => rpc_block_limits_by_fork(fork_context.current_fork_name()), Protocol::PayloadEnvelopesByRange => rpc_payload_limits(), Protocol::PayloadEnvelopesByRoot => rpc_payload_limits(), Protocol::BlobsByRange => rpc_blob_limits::(), @@ -648,6 +667,7 @@ impl ProtocolId { match self.versioned_protocol { SupportedProtocol::BlocksByRangeV2 | SupportedProtocol::BlocksByRootV2 + | SupportedProtocol::BlocksByHeadV1 | SupportedProtocol::PayloadEnvelopesByRangeV1 | SupportedProtocol::PayloadEnvelopesByRootV1 | SupportedProtocol::BlobsByRangeV1 @@ -801,6 +821,7 @@ pub enum RequestType { Goodbye(GoodbyeReason), BlocksByRange(OldBlocksByRangeRequest), BlocksByRoot(BlocksByRootRequest), + BlocksByHead(BlocksByHeadRequest), PayloadEnvelopesByRange(PayloadEnvelopesByRangeRequest), PayloadEnvelopesByRoot(PayloadEnvelopesByRootRequest), BlobsByRange(BlobsByRangeRequest), @@ -826,6 +847,7 @@ impl RequestType { RequestType::Goodbye(_) => 0, RequestType::BlocksByRange(req) => *req.count(), RequestType::BlocksByRoot(req) => req.block_roots().len() as u64, + RequestType::BlocksByHead(req) => req.count, RequestType::PayloadEnvelopesByRange(req) => req.count, RequestType::PayloadEnvelopesByRoot(req) => req.beacon_block_roots.len() as u64, RequestType::BlobsByRange(req) => req.max_blobs_requested(digest_epoch, spec), @@ -857,6 +879,7 @@ impl RequestType { BlocksByRootRequest::V1(_) => SupportedProtocol::BlocksByRootV1, BlocksByRootRequest::V2(_) => SupportedProtocol::BlocksByRootV2, }, + RequestType::BlocksByHead(_) => SupportedProtocol::BlocksByHeadV1, RequestType::PayloadEnvelopesByRange(_) => SupportedProtocol::PayloadEnvelopesByRangeV1, RequestType::PayloadEnvelopesByRoot(_) => SupportedProtocol::PayloadEnvelopesByRootV1, RequestType::BlobsByRange(_) => SupportedProtocol::BlobsByRangeV1, @@ -890,6 +913,7 @@ impl RequestType { // variants that have `multiple_responses()` can have values. RequestType::BlocksByRange(_) => ResponseTermination::BlocksByRange, RequestType::BlocksByRoot(_) => ResponseTermination::BlocksByRoot, + RequestType::BlocksByHead(_) => ResponseTermination::BlocksByHead, RequestType::PayloadEnvelopesByRange(_) => ResponseTermination::PayloadEnvelopesByRange, RequestType::PayloadEnvelopesByRoot(_) => ResponseTermination::PayloadEnvelopesByRoot, RequestType::BlobsByRange(_) => ResponseTermination::BlobsByRange, @@ -926,6 +950,10 @@ impl RequestType { ProtocolId::new(SupportedProtocol::BlocksByRootV2, Encoding::SSZSnappy), ProtocolId::new(SupportedProtocol::BlocksByRootV1, Encoding::SSZSnappy), ], + RequestType::BlocksByHead(_) => vec![ProtocolId::new( + SupportedProtocol::BlocksByHeadV1, + Encoding::SSZSnappy, + )], RequestType::PayloadEnvelopesByRange(_) => vec![ProtocolId::new( SupportedProtocol::PayloadEnvelopesByRangeV1, Encoding::SSZSnappy, @@ -984,6 +1012,7 @@ impl RequestType { RequestType::Goodbye(_) => false, RequestType::BlocksByRange(_) => false, RequestType::BlocksByRoot(_) => false, + RequestType::BlocksByHead(_) => false, RequestType::BlobsByRange(_) => false, RequestType::PayloadEnvelopesByRange(_) => false, RequestType::PayloadEnvelopesByRoot(_) => false, @@ -1097,6 +1126,7 @@ impl std::fmt::Display for RequestType { RequestType::Goodbye(reason) => write!(f, "Goodbye: {}", reason), RequestType::BlocksByRange(req) => write!(f, "Blocks by range: {}", req), RequestType::BlocksByRoot(req) => write!(f, "Blocks by root: {:?}", req), + RequestType::BlocksByHead(req) => write!(f, "Blocks by head: {}", req), RequestType::PayloadEnvelopesByRange(req) => { write!(f, "Payload envelopes by range: {:?}", req) } @@ -1171,6 +1201,8 @@ mod tests { fork_context.fork_exists(ForkName::Gloas) } + BlocksByHeadV1 => fork_context.fork_exists(ForkName::Fulu), + // Light client protocols are not in currently_supported() LightClientBootstrapV1 | LightClientOptimisticUpdateV1 diff --git a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs index ebdca386d8..a5c98a4d30 100644 --- a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs @@ -105,6 +105,8 @@ pub struct RPCRateLimiter { bbrange_rl: Limiter, /// BlocksByRoot rate limiter. bbroots_rl: Limiter, + /// BlocksByHead rate limiter. + bbhead_rl: Limiter, /// BlobsByRange rate limiter. blbrange_rl: Limiter, /// BlobsByRoot rate limiter. @@ -152,6 +154,8 @@ pub struct RPCRateLimiterBuilder { bbrange_quota: Option, /// Quota for the BlocksByRoot protocol. bbroots_quota: Option, + /// Quota for the BlocksByHead protocol. + bbhead_quota: Option, /// Quota for the ExecutionPayloadEnvelopesByRange protocol. perange_quota: Option, /// Quota for the ExecutionPayloadEnvelopesByRoot protocol. @@ -185,6 +189,7 @@ impl RPCRateLimiterBuilder { Protocol::Goodbye => self.goodbye_quota = q, Protocol::BlocksByRange => self.bbrange_quota = q, Protocol::BlocksByRoot => self.bbroots_quota = q, + Protocol::BlocksByHead => self.bbhead_quota = q, Protocol::PayloadEnvelopesByRange => self.perange_quota = q, Protocol::PayloadEnvelopesByRoot => self.peroots_quota = q, Protocol::BlobsByRange => self.blbrange_quota = q, @@ -211,6 +216,9 @@ impl RPCRateLimiterBuilder { let bbrange_quota = self .bbrange_quota .ok_or("BlocksByRange quota not specified")?; + let bbhead_quota = self + .bbhead_quota + .ok_or("BlocksByHead quota not specified")?; let perange_quota = self .perange_quota .ok_or("PayloadEnvelopesByRange quota not specified")?; @@ -252,6 +260,7 @@ impl RPCRateLimiterBuilder { let goodbye_rl = Limiter::from_quota(goodbye_quota)?; let bbroots_rl = Limiter::from_quota(bbroots_quota)?; let bbrange_rl = Limiter::from_quota(bbrange_quota)?; + let bbhead_rl = Limiter::from_quota(bbhead_quota)?; let envrange_rl = Limiter::from_quota(perange_quota)?; let envroots_rl = Limiter::from_quota(peroots_quota)?; let blbrange_rl = Limiter::from_quota(blbrange_quota)?; @@ -277,6 +286,7 @@ impl RPCRateLimiterBuilder { goodbye_rl, bbroots_rl, bbrange_rl, + bbhead_rl, envrange_rl, envroots_rl, blbrange_rl, @@ -332,6 +342,7 @@ impl RPCRateLimiter { goodbye_quota, blocks_by_range_quota, blocks_by_root_quota, + blocks_by_head_quota, payload_envelopes_by_range_quota, payload_envelopes_by_root_quota, blobs_by_range_quota, @@ -351,6 +362,7 @@ impl RPCRateLimiter { .set_quota(Protocol::Goodbye, goodbye_quota) .set_quota(Protocol::BlocksByRange, blocks_by_range_quota) .set_quota(Protocol::BlocksByRoot, blocks_by_root_quota) + .set_quota(Protocol::BlocksByHead, blocks_by_head_quota) .set_quota( Protocol::PayloadEnvelopesByRange, payload_envelopes_by_range_quota, @@ -406,6 +418,7 @@ impl RPCRateLimiter { Protocol::Goodbye => &mut self.goodbye_rl, Protocol::BlocksByRange => &mut self.bbrange_rl, Protocol::BlocksByRoot => &mut self.bbroots_rl, + Protocol::BlocksByHead => &mut self.bbhead_rl, Protocol::PayloadEnvelopesByRange => &mut self.envrange_rl, Protocol::PayloadEnvelopesByRoot => &mut self.envroots_rl, Protocol::BlobsByRange => &mut self.blbrange_rl, @@ -432,6 +445,7 @@ impl RPCRateLimiter { status_rl, bbrange_rl, bbroots_rl, + bbhead_rl, envrange_rl, envroots_rl, blbrange_rl, @@ -451,6 +465,7 @@ impl RPCRateLimiter { status_rl.prune(time_since_start); bbrange_rl.prune(time_since_start); bbroots_rl.prune(time_since_start); + bbhead_rl.prune(time_since_start); envrange_rl.prune(time_since_start); envroots_rl.prune(time_since_start); blbrange_rl.prune(time_since_start); diff --git a/beacon_node/lighthouse_network/src/service/api_types.rs b/beacon_node/lighthouse_network/src/service/api_types.rs index 486a443857..f598f59aee 100644 --- a/beacon_node/lighthouse_network/src/service/api_types.rs +++ b/beacon_node/lighthouse_network/src/service/api_types.rs @@ -161,6 +161,9 @@ pub enum Response { DataColumnsByRange(Option>>), /// A response to a get BLOCKS_BY_ROOT request. BlocksByRoot(Option>>), + /// A response to a get BEACON_BLOCKS_BY_HEAD request. A None response signals the end of the + /// batch. + BlocksByHead(Option>>), /// A response to a get `EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT` request. PayloadEnvelopesByRoot(Option>>), /// A response to a get `EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE` request. @@ -186,6 +189,10 @@ impl std::convert::From> for RpcResponse { Some(b) => RpcResponse::Success(RpcSuccessResponse::BlocksByRoot(b)), None => RpcResponse::StreamTermination(ResponseTermination::BlocksByRoot), }, + Response::BlocksByHead(r) => match r { + Some(b) => RpcResponse::Success(RpcSuccessResponse::BlocksByHead(b)), + None => RpcResponse::StreamTermination(ResponseTermination::BlocksByHead), + }, Response::BlocksByRange(r) => match r { Some(b) => RpcResponse::Success(RpcSuccessResponse::BlocksByRange(b)), None => RpcResponse::StreamTermination(ResponseTermination::BlocksByRange), diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index f0c1567cb0..41d937e324 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1691,6 +1691,14 @@ impl Network { request_type, }) } + RequestType::BlocksByHead(_) => { + metrics::inc_counter_vec(&metrics::TOTAL_RPC_REQUESTS, &["blocks_by_head"]); + Some(NetworkEvent::RequestReceived { + peer_id, + inbound_request_id, + request_type, + }) + } RequestType::PayloadEnvelopesByRange(_) => { metrics::inc_counter_vec( &metrics::TOTAL_RPC_REQUESTS, @@ -1827,6 +1835,9 @@ impl Network { RpcSuccessResponse::BlocksByRoot(resp) => { self.build_response(id, peer_id, Response::BlocksByRoot(Some(resp))) } + RpcSuccessResponse::BlocksByHead(resp) => { + self.build_response(id, peer_id, Response::BlocksByHead(Some(resp))) + } RpcSuccessResponse::PayloadEnvelopesByRange(resp) => self.build_response( id, peer_id, @@ -1871,6 +1882,7 @@ impl Network { let response = match termination { ResponseTermination::BlocksByRange => Response::BlocksByRange(None), ResponseTermination::BlocksByRoot => Response::BlocksByRoot(None), + ResponseTermination::BlocksByHead => Response::BlocksByHead(None), ResponseTermination::PayloadEnvelopesByRange => { Response::PayloadEnvelopesByRange(None) } diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index e089159eb8..6a3ccbcd65 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -14,8 +14,8 @@ use beacon_processor::{ }; use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, - LightClientUpdatesByRangeRequest, PayloadEnvelopesByRangeRequest, + BlobsByRangeRequest, BlobsByRootRequest, BlocksByHeadRequest, DataColumnsByRangeRequest, + DataColumnsByRootRequest, LightClientUpdatesByRangeRequest, PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, }; use lighthouse_network::service::api_types::CustodyBackfillBatchId; @@ -699,6 +699,26 @@ impl NetworkBeaconProcessor { }) } + /// Create a new work event to process `BlocksByHeadRequest`s from the RPC network. + pub fn send_blocks_by_head_request( + self: &Arc, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + request: BlocksByHeadRequest, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = async move { + processor + .handle_blocks_by_head_request(peer_id, inbound_request_id, request) + .await; + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::BlocksByHeadRequest(Box::pin(process_fn)), + }) + } + /// Create a new work event to process `BlocksByRootRequest`s from the RPC network. pub fn send_blocks_by_roots_request( self: &Arc, diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index 8b31b67acb..37a6f3779a 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -7,8 +7,8 @@ use beacon_chain::payload_envelope_streamer::EnvelopeRequestSource; use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessStatus, WhenSlotSkipped}; use itertools::{Itertools, process_results}; use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, - PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, + BlobsByRangeRequest, BlobsByRootRequest, BlocksByHeadRequest, DataColumnsByRangeRequest, + DataColumnsByRootRequest, PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, }; use lighthouse_network::rpc::*; use lighthouse_network::{PeerId, ReportSource, Response, SyncInfo}; @@ -256,6 +256,266 @@ impl NetworkBeaconProcessor { Ok(()) } + /// Handle a `BeaconBlocksByHead` request from the peer. + /// + /// Walks the parent chain of `request.beacon_root` (inclusive) and emits up to + /// `min(request.count, MAX_REQUEST_BLOCKS_DENEB)` blocks in descending slot order. + /// See consensus-specs PR 5181. + #[instrument( + name = "lh_handle_blocks_by_head_request", + parent = None, + level = "debug", + skip_all, + fields(peer_id = %peer_id, client = tracing::field::Empty) + )] + pub async fn handle_blocks_by_head_request( + self: Arc, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + request: BlocksByHeadRequest, + ) { + let client = self.network_globals.client(&peer_id); + Span::current().record("client", field::display(client.kind)); + + self.terminate_response_stream( + peer_id, + inbound_request_id, + self.clone() + .handle_blocks_by_head_request_inner(peer_id, inbound_request_id, request) + .await, + Response::BlocksByHead, + ); + } + + async fn handle_blocks_by_head_request_inner( + self: Arc, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + request: BlocksByHeadRequest, + ) -> Result<(), (RpcErrorResponse, &'static str)> { + let spec = &self.chain.spec; + // Cap the response at MAX_REQUEST_BLOCKS_DENEB regardless of what the peer asked for, + // matching the spec. + let max_request_blocks = spec.max_request_blocks(types::ForkName::Deneb) as u64; + let cap = request.count.min(max_request_blocks); + let beacon_root = request.beacon_root; + + debug!( + %peer_id, + beacon_root = ?beacon_root, + count = request.count, + cap, + "Received BlocksByHead Request" + ); + + if cap == 0 { + return Ok(()); + } + + // Walk the parent chain on a blocking thread because `get_blinded_block` hits the store + // synchronously and we may walk up to MAX_REQUEST_BLOCKS_DENEB ancestors. + let network_beacon_processor = self.clone(); + let block_roots = self + .executor + .spawn_blocking_handle( + move || network_beacon_processor.get_block_roots_ancestor_of_head(beacon_root, cap), + "get_block_roots_ancestor_of_head", + ) + .ok_or((RpcErrorResponse::ServerError, "shutting down"))? + .await + .map_err(|_| (RpcErrorResponse::ServerError, "tokio join"))??; + + let requested_blocks = block_roots.len(); + + let log_results = |peer_id, blocks_sent| { + debug!( + %peer_id, + requested = requested_blocks, + returned = blocks_sent, + "BlocksByHead outgoing response processed" + ); + }; + + let mut block_stream = match self.chain.get_blocks(block_roots) { + Ok(block_stream) => block_stream, + Err(e) => { + error!(error = ?e, "Error getting block stream"); + return Err((RpcErrorResponse::ServerError, "Iterator error")); + } + }; + + // Fetching blocks is async because it may have to hit the execution layer for payloads. + let mut blocks_sent = 0; + while let Some((root, result)) = block_stream.next().await { + match result.as_ref() { + Ok(Some(block)) => { + blocks_sent += 1; + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + inbound_request_id, + response: Response::BlocksByHead(Some(block.clone())), + }); + } + Ok(None) => { + error!( + %peer_id, + request_root = ?root, + "Block in the chain is not in the store" + ); + log_results(peer_id, blocks_sent); + return Err((RpcErrorResponse::ServerError, "Database inconsistency")); + } + Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => { + debug!( + block_root = ?root, + reason = "execution layer not synced", + "Failed to fetch execution payload for blocks by head request" + ); + log_results(peer_id, blocks_sent); + return Err(( + RpcErrorResponse::ResourceUnavailable, + "Execution layer not synced", + )); + } + Err(e) => { + if matches!( + e, + BeaconChainError::ExecutionLayerErrorPayloadReconstruction(_block_hash, boxed_error) + if matches!(**boxed_error, execution_layer::Error::EngineError(_)) + ) { + warn!( + info = "this may occur occasionally when the EE is busy", + block_root = ?root, + error = ?e, + "Error rebuilding payload for peer" + ); + } else { + error!( + block_root = ?root, + error = ?e, + "Error fetching block for peer" + ); + } + log_results(peer_id, blocks_sent); + return Err((RpcErrorResponse::ServerError, "Failed fetching blocks")); + } + } + } + + log_results(peer_id, blocks_sent); + Ok(()) + } + + /// Walks the parent chain of `head_root` (inclusive) and returns up to `count` block roots + /// in descending slot order. Synchronous so it can be run on a blocking thread. + /// + /// Two regimes are handled: + /// 1. Above finalization → fork-choice's in-memory proto-array supplies the roots + /// (zero DB reads). + /// 2. At or below finalization → the freezer DB's `BeaconBlockRoots` column (the + /// canonical slot→root index for finalized blocks, populated for + /// `[oldest_block_slot, split.slot)` with skip slots reusing the prior block's + /// root) supplies the roots. The head state is never consulted: its 8192-slot + /// `block_roots` bucket would silently truncate deep walks and is the wrong + /// source of truth for canonical history below finalization. + /// + /// Returns `ResourceUnavailable` if `head_root` is not known to the node. + fn get_block_roots_ancestor_of_head( + &self, + head_root: Hash256, + count: u64, + ) -> Result, (RpcErrorResponse, &'static str)> { + if count == 0 { + return Ok(vec![]); + } + + // 1. Walk ancestors in proto-array (in-memory, zero DB reads). Track the + // deepest slot we collected — that's where the freezer walk picks up. + let mut roots: Vec = Vec::with_capacity(count as usize); + let mut deepest_slot: Option = None; + { + let fork_choice = self.chain.canonical_head.fork_choice_read_lock(); + for (root, slot) in fork_choice + .proto_array() + .iter_block_roots(&head_root) + .take(count as usize) + { + roots.push(root); + deepest_slot = Some(slot); + } + } + + let store = &self.chain.store; + + // 2. Fallback: `head_root` is at or below finalization (proto-array doesn't + // track it). Look up its slot in the store, then verify it is the canonical + // block at that slot via the freezer index — a non-canonical hot-DB block at + // slot < split.slot can shadow the finalized chain. If the freezer + // disagrees (or doesn't have that slot), serve just the single block we + // found, satisfying the spec's "MUST return at least one block if you have + // it" clause. + let mut current_slot = if let Some(slot) = deepest_slot { + slot + } else { + let block = self + .chain + .get_blinded_block(&head_root) + .map_err(|e| { + error!(error = ?e, "Error reading blinded block for BlocksByHead beacon_root"); + (RpcErrorResponse::ServerError, "Database error") + })? + .ok_or((RpcErrorResponse::ResourceUnavailable, "Unknown beacon_root"))?; + let block_slot = block.slot(); + roots.push(head_root); + + match store.get_cold_block_root(block_slot) { + Ok(Some(r)) if r == head_root => {} // canonical, OK to walk back + Ok(_) => return Ok(roots), + Err(e) => { + error!(error = ?e, "Error reading freezer block_root for BlocksByHead"); + return Err((RpcErrorResponse::ServerError, "Database error")); + } + } + + block_slot + }; + + if (roots.len() as u64) >= count { + return Ok(roots); + } + + // 3. Spillover via the freezer DB's `BeaconBlockRoots` index (the canonical + // slot→root mapping for finalized blocks). Skip slots reuse the prior + // block's root; dedup on insert. + let oldest_block_slot = store.get_oldest_block_slot(); + let mut last_root = roots.last().copied(); + while (roots.len() as u64) < count && current_slot > oldest_block_slot { + current_slot = match current_slot.as_u64().checked_sub(1) { + Some(s) => Slot::from(s), + None => break, + }; + match store.get_cold_block_root(current_slot) { + Ok(Some(root)) => { + if Some(root) != last_root { + roots.push(root); + last_root = Some(root); + } + } + Ok(None) => { + // Hole in the freezer index (e.g. before `oldest_block_slot` on a + // checkpoint-synced node). Stop walking. + break; + } + Err(e) => { + error!(error = ?e, "Error walking freezer block_roots"); + return Err((RpcErrorResponse::ServerError, "Database error")); + } + } + } + + Ok(roots) + } + /// Handle a `ExecutionPayloadEnvelopesByRoot` request from the peer. #[instrument( name = "lh_handle_payload_envelopes_by_root_request", diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index c4e7f8f8d1..f13815f7b6 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -24,8 +24,8 @@ use itertools::Itertools; use libp2p::gossipsub::MessageAcceptance; use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, MetaDataV3, - PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, + BlobsByRangeRequest, BlobsByRootRequest, BlocksByHeadRequest, DataColumnsByRangeRequest, + MetaDataV3, PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, }; use lighthouse_network::{ Client, MessageId, NetworkConfig, NetworkGlobals, PeerId, Response, @@ -501,6 +501,16 @@ impl TestRig { .unwrap(); } + pub fn enqueue_blocks_by_head_request(&self, beacon_root: Hash256, count: u64) { + self.network_beacon_processor + .send_blocks_by_head_request( + PeerId::random(), + InboundRequestId::new_unchecked(42, 24), + BlocksByHeadRequest { beacon_root, count }, + ) + .unwrap(); + } + pub fn enqueue_blobs_by_root_request(&self, blob_ids: RuntimeVariableList) { self.network_beacon_processor .send_blobs_by_roots_request( @@ -2346,3 +2356,153 @@ async fn test_payload_envelopes_by_range_no_duplicates_with_skip_slots() { // 1. Gossip envelope arrives before its block → queued via UnknownBlockForEnvelope // 2. Block imported → envelope released and processed successfully // 3. Timeout path → envelope released and re-verified + +/// Drain `network_rx` collecting `Response::BlocksByHead(Some(_))` block roots until the +/// stream terminator (`None`) arrives. Panics on any other message type so tests fail +/// loudly if an error response sneaks in. +async fn drain_blocks_by_head_response(rig: &mut TestRig) -> Vec { + let mut roots = Vec::new(); + while let Some(msg) = rig.network_rx.recv().await { + match msg { + NetworkMessage::SendResponse { + response: Response::BlocksByHead(Some(block)), + .. + } => roots.push(block.canonical_root()), + NetworkMessage::SendResponse { + response: Response::BlocksByHead(None), + .. + } => return roots, + other => panic!("unexpected message: {:?}", other), + } + } + roots +} + +// `BlocksByHead` request that crosses the finalized boundary: proto-array supplies +// the unfinalized head + ancestors down to the finalized root, then the freezer's +// `BeaconBlockRoots` index supplies the rest. Verifies the spillover path +// `get_block_roots_ancestor_of_head` takes when count > proto-array depth. +#[tokio::test] +async fn test_blocks_by_head_spillover_into_freezer() { + // Long enough for finalization + state migration to populate the freezer. + let mut rig = TestRig::new(SLOTS_PER_EPOCH * 4).await; + + // Sanity-check the precondition: finalization advanced past genesis and the split + // slot is non-zero, so the freezer's `BeaconBlockRoots` column has entries. + assert!( + rig.chain + .canonical_head + .cached_head() + .finalized_checkpoint() + .epoch + > Epoch::new(0), + "test precondition: chain must have finalized past epoch 0", + ); + assert!( + rig.chain.store.get_split_slot() > Slot::new(0), + "test precondition: state migration must have populated the freezer", + ); + + let head_slot = rig.chain.canonical_head.cached_head().head_slot(); + let head_root = rig.chain.canonical_head.cached_head().head_block_root(); + + // Walk all the way back to slot 1: exercises both proto-array (above finalization) + // and freezer (at/below finalization). + let count = head_slot.as_u64(); + rig.enqueue_blocks_by_head_request(head_root, count); + let actual = drain_blocks_by_head_response(&mut rig).await; + + // Build the canonical descending root list independently. The harness has no skip + // slots so every slot in [1, head_slot] has a unique block, but we still dedup + // defensively to mirror the function under test. + let mut expected: Vec = Vec::new(); + let mut last: Option = None; + for offset in 0..count { + let slot = Slot::new(head_slot.as_u64() - offset); + if let Some(root) = rig + .chain + .block_root_at_slot(slot, WhenSlotSkipped::Prev) + .unwrap() + && Some(root) != last + { + expected.push(root); + last = Some(root); + } + } + + assert_eq!( + actual, expected, + "BlocksByHead must serve the full canonical parent chain across the finalized boundary", + ); + assert_eq!(actual.first(), Some(&head_root), "first root must be head"); +} + +// `BlocksByHead` with `beacon_root` set to a finalized block root (case-2 fallback in +// `get_block_roots_ancestor_of_head`): proto-array doesn't track it, so we +// `get_blinded_block` for its slot, verify canonicity via the freezer index, and walk +// back from there. +#[tokio::test] +async fn test_blocks_by_head_finalized_root() { + let mut rig = TestRig::new(SLOTS_PER_EPOCH * 4).await; + + let finalized_root = rig + .chain + .canonical_head + .cached_head() + .finalized_checkpoint() + .root; + let finalized_slot = rig + .chain + .get_blinded_block(&finalized_root) + .unwrap() + .expect("finalized block exists in store") + .slot(); + assert!( + finalized_slot > Slot::new(0), + "test precondition: finalized block must not be genesis", + ); + + let count = 8u64.min(finalized_slot.as_u64()); + rig.enqueue_blocks_by_head_request(finalized_root, count); + let actual = drain_blocks_by_head_response(&mut rig).await; + + let mut expected: Vec = Vec::new(); + let mut last: Option = None; + for offset in 0..count { + let slot = Slot::new(finalized_slot.as_u64() - offset); + if let Some(root) = rig + .chain + .block_root_at_slot(slot, WhenSlotSkipped::Prev) + .unwrap() + && Some(root) != last + { + expected.push(root); + last = Some(root); + } + } + + assert_eq!(actual, expected); + assert_eq!( + actual.first(), + Some(&finalized_root), + "first root must be the requested finalized root", + ); +} + +// `BlocksByHead` for a `beacon_root` we don't have. Spec says we MUST return an error +// (we map this to `ResourceUnavailable`). +#[tokio::test] +async fn test_blocks_by_head_unknown_root() { + let mut rig = TestRig::new(SLOTS_PER_EPOCH).await; + rig.enqueue_blocks_by_head_request(Hash256::repeat_byte(0xab), 4); + + match rig.network_rx.recv().await.expect("a network message") { + NetworkMessage::SendErrorResponse { error, .. } => { + assert_matches!( + error, + lighthouse_network::rpc::RpcErrorResponse::ResourceUnavailable + ); + } + other => panic!("expected SendErrorResponse, got {:?}", other), + } +} diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 443fa51cc6..a718997e0a 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -243,6 +243,13 @@ impl Router { request, ), ), + RequestType::BlocksByHead(request) => self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_blocks_by_head_request( + peer_id, + inbound_request_id, + request, + ), + ), RequestType::PayloadEnvelopesByRoot(request) => self .handle_beacon_processor_send_result( self.network_beacon_processor @@ -346,6 +353,11 @@ impl Router { Response::PayloadEnvelopesByRoot(_) | Response::PayloadEnvelopesByRange(_) => { debug!("Requesting envelopes by root and by range not supported yet"); } + // Lighthouse currently only serves BlocksByHead and does not issue it as a client, + // so receiving a response is unexpected. Drop it without crashing. + Response::BlocksByHead(_) => { + debug!("BlocksByHead response received but not requested by lighthouse"); + } // Light client responses should not be received Response::LightClientBootstrap(_) | Response::LightClientOptimisticUpdate(_) From 1b921a64e68e8b7878385041289d2cc4d1e5fb48 Mon Sep 17 00:00:00 2001 From: chonghe <44791194+chong-he@users.noreply.github.com> Date: Fri, 8 May 2026 16:12:38 +0800 Subject: [PATCH 06/18] Fix execution integration test CI failure (#9277) Co-Authored-By: Tan Chee Keong --- testing/execution_engine_integration/src/nethermind.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/execution_engine_integration/src/nethermind.rs b/testing/execution_engine_integration/src/nethermind.rs index 6a336161bd..e606f5558c 100644 --- a/testing/execution_engine_integration/src/nethermind.rs +++ b/testing/execution_engine_integration/src/nethermind.rs @@ -20,6 +20,7 @@ fn build_result(repo_dir: &Path) -> Output { .arg("src/Nethermind/Nethermind.sln") .arg("-c") .arg("Release") + .arg("-p:TreatWarningsAsErrors=false") .current_dir(repo_dir) .output() .expect("failed to make nethermind") From 2208e179376b81744fedf1ded59729701a5b2e8f Mon Sep 17 00:00:00 2001 From: Shane K Moore <41407272+shane-moore@users.noreply.github.com> Date: Mon, 11 May 2026 08:27:41 -0700 Subject: [PATCH 07/18] chore: remove builder_index from produce_block_v4 (#9267) Part of #8828 for the stateful path and helps align gloas `produceBlockV4` with beacon-APIs [PR](https://github.com/ethereum/beacon-APIs/pull/580) - Plumb `include_payload` query through the handler. Ignored for now since stateless mode isn't wired up yet - Add `execution_payload_included` metadata field + `Eth-Execution-Payload-Included` header per spec. Both `false` until stateless lands - Drop the `{builder_index}` segment from the envelope GET URL since no longer included in spec Co-Authored-By: shane-moore Co-Authored-By: Eitan Seri-Levi --- beacon_node/http_api/src/produce_block.rs | 22 +++++++++++--- .../validator/execution_payload_envelope.rs | 11 +------ beacon_node/http_api/src/version.rs | 16 +++++++++- beacon_node/http_api/tests/tests.rs | 13 ++++---- common/eth2/src/lib.rs | 30 ++++++++++++++----- common/eth2/src/types.rs | 10 ++++++- .../validator_services/src/block_service.rs | 10 +++---- 7 files changed, 76 insertions(+), 36 deletions(-) diff --git a/beacon_node/http_api/src/produce_block.rs b/beacon_node/http_api/src/produce_block.rs index 7173eb698f..ed1ecb9456 100644 --- a/beacon_node/http_api/src/produce_block.rs +++ b/beacon_node/http_api/src/produce_block.rs @@ -2,8 +2,9 @@ use crate::{ build_block_contents, version::{ ResponseIncludesVersion, add_consensus_block_value_header, add_consensus_version_header, - add_execution_payload_blinded_header, add_execution_payload_value_header, - add_ssz_content_type_header, beacon_response, inconsistent_fork_rejection, + add_execution_payload_blinded_header, add_execution_payload_included_header, + add_execution_payload_value_header, add_ssz_content_type_header, beacon_response, + inconsistent_fork_rejection, }, }; use beacon_chain::graffiti_calculator::GraffitiSettings; @@ -83,7 +84,16 @@ pub async fn produce_block_v4( warp_utils::reject::custom_bad_request(format!("failed to fetch a block: {:?}", e)) })?; - build_response_v4::(block, consensus_block_value, accept_header, &chain.spec) + // TODO(gloas): wire up for stateless mode (#8828). + let execution_payload_included = false; + + build_response_v4::( + block, + consensus_block_value, + execution_payload_included, + accept_header, + &chain.spec, + ) } #[instrument( @@ -133,6 +143,7 @@ pub async fn produce_block_v3( pub fn build_response_v4( block: BeaconBlock>, consensus_block_value: u64, + execution_payload_included: bool, accept_header: Option, spec: &ChainSpec, ) -> Result, warp::Rejection> { @@ -146,6 +157,7 @@ pub fn build_response_v4( let metadata = ProduceBlockV4Metadata { consensus_version: fork_name, consensus_block_value: consensus_block_value_wei, + execution_payload_included, }; match accept_header { @@ -155,6 +167,7 @@ pub fn build_response_v4( .map(|res: Response| add_ssz_content_type_header(res)) .map(|res: Response| add_consensus_version_header(res, fork_name)) .map(|res| add_consensus_block_value_header(res, consensus_block_value_wei)) + .map(|res| add_execution_payload_included_header(res, execution_payload_included)) .map_err(|e| -> warp::Rejection { warp_utils::reject::custom_server_error(format!("failed to create response: {}", e)) }), @@ -165,7 +178,8 @@ pub fn build_response_v4( }) .into_response()) .map(|res| add_consensus_version_header(res, fork_name)) - .map(|res| add_consensus_block_value_header(res, consensus_block_value_wei)), + .map(|res| add_consensus_block_value_header(res, consensus_block_value_wei)) + .map(|res| add_execution_payload_included_header(res, execution_payload_included)), } } diff --git a/beacon_node/http_api/src/validator/execution_payload_envelope.rs b/beacon_node/http_api/src/validator/execution_payload_envelope.rs index c40b375e49..7a7a430414 100644 --- a/beacon_node/http_api/src/validator/execution_payload_envelope.rs +++ b/beacon_node/http_api/src/validator/execution_payload_envelope.rs @@ -12,7 +12,7 @@ use types::Slot; use warp::http::Response; use warp::{Filter, Rejection}; -// GET validator/execution_payload_envelope/{slot}/{builder_index} +// GET validator/execution_payload_envelope/{slot} pub fn get_validator_execution_payload_envelope( eth_v1: EthV1Filter, chain_filter: ChainFilter, @@ -27,11 +27,6 @@ pub fn get_validator_execution_payload_envelope( "Invalid slot".to_string(), )) })) - .and(warp::path::param::().or_else(|_| async { - Err(warp_utils::reject::custom_bad_request( - "Invalid builder_index".to_string(), - )) - })) .and(warp::path::end()) .and(warp::header::optional::("accept")) .and(not_while_syncing_filter) @@ -39,10 +34,6 @@ pub fn get_validator_execution_payload_envelope( .and(chain_filter) .then( |slot: Slot, - // TODO(gloas) we're only doing local building - // we'll need to implement builder index logic - // eventually. - _builder_index: u64, accept_header: Option, not_synced_filter: Result<(), Rejection>, task_spawner: TaskSpawner, diff --git a/beacon_node/http_api/src/version.rs b/beacon_node/http_api/src/version.rs index 371064c886..bba1641416 100644 --- a/beacon_node/http_api/src/version.rs +++ b/beacon_node/http_api/src/version.rs @@ -5,7 +5,8 @@ use eth2::beacon_response::{ }; use eth2::{ CONSENSUS_BLOCK_VALUE_HEADER, CONSENSUS_VERSION_HEADER, CONTENT_TYPE_HEADER, - EXECUTION_PAYLOAD_BLINDED_HEADER, EXECUTION_PAYLOAD_VALUE_HEADER, SSZ_CONTENT_TYPE_HEADER, + EXECUTION_PAYLOAD_BLINDED_HEADER, EXECUTION_PAYLOAD_INCLUDED_HEADER, + EXECUTION_PAYLOAD_VALUE_HEADER, SSZ_CONTENT_TYPE_HEADER, }; use serde::Serialize; use types::{ForkName, InconsistentFork, Uint256}; @@ -88,6 +89,19 @@ pub fn add_execution_payload_blinded_header( .into_response() } +/// Add the `Eth-Execution-Payload-Included` header to a response. +pub fn add_execution_payload_included_header( + reply: T, + execution_payload_included: bool, +) -> Response { + reply::with_header( + reply, + EXECUTION_PAYLOAD_INCLUDED_HEADER, + execution_payload_included.to_string(), + ) + .into_response() +} + /// Add the `Eth-Execution-Payload-Value` header to a response. pub fn add_execution_payload_value_header( reply: T, diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 0d6735ff61..a7fe34593a 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -4288,6 +4288,7 @@ impl ApiTester { ); // TODO(gloas): check why consensus block value is 0 // assert!(!metadata.consensus_block_value.is_zero()); + assert!(!metadata.execution_payload_included); let block_root = block.tree_hash_root(); let envelope = self @@ -4360,7 +4361,7 @@ impl ApiTester { let (response, metadata) = self .client - .get_validator_blocks_v4::(slot, &randao_reveal, None, None, None) + .get_validator_blocks_v4::(slot, &randao_reveal, None, None, None, None) .await .unwrap(); let block = response.data; @@ -4369,7 +4370,7 @@ impl ApiTester { let envelope = self .client - .get_validator_execution_payload_envelope::(slot, BUILDER_INDEX_SELF_BUILD) + .get_validator_execution_payload_envelope::(slot) .await .unwrap() .data; @@ -4423,7 +4424,7 @@ impl ApiTester { let (block, metadata) = self .client - .get_validator_blocks_v4_ssz::(slot, &randao_reveal, None, None, None) + .get_validator_blocks_v4_ssz::(slot, &randao_reveal, None, None, None, None) .await .unwrap(); @@ -4431,7 +4432,7 @@ impl ApiTester { let envelope = self .client - .get_validator_execution_payload_envelope_ssz::(slot, BUILDER_INDEX_SELF_BUILD) + .get_validator_execution_payload_envelope_ssz::(slot) .await .unwrap(); @@ -4861,7 +4862,7 @@ impl ApiTester { // Produce and publish a block. let (response, _metadata) = self .client - .get_validator_blocks_v4::(slot, &randao_reveal, None, None, None) + .get_validator_blocks_v4::(slot, &randao_reveal, None, None, None, None) .await .unwrap(); let block = response.data; @@ -4878,7 +4879,7 @@ impl ApiTester { // Retrieve and publish the envelope. let envelope = self .client - .get_validator_execution_payload_envelope::(slot, BUILDER_INDEX_SELF_BUILD) + .get_validator_execution_payload_envelope::(slot) .await .unwrap() .data; diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index becbe550a6..e9fb44209b 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -56,6 +56,7 @@ pub const V4: EndpointVersion = EndpointVersion(4); pub const CONSENSUS_VERSION_HEADER: &str = "Eth-Consensus-Version"; pub const EXECUTION_PAYLOAD_BLINDED_HEADER: &str = "Eth-Execution-Payload-Blinded"; pub const EXECUTION_PAYLOAD_VALUE_HEADER: &str = "Eth-Execution-Payload-Value"; +pub const EXECUTION_PAYLOAD_INCLUDED_HEADER: &str = "Eth-Execution-Payload-Included"; pub const CONSENSUS_BLOCK_VALUE_HEADER: &str = "Eth-Consensus-Block-Value"; pub const CONTENT_TYPE_HEADER: &str = "Content-Type"; @@ -2554,12 +2555,14 @@ impl BeaconNodeHttpClient { } /// returns `GET v4/validator/blocks/{slot}` URL path + #[allow(clippy::too_many_arguments)] pub async fn get_validator_blocks_v4_path( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, + include_payload: Option, builder_booster_factor: Option, graffiti_policy: Option, ) -> Result { @@ -2584,6 +2587,11 @@ impl BeaconNodeHttpClient { .append_pair("skip_randao_verification", ""); } + if let Some(include_payload) = include_payload { + path.query_pairs_mut() + .append_pair("include_payload", &include_payload.to_string()); + } + if let Some(builder_booster_factor) = builder_booster_factor { path.query_pairs_mut() .append_pair("builder_boost_factor", &builder_booster_factor.to_string()); @@ -2603,6 +2611,7 @@ impl BeaconNodeHttpClient { slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, + include_payload: Option, builder_booster_factor: Option, graffiti_policy: Option, ) -> Result< @@ -2617,6 +2626,7 @@ impl BeaconNodeHttpClient { randao_reveal, graffiti, SkipRandaoVerification::No, + include_payload, builder_booster_factor, graffiti_policy, ) @@ -2624,12 +2634,14 @@ impl BeaconNodeHttpClient { } /// `GET v4/validator/blocks/{slot}` + #[allow(clippy::too_many_arguments)] pub async fn get_validator_blocks_v4_modular( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, + include_payload: Option, builder_booster_factor: Option, graffiti_policy: Option, ) -> Result< @@ -2645,6 +2657,7 @@ impl BeaconNodeHttpClient { randao_reveal, graffiti, skip_randao_verification, + include_payload, builder_booster_factor, graffiti_policy, ) @@ -2675,6 +2688,7 @@ impl BeaconNodeHttpClient { slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, + include_payload: Option, builder_booster_factor: Option, graffiti_policy: Option, ) -> Result<(BeaconBlock, ProduceBlockV4Metadata), Error> { @@ -2683,6 +2697,7 @@ impl BeaconNodeHttpClient { randao_reveal, graffiti, SkipRandaoVerification::No, + include_payload, builder_booster_factor, graffiti_policy, ) @@ -2690,12 +2705,14 @@ impl BeaconNodeHttpClient { } /// `GET v4/validator/blocks/{slot}` in ssz format + #[allow(clippy::too_many_arguments)] pub async fn get_validator_blocks_v4_modular_ssz( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, + include_payload: Option, builder_booster_factor: Option, graffiti_policy: Option, ) -> Result<(BeaconBlock, ProduceBlockV4Metadata), Error> { @@ -2705,6 +2722,7 @@ impl BeaconNodeHttpClient { randao_reveal, graffiti, skip_randao_verification, + include_payload, builder_booster_factor, graffiti_policy, ) @@ -2734,11 +2752,10 @@ impl BeaconNodeHttpClient { opt_response.ok_or(Error::StatusCode(StatusCode::NOT_FOUND)) } - /// `GET v1/validator/execution_payload_envelope/{slot}/{builder_index}` + /// `GET v1/validator/execution_payload_envelope/{slot}` pub async fn get_validator_execution_payload_envelope( &self, slot: Slot, - builder_index: u64, ) -> Result>, Error> { let mut path = self.eth_path(V1)?; @@ -2746,17 +2763,15 @@ impl BeaconNodeHttpClient { .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("validator") .push("execution_payload_envelope") - .push(&slot.to_string()) - .push(&builder_index.to_string()); + .push(&slot.to_string()); self.get(path).await } - /// `GET v1/validator/execution_payload_envelope/{slot}/{builder_index}` in SSZ format + /// `GET v1/validator/execution_payload_envelope/{slot}` in SSZ format pub async fn get_validator_execution_payload_envelope_ssz( &self, slot: Slot, - builder_index: u64, ) -> Result, Error> { let mut path = self.eth_path(V1)?; @@ -2764,8 +2779,7 @@ impl BeaconNodeHttpClient { .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("validator") .push("execution_payload_envelope") - .push(&slot.to_string()) - .push(&builder_index.to_string()); + .push(&slot.to_string()); let opt_response = self .get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.get_validator_block) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index dfa0fbd87d..449ea88685 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -5,7 +5,7 @@ pub use types::*; use crate::{ CONSENSUS_BLOCK_VALUE_HEADER, CONSENSUS_VERSION_HEADER, EXECUTION_PAYLOAD_BLINDED_HEADER, - EXECUTION_PAYLOAD_VALUE_HEADER, Error as ServerError, + EXECUTION_PAYLOAD_INCLUDED_HEADER, EXECUTION_PAYLOAD_VALUE_HEADER, Error as ServerError, }; use bls::{PublicKeyBytes, SecretKey, Signature, SignatureBytes}; use context_deserialize::ContextDeserialize; @@ -778,6 +778,7 @@ pub struct ValidatorBlocksQuery { pub randao_reveal: SignatureBytes, pub graffiti: Option, pub skip_randao_verification: SkipRandaoVerification, + pub include_payload: Option, pub builder_boost_factor: Option, pub graffiti_policy: Option, } @@ -1848,6 +1849,7 @@ pub struct ProduceBlockV4Metadata { pub consensus_version: ForkName, #[serde(with = "serde_utils::u256_dec")] pub consensus_block_value: Uint256, + pub execution_payload_included: bool, } impl FullBlockContents { @@ -2021,10 +2023,16 @@ impl TryFrom<&HeaderMap> for ProduceBlockV4Metadata { Uint256::from_str_radix(s, 10) .map_err(|e| format!("invalid {CONSENSUS_BLOCK_VALUE_HEADER}: {e:?}")) })?; + let execution_payload_included = + parse_required_header(headers, EXECUTION_PAYLOAD_INCLUDED_HEADER, |s| { + s.parse::() + .map_err(|e| format!("invalid {EXECUTION_PAYLOAD_INCLUDED_HEADER}: {e:?}")) + })?; Ok(ProduceBlockV4Metadata { consensus_version, consensus_block_value, + execution_payload_included, }) } } diff --git a/validator_client/validator_services/src/block_service.rs b/validator_client/validator_services/src/block_service.rs index 99e53b0100..1dd1878f4c 100644 --- a/validator_client/validator_services/src/block_service.rs +++ b/validator_client/validator_services/src/block_service.rs @@ -14,7 +14,6 @@ use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc; use tracing::{Instrument, debug, error, info, info_span, instrument, trace, warn}; -use types::consts::gloas::BUILDER_INDEX_SELF_BUILD; use types::{BlockType, ChainSpec, EthSpec, Graffiti, Slot}; use validator_store::{Error as ValidatorStoreError, SignedBlock, UnsignedBlock, ValidatorStore}; @@ -479,6 +478,7 @@ impl BlockService { slot, randao_reveal_ref, graffiti.as_ref(), + None, builder_boost_factor, self_ref.graffiti_policy, ) @@ -506,6 +506,7 @@ impl BlockService { slot, randao_reveal_ref, graffiti.as_ref(), + None, builder_boost_factor, self_ref.graffiti_policy, ) @@ -652,16 +653,13 @@ impl BlockService { ) -> Result<(), BlockError> { info!(slot = slot.as_u64(), "Fetching execution payload envelope"); - // Fetch the envelope from the beacon node. Use builder_index=BUILDER_INDEX_SELF_BUILD for local building. + // Fetch the envelope from the beacon node. // TODO(gloas): Use proposer_fallback once multi-BN is supported. let envelope = self .beacon_nodes .first_success(|beacon_node| async move { beacon_node - .get_validator_execution_payload_envelope_ssz::( - slot, - BUILDER_INDEX_SELF_BUILD, - ) + .get_validator_execution_payload_envelope_ssz::(slot) .await .map_err(|e| { BlockError::Recoverable(format!( From f968c7e5bbc9e55931a3fc1d1dc62138dca50622 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Tue, 12 May 2026 04:59:54 +0300 Subject: [PATCH 08/18] Dont penalize payload envelope peers after gossip verification (#9283) We got in a little bit of trouble in devnet-3. After gossip verifying an envelope and before importing it, we got the following error ``` May 07 08:04:24.383 WARN Execution payload envelope rejected reason: "EnvelopeProcessingError(WithdrawalsRootMismatch { state: 0x852d38aaecc9f4e2e309919f74020c7bbcf050fea4a20edf3304f171e44ee9d5, payload: ``` The envelope had already passed gossip verification checks and was correctly propagated to the network, we should not penalize peers for doing this. This caused our node to isolate itself from the rest of the network. This PR removes peer penalties for any envelope that passes gossip validation Co-Authored-By: Eitan Seri-Levi --- .../gossip_methods.rs | 45 +++---------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 0135d7f5dd..57871a2009 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -3977,44 +3977,13 @@ impl NetworkBeaconProcessor { // TODO(gloas) metrics // register_process_result_metrics(&result, metrics::BlockSource::Gossip, "envelope"); - match &result { - Ok(AvailabilityProcessingStatus::Imported(_)) - | Ok(AvailabilityProcessingStatus::MissingComponents(_, _)) => { - // Nothing to do - } - Err(e) => match e { - EnvelopeError::ExecutionPayloadError(epe) if !epe.penalize_peer() => {} - EnvelopeError::BadSignature - | EnvelopeError::BuilderIndexMismatch { .. } - | EnvelopeError::SlotMismatch { .. } - | EnvelopeError::BlockHashMismatch { .. } - | EnvelopeError::UnknownValidator { .. } - | EnvelopeError::IncorrectBlockProposer { .. } - | EnvelopeError::ExecutionPayloadError(_) => { - self.gossip_penalize_peer( - peer_id, - PeerAction::LowToleranceError, - "gossip_envelope_processing_low", - ); - } - - EnvelopeError::EnvelopeProcessingError(_) - | EnvelopeError::BlockError(_) - | EnvelopeError::BlockRootUnknown { .. } => { - self.gossip_penalize_peer( - peer_id, - PeerAction::LowToleranceError, - "gossip_envelope_processing_error", - ); - } - - EnvelopeError::PriorToFinalization { .. } - | EnvelopeError::OptimisticSyncNotSupported { .. } - | EnvelopeError::BeaconChainError(_) - | EnvelopeError::BeaconStateError(_) - | EnvelopeError::BlockProcessingError(_) - | EnvelopeError::InternalError(_) => {} - }, + if let Err(e) = &result { + debug!( + ?beacon_block_root, + %peer_id, + error = ?e, + "Execution payload envelope processing failed" + ); } } From 757873200ba78d7c84eea6f77963c5e4ca8b12e3 Mon Sep 17 00:00:00 2001 From: hopinheimer <48147533+hopinheimer@users.noreply.github.com> Date: Tue, 12 May 2026 02:24:18 -0400 Subject: [PATCH 09/18] Fix stale `beacon_state_root` in test helpers (#9289) Test helpers `add_attested_block_at_slot` and `add_attested_blocks_at_slot` accepted `state_root` argument which was computed before applying the block. Co-Authored-By: hopinheimer --- .../beacon_chain/src/shuffling_cache.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 57 +++--- beacon_node/beacon_chain/tests/rewards.rs | 4 +- beacon_node/beacon_chain/tests/store_tests.rs | 186 ++++++------------ .../tests/sync_committee_verification.rs | 6 +- .../beacon_chain/tests/validator_monitor.rs | 5 +- beacon_node/http_api/tests/fork_tests.rs | 28 +-- beacon_node/operation_pool/src/lib.rs | 46 ++--- .../src/per_block_processing/tests.rs | 2 - .../src/per_epoch_processing/tests.rs | 2 - consensus/types/tests/committee_cache.rs | 1 - consensus/types/tests/state.rs | 1 - testing/state_transition_vectors/src/main.rs | 4 +- 13 files changed, 118 insertions(+), 226 deletions(-) diff --git a/beacon_node/beacon_chain/src/shuffling_cache.rs b/beacon_node/beacon_chain/src/shuffling_cache.rs index 3d0fd80cf6..0377b553e3 100644 --- a/beacon_node/beacon_chain/src/shuffling_cache.rs +++ b/beacon_node/beacon_chain/src/shuffling_cache.rs @@ -325,7 +325,7 @@ mod test { .deterministic_keypairs(8) .fresh_ephemeral_store() .build(); - let (mut state, _) = harness.get_current_state_and_root(); + let mut state = harness.get_current_state(); state .build_committee_cache(RelativeEpoch::Current, &harness.chain.spec) .unwrap(); diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index ca55811a70..4378da8405 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1570,6 +1570,15 @@ where mut state: Cow>, state_root: Hash256, ) -> Result, BeaconChainError> { + assert_eq!( + state.get_latest_block_root(state_root), + beacon_block_root, + "State must match beacon block root, state slot {:?} attestation slot {:?} state root {:?}", + state.latest_block_header().slot, + slot, + state_root, + ); + let epoch = slot.epoch(E::slots_per_epoch()); if state.slot() > slot { @@ -1594,6 +1603,13 @@ where *state.get_block_root(target_slot)? }; + let payload_present = state.fork_name_unchecked().gloas_enabled() + && state.latest_block_header().slot != slot + && self + .chain + .canonical_head + .block_has_canonical_payload(&beacon_block_root, &self.spec)?; + Ok(Attestation::empty_for_signing( index, committee_len, @@ -1604,7 +1620,7 @@ where epoch, root: target_root, }, - false, + payload_present, &self.spec, )?) } @@ -3111,13 +3127,11 @@ where &self, slot: Slot, state: BeaconState, - state_root: Hash256, validators: &[usize], ) -> Result<(SignedBeaconBlockHash, BeaconState), BlockError> { self.add_attested_block_at_slot_with_sync( slot, state, - state_root, validators, SyncCommitteeStrategy::NoValidators, ) @@ -3128,18 +3142,18 @@ where &self, slot: Slot, state: BeaconState, - state_root: Hash256, validators: &[usize], sync_committee_strategy: SyncCommitteeStrategy, ) -> Result<(SignedBeaconBlockHash, BeaconState), BlockError> { - let (block_hash, block, state) = self.add_block_at_slot(slot, state).await?; - self.attest_block(&state, state_root, block_hash, &block.0, validators); + let (block_hash, block, mut new_state) = self.add_block_at_slot(slot, state).await?; + let new_state_root = new_state.canonical_root().unwrap(); + self.attest_block(&new_state, new_state_root, block_hash, &block.0, validators); if sync_committee_strategy == SyncCommitteeStrategy::AllValidators - && state.current_sync_committee().is_ok() + && new_state.current_sync_committee().is_ok() { self.sync_committee_sign_block( - &state, + &new_state, block_hash.into(), slot, if (slot + 1).epoch(E::slots_per_epoch()) @@ -3153,19 +3167,17 @@ where ); } - Ok((block_hash, state)) + Ok((block_hash, new_state)) } pub async fn add_attested_blocks_at_slots( &self, state: BeaconState, - state_root: Hash256, slots: &[Slot], validators: &[usize], ) -> AddBlocksResult { self.add_attested_blocks_at_slots_with_sync( state, - state_root, slots, validators, SyncCommitteeStrategy::NoValidators, @@ -3176,7 +3188,6 @@ where pub async fn add_attested_blocks_at_slots_with_sync( &self, state: BeaconState, - state_root: Hash256, slots: &[Slot], validators: &[usize], sync_committee_strategy: SyncCommitteeStrategy, @@ -3184,7 +3195,6 @@ where assert!(!slots.is_empty()); self.add_attested_blocks_at_slots_given_lbh( state, - state_root, slots, validators, None, @@ -3241,7 +3251,6 @@ where pub async fn add_attested_blocks_at_slots_with_lc_data( &self, mut state: BeaconState, - state_root: Hash256, slots: &[Slot], validators: &[usize], mut latest_block_hash: Option, @@ -3255,7 +3264,6 @@ where .add_attested_block_at_slot_with_sync( *slot, state, - state_root, validators, sync_committee_strategy, ) @@ -3281,7 +3289,6 @@ where async fn add_attested_blocks_at_slots_given_lbh( &self, mut state: BeaconState, - state_root: Hash256, slots: &[Slot], validators: &[usize], mut latest_block_hash: Option, @@ -3295,7 +3302,6 @@ where let (block_hash, new_state) = Box::pin(self.add_attested_block_at_slot_with_sync( *slot, state, - state_root, validators, sync_committee_strategy, )) @@ -3359,14 +3365,8 @@ where for epoch in min_epoch.as_u64()..=max_epoch.as_u64() { let mut new_chains = vec![]; - for ( - mut head_state, - slots, - validators, - mut block_hashes, - mut state_hashes, - head_block, - ) in chains + for (head_state, slots, validators, mut block_hashes, mut state_hashes, head_block) in + chains { let epoch_slots = slots .iter() @@ -3374,11 +3374,9 @@ where .copied() .collect::>(); - let head_state_root = head_state.update_tree_hash_cache().unwrap(); let (new_block_hashes, new_state_hashes, new_head_block, new_head_state) = self .add_attested_blocks_at_slots_given_lbh( head_state, - head_state_root, &epoch_slots, &validators, Some(head_block), @@ -3540,7 +3538,7 @@ where sync_committee_strategy: SyncCommitteeStrategy, light_client_strategy: LightClientStrategy, ) -> Hash256 { - let (mut state, slots) = match block_strategy { + let (state, slots) = match block_strategy { BlockStrategy::OnCanonicalHead => { let current_slot: u64 = self.get_current_slot().into(); let slots: Vec = (current_slot..(current_slot + (num_blocks as u64))) @@ -3569,12 +3567,10 @@ where AttestationStrategy::SomeValidators(vals) => vals, }; - let state_root = state.update_tree_hash_cache().unwrap(); let (_, _, last_produced_block_hash, _) = match light_client_strategy { LightClientStrategy::Enabled => { self.add_attested_blocks_at_slots_with_lc_data( state, - state_root, &slots, &validators, None, @@ -3585,7 +3581,6 @@ where LightClientStrategy::Disabled => { self.add_attested_blocks_at_slots_with_sync( state, - state_root, &slots, &validators, sync_committee_strategy, diff --git a/beacon_node/beacon_chain/tests/rewards.rs b/beacon_node/beacon_chain/tests/rewards.rs index bc7c98041f..0c8815995e 100644 --- a/beacon_node/beacon_chain/tests/rewards.rs +++ b/beacon_node/beacon_chain/tests/rewards.rs @@ -105,11 +105,11 @@ async fn test_sync_committee_rewards() { // Add block let chain = &harness.chain; - let (head_state, head_state_root) = harness.get_current_state_and_root(); + let head_state = harness.get_current_state(); let target_slot = harness.get_current_slot() + 1; let (block_root, mut state) = harness - .add_attested_block_at_slot(target_slot, head_state, head_state_root, &[]) + .add_attested_block_at_slot(target_slot, head_state, &[]) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 1576092c81..cfdd54857a 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -198,11 +198,10 @@ async fn light_client_bootstrap_test() { let num_initial_slots = E::slots_per_epoch() * 7; let slots: Vec = (1..num_initial_slots).map(Slot::new).collect(); - let (genesis_state, genesis_state_root) = harness.get_current_state_and_root(); + let genesis_state = harness.get_current_state(); harness .add_attested_blocks_at_slots_with_lc_data( genesis_state.clone(), - genesis_state_root, &slots, &all_validators, None, @@ -258,14 +257,9 @@ async fn light_client_updates_test() { let num_initial_slots = E::slots_per_epoch() * 10; let slots: Vec = (1..num_initial_slots).map(Slot::new).collect(); - let (genesis_state, genesis_state_root) = harness.get_current_state_and_root(); + let genesis_state = harness.get_current_state(); harness - .add_attested_blocks_at_slots( - genesis_state.clone(), - genesis_state_root, - &slots, - &all_validators, - ) + .add_attested_blocks_at_slots(genesis_state.clone(), &slots, &all_validators) .await; harness.advance_slot(); @@ -639,7 +633,7 @@ async fn forwards_iter_block_and_state_roots_until() { for slot in (1..=num_blocks_produced).map(Slot::from) { let (block_root, mut state) = harness - .add_attested_block_at_slot(slot, head_state, head_state_root, all_validators) + .add_attested_block_at_slot(slot, head_state, all_validators) .await .unwrap(); head_state_root = state.update_tree_hash_cache().unwrap(); @@ -714,10 +708,10 @@ async fn block_replayer_hooks() { let max_slot = *block_slots.last().unwrap(); let all_slots = (0..=max_slot.as_u64()).map(Slot::new).collect::>(); - let (state, state_root) = harness.get_current_state_and_root(); + let state = harness.get_current_state(); let all_validators = harness.get_all_validators(); let (_, _, end_block_root, mut end_state) = harness - .add_attested_blocks_at_slots(state.clone(), state_root, &block_slots, &all_validators) + .add_attested_blocks_at_slots(state.clone(), &block_slots, &all_validators) .await; let blocks = store @@ -786,10 +780,10 @@ async fn delete_blocks_and_states() { // Finalize an initial portion of the chain. let initial_slots: Vec = (1..=unforked_blocks).map(Into::into).collect(); - let (state, state_root) = harness.get_current_state_and_root(); + let state = harness.get_current_state(); let all_validators = harness.get_all_validators(); harness - .add_attested_blocks_at_slots(state, state_root, &initial_slots, &all_validators) + .add_attested_blocks_at_slots(state, &initial_slots, &all_validators) .await; // Create a fork post-finalization. @@ -924,10 +918,10 @@ async fn multi_epoch_fork_valid_blocks_test( // Create the initial portion of the chain if initial_blocks > 0 { let initial_slots: Vec = (1..=initial_blocks).map(Into::into).collect(); - let (state, state_root) = harness.get_current_state_and_root(); + let state = harness.get_current_state(); let all_validators = harness.get_all_validators(); harness - .add_attested_blocks_at_slots(state, state_root, &initial_slots, &all_validators) + .add_attested_blocks_at_slots(state, &initial_slots, &all_validators) .await; } @@ -1269,17 +1263,17 @@ async fn proposer_shuffling_root_consistency_test( // Build chain out to parent block. let initial_slots: Vec = (1..=parent_slot).map(Into::into).collect(); - let (state, state_root) = harness.get_current_state_and_root(); + let state = harness.get_current_state(); let all_validators = harness.get_all_validators(); let (_, _, parent_root, _) = harness - .add_attested_blocks_at_slots(state, state_root, &initial_slots, &all_validators) + .add_attested_blocks_at_slots(state, &initial_slots, &all_validators) .await; // Add the child block. - let (state, state_root) = harness.get_current_state_and_root(); + let state = harness.get_current_state(); let all_validators = harness.get_all_validators(); let (_, _, child_root, child_block_state) = harness - .add_attested_blocks_at_slots(state, state_root, &[child_slot], &all_validators) + .add_attested_blocks_at_slots(state, &[child_slot], &all_validators) .await; let child_block_epoch = child_slot.epoch(E::slots_per_epoch()); @@ -1591,10 +1585,10 @@ async fn proposer_duties_from_head_fulu() { // Build chain out to parent block. let initial_slots: Vec = (1..=initial_blocks).map(Into::into).collect(); - let (state, state_root) = harness.get_current_state_and_root(); + let state = harness.get_current_state(); let all_validators = harness.get_all_validators(); let (_, _, head_block_root, head_state) = harness - .add_attested_blocks_at_slots(state, state_root, &initial_slots, &all_validators) + .add_attested_blocks_at_slots(state, &initial_slots, &all_validators) .await; // Compute the proposer duties at the next epoch from the head @@ -1642,10 +1636,10 @@ async fn proposer_lookahead_gloas_fork_epoch() { // Build chain out to parent block. let initial_slots: Vec = (1..=initial_blocks).map(Into::into).collect(); - let (state, state_root) = harness.get_current_state_and_root(); + let state = harness.get_current_state(); let all_validators = harness.get_all_validators(); let (_, _, head_block_root, mut head_state) = harness - .add_attested_blocks_at_slots(state, state_root, &initial_slots, &all_validators) + .add_attested_blocks_at_slots(state, &initial_slots, &all_validators) .await; let head_state_root = head_state.canonical_root().unwrap(); @@ -1681,7 +1675,7 @@ async fn proposer_lookahead_gloas_fork_epoch() { // Build a block in the Gloas fork epoch and assert that the shuffling does not change. let gloas_slots = vec![gloas_fork_epoch.start_slot(E::slots_per_epoch())]; let (_, _, _, _) = harness - .add_attested_blocks_at_slots(head_state, head_state_root, &gloas_slots, &all_validators) + .add_attested_blocks_at_slots(head_state, &gloas_slots, &all_validators) .await; let (no_lookahead_indices, no_lookahead_dependent_root, _, _, no_lookahead_fork) = @@ -1704,16 +1698,11 @@ async fn prunes_abandoned_fork_between_two_finalized_checkpoints() { let store = get_store(&db_path); let rig = get_harness(store.clone(), VALIDATOR_COUNT); let slots_per_epoch = rig.slots_per_epoch(); - let (mut state, state_root) = rig.get_current_state_and_root(); + let mut state = rig.get_current_state(); let canonical_chain_slots: Vec = (1..=rig.epoch_start_slot(1)).map(Slot::new).collect(); let (canonical_chain_blocks_pre_finalization, _, _, new_state) = rig - .add_attested_blocks_at_slots( - state, - state_root, - &canonical_chain_slots, - &honest_validators, - ) + .add_attested_blocks_at_slots(state, &canonical_chain_slots, &honest_validators) .await; state = new_state; let canonical_chain_slot: u64 = rig.get_current_slot().into(); @@ -1721,14 +1710,9 @@ async fn prunes_abandoned_fork_between_two_finalized_checkpoints() { let stray_slots: Vec = (canonical_chain_slot + 1..rig.epoch_start_slot(2)) .map(Slot::new) .collect(); - let (current_state, current_state_root) = rig.get_current_state_and_root(); + let current_state = rig.get_current_state(); let (stray_blocks, stray_states, stray_head, _) = rig - .add_attested_blocks_at_slots( - current_state, - current_state_root, - &stray_slots, - &adversarial_validators, - ) + .add_attested_blocks_at_slots(current_state, &stray_slots, &adversarial_validators) .await; // Precondition: Ensure all stray_blocks blocks are still known @@ -1758,9 +1742,8 @@ async fn prunes_abandoned_fork_between_two_finalized_checkpoints() { ..=(canonical_chain_slot + slots_per_epoch * 5)) .map(Slot::new) .collect(); - let state_root = state.update_tree_hash_cache().unwrap(); let (canonical_chain_blocks_post_finalization, _, _, _) = rig - .add_attested_blocks_at_slots(state, state_root, &finalization_slots, &honest_validators) + .add_attested_blocks_at_slots(state, &finalization_slots, &honest_validators) .await; // Postcondition: New blocks got finalized @@ -1815,15 +1798,14 @@ async fn pruning_does_not_touch_abandoned_block_shared_with_canonical_chain() { let store = get_store(&db_path); let rig = get_harness(store.clone(), VALIDATOR_COUNT); let slots_per_epoch = rig.slots_per_epoch(); - let (state, state_root) = rig.get_current_state_and_root(); + let state = rig.get_current_state(); // Fill up 0th epoch let canonical_chain_slots_zeroth_epoch: Vec = (1..rig.epoch_start_slot(1)).map(Slot::new).collect(); - let (_, _, _, mut state) = rig + let (_, _, _, state) = rig .add_attested_blocks_at_slots( state, - state_root, &canonical_chain_slots_zeroth_epoch, &honest_validators, ) @@ -1834,11 +1816,9 @@ async fn pruning_does_not_touch_abandoned_block_shared_with_canonical_chain() { ..=rig.epoch_start_slot(1) + 1) .map(Slot::new) .collect(); - let state_root = state.update_tree_hash_cache().unwrap(); - let (canonical_chain_blocks_first_epoch, _, shared_head, mut state) = rig + let (canonical_chain_blocks_first_epoch, _, shared_head, state) = rig .add_attested_blocks_at_slots( state.clone(), - state_root, &canonical_chain_slots_first_epoch, &honest_validators, ) @@ -1849,11 +1829,9 @@ async fn pruning_does_not_touch_abandoned_block_shared_with_canonical_chain() { ..=rig.epoch_start_slot(1) + 2) .map(Slot::new) .collect(); - let state_root = state.update_tree_hash_cache().unwrap(); let (stray_blocks, stray_states, stray_head, _) = rig .add_attested_blocks_at_slots( state.clone(), - state_root, &stray_chain_slots_first_epoch, &adversarial_validators, ) @@ -1890,9 +1868,8 @@ async fn pruning_does_not_touch_abandoned_block_shared_with_canonical_chain() { ..=(canonical_chain_slot + slots_per_epoch * 5)) .map(Slot::new) .collect(); - let state_root = state.update_tree_hash_cache().unwrap(); let (canonical_chain_blocks, _, _, _) = rig - .add_attested_blocks_at_slots(state, state_root, &finalization_slots, &honest_validators) + .add_attested_blocks_at_slots(state, &finalization_slots, &honest_validators) .await; // Postconditions @@ -1945,12 +1922,12 @@ async fn pruning_does_not_touch_blocks_prior_to_finalization() { let store = get_store(&db_path); let rig = get_harness(store.clone(), VALIDATOR_COUNT); let slots_per_epoch = rig.slots_per_epoch(); - let (mut state, state_root) = rig.get_current_state_and_root(); + let mut state = rig.get_current_state(); // Fill up 0th epoch with canonical chain blocks let zeroth_epoch_slots: Vec = (1..=rig.epoch_start_slot(1)).map(Slot::new).collect(); let (canonical_chain_blocks, _, _, new_state) = rig - .add_attested_blocks_at_slots(state, state_root, &zeroth_epoch_slots, &honest_validators) + .add_attested_blocks_at_slots(state, &zeroth_epoch_slots, &honest_validators) .await; state = new_state; let canonical_chain_slot: u64 = rig.get_current_slot().into(); @@ -1959,14 +1936,8 @@ async fn pruning_does_not_touch_blocks_prior_to_finalization() { let first_epoch_slots: Vec = ((rig.epoch_start_slot(1) + 1)..(rig.epoch_start_slot(2))) .map(Slot::new) .collect(); - let state_root = state.update_tree_hash_cache().unwrap(); let (stray_blocks, stray_states, stray_head, _) = rig - .add_attested_blocks_at_slots( - state.clone(), - state_root, - &first_epoch_slots, - &adversarial_validators, - ) + .add_attested_blocks_at_slots(state.clone(), &first_epoch_slots, &adversarial_validators) .await; // Preconditions @@ -1994,9 +1965,8 @@ async fn pruning_does_not_touch_blocks_prior_to_finalization() { ..=(canonical_chain_slot + slots_per_epoch * 4)) .map(Slot::new) .collect(); - let state_root = state.update_tree_hash_cache().unwrap(); let (_, _, _, _) = rig - .add_attested_blocks_at_slots(state, state_root, &slots, &honest_validators) + .add_attested_blocks_at_slots(state, &slots, &honest_validators) .await; // Postconditions @@ -2037,29 +2007,23 @@ async fn prunes_fork_growing_past_youngest_finalized_checkpoint() { let db_path = tempdir().unwrap(); let store = get_store(&db_path); let rig = get_harness(store.clone(), VALIDATOR_COUNT); - let (state, state_root) = rig.get_current_state_and_root(); + let state = rig.get_current_state(); // Fill up 0th epoch with canonical chain blocks let zeroth_epoch_slots: Vec = (1..=rig.epoch_start_slot(1)).map(Slot::new).collect(); - let (canonical_blocks_zeroth_epoch, _, _, mut state) = rig - .add_attested_blocks_at_slots(state, state_root, &zeroth_epoch_slots, &honest_validators) + let (canonical_blocks_zeroth_epoch, _, _, state) = rig + .add_attested_blocks_at_slots(state, &zeroth_epoch_slots, &honest_validators) .await; // Fill up 1st epoch. Contains a fork. let slots_first_epoch: Vec = (rig.epoch_start_slot(1) + 1..rig.epoch_start_slot(2)) .map(Into::into) .collect(); - let state_root = state.update_tree_hash_cache().unwrap(); - let (stray_blocks_first_epoch, stray_states_first_epoch, _, mut stray_state) = rig - .add_attested_blocks_at_slots( - state.clone(), - state_root, - &slots_first_epoch, - &adversarial_validators, - ) + let (stray_blocks_first_epoch, stray_states_first_epoch, _, stray_state) = rig + .add_attested_blocks_at_slots(state.clone(), &slots_first_epoch, &adversarial_validators) .await; - let (canonical_blocks_first_epoch, _, _, mut canonical_state) = rig - .add_attested_blocks_at_slots(state, state_root, &slots_first_epoch, &honest_validators) + let (canonical_blocks_first_epoch, _, _, canonical_state) = rig + .add_attested_blocks_at_slots(state, &slots_first_epoch, &honest_validators) .await; // Fill up 2nd epoch. Extends both the canonical chain and the fork. @@ -2067,11 +2031,9 @@ async fn prunes_fork_growing_past_youngest_finalized_checkpoint() { ..=rig.epoch_start_slot(2) + 1) .map(Into::into) .collect(); - let stray_state_root = stray_state.update_tree_hash_cache().unwrap(); let (stray_blocks_second_epoch, stray_states_second_epoch, stray_head, _) = rig .add_attested_blocks_at_slots( stray_state, - stray_state_root, &stray_slots_second_epoch, &adversarial_validators, ) @@ -2114,10 +2076,8 @@ async fn prunes_fork_growing_past_youngest_finalized_checkpoint() { let canonical_slots: Vec = (rig.epoch_start_slot(2)..=rig.epoch_start_slot(6)) .map(Into::into) .collect(); - let canonical_state_root = canonical_state.update_tree_hash_cache().unwrap(); let (canonical_blocks, _, _, _) = Box::pin(rig.add_attested_blocks_at_slots( canonical_state, - canonical_state_root, &canonical_slots, &honest_validators, )) @@ -2179,14 +2139,13 @@ async fn prunes_skipped_slots_states() { let db_path = tempdir().unwrap(); let store = get_store(&db_path); let rig = get_harness(store.clone(), VALIDATOR_COUNT); - let (state, state_root) = rig.get_current_state_and_root(); + let state = rig.get_current_state(); let canonical_slots_zeroth_epoch: Vec = (1..=rig.epoch_start_slot(1)).map(Into::into).collect(); - let (canonical_blocks_zeroth_epoch, _, _, mut canonical_state) = rig + let (canonical_blocks_zeroth_epoch, _, _, canonical_state) = rig .add_attested_blocks_at_slots( state.clone(), - state_root, &canonical_slots_zeroth_epoch, &honest_validators, ) @@ -2197,11 +2156,9 @@ async fn prunes_skipped_slots_states() { let stray_slots: Vec = ((skipped_slot + 1).into()..rig.epoch_start_slot(2)) .map(Into::into) .collect(); - let canonical_state_root = canonical_state.update_tree_hash_cache().unwrap(); let (stray_blocks, stray_states, _, stray_state) = rig .add_attested_blocks_at_slots( canonical_state.clone(), - canonical_state_root, &stray_slots, &adversarial_validators, ) @@ -2242,14 +2199,8 @@ async fn prunes_skipped_slots_states() { let canonical_slots: Vec = ((skipped_slot + 1).into()..rig.epoch_start_slot(7)) .map(Into::into) .collect(); - let canonical_state_root = canonical_state.update_tree_hash_cache().unwrap(); let (canonical_blocks_post_finalization, _, _, _) = rig - .add_attested_blocks_at_slots( - canonical_state, - canonical_state_root, - &canonical_slots, - &honest_validators, - ) + .add_attested_blocks_at_slots(canonical_state, &canonical_slots, &honest_validators) .await; // Postconditions @@ -2304,14 +2255,13 @@ async fn finalizes_non_epoch_start_slot() { let db_path = tempdir().unwrap(); let store = get_store(&db_path); let rig = get_harness(store.clone(), VALIDATOR_COUNT); - let (state, state_root) = rig.get_current_state_and_root(); + let state = rig.get_current_state(); let canonical_slots_zeroth_epoch: Vec = (1..rig.epoch_start_slot(1)).map(Into::into).collect(); - let (canonical_blocks_zeroth_epoch, _, _, mut canonical_state) = rig + let (canonical_blocks_zeroth_epoch, _, _, canonical_state) = rig .add_attested_blocks_at_slots( state.clone(), - state_root, &canonical_slots_zeroth_epoch, &honest_validators, ) @@ -2322,11 +2272,9 @@ async fn finalizes_non_epoch_start_slot() { let stray_slots: Vec = ((skipped_slot + 1).into()..rig.epoch_start_slot(2)) .map(Into::into) .collect(); - let canonical_state_root = canonical_state.update_tree_hash_cache().unwrap(); let (stray_blocks, stray_states, _, stray_state) = rig .add_attested_blocks_at_slots( canonical_state.clone(), - canonical_state_root, &stray_slots, &adversarial_validators, ) @@ -2367,14 +2315,8 @@ async fn finalizes_non_epoch_start_slot() { let canonical_slots: Vec = ((skipped_slot + 1).into()..rig.epoch_start_slot(7)) .map(Into::into) .collect(); - let canonical_state_root = canonical_state.update_tree_hash_cache().unwrap(); let (canonical_blocks_post_finalization, _, _, _) = rig - .add_attested_blocks_at_slots( - canonical_state, - canonical_state_root, - &canonical_slots, - &honest_validators, - ) + .add_attested_blocks_at_slots(canonical_state, &canonical_slots, &honest_validators) .await; // Postconditions @@ -2597,11 +2539,10 @@ async fn pruning_test( let start_slot = Slot::new(1); let divergence_slot = start_slot + num_initial_blocks; - let (state, state_root) = harness.get_current_state_and_root(); + let state = harness.get_current_state(); let (_, _, _, divergence_state) = harness .add_attested_blocks_at_slots( state, - state_root, &slots(start_slot, num_initial_blocks)[..], &honest_validators, ) @@ -2626,7 +2567,7 @@ async fn pruning_test( ), ]) .await; - let (_, _, _, mut canonical_state) = chains.remove(0); + let (_, _, _, canonical_state) = chains.remove(0); let (stray_blocks, stray_states, _, stray_head_state) = chains.remove(0); let stray_head_slot = divergence_slot + num_fork_skips + num_fork_blocks - 1; @@ -2650,11 +2591,9 @@ async fn pruning_test( // Trigger finalization let num_finalization_blocks = 4 * E::slots_per_epoch(); let canonical_slot = divergence_slot + num_canonical_skips + num_canonical_middle_blocks; - let canonical_state_root = canonical_state.update_tree_hash_cache().unwrap(); harness .add_attested_blocks_at_slots( canonical_state, - canonical_state_root, &slots(canonical_slot, num_finalization_blocks), &honest_validators, ) @@ -2862,14 +2801,9 @@ async fn reproduction_unaligned_checkpoint_sync_pruned_payload() { let harness = get_harness_import_all_data_columns(full_store.clone(), LOW_VALIDATOR_COUNT); let all_validators = (0..LOW_VALIDATOR_COUNT).collect::>(); - let (genesis_state, genesis_state_root) = harness.get_current_state_and_root(); + let genesis_state = harness.get_current_state(); harness - .add_attested_blocks_at_slots( - genesis_state.clone(), - genesis_state_root, - &slots, - &all_validators, - ) + .add_attested_blocks_at_slots(genesis_state.clone(), &slots, &all_validators) .await; // Extract snapshot data from the harness. @@ -3016,14 +2950,9 @@ async fn weak_subjectivity_sync_test( let all_validators = (0..LOW_VALIDATOR_COUNT).collect::>(); - let (genesis_state, genesis_state_root) = harness.get_current_state_and_root(); + let genesis_state = harness.get_current_state(); harness - .add_attested_blocks_at_slots( - genesis_state.clone(), - genesis_state_root, - &slots, - &all_validators, - ) + .add_attested_blocks_at_slots(genesis_state.clone(), &slots, &all_validators) .await; let wss_block_root = harness @@ -3831,14 +3760,9 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { .map(Slot::new) .collect::>(); - let (genesis_state, genesis_state_root) = harness.get_current_state_and_root(); + let genesis_state = harness.get_current_state(); harness - .add_attested_blocks_at_slots( - genesis_state.clone(), - genesis_state_root, - &slots, - &all_validators, - ) + .add_attested_blocks_at_slots(genesis_state.clone(), &slots, &all_validators) .await; // Before the split slot becomes finalized, create two forking blocks that build on the split @@ -5706,7 +5630,7 @@ async fn test_gloas_block_replay_with_envelopes() { let harness = get_harness(store.clone(), LOW_VALIDATOR_COUNT); let num_blocks = 16u64; - let (genesis_state, _genesis_state_root) = harness.get_current_state_and_root(); + let genesis_state = harness.get_current_state(); let mut state = genesis_state.clone(); let mut last_block_root = Hash256::zero(); @@ -5782,7 +5706,7 @@ async fn test_gloas_hot_state_hierarchy() { let num_blocks = E::slots_per_epoch() * 5; let all_validators = (0..LOW_VALIDATOR_COUNT).collect::>(); - let (genesis_state, _genesis_state_root) = harness.get_current_state_and_root(); + let genesis_state = harness.get_current_state(); // Use manual block building with envelopes for the first few blocks, // then use the standard attested-blocks path once we've verified envelope handling. diff --git a/beacon_node/beacon_chain/tests/sync_committee_verification.rs b/beacon_node/beacon_chain/tests/sync_committee_verification.rs index d2124c6641..b01084c6aa 100644 --- a/beacon_node/beacon_chain/tests/sync_committee_verification.rs +++ b/beacon_node/beacon_chain/tests/sync_committee_verification.rs @@ -185,7 +185,6 @@ async fn aggregated_gossip_verification() { harness .add_attested_blocks_at_slots( state, - Hash256::zero(), &[Slot::new(1), Slot::new(2)], (0..VALIDATOR_COUNT).collect::>().as_slice(), ) @@ -495,7 +494,7 @@ async fn aggregated_gossip_verification() { ); harness - .add_attested_block_at_slot(target_slot, state, Hash256::zero(), &[]) + .add_attested_block_at_slot(target_slot, state, &[]) .await .expect("should add block"); @@ -519,7 +518,6 @@ async fn unaggregated_gossip_verification() { harness .add_attested_blocks_at_slots( state, - Hash256::zero(), &[Slot::new(1), Slot::new(2)], (0..VALIDATOR_COUNT).collect::>().as_slice(), ) @@ -801,7 +799,7 @@ async fn unaggregated_gossip_verification() { ); harness - .add_attested_block_at_slot(target_slot, state, Hash256::zero(), &[]) + .add_attested_block_at_slot(target_slot, state, &[]) .await .expect("should add block"); diff --git a/beacon_node/beacon_chain/tests/validator_monitor.rs b/beacon_node/beacon_chain/tests/validator_monitor.rs index a37ab6458f..9e3973d0d1 100644 --- a/beacon_node/beacon_chain/tests/validator_monitor.rs +++ b/beacon_node/beacon_chain/tests/validator_monitor.rs @@ -46,8 +46,7 @@ async fn missed_blocks_across_epochs() { let harness = get_harness(VALIDATOR_COUNT, vec![]); let validator_monitor = &harness.chain.validator_monitor; - let mut genesis_state = harness.get_current_state(); - let genesis_state_root = genesis_state.update_tree_hash_cache().unwrap(); + let genesis_state = harness.get_current_state(); let genesis_block_root = harness.head_block_root(); // Skip a slot in the first epoch (to prime the cache inside the missed block function) and then @@ -64,7 +63,7 @@ async fn missed_blocks_across_epochs() { .collect::>(); let (block_roots_by_slot, state_roots_by_slot, _, head_state) = harness - .add_attested_blocks_at_slots(genesis_state, genesis_state_root, &slots, &all_validators) + .add_attested_blocks_at_slots(genesis_state, &slots, &all_validators) .await; // Prime the proposer shuffling cache. diff --git a/beacon_node/http_api/tests/fork_tests.rs b/beacon_node/http_api/tests/fork_tests.rs index 4ba35c238c..0ff8ebc452 100644 --- a/beacon_node/http_api/tests/fork_tests.rs +++ b/beacon_node/http_api/tests/fork_tests.rs @@ -57,14 +57,9 @@ async fn sync_committee_duties_across_fork() { // If there's a skip slot at the fork slot, the endpoint should return duties, even // though the head state hasn't transitioned yet. let fork_slot = fork_epoch.start_slot(E::slots_per_epoch()); - let (genesis_state, genesis_state_root) = harness.get_current_state_and_root(); - let (_, mut state) = harness - .add_attested_block_at_slot( - fork_slot - 1, - genesis_state, - genesis_state_root, - &all_validators, - ) + let genesis_state = harness.get_current_state(); + let (_, state) = harness + .add_attested_block_at_slot(fork_slot - 1, genesis_state, &all_validators) .await .unwrap(); @@ -79,9 +74,8 @@ async fn sync_committee_duties_across_fork() { assert_eq!(sync_duties.len(), E::sync_committee_size()); // After applying a block at the fork slot the duties should remain unchanged. - let state_root = state.canonical_root().unwrap(); harness - .add_attested_block_at_slot(fork_slot, state, state_root, &all_validators) + .add_attested_block_at_slot(fork_slot, state, &all_validators) .await .unwrap(); @@ -295,14 +289,9 @@ async fn sync_committee_indices_across_fork() { // If there's a skip slot at the fork slot, the endpoint will return a 400 until a block is // applied. let fork_slot = fork_epoch.start_slot(E::slots_per_epoch()); - let (genesis_state, genesis_state_root) = harness.get_current_state_and_root(); - let (_, mut state) = harness - .add_attested_block_at_slot( - fork_slot - 1, - genesis_state, - genesis_state_root, - &all_validators, - ) + let genesis_state = harness.get_current_state(); + let (_, state) = harness + .add_attested_block_at_slot(fork_slot - 1, genesis_state, &all_validators) .await .unwrap(); @@ -334,9 +323,8 @@ async fn sync_committee_indices_across_fork() { // Once the head is updated it should be useable for requests, including in the next sync // committee period. - let state_root = state.canonical_root().unwrap(); harness - .add_attested_block_at_slot(fork_slot + 1, state, state_root, &all_validators) + .add_attested_block_at_slot(fork_slot + 1, state, &all_validators) .await .unwrap(); diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index de5fe9a098..a1789e3b19 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -897,7 +897,6 @@ mod release_tests { BeaconChainHarness, EphemeralHarnessType, RelativeSyncCommittee, test_spec, }; use bls::Keypair; - use fixed_bytes::FixedBytesExtended; use maplit::hashset; use state_processing::epoch_cache::initialize_epoch_cache; use state_processing::{VerifyOperation, common::get_attesting_indices_from_state}; @@ -944,10 +943,10 @@ mod release_tests { fn get_current_state_initialize_epoch_cache( harness: &BeaconChainHarness>, spec: &ChainSpec, - ) -> BeaconState { - let mut state = harness.get_current_state(); + ) -> (BeaconState, Hash256) { + let (mut state, state_root) = harness.get_current_state_and_root(); initialize_epoch_cache(&mut state, spec).unwrap(); - state + (state, state_root) } /// Test state for sync contribution-related tests. @@ -965,7 +964,6 @@ mod release_tests { harness .add_attested_blocks_at_slots( state, - Hash256::zero(), &[Slot::new(1)], (0..num_validators).collect::>().as_slice(), ) @@ -983,7 +981,7 @@ mod release_tests { return; } - let mut state = get_current_state_initialize_epoch_cache(&harness, spec); + let (mut state, state_root) = get_current_state_initialize_epoch_cache(&harness, spec); let slot = state.slot(); let committees = state .get_beacon_committees_at_slot(slot) @@ -998,8 +996,8 @@ mod release_tests { let attestations = harness.make_attestations( (0..num_validators).collect::>().as_slice(), &state, - Hash256::zero(), - SignedBeaconBlockHash::from(Hash256::zero()), + state_root, + harness.head_block_root().into(), slot, ); @@ -1065,7 +1063,7 @@ mod release_tests { let (harness, ref spec) = attestation_test_state::(1); let op_pool = OperationPool::::new(); - let mut state = get_current_state_initialize_epoch_cache(&harness, spec); + let (mut state, state_root) = get_current_state_initialize_epoch_cache(&harness, spec); let slot = state.slot(); let committees = state @@ -1087,8 +1085,8 @@ mod release_tests { let attestations = harness.make_attestations( (0..num_validators).collect::>().as_slice(), &state, - Hash256::zero(), - SignedBeaconBlockHash::from(Hash256::zero()), + state_root, + harness.head_block_root().into(), slot, ); @@ -1141,7 +1139,7 @@ mod release_tests { fn attestation_duplicate() { let (harness, ref spec) = attestation_test_state::(1); - let state = get_current_state_initialize_epoch_cache(&harness, spec); + let (state, state_root) = get_current_state_initialize_epoch_cache(&harness, spec); let op_pool = OperationPool::::new(); @@ -1158,8 +1156,8 @@ mod release_tests { let attestations = harness.make_attestations( (0..num_validators).collect::>().as_slice(), &state, - Hash256::zero(), - SignedBeaconBlockHash::from(Hash256::zero()), + state_root, + harness.head_block_root().into(), slot, ); @@ -1184,7 +1182,7 @@ mod release_tests { fn attestation_pairwise_overlapping() { let (harness, ref spec) = attestation_test_state::(1); - let state = get_current_state_initialize_epoch_cache(&harness, spec); + let (state, state_root) = get_current_state_initialize_epoch_cache(&harness, spec); let op_pool = OperationPool::::new(); @@ -1202,8 +1200,8 @@ mod release_tests { let attestations = harness.make_attestations( (0..num_validators).collect::>().as_slice(), &state, - Hash256::zero(), - SignedBeaconBlockHash::from(Hash256::zero()), + state_root, + harness.head_block_root().into(), slot, ); @@ -1279,7 +1277,7 @@ mod release_tests { let (harness, ref spec) = attestation_test_state::(num_committees); - let mut state = get_current_state_initialize_epoch_cache(&harness, spec); + let (mut state, state_root) = get_current_state_initialize_epoch_cache(&harness, spec); let op_pool = OperationPool::::new(); @@ -1300,8 +1298,8 @@ mod release_tests { let attestations = harness.make_attestations( (0..num_validators).collect::>().as_slice(), &state, - Hash256::zero(), - SignedBeaconBlockHash::from(Hash256::zero()), + state_root, + harness.head_block_root().into(), slot, ); @@ -1385,7 +1383,7 @@ mod release_tests { let (harness, ref spec) = attestation_test_state::(num_committees); - let mut state = get_current_state_initialize_epoch_cache(&harness, spec); + let (mut state, state_root) = get_current_state_initialize_epoch_cache(&harness, spec); let op_pool = OperationPool::::new(); let slot = state.slot(); @@ -1411,8 +1409,8 @@ mod release_tests { let attestations = harness.make_attestations( (0..num_validators).collect::>().as_slice(), &state, - Hash256::zero(), - SignedBeaconBlockHash::from(Hash256::zero()), + state_root, + harness.head_block_root().into(), slot, ); @@ -2275,7 +2273,6 @@ mod release_tests { harness .add_attested_blocks_at_slots( harness.get_current_state(), - Hash256::zero(), &[Slot::new(1)], (0..num_validators).collect::>().as_slice(), ) @@ -2332,7 +2329,6 @@ mod release_tests { harness .add_attested_blocks_at_slots( harness.get_current_state(), - Hash256::zero(), &[Slot::new(1)], (0..num_validators).collect::>().as_slice(), ) diff --git a/consensus/state_processing/src/per_block_processing/tests.rs b/consensus/state_processing/src/per_block_processing/tests.rs index 96610c2010..593a2557e8 100644 --- a/consensus/state_processing/src/per_block_processing/tests.rs +++ b/consensus/state_processing/src/per_block_processing/tests.rs @@ -12,7 +12,6 @@ use crate::{ }; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use bls::{AggregateSignature, Keypair, PublicKeyBytes, Signature, SignatureBytes}; -use fixed_bytes::FixedBytesExtended; use ssz_types::Bitfield; use ssz_types::VariableList; use std::sync::{Arc, LazyLock}; @@ -52,7 +51,6 @@ async fn get_harness( harness .add_attested_blocks_at_slots( state, - Hash256::zero(), (1..last_slot_of_epoch.as_u64()) .map(Slot::new) .collect::>() diff --git a/consensus/state_processing/src/per_epoch_processing/tests.rs b/consensus/state_processing/src/per_epoch_processing/tests.rs index c04b7f843d..29716866b5 100644 --- a/consensus/state_processing/src/per_epoch_processing/tests.rs +++ b/consensus/state_processing/src/per_epoch_processing/tests.rs @@ -2,7 +2,6 @@ use crate::per_epoch_processing::process_epoch; use beacon_chain::test_utils::BeaconChainHarness; use beacon_chain::types::{EthSpec, MinimalEthSpec}; -use bls::{FixedBytesExtended, Hash256}; use types::Slot; #[tokio::test] @@ -22,7 +21,6 @@ async fn runs_without_error() { harness .add_attested_blocks_at_slots( state, - Hash256::zero(), (1..target_slot.as_u64()) .map(Slot::new) .collect::>() diff --git a/consensus/types/tests/committee_cache.rs b/consensus/types/tests/committee_cache.rs index 5c1962276f..5205446c71 100644 --- a/consensus/types/tests/committee_cache.rs +++ b/consensus/types/tests/committee_cache.rs @@ -47,7 +47,6 @@ async fn new_state(validator_count: usize, slot: Slot) -> BeaconStat harness .add_attested_blocks_at_slots( head_state, - Hash256::zero(), (1..=slot.as_u64()) .map(Slot::new) .collect::>() diff --git a/consensus/types/tests/state.rs b/consensus/types/tests/state.rs index 2168da9afc..8e05b8ecb1 100644 --- a/consensus/types/tests/state.rs +++ b/consensus/types/tests/state.rs @@ -39,7 +39,6 @@ async fn get_harness( harness .add_attested_blocks_at_slots( state, - Hash256::zero(), slots.as_slice(), (0..validator_count).collect::>().as_slice(), ) diff --git a/testing/state_transition_vectors/src/main.rs b/testing/state_transition_vectors/src/main.rs index 6a212f034d..68c686649a 100644 --- a/testing/state_transition_vectors/src/main.rs +++ b/testing/state_transition_vectors/src/main.rs @@ -4,7 +4,6 @@ mod exit; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use bls::Keypair; -use fixed_bytes::FixedBytesExtended; use ssz::Encode; use std::env; use std::fs::{self, File}; @@ -13,7 +12,7 @@ use std::path::{Path, PathBuf}; use std::process::exit; use std::sync::LazyLock; use types::{BeaconState, EthSpec, SignedBeaconBlock, test_utils::generate_deterministic_keypairs}; -use types::{Hash256, MainnetEthSpec, Slot}; +use types::{MainnetEthSpec, Slot}; type E = MainnetEthSpec; @@ -65,7 +64,6 @@ async fn get_harness( harness .add_attested_blocks_at_slots( state, - Hash256::zero(), (skip_to_slot.as_u64()..slot.as_u64()) .map(Slot::new) .collect::>() From 9101ddc69dad2c04d8bc5e7a58a6c2bea8680efa Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 13 May 2026 07:20:08 +0300 Subject: [PATCH 10/18] `ignore-ws-check` flag doesnt allow the node to start outside the weak subjectivity period (#9290) Using the `ignore-ws-check` doesn't actually let you start up a node thats outside the weak subjectivity period Co-Authored-By: Eitan Seri-Levi --- beacon_node/beacon_chain/src/builder.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index d70561db9b..13dac087ef 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -849,12 +849,13 @@ where It is highly recommended to purge your db and checkpoint sync. For more information please \ read this blog post: https://blog.ethereum.org/2014/11/25/proof-stake-learned-love-weak-subjectivity" ) + } else { + return Err( + "The current head state is outside the weak subjectivity period. A node in this state is susceptible to long range attacks. You should purge your db and \ + checkpoint sync. For more information please read this blog post: https://blog.ethereum.org/2014/11/25/proof-stake-learned-love-weak-subjectivity \ + If you understand the risks, it is possible to ignore this error with the --ignore-ws-check flag.".to_string() + ); } - return Err( - "The current head state is outside the weak subjectivity period. A node in this state is susceptible to long range attacks. You should purge your db and \ - checkpoint sync. For more information please read this blog post: https://blog.ethereum.org/2014/11/25/proof-stake-learned-love-weak-subjectivity \ - If you understand the risks, it is possible to ignore this error with the --ignore-ws-check flag.".to_string() - ); } let validator_pubkey_cache = self From 1a686311803c7a8123c4fa576f2ea7c0283005dc Mon Sep 17 00:00:00 2001 From: Daniel Knopik <107140945+dknopik@users.noreply.github.com> Date: Wed, 13 May 2026 09:03:34 +0200 Subject: [PATCH 11/18] Gloas payload cache (#9209) In Gloas, beacon blocks are imported into fork choice immediately - the payload envelope and data columns arrive separately. KZG commitments moved from the column sidecar into the execution payload bid, so the existing `DataAvailabilityChecker` (which assumes block and data are coupled) can't be used for Gloas. * Introduced `PendingPayloadCache` to keep track of payload and data columns per block root. * Added gossip column verification * Added support for Gloas data column reconstruction * Payload envelope verification simplified: removed `MaybeAvailableEnvelope`, `ExecutedEnvelope`, `EnvelopeImportData` Not yet implemented (tracked with TODOs): - Proper lookup sync for Gloas columns arriving before blocks - Partial column merging for Gloas - Moving `load_gloas_payload_bid` disk reads off the async runtime - Backfill/range sync for Gloas Based on @eserilev's PR and work in progress. See also #9202 for verification. Co-Authored-By: Eitan Seri-Levi Co-Authored-By: Eitan Seri- Levi Co-Authored-By: Daniel Knopik Co-Authored-By: Daniel Knopik <107140945+dknopik@users.noreply.github.com> Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> Co-Authored-By: Jimmy Chen --- beacon_node/beacon_chain/benches/benches.rs | 8 + beacon_node/beacon_chain/src/beacon_chain.rs | 418 ++++++---- .../beacon_chain/src/block_verification.rs | 7 +- beacon_node/beacon_chain/src/builder.rs | 16 +- .../beacon_chain/src/canonical_head.rs | 19 + .../src/data_availability_checker.rs | 75 +- .../src/data_availability_checker/error.rs | 6 +- .../overflow_lru_cache.rs | 2 +- .../src/data_column_verification.rs | 388 +++++++-- .../fetch_blobs/fetch_blobs_beacon_adapter.rs | 10 +- .../beacon_chain/src/fetch_blobs/mod.rs | 2 +- .../beacon_chain/src/fetch_blobs/tests.rs | 2 +- beacon_node/beacon_chain/src/kzg_utils.rs | 195 ++++- beacon_node/beacon_chain/src/lib.rs | 1 + beacon_node/beacon_chain/src/metrics.rs | 10 + .../execution_pending_envelope.rs | 26 +- .../payload_envelope_verification/import.rs | 95 ++- .../src/payload_envelope_verification/mod.rs | 114 +-- .../payload_notifier.rs | 3 +- .../src/pending_payload_cache/mod.rs | 781 ++++++++++++++++++ .../pending_payload_cache/pending_column.rs | 63 ++ .../pending_components.rs | 180 ++++ beacon_node/beacon_chain/src/test_utils.rs | 67 +- .../beacon_chain/tests/block_verification.rs | 51 +- beacon_node/beacon_chain/tests/events.rs | 12 +- beacon_node/beacon_chain/tests/store_tests.rs | 110 ++- .../src/beacon/execution_payload_envelope.rs | 5 +- beacon_node/http_api/src/publish_blocks.rs | 2 +- .../http_api/tests/interactive_tests.rs | 1 + beacon_node/network/src/metrics.rs | 4 +- .../gossip_methods.rs | 122 ++- .../src/network_beacon_processor/mod.rs | 5 +- .../network_beacon_processor/sync_methods.rs | 2 + .../src/network_beacon_processor/tests.rs | 10 +- .../src/sync/block_sidecar_coupling.rs | 15 +- beacon_node/network/src/sync/manager.rs | 12 +- .../network/src/sync/network_context.rs | 16 +- beacon_node/network/src/sync/tests/lookups.rs | 10 +- consensus/fork_choice/src/fork_choice.rs | 8 + .../types/src/block/signed_beacon_block.rs | 6 + .../execution/signed_execution_payload_bid.rs | 8 + 41 files changed, 2351 insertions(+), 536 deletions(-) create mode 100644 beacon_node/beacon_chain/src/pending_payload_cache/mod.rs create mode 100644 beacon_node/beacon_chain/src/pending_payload_cache/pending_column.rs create mode 100644 beacon_node/beacon_chain/src/pending_payload_cache/pending_components.rs diff --git a/beacon_node/beacon_chain/benches/benches.rs b/beacon_node/beacon_chain/benches/benches.rs index e71a19d8c1..de0bd05a1f 100644 --- a/beacon_node/beacon_chain/benches/benches.rs +++ b/beacon_node/beacon_chain/benches/benches.rs @@ -53,6 +53,13 @@ fn all_benches(c: &mut Criterion) { ) .unwrap(); + let kzg_commitments = signed_block + .message() + .body() + .blob_kzg_commitments() + .unwrap() + .clone(); + let spec = spec.clone(); c.bench_function(&format!("reconstruct_{}", blob_count), |b| { @@ -60,6 +67,7 @@ fn all_benches(c: &mut Criterion) { black_box(reconstruct_data_columns( &kzg, column_sidecars.iter().as_slice()[0..column_sidecars.len() / 2].to_vec(), + &kzg_commitments, spec.as_ref(), )) }) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index f618cf6321..af8cd477d6 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -19,9 +19,11 @@ pub use crate::canonical_head::CanonicalHead; use crate::chain_config::ChainConfig; use crate::custody_context::CustodyContextSsz; use crate::data_availability_checker::{ - Availability, AvailabilityCheckError, AvailableBlock, AvailableBlockData, - DataAvailabilityChecker, DataColumnReconstructionResult, + Availability as BlockAvailability, AvailabilityCheckError, AvailableBlock, AvailableBlockData, + DataColumnReconstructionResult as DataColumnReconstructionResultV1, }; + +use crate::data_availability_checker::DataAvailabilityChecker; use crate::data_column_verification::{ GossipDataColumnError, GossipPartialDataColumnError, GossipVerifiedDataColumn, GossipVerifiedPartialDataColumnHeader, KzgVerifiedCustodyPartialDataColumn, @@ -36,7 +38,6 @@ use crate::execution_payload::{NotifyExecutionLayer, PreparePayloadHandle, get_e use crate::fetch_blobs::EngineGetBlobsOutput; use crate::fork_choice_signal::{ForkChoiceSignalRx, ForkChoiceSignalTx}; use crate::graffiti_calculator::{GraffitiCalculator, GraffitiSettings}; -use crate::kzg_utils::reconstruct_blobs; use crate::light_client_finality_update_verification::{ Error as LightClientFinalityUpdateError, VerifiedLightClientFinalityUpdate, }; @@ -65,6 +66,11 @@ use crate::payload_attestation_verification::VerifiedPayloadAttestationMessage; use crate::payload_bid_verification::payload_bid_cache::GossipVerifiedPayloadBidCache; #[cfg(not(test))] use crate::payload_envelope_streamer::{EnvelopeRequestSource, launch_payload_envelope_stream}; +use crate::pending_payload_cache::PendingPayloadCache; +use crate::pending_payload_cache::{ + Availability as PayloadAvailability, + DataColumnReconstructionResult as DataColumnReconstructionResultGloas, +}; use crate::pending_payload_envelopes::PendingPayloadEnvelopes; use crate::persisted_beacon_chain::PersistedBeaconChain; use crate::persisted_custody::persist_custody_context; @@ -498,9 +504,10 @@ pub struct BeaconChain { pub validator_monitor: RwLock>, /// The slot at which blocks are downloaded back to. pub genesis_backfill_slot: Slot, - /// Provides a KZG verification and temporary storage for blocks and blobs as - /// they are collected and combined. + /// Provides KZG verification and temporary storage for pre-Gloas blocks and blobs. pub data_availability_checker: Arc>, + /// Provides KZG verification and temporary storage for post-Gloas payload envelopes. + pub pending_payload_cache: Arc>, /// The KZG trusted setup used by this chain. pub kzg: Arc, /// RNG instance used by the chain. Currently used for shuffling column sidecars in block publishing. @@ -1180,6 +1187,7 @@ impl BeaconChain { let all_cached_columns_opt = self .data_availability_checker .get_data_columns(block_root) + .or_else(|| self.pending_payload_cache.get_data_columns(block_root)) .or_else(|| self.early_attester_cache.get_data_columns(block_root)); if let Some(mut all_cached_columns) = all_cached_columns_opt { @@ -1198,6 +1206,24 @@ impl BeaconChain { } } + pub fn cached_data_column_indexes( + &self, + block_root: &Hash256, + slot: Slot, + ) -> Option> { + if self + .spec + .fork_name_at_slot::(slot) + .gloas_enabled() + { + self.pending_payload_cache + .cached_data_column_indexes(block_root) + } else { + self.data_availability_checker + .cached_data_column_indexes(block_root) + } + } + /// Returns the block at the given root, if any. /// /// ## Errors @@ -1286,45 +1312,6 @@ impl BeaconChain { .map_err(Error::from) } - /// Returns the blobs at the given root, if any. - /// - /// Uses the `block.epoch()` to determine whether to retrieve blobs or columns from the store. - /// - /// If at least 50% of columns are retrieved, blobs will be reconstructed and returned, - /// otherwise an error `InsufficientColumnsToReconstructBlobs` is returned. - /// - /// ## Errors - /// May return a database error. - pub fn get_or_reconstruct_blobs( - &self, - block_root: &Hash256, - ) -> Result>, Error> { - let Some(block) = self.store.get_blinded_block(block_root)? else { - return Ok(None); - }; - - if self.spec.is_peer_das_enabled_for_epoch(block.epoch()) { - let fork_name = self.spec.fork_name_at_epoch(block.epoch()); - if let Some(columns) = self.store.get_data_columns(block_root, fork_name)? { - let num_required_columns = T::EthSpec::number_of_columns() / 2; - let reconstruction_possible = columns.len() >= num_required_columns; - if reconstruction_possible { - reconstruct_blobs(&self.kzg, columns, None, &block, &self.spec) - .map(Some) - .map_err(Error::FailedToReconstructBlobs) - } else { - Err(Error::InsufficientColumnsToReconstructBlobs { - columns_found: columns.len(), - }) - } - } else { - Ok(None) - } - } else { - Ok(self.get_blobs(block_root)?.blobs()) - } - } - /// Returns the data columns at the given root, if any. /// /// ## Errors @@ -3306,13 +3293,7 @@ impl BeaconChain { )); }; - // If this block has already been imported to forkchoice it must have been available, so - // we don't need to process its samples again. - if self - .canonical_head - .fork_choice_read_lock() - .contains_block(&block_root) - { + if self.is_block_data_imported(block_root, slot) { return Err(BlockError::DuplicateFullyImported(block_root)); } @@ -3357,12 +3338,7 @@ impl BeaconChain { return Ok(None); }; - // If this block has already been imported to forkchoice it must have been available - if self - .canonical_head - .fork_choice_read_lock() - .contains_block(&block_root) - { + if self.is_block_data_imported(block_root, slot) { return Err(BlockError::DuplicateFullyImported(block_root)); } @@ -3401,15 +3377,28 @@ impl BeaconChain { .map(|column| column.as_data_column()), ); - let availability = self - .data_availability_checker - .put_kzg_verified_custody_data_columns( - block_root, - merge_result.full_columns.clone(), - )?; - - self.process_availability(slot, availability, || Ok(())) - .await? + if self + .spec + .fork_name_at_slot::(slot) + .gloas_enabled() + { + let availability = self + .pending_payload_cache + .put_kzg_verified_custody_data_columns(block_root, &merge_result.full_columns) + .map_err(BlockError::from)?; + self.process_payload_envelope_availability(slot, availability, || Ok(())) + .await? + } else { + let availability = self + .data_availability_checker + .put_kzg_verified_custody_data_columns( + block_root, + merge_result.full_columns.clone(), + ) + .map_err(BlockError::from)?; + self.process_availability(slot, availability, || Ok(())) + .await? + } } else { AvailabilityProcessingStatus::MissingComponents(slot, block_root) }; @@ -3426,13 +3415,7 @@ impl BeaconChain { block_root: Hash256, blobs: FixedBlobSidecarList, ) -> Result { - // If this block has already been imported to forkchoice it must have been available, so - // we don't need to process its blobs again. - if self - .canonical_head - .fork_choice_read_lock() - .contains_block(&block_root) - { + if self.is_block_data_imported(block_root, slot) { return Err(BlockError::DuplicateFullyImported(block_root)); } @@ -3521,9 +3504,12 @@ impl BeaconChain { if let Some(event_handler) = self.event_handler.as_ref() && event_handler.has_data_column_sidecar_subscribers() { + let mut data_columns_iter = data_columns_iter.peekable(); + let Some(slot) = data_columns_iter.peek().map(|col| col.slot()) else { + return; + }; let imported_data_columns = self - .data_availability_checker - .cached_data_column_indexes(block_root) + .cached_data_column_indexes(block_root, slot) .unwrap_or_default(); let new_data_columns = data_columns_iter.filter(|b| !imported_data_columns.contains(b.index())); @@ -3554,15 +3540,7 @@ impl BeaconChain { )); }; - // If this block has already been imported to forkchoice it must have been available, so - // we don't need to process its columns again. - // TODO(gloas) the block will be available in fork choice for gloas. This does not indicate availability - // anymore. - if self - .canonical_head - .fork_choice_read_lock() - .contains_block(&block_root) - { + if self.is_block_data_imported(block_root, slot) { return Err(BlockError::DuplicateFullyImported(block_root)); } @@ -3596,6 +3574,7 @@ impl BeaconChain { pub async fn reconstruct_data_columns( self: &Arc, + slot: Slot, block_root: Hash256, ) -> Result< Option<( @@ -3604,48 +3583,84 @@ impl BeaconChain { )>, BlockError, > { - // As of now we only reconstruct data columns on supernodes, so if the block is already - // available on a supernode, there's no need to reconstruct as the node must already have - // all columns. - if self - .canonical_head - .fork_choice_read_lock() - .contains_block(&block_root) - { + // As of now we only reconstruct data columns on supernodes, so if all availability data + // for the block is already imported, there's nothing left to reconstruct. + if self.is_block_data_imported(block_root, slot) { return Ok(None); } - let data_availability_checker = self.data_availability_checker.clone(); + let is_gloas = self + .spec + .fork_name_at_slot::(slot) + .gloas_enabled(); - let result = self - .task_executor - .spawn_blocking_with_rayon_async(RayonPoolType::HighPriority, move || { - data_availability_checker.reconstruct_data_columns(&block_root) - }) - .await - .map_err(|_| BeaconChainError::RuntimeShutdown)??; + if is_gloas { + let pending_payload_cache = self.pending_payload_cache.clone(); + let result = self + .task_executor + .spawn_blocking_with_rayon_async(RayonPoolType::HighPriority, move || { + pending_payload_cache.reconstruct_data_columns(&block_root) + }) + .await + .map_err(|_| BlockError::from(BeaconChainError::RuntimeShutdown))? + .map_err(BlockError::from)?; - match result { - DataColumnReconstructionResult::Success((availability, data_columns_to_publish)) => { - let Some(slot) = data_columns_to_publish.first().map(|d| d.slot()) else { - // This should be unreachable because empty result would return `RecoveredColumnsNotImported` instead of success. - return Ok(None); - }; + match result { + DataColumnReconstructionResultGloas::Success(( + availability, + data_columns_to_publish, + )) => { + let Some(slot) = data_columns_to_publish.first().map(|d| d.slot()) else { + return Ok(None); + }; - self.process_availability(slot, availability, || Ok(())) - .await - .map(|availability_processing_status| { - Some((availability_processing_status, data_columns_to_publish)) - }) + Ok(self + .process_payload_envelope_availability(slot, availability, || Ok(())) + .await + .map(|status| Some((status, data_columns_to_publish)))?) + } + DataColumnReconstructionResultGloas::NotStarted(reason) + | DataColumnReconstructionResultGloas::RecoveredColumnsNotImported(reason) => { + metrics::inc_counter_vec( + &metrics::KZG_DATA_COLUMN_RECONSTRUCTION_INCOMPLETE_TOTAL, + &[reason], + ); + Ok(None) + } } - DataColumnReconstructionResult::NotStarted(reason) - | DataColumnReconstructionResult::RecoveredColumnsNotImported(reason) => { - // We use metric here because logging this would be *very* noisy. - metrics::inc_counter_vec( - &metrics::KZG_DATA_COLUMN_RECONSTRUCTION_INCOMPLETE_TOTAL, - &[reason], - ); - Ok(None) + } else { + let data_availability_checker = self.data_availability_checker.clone(); + let result = self + .task_executor + .spawn_blocking_with_rayon_async(RayonPoolType::HighPriority, move || { + data_availability_checker.reconstruct_data_columns(&block_root) + }) + .await + .map_err(|_| BlockError::from(BeaconChainError::RuntimeShutdown))? + .map_err(BlockError::from)?; + + match result { + DataColumnReconstructionResultV1::Success(( + availability, + data_columns_to_publish, + )) => { + let Some(slot) = data_columns_to_publish.first().map(|d| d.slot()) else { + return Ok(None); + }; + + Ok(self + .process_availability(slot, availability, || Ok(())) + .await + .map(|status| Some((status, data_columns_to_publish)))?) + } + DataColumnReconstructionResultV1::NotStarted(reason) + | DataColumnReconstructionResultV1::RecoveredColumnsNotImported(reason) => { + metrics::inc_counter_vec( + &metrics::KZG_DATA_COLUMN_RECONSTRUCTION_INCOMPLETE_TOTAL, + &[reason], + ); + Ok(None) + } } } } @@ -3659,6 +3674,32 @@ impl BeaconChain { } } + /// Returns true when no further availability data for `block_root` should be processed. + /// + /// Pre-Gloas: + /// - true once the block is fully imported into fork choice. + /// + /// Gloas: + /// - true only once the payload envelope and required data columns are fully imported. + /// The beacon block itself may already be present in fork choice before this is true. + fn is_block_data_imported(&self, block_root: Hash256, slot: Slot) -> bool { + let is_gloas = self + .spec + .fork_name_at_slot::(slot) + .gloas_enabled(); + + let fork_choice = self.canonical_head.fork_choice_read_lock(); + if !fork_choice.contains_block(&block_root) { + return false; + } + + if !is_gloas { + return true; + } + + fork_choice.is_payload_received(&block_root) + } + /// Returns `Ok(block_root)` if the given `unverified_block` was successfully verified and /// imported into the chain. /// @@ -3723,6 +3764,19 @@ impl BeaconChain { &chain, notify_execution_layer, )?; + + let block = execution_pending.block.block_cloned(); + if block.fork_name_unchecked().gloas_enabled() { + let bid = Arc::new( + block + .message() + .body() + .signed_execution_payload_bid()? + .clone(), + ); + chain.pending_payload_cache.insert_bid(block_root, bid); + } + publish_fn()?; // Record the time it took to complete consensus verification. @@ -3891,12 +3945,25 @@ impl BeaconChain { } } - let availability = self - .data_availability_checker - .put_gossip_verified_data_columns(block_root, slot, data_columns)?; - - self.process_availability(slot, availability, publish_fn) - .await + if self + .spec + .fork_name_at_slot::(slot) + .gloas_enabled() + { + let availability = self + .pending_payload_cache + .put_gossip_verified_data_columns(block_root, data_columns)?; + Ok(self + .process_payload_envelope_availability(slot, availability, publish_fn) + .await?) + } else { + let availability = self + .data_availability_checker + .put_gossip_verified_data_columns(block_root, slot, data_columns)?; + Ok(self + .process_availability(slot, availability, publish_fn) + .await?) + } } fn check_blob_header_signature_and_slashability<'a>( @@ -3943,7 +4010,8 @@ impl BeaconChain { )?; let availability = self .data_availability_checker - .put_rpc_blobs(block_root, blobs)?; + .put_rpc_blobs(block_root, blobs) + .map_err(BlockError::from)?; self.process_availability(slot, availability, || Ok(())) .await @@ -3955,14 +4023,20 @@ impl BeaconChain { block_root: Hash256, engine_get_blobs_output: EngineGetBlobsOutput, ) -> Result { - let availability = match engine_get_blobs_output { + match engine_get_blobs_output { EngineGetBlobsOutput::Blobs(blobs) => { self.check_blob_header_signature_and_slashability( block_root, blobs.iter().map(|b| b.as_blob()), )?; - self.data_availability_checker - .put_kzg_verified_blobs(block_root, blobs)? + let availability = self + .data_availability_checker + .put_kzg_verified_blobs(block_root, blobs) + .map_err(BlockError::from)?; + + Ok(self + .process_availability(slot, availability, || Ok(())) + .await?) } EngineGetBlobsOutput::CustodyColumns(data_columns) => { // TODO(gloas) verify that this check is no longer relevant for gloas @@ -3975,13 +4049,29 @@ impl BeaconChain { _ => None, }), )?; - self.data_availability_checker - .put_kzg_verified_custody_data_columns(block_root, data_columns)? + if self + .spec + .fork_name_at_slot::(slot) + .gloas_enabled() + { + let availability = self + .pending_payload_cache + .put_kzg_verified_custody_data_columns(block_root, &data_columns) + .map_err(BlockError::from)?; + Ok(self + .process_payload_envelope_availability(slot, availability, || Ok(())) + .await?) + } else { + let availability = self + .data_availability_checker + .put_kzg_verified_custody_data_columns(block_root, data_columns) + .map_err(BlockError::from)?; + Ok(self + .process_availability(slot, availability, || Ok(())) + .await?) + } } - }; - - self.process_availability(slot, availability, || Ok(())) - .await + } } /// Checks if the provided columns can make any cached blocks available, and imports immediately @@ -4001,16 +4091,27 @@ impl BeaconChain { }), )?; - // This slot value is purely informative for the consumers of - // `AvailabilityProcessingStatus::MissingComponents` to log an error with a slot. - let availability = self.data_availability_checker.put_rpc_custody_columns( - block_root, - slot, - custody_columns, - )?; - - self.process_availability(slot, availability, || Ok(())) - .await + if self + .spec + .fork_name_at_slot::(slot) + .gloas_enabled() + { + let availability = self + .pending_payload_cache + .put_rpc_custody_columns(block_root, custody_columns) + .map_err(BlockError::from)?; + Ok(self + .process_payload_envelope_availability(slot, availability, || Ok(())) + .await?) + } else { + let availability = self + .data_availability_checker + .put_rpc_custody_columns(block_root, slot, custody_columns) + .map_err(BlockError::from)?; + Ok(self + .process_availability(slot, availability, || Ok(())) + .await?) + } } fn check_data_column_sidecar_header_signature_and_slashability<'a>( @@ -4053,16 +4154,33 @@ impl BeaconChain { async fn process_availability( self: &Arc, slot: Slot, - availability: Availability, + availability: BlockAvailability, publish_fn: impl FnOnce() -> Result<(), BlockError>, ) -> Result { match availability { - Availability::Available(block) => { + BlockAvailability::Available(block) => { publish_fn()?; - // Block is fully available, import into fork choice self.import_available_block(block).await } - Availability::MissingComponents(block_root) => Ok( + BlockAvailability::MissingComponents(block_root) => Ok( + AvailabilityProcessingStatus::MissingComponents(slot, block_root), + ), + } + } + + pub(crate) async fn process_payload_envelope_availability( + self: &Arc, + slot: Slot, + availability: PayloadAvailability, + publish_fn: impl FnOnce() -> Result<(), BlockError>, + ) -> Result { + match availability { + PayloadAvailability::Available(available_envelope) => { + publish_fn()?; + self.import_available_execution_payload_envelope(available_envelope) + .await + } + PayloadAvailability::MissingComponents(block_root) => Ok( AvailabilityProcessingStatus::MissingComponents(slot, block_root), ), } @@ -7572,7 +7690,7 @@ impl BeaconChain { ) } - pub(crate) fn get_blobs_or_columns_store_op( + pub fn get_blobs_or_columns_store_op( &self, block_root: Hash256, block_slot: Slot, diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 9a43147233..24f971f736 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -286,6 +286,10 @@ pub enum BlockError { /// TODO: We may need to penalize the peer that gave us a potentially invalid rpc blob. /// https://github.com/sigp/lighthouse/issues/4546 AvailabilityCheck(AvailabilityCheckError), + /// The payload envelope's block root is unknown. + EnvelopeBlockRootUnknown(Hash256), + /// Optimistic sync is not supported for Gloas payload envelopes. + OptimisticSyncNotSupported { block_root: Hash256 }, /// A Blob with a slot after PeerDAS is received and is not required to be imported. /// This can happen because we stay subscribed to the blob subnet after 2 epochs, as we could /// still receive valid blobs from a Deneb epoch after PeerDAS is activated. @@ -624,7 +628,8 @@ pub fn signature_verify_chain_segment( consensus_context, }); } - + // TODO(gloas) When implementing range and backfill sync for gloas + // we need a batch verify kzg function in the new da checker as well. chain .data_availability_checker .batch_verify_kzg_for_available_blocks(&available_blocks)?; diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 13dac087ef..e668bef7c0 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -12,6 +12,7 @@ use crate::kzg_utils::{build_data_column_sidecars_fulu, build_data_column_sideca use crate::light_client_server_cache::LightClientServerCache; use crate::migrate::{BackgroundMigrator, MigratorConfig}; use crate::observed_data_sidecars::ObservedDataSidecars; +use crate::pending_payload_cache::PendingPayloadCache; use crate::persisted_beacon_chain::PersistedBeaconChain; use crate::persisted_custody::load_custody_context; use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache}; @@ -987,6 +988,7 @@ where ) }; debug!(?custody_context, "Loaded persisted custody context"); + let custody_context = Arc::new(custody_context); let beacon_chain = BeaconChain { spec: self.spec.clone(), @@ -1062,14 +1064,22 @@ where data_availability_checker: Arc::new( DataAvailabilityChecker::new( complete_blob_backfill, - slot_clock, + slot_clock.clone(), self.kzg.clone(), - Arc::new(custody_context), - self.spec, + custody_context.clone(), + self.spec.clone(), enable_partial_columns, ) .map_err(|e| format!("Error initializing DataAvailabilityChecker: {:?}", e))?, ), + pending_payload_cache: Arc::new( + PendingPayloadCache::new( + self.kzg.clone(), + custody_context.clone(), + self.spec.clone(), + ) + .map_err(|e| format!("Error initializing PendingPayloadCache: {:?}", e))?, + ), kzg: self.kzg.clone(), rng: Arc::new(Mutex::new(rng)), gossip_verified_payload_bid_cache: <_>::default(), diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 0e6515ebbd..b3ab2e6975 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -988,6 +988,25 @@ impl BeaconChain { .start_slot(T::EthSpec::slots_per_epoch()), ); + // Prune the Gloas pending-payload cache. Anything older than the data-availability + // boundary cannot still be in flight; finalised entries are also safe to drop. + if self.spec.gloas_fork_epoch.is_some() { + let finalized_epoch = new_view.finalized_checkpoint.epoch; + let current_epoch = new_snapshot + .beacon_state + .slot() + .epoch(T::EthSpec::slots_per_epoch()); + if let Some(min_epochs_for_blobs) = self + .spec + .min_epoch_data_availability_boundary(current_epoch) + { + let cutoff_epoch = std::cmp::max(finalized_epoch + 1, min_epochs_for_blobs); + if let Err(e) = self.pending_payload_cache.do_maintenance(cutoff_epoch) { + error!(error = ?e, "Failed to prune pending payload cache on finalization"); + } + } + } + if let Some(event_handler) = self.event_handler.as_ref() && event_handler.has_finalized_subscribers() { diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index f0fa9c7794..cfd8ee7d34 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -33,6 +33,7 @@ use crate::data_column_verification::{ GossipVerifiedDataColumn, KzgVerifiedCustodyDataColumn, KzgVerifiedDataColumn, verify_kzg_for_data_column_list, }; +use crate::kzg_utils::validate_data_columns_with_commitments; use crate::metrics::{ KZG_DATA_COLUMN_RECONSTRUCTION_ATTEMPTS, KZG_DATA_COLUMN_RECONSTRUCTION_FAILURES, }; @@ -490,8 +491,7 @@ impl DataAvailabilityChecker { AvailableBlockData::Blobs(blobs) => verify_kzg_for_blob_list(blobs.iter(), &self.kzg) .map_err(AvailabilityCheckError::InvalidBlobs), AvailableBlockData::DataColumns(columns) => { - verify_kzg_for_data_column_list(columns.iter(), &self.kzg) - .map_err(AvailabilityCheckError::InvalidColumn) + verify_columns_against_block(&self.kzg, available_block.block(), columns) } } } @@ -504,13 +504,17 @@ impl DataAvailabilityChecker { available_blocks: &[AvailableBlock], ) -> Result<(), AvailabilityCheckError> { let mut all_blobs = Vec::new(); - let mut all_data_columns = Vec::new(); for available_block in available_blocks { - match available_block.data().to_owned() { + match available_block.data() { AvailableBlockData::NoData => {} - AvailableBlockData::Blobs(blobs) => all_blobs.extend(blobs), - AvailableBlockData::DataColumns(columns) => all_data_columns.extend(columns), + AvailableBlockData::Blobs(blobs) => all_blobs.extend(blobs.iter().cloned()), + AvailableBlockData::DataColumns(columns) => { + // Each block has its own commitments. For Gloas they live in the bid; for + // Fulu they live inline on the column. Verify per block and let the helper + // pick the right path. + verify_columns_against_block(&self.kzg, available_block.block(), columns)?; + } } } @@ -519,11 +523,6 @@ impl DataAvailabilityChecker { .map_err(AvailabilityCheckError::InvalidBlobs)?; } - if !all_data_columns.is_empty() { - verify_kzg_for_data_column_list(all_data_columns.iter(), &self.kzg) - .map_err(AvailabilityCheckError::InvalidColumn)?; - } - Ok(()) } @@ -605,9 +604,21 @@ impl DataAvailabilityChecker { metrics::inc_counter(&KZG_DATA_COLUMN_RECONSTRUCTION_ATTEMPTS); let timer = metrics::start_timer(&metrics::DATA_AVAILABILITY_RECONSTRUCTION_TIME); + let columns: Vec<_> = verified_data_columns + .into_iter() + .map(|c| c.into_inner()) + .collect(); + // Fulu columns carry their commitments; reconstruction needs the count to drive the + // per-blob recovery loop. + let kzg_commitments = columns + .first() + .and_then(|c| c.kzg_commitments().ok().cloned()) + .ok_or(AvailabilityCheckError::InvalidVariant)?; + let all_data_columns = KzgVerifiedCustodyDataColumn::reconstruct_columns( &self.kzg, - &verified_data_columns, + columns, + &kzg_commitments, &self.spec, ) .map_err(|e| { @@ -676,6 +687,35 @@ impl DataAvailabilityChecker { } } +/// Verify a batch of data columns belonging to a single block, picking the right commitment +/// source for the block's fork (Fulu: inline on column; Gloas: from the embedded payload bid). +fn verify_columns_against_block( + kzg: &Kzg, + block: &SignedBeaconBlock, + columns: &[Arc>], +) -> Result<(), AvailabilityCheckError> { + if columns.is_empty() { + return Ok(()); + } + if block.fork_name_unchecked().gloas_enabled() { + let commitments = block + .message() + .body() + .signed_execution_payload_bid() + .map(|bid| bid.message.blob_kzg_commitments.clone()) + .map_err(|_| { + AvailabilityCheckError::Unexpected( + "Gloas block missing signed_execution_payload_bid".to_string(), + ) + })?; + validate_data_columns_with_commitments(kzg, columns.iter(), commitments.as_ref()) + .map_err(AvailabilityCheckError::InvalidColumn) + } else { + verify_kzg_for_data_column_list(columns.iter(), kzg) + .map_err(AvailabilityCheckError::InvalidColumn) + } +} + /// Helper struct to group data availability checker metrics. pub struct DataAvailabilityCheckerMetrics { pub block_cache_size: usize, @@ -874,10 +914,13 @@ impl AvailableBlock { match &block_data { AvailableBlockData::NoData => { - if columns_required { - return Err(AvailabilityCheckError::MissingCustodyColumns); - } else if blobs_required { - return Err(AvailabilityCheckError::MissingBlobs); + // For Gloas, DA is checked for the PayloadEnvelope, not for the block. + if !block.fork_name_unchecked().gloas_enabled() { + if columns_required { + return Err(AvailabilityCheckError::MissingCustodyColumns); + } else if blobs_required { + return Err(AvailabilityCheckError::MissingBlobs); + } } } AvailableBlockData::Blobs(blobs) => { diff --git a/beacon_node/beacon_chain/src/data_availability_checker/error.rs b/beacon_node/beacon_chain/src/data_availability_checker/error.rs index af3cb72c03..ab69a62985 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/error.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/error.rs @@ -4,6 +4,7 @@ use types::{BeaconStateError, ColumnIndex, Hash256}; #[derive(Debug)] pub enum Error { InvalidBlobs(KzgError), + MissingBid(Hash256), InvalidColumn((Option, KzgError)), ReconstructColumnsError(KzgError), KzgCommitmentMismatch { @@ -23,6 +24,7 @@ pub enum Error { RebuildingStateCaches(BeaconStateError), SlotClockError, InvalidAvailableBlockData, + InvalidVariant, } #[derive(PartialEq, Eq)] @@ -38,6 +40,7 @@ impl Error { match self { Error::SszTypes(_) | Error::MissingBlobs + | Error::MissingBid(_) | Error::MissingCustodyColumns | Error::StoreError(_) | Error::DecodeError(_) @@ -46,7 +49,8 @@ impl Error { | Error::BlockReplayError(_) | Error::RebuildingStateCaches(_) | Error::SlotClockError - | Error::InvalidAvailableBlockData => ErrorCategory::Internal, + | Error::InvalidAvailableBlockData + | Error::InvalidVariant => ErrorCategory::Internal, Error::InvalidBlobs { .. } | Error::InvalidColumn { .. } | Error::ReconstructColumnsError { .. } diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 7d1bba2de9..3034e196b9 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -109,7 +109,7 @@ impl PendingComponents { .unwrap_or(false) } - /// Returns the indices of cached custody columns + /// Returns the indices of cached sampling columns pub fn get_cached_data_columns_indices(&self) -> Vec { self.verified_data_columns .iter() diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 8ea3c792f4..71562b376b 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -3,7 +3,8 @@ use crate::block_verification::{ }; use crate::data_availability_checker::MissingCellsError; use crate::kzg_utils::{ - reconstruct_data_columns, validate_full_data_columns, validate_partial_data_columns, + reconstruct_data_columns, validate_data_columns_with_commitments, validate_full_data_columns, + validate_partial_data_columns, }; use crate::observed_data_sidecars::{ Error as ObservedDataSidecarsError, ObservationKey, ObservationStrategy, Observe, @@ -20,6 +21,7 @@ use std::iter; use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; +use store::DatabaseBlock; use tracing::{debug, instrument}; use tree_hash::TreeHash; use types::data::{ @@ -27,13 +29,16 @@ use types::data::{ PartialDataColumnSidecarError, }; use types::{ - BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, - EthSpec, Hash256, PartialDataColumnSidecarRef, SignedBeaconBlockHeader, Slot, + BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, + KzgCommitment, PartialDataColumnSidecarRef, SignedBeaconBlockHeader, SignedExecutionPayloadBid, + Slot, }; /// An error occurred while validating a gossip data column. #[derive(Debug)] pub enum GossipDataColumnError { + /// Internal logic error: the column sidecar variant does not match the expected fork. + /// This is not a peer fault and should not be used to penalize peers. InvalidVariant, /// There was an error whilst processing the data column. It is not known if it is /// valid or invalid. @@ -82,10 +87,7 @@ pub enum GossipDataColumnError { /// ## Peer scoring /// /// The column is invalid or the peer is faulty. - InvalidSubnetId { - received: u64, - expected: u64, - }, + InvalidSubnetId { received: u64, expected: u64 }, /// The column sidecar is from a slot that is later than the current slot (with respect to the /// gossip clock disparity). /// @@ -118,18 +120,27 @@ pub enum GossipDataColumnError { /// ## Peer scoring /// /// The column is invalid and the peer is faulty. - ProposerIndexMismatch { - sidecar: usize, - local: usize, - }, + ProposerIndexMismatch { sidecar: usize, local: usize }, /// The provided columns's parent block is unknown. /// /// ## Peer scoring /// /// We cannot process the columns without validating its parent, the peer isn't necessarily faulty. - ParentUnknown { - parent_root: Hash256, - slot: Slot, + ParentUnknown { parent_root: Hash256, slot: Slot }, + /// The block referenced by the data column is unknown. + /// + /// ## Peer scoring + /// + /// We cannot process the column without the referenced block, the peer isn't necessarily faulty. + BlockRootUnknown { block_root: Hash256, slot: Slot }, + /// The data column slot does not match its referenced block slot. + /// + /// ## Peer scoring + /// + /// The column sidecar is invalid and the peer is faulty. + BlockSlotMismatch { + block_slot: Slot, + data_column_slot: Slot, }, /// The column conflicts with finalization, no need to propagate. /// @@ -137,9 +148,7 @@ pub enum GossipDataColumnError { /// /// It's unclear if this column is valid, but it conflicts with finality and shouldn't be /// imported. - NotFinalizedDescendant { - block_parent_root: Hash256, - }, + NotFinalizedDescendant { block_parent_root: Hash256 }, /// Invalid kzg commitment inclusion proof /// /// ## Peer scoring @@ -187,10 +196,7 @@ pub enum GossipDataColumnError { /// ## Peer scoring /// /// The column sidecar is invalid and the peer is faulty - InconsistentProofsLength { - cells_len: usize, - proofs_len: usize, - }, + InconsistentProofsLength { cells_len: usize, proofs_len: usize }, /// The number of KZG commitments exceeds the maximum number of blobs allowed for the fork. The /// sidecar is invalid. /// @@ -200,6 +206,12 @@ pub enum GossipDataColumnError { max_blobs_per_block: usize, commitments_len: usize, }, + + /// An internal error occurred. + /// + /// ## Peer scoring + /// This is an internal issue, the peer isn't at fault. + InternalError(String), } impl From for GossipDataColumnError { @@ -302,26 +314,35 @@ impl GossipVerifiedDataColumn subnet_id: DataColumnSubnetId, chain: &BeaconChain, ) -> Result { - match column_sidecar.as_ref() { - DataColumnSidecar::Fulu(c) => { - let header = c.signed_block_header.clone(); + let data_column = match column_sidecar.as_ref() { + DataColumnSidecar::Fulu(column_sidecar_fulu) => { + let header = &column_sidecar_fulu.signed_block_header; // We only process slashing info if the gossip verification failed // since we do not process the data column any further in that case. validate_data_column_sidecar_for_gossip_fulu::( - column_sidecar, + column_sidecar.clone(), subnet_id, chain, ) .map_err(|e| { process_block_slash_info::<_, GossipDataColumnError>( chain, - BlockSlashInfo::from_early_error_data_column(header, e), + BlockSlashInfo::from_early_error_data_column(header.clone(), e), ) - }) + })? } - // TODO(gloas) support gloas data column variant - DataColumnSidecar::Gloas(_) => Err(GossipDataColumnError::InvalidVariant), - } + DataColumnSidecar::Gloas(_) => validate_data_column_sidecar_for_gossip_gloas::( + column_sidecar.clone(), + subnet_id, + chain, + )?, + }; + + Ok(GossipVerifiedDataColumn { + block_root: column_sidecar.block_root(), + data_column, + _phantom: PhantomData, + }) } /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for block production ONLY. @@ -331,7 +352,28 @@ impl GossipVerifiedDataColumn column_sidecar: Arc>, chain: &BeaconChain, ) -> Result { - verify_data_column_sidecar(&column_sidecar, &chain.spec)?; + match column_sidecar.as_ref() { + DataColumnSidecar::Fulu(data_column_fulu) => { + verify_data_column_sidecar_with_commitments_len( + &column_sidecar, + data_column_fulu.kzg_commitments.len(), + &chain.spec, + )?; + } + DataColumnSidecar::Gloas(_) => { + let bid = load_gloas_payload_bid(column_sidecar.block_root(), chain)?.ok_or( + GossipDataColumnError::BlockRootUnknown { + block_root: column_sidecar.block_root(), + slot: column_sidecar.slot(), + }, + )?; + verify_data_column_sidecar_with_commitments_len( + &column_sidecar, + bid.message.blob_kzg_commitments.len(), + &chain.spec, + )?; + } + } // Check if the data column is already in the DA checker cache. This happens when data columns // are made available through the `engine_getBlobs` method. If it exists in the cache, we know @@ -340,28 +382,20 @@ impl GossipVerifiedDataColumn // In this case, we should accept it for gossip propagation. verify_is_unknown_sidecar(chain, &column_sidecar)?; - match chain - .data_availability_checker - .missing_cells_for_column_sidecar(&column_sidecar) - { - Ok(Some(_)) => Ok(Self { + // Check if this column contains any cells not already in the cache. If all cells are + // already cached, reject as `PriorKnownUnpublished` to avoid redundant processing. + match missing_cells_for_column_sidecar(chain, &column_sidecar)? { + Some(_) => Ok(Self { block_root: column_sidecar.block_root(), data_column: KzgVerifiedDataColumn::from_execution_verified(column_sidecar), _phantom: Default::default(), }), - Ok(None) => { - // Observe this data column so we don't process it again. + None => { if O::observe() { observe_gossip_data_column(&column_sidecar, chain)?; } Err(GossipDataColumnError::PriorKnownUnpublished) } - Err(MissingCellsError::MismatchesCachedColumn) => { - Err(GossipDataColumnError::MismatchesCachedColumn) - } - Err(MissingCellsError::UnexpectedError(_)) => { - todo!("handle unexpected error") - } } } @@ -430,12 +464,30 @@ impl KzgVerifiedDataColumn { data_columns: Vec>>, kzg: &Kzg, ) -> Result, (Option, KzgError)> { + let seen_timestamp = timestamp_now(); verify_kzg_for_data_column_list(data_columns.iter(), kzg)?; Ok(data_columns .into_iter() .map(|column| Self { data: column, - seen_timestamp: timestamp_now(), + seen_timestamp, + }) + .collect()) + } + + pub fn from_batch_with_scoring_and_commitments( + data_columns: Vec>>, + kzg_commitments: &[KzgCommitment], + kzg: &Kzg, + ) -> Result, (Option, KzgError)> { + let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_BATCH_TIMES); + let seen_timestamp = timestamp_now(); + validate_data_columns_with_commitments(kzg, data_columns.iter(), kzg_commitments)?; + Ok(data_columns + .into_iter() + .map(|column| Self { + data: column, + seen_timestamp, }) .collect()) } @@ -635,17 +687,12 @@ impl KzgVerifiedCustodyDataColumn { pub fn reconstruct_columns( kzg: &Kzg, - partial_set_of_columns: &[Self], + partial_set_of_columns: Vec>>, + kzg_commitments: &[KzgCommitment], spec: &ChainSpec, ) -> Result>, KzgError> { - let all_data_columns = reconstruct_data_columns( - kzg, - partial_set_of_columns - .iter() - .map(|d| d.clone_arc()) - .collect::>(), - spec, - )?; + let all_data_columns = + reconstruct_data_columns(kzg, partial_set_of_columns, kzg_commitments, spec)?; let seen_timestamp = timestamp_now(); @@ -860,6 +907,26 @@ pub fn verify_kzg_for_data_column( }) } +#[instrument(skip_all, level = "debug")] +pub fn verify_kzg_for_data_column_with_commitments( + data_column: Arc>, + cells_to_verify: PartialDataColumnSidecarRef, + kzg_commitments: &[KzgCommitment], + kzg: &Kzg, + seen_timestamp: Duration, +) -> Result, (Option, KzgError)> { + let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); + validate_partial_data_columns( + kzg, + iter::once((*data_column.index(), cells_to_verify)), + kzg_commitments, + )?; + Ok(KzgVerifiedDataColumn { + data: data_column, + seen_timestamp, + }) +} + /// Complete kzg verification for a `VerifiablePartialDataColumn`. /// /// Returns an error if the kzg verification check fails. @@ -910,13 +977,18 @@ pub fn validate_data_column_sidecar_for_gossip_fulu>, subnet: DataColumnSubnetId, chain: &BeaconChain, -) -> Result, GossipDataColumnError> { +) -> Result, GossipDataColumnError> { let DataColumnSidecar::Fulu(data_column_fulu) = data_column.as_ref() else { return Err(GossipDataColumnError::InvalidVariant); }; let column_slot = data_column.slot(); - verify_data_column_sidecar(&data_column, &chain.spec)?; + + verify_data_column_sidecar_with_commitments_len( + &data_column, + data_column_fulu.kzg_commitments.len(), + &chain.spec, + )?; verify_index_matches_subnet(&data_column, subnet, &chain.spec)?; verify_sidecar_not_from_future_slot(chain, column_slot)?; verify_slot_greater_than_latest_finalized_slot(chain, column_slot)?; @@ -935,7 +1007,10 @@ pub fn validate_data_column_sidecar_for_gossip_fulu { GossipDataColumnError::MismatchesCachedColumn } - MissingCellsError::UnexpectedError(_) => todo!("handle unexpected error"), + MissingCellsError::UnexpectedError(e) => GossipDataColumnError::InternalError(format!( + "An unexpected error occurred while validating fulu data columns. {:?}", + e + )), })? else { // Observe this data column so we don't process it again. @@ -945,7 +1020,7 @@ pub fn validate_data_column_sidecar_for_gossip_fulu( + data_column: Arc>, + subnet: DataColumnSubnetId, + chain: &BeaconChain, +) -> Result, GossipDataColumnError> { + let DataColumnSidecar::Gloas(_) = data_column.as_ref() else { + return Err(GossipDataColumnError::InvalidVariant); + }; + + let column_slot = data_column.slot(); + + if *data_column.index() >= T::EthSpec::number_of_columns() as u64 { + return Err(GossipDataColumnError::InvalidColumnIndex( + *data_column.index(), + )); + } + verify_index_matches_subnet(&data_column, subnet, &chain.spec)?; + verify_sidecar_not_from_future_slot(chain, column_slot)?; + verify_slot_greater_than_latest_finalized_slot(chain, column_slot)?; + verify_is_unknown_sidecar(chain, &data_column)?; + + let bid = load_gloas_payload_bid(data_column.block_root(), chain)?.ok_or( + GossipDataColumnError::BlockRootUnknown { + block_root: data_column.block_root(), + slot: column_slot, + }, + )?; + if bid.message.slot != column_slot { + return Err(GossipDataColumnError::BlockSlotMismatch { + block_slot: bid.message.slot, + data_column_slot: column_slot, + }); + } + let kzg_commitments = &bid.message.blob_kzg_commitments; + verify_data_column_sidecar_with_commitments_len( + &data_column, + kzg_commitments.len(), + &chain.spec, + )?; + + let Some(cells_to_kzg_verify) = missing_cells_for_column_sidecar(chain, &data_column)? else { + // Observe this data column so we don't process it again. + if O::observe() { + observe_gossip_data_column(&data_column, chain)?; + } + return Err(GossipDataColumnError::PriorKnownUnpublished); + }; + + let kzg = &chain.kzg; + let seen_timestamp = chain.slot_clock.now_duration().unwrap_or_default(); + let kzg_verified = verify_kzg_for_data_column_with_commitments( + data_column.clone(), + cells_to_kzg_verify, + kzg_commitments.as_ref(), + kzg, + seen_timestamp, + ) + .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; + + if O::observe() { + observe_gossip_data_column(&data_column, chain)?; + } + + Ok(kzg_verified) } #[instrument(skip_all, level = "debug")] @@ -1115,9 +1259,9 @@ pub enum PartialColumnVerificationResult { Err(GossipPartialDataColumnError), } -/// Verify if the data column sidecar is valid. -fn verify_data_column_sidecar( +fn verify_data_column_sidecar_with_commitments_len( data_column: &DataColumnSidecar, + commitments_len: usize, spec: &ChainSpec, ) -> Result<(), GossipDataColumnError> { if *data_column.index() >= E::number_of_columns() as u64 { @@ -1126,12 +1270,6 @@ fn verify_data_column_sidecar( )); } - // TODO(gloas): implement Gloas verification that takes kzg_commitments from block as parameter - let commitments_len = match data_column { - DataColumnSidecar::Fulu(dc) => dc.kzg_commitments.len(), - DataColumnSidecar::Gloas(_) => return Err(GossipDataColumnError::InvalidVariant), - }; - if commitments_len == 0 { return Err(GossipDataColumnError::UnexpectedDataColumn); } @@ -1164,6 +1302,93 @@ fn verify_data_column_sidecar( Ok(()) } +/// Loads the Gloas payload bid for `block_root` from the `pending_payload_cache`, the +/// `early_attester_cache`, or the on-disk store (in that order). +/// +/// TODO(gloas): the store fallback is a synchronous disk read and several callers run inside +/// `async` gossip / RPC validation paths. Move the disk path off the async runtime (e.g. behind +/// `spawn_blocking`) — or restructure callers to fetch the bid before entering async — once the +/// gossip pipeline is reworked for Gloas. The cache and early-attester paths are short +/// in-memory locks and acceptable as-is. +pub(crate) fn load_gloas_payload_bid( + block_root: Hash256, + chain: &BeaconChain, +) -> Result>>, BeaconChainError> { + if let Some(bid) = chain.pending_payload_cache.get_bid(&block_root) { + return Ok(Some(bid)); + } + + let bid = if let Some(block) = chain.early_attester_cache.get_block(block_root) { + Arc::new( + block + .message() + .body() + .signed_execution_payload_bid() + .map_err(BeaconChainError::BeaconStateError)? + .clone(), + ) + } else { + match chain + .store + .try_get_full_block(&block_root) + .map_err(BeaconChainError::DBError)? + { + Some(DatabaseBlock::Full(block)) => Arc::new( + block + .message() + .body() + .signed_execution_payload_bid() + .map_err(BeaconChainError::BeaconStateError)? + .clone(), + ), + Some(DatabaseBlock::Blinded(block)) => Arc::new( + block + .message() + .body() + .signed_execution_payload_bid() + .map_err(BeaconChainError::BeaconStateError)? + .clone(), + ), + None => { + return Ok(None); + } + } + }; + + chain + .pending_payload_cache + .insert_bid(block_root, bid.clone()); + + Ok(Some(bid)) +} + +fn missing_cells_for_column_sidecar<'a, T: BeaconChainTypes>( + chain: &'_ BeaconChain, + data_column: &'a DataColumnSidecar, +) -> Result>, GossipDataColumnError> { + let result = if chain + .spec + .fork_name_at_slot::(data_column.slot()) + .gloas_enabled() + { + chain + .pending_payload_cache + .missing_cells_for_column_sidecar(data_column) + } else { + chain + .data_availability_checker + .missing_cells_for_column_sidecar(data_column) + }; + + result.map_err(|err| match err { + MissingCellsError::MismatchesCachedColumn => GossipDataColumnError::MismatchesCachedColumn, + MissingCellsError::UnexpectedError(e) => GossipDataColumnError::InternalError(format!( + "An unexpected error occurred while calculating missing partial cells {:?}", + e + )), + }) +} + /// Verify that `column_sidecar` is not yet known, i.e. this is the first time `column_sidecar` has been received for the tuple: /// `(block_header.slot, block_header.proposer_index, column_sidecar.index)` fn verify_is_unknown_sidecar( @@ -1187,10 +1412,15 @@ fn verify_is_unknown_sidecar( } fn verify_column_inclusion_proof( - data_column: &DataColumnSidecarFulu, + data_column: &DataColumnSidecar, ) -> Result<(), GossipDataColumnError> { let _timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_INCLUSION_PROOF_VERIFICATION); - if !data_column.verify_inclusion_proof() { + + let DataColumnSidecar::Fulu(data_column_fulu) = data_column else { + return Err(GossipDataColumnError::InvalidVariant); + }; + + if !data_column_fulu.verify_inclusion_proof() { return Err(GossipDataColumnError::InvalidInclusionProof); } @@ -1447,7 +1677,7 @@ mod test { let verify_fn = |column_sidecar: DataColumnSidecar| { let col_index = *column_sidecar.index(); validate_data_column_sidecar_for_gossip_fulu::<_, Observe>( - column_sidecar.into(), + Arc::new(column_sidecar), DataColumnSubnetId::from_column_index(col_index, &harness.spec), &harness.chain, ) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index c94fb036f8..f5ba647fce 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -119,10 +119,12 @@ impl FetchBlobsBeaconAdapter { .cached_blob_indexes(block_root) } - pub(crate) fn cached_data_column_indexes(&self, block_root: &Hash256) -> Option> { - self.chain - .data_availability_checker - .cached_data_column_indexes(block_root) + pub(crate) fn cached_data_column_indexes( + &self, + block_root: &Hash256, + slot: Slot, + ) -> Option> { + self.chain.cached_data_column_indexes(block_root, slot) } pub(crate) async fn process_engine_blobs( diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index f7b4b8a29e..351e35666a 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -445,7 +445,7 @@ async fn compute_custody_columns_to_import( // Only consider columns that are not already known to data availability. if let Some(known_columns) = - chain_adapter_cloned.cached_data_column_indexes(&block_root) + chain_adapter_cloned.cached_data_column_indexes(&block_root, header.slot()) { custody_columns.retain(|col| !known_columns.contains(&col.index())); if custody_columns.is_empty() { diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index ef282a3eaa..37d40f3a27 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -199,7 +199,7 @@ mod get_blobs_v2 { .returning(|_| None); mock_adapter .expect_cached_data_column_indexes() - .returning(|_| None); + .returning(|_, _| None); mock_process_engine_blobs_result( &mut mock_adapter, Ok(AvailabilityProcessingStatus::Imported(block_root)), diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index b05a896777..bc803efe93 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -111,6 +111,57 @@ pub fn validate_full_data_columns<'a, E: EthSpec>( kzg.verify_cell_proof_batch(&cells, &proofs, column_indices, &commitments) } +/// Validate a batch of full `DataColumnSidecar`s against commitments supplied out-of-band. +/// +/// Gloas sidecars do not carry commitments. Their commitments come from the block's +/// `ExecutionPayloadBid`. +pub fn validate_data_columns_with_commitments<'a, E: EthSpec>( + kzg: &Kzg, + data_column_iter: impl Iterator>>, + kzg_commitments: &[KzgCommitment], +) -> Result<(), (Option, KzgError)> { + let mut cells = Vec::new(); + let mut proofs = Vec::new(); + let mut column_indices = Vec::new(); + let mut commitments = Vec::new(); + + for data_column in data_column_iter { + let col_index = *data_column.index(); + + if data_column.column().is_empty() { + return Err((Some(col_index), KzgError::KzgVerificationFailed)); + } + + for cell in data_column.column() { + cells.push(ssz_cell_to_crypto_cell::(cell).map_err(|e| (Some(col_index), e))?); + column_indices.push(col_index); + } + + for &proof in data_column.kzg_proofs() { + proofs.push(proof.0); + } + + for &commitment in kzg_commitments { + commitments.push(commitment.0); + } + + let expected_len = column_indices.len(); + + // We make this check at each iteration so that the error is attributable to a specific column. + if cells.len() != expected_len + || proofs.len() != expected_len + || commitments.len() != expected_len + { + return Err(( + Some(col_index), + KzgError::InconsistentArrayLength("Invalid data column".to_string()), + )); + } + } + + kzg.verify_cell_proof_batch(&cells, &proofs, column_indices, &commitments) +} + /// Validate a batch of partial `VerifiablePartialDataColumn`s. /// /// Partial columns may have missing cells, indicated by a bitmap. We only verify present cells. @@ -618,19 +669,17 @@ pub fn reconstruct_blobs( // Sort data columns by index to ensure ascending order for KZG operations data_columns.sort_unstable_by_key(|dc| *dc.index()); - let first_data_column = data_columns - .first() - .ok_or("data_columns should have at least one element".to_string())?; + if data_columns.is_empty() { + return Err("data_columns should have at least one element".to_string()); + } let blob_indices: Vec = match blob_indices_opt { Some(indices) => indices.into_iter().map(|i| i as usize).collect(), None => { - // TODO(gloas): support blob reconstruction for Gloas - // https://github.com/sigp/lighthouse/issues/7413 - let num_of_blobs = first_data_column - .kzg_commitments() - .map_err(|_| "Gloas blob reconstruction not yet supported".to_string())? - .len(); + let num_of_blobs = signed_block + .message() + .blob_kzg_commitments_len() + .ok_or_else(|| "Block does not have blob KZG commitments".to_string())?; (0..num_of_blobs).collect() } }; @@ -689,9 +738,14 @@ pub fn reconstruct_blobs( } /// Reconstruct all data columns from a subset of data column sidecars (requires at least 50%). +/// +/// `kzg_commitments` are the commitments for the underlying blobs. For Fulu they live in the +/// column itself; for Gloas they live in the bid. We take them as a parameter so this function +/// works for both forks (mirroring `validate_data_columns_with_commitments`). pub fn reconstruct_data_columns( kzg: &Kzg, mut data_columns: Vec>>, + kzg_commitments: &[KzgCommitment], spec: &ChainSpec, ) -> Result, KzgError> { // Sort data columns by index to ensure ascending order for KZG operations @@ -703,16 +757,7 @@ pub fn reconstruct_data_columns( "data_columns should have at least one element".to_string(), ))?; - // TODO(gloas): support data column reconstruction for Gloas - // https://github.com/sigp/lighthouse/issues/7413 - let num_of_blobs = first_data_column - .kzg_commitments() - .map_err(|_| { - KzgError::InconsistentArrayLength( - "Gloas data column reconstruction not yet supported".to_string(), - ) - })? - .len(); + let num_of_blobs = kzg_commitments.len(); let blob_cells_and_proofs_vec = (0..num_of_blobs) .into_par_iter() @@ -757,8 +802,9 @@ pub fn reconstruct_data_columns( #[cfg(test)] mod test { use crate::kzg_utils::{ - blobs_to_data_column_sidecars, blobs_to_data_column_sidecars_gloas, reconstruct_blobs, - reconstruct_data_columns, validate_full_data_columns, + blob_to_kzg_commitment, blobs_to_data_column_sidecars, blobs_to_data_column_sidecars_gloas, + reconstruct_blobs, reconstruct_data_columns, validate_data_columns_with_commitments, + validate_full_data_columns, }; use bls::Signature; use eth2::types::BlobsBundle; @@ -787,9 +833,13 @@ mod test { test_reconstruct_blobs_from_data_columns_unordered(&kzg, &fulu_spec); test_validate_data_columns(&kzg, &fulu_spec); + test_validate_data_columns_with_commitments(&kzg, &fulu_spec); + let gloas_spec = ForkName::Gloas.make_genesis_spec(E::default_spec()); test_build_data_columns_gloas(&kzg, &gloas_spec); test_build_data_columns_gloas_empty(&kzg, &gloas_spec); + test_reconstruct_data_columns_gloas(&kzg, &gloas_spec); + test_validate_data_columns_with_commitments_gloas(&kzg, &gloas_spec); } #[track_caller] @@ -806,6 +856,63 @@ mod test { assert!(result.is_ok()); } + #[track_caller] + fn test_validate_data_columns_with_commitments(kzg: &Kzg, spec: &ChainSpec) { + let num_of_blobs = 2; + let (signed_block, blobs, proofs) = + create_test_fulu_block_and_blobs::(num_of_blobs, spec); + let blob_refs = blobs.iter().collect::>(); + let column_sidecars = + blobs_to_data_column_sidecars(&blob_refs, proofs.to_vec(), &signed_block, kzg, spec) + .unwrap(); + + let commitments = signed_block + .message() + .body() + .blob_kzg_commitments() + .unwrap(); + + let result = + validate_data_columns_with_commitments(kzg, column_sidecars.iter(), commitments); + assert!(result.is_ok()); + + // Verify that wrong commitments cause a failure + let bad_commitments = vec![KzgCommitment::empty_for_testing(); num_of_blobs]; + let result = + validate_data_columns_with_commitments(kzg, column_sidecars.iter(), &bad_commitments); + assert!(result.is_err()); + } + + #[track_caller] + fn test_validate_data_columns_with_commitments_gloas(kzg: &Kzg, spec: &ChainSpec) { + let num_of_blobs = 2; + let (blobs, _proofs) = create_test_gloas_blobs::(num_of_blobs); + let blob_refs: Vec<_> = blobs.iter().collect(); + let column_sidecars = blobs_to_data_column_sidecars_gloas::( + &blob_refs, + Hash256::random(), + Slot::new(0), + kzg, + spec, + ) + .unwrap(); + + let commitments: Vec = blobs + .iter() + .map(|blob| blob_to_kzg_commitment::(kzg, blob).unwrap()) + .collect(); + + let result = + validate_data_columns_with_commitments(kzg, column_sidecars.iter(), &commitments); + assert!(result.is_ok()); + + // Verify that wrong commitments cause a failure + let bad_commitments = vec![KzgCommitment::empty_for_testing(); num_of_blobs]; + let result = + validate_data_columns_with_commitments(kzg, column_sidecars.iter(), &bad_commitments); + assert!(result.is_err()); + } + #[track_caller] fn test_build_data_columns_empty(kzg: &Kzg, spec: &ChainSpec) { let num_of_blobs = 0; @@ -918,11 +1025,18 @@ mod test { let column_sidecars = blobs_to_data_column_sidecars(&blob_refs, proofs.to_vec(), &signed_block, kzg, spec) .unwrap(); + let commitments = signed_block + .message() + .body() + .blob_kzg_commitments() + .unwrap() + .clone(); // Now reconstruct let reconstructed_columns = reconstruct_data_columns( kzg, column_sidecars.iter().as_slice()[0..column_sidecars.len() / 2].to_vec(), + &commitments, spec, ) .unwrap(); @@ -942,12 +1056,49 @@ mod test { let column_sidecars = blobs_to_data_column_sidecars(&blob_refs, proofs.to_vec(), &signed_block, kzg, spec) .unwrap(); + let commitments = signed_block + .message() + .body() + .blob_kzg_commitments() + .unwrap() + .clone(); // Test reconstruction with columns in reverse order (non-ascending) let mut subset_columns: Vec<_> = column_sidecars.iter().as_slice()[0..column_sidecars.len() / 2].to_vec(); subset_columns.reverse(); // This would fail without proper sorting in reconstruct_data_columns - let reconstructed_columns = reconstruct_data_columns(kzg, subset_columns, spec).unwrap(); + let reconstructed_columns = + reconstruct_data_columns(kzg, subset_columns, &commitments, spec).unwrap(); + + for i in 0..E::number_of_columns() { + assert_eq!(reconstructed_columns.get(i), column_sidecars.get(i), "{i}"); + } + } + + /// Reconstruct a full Gloas column set from a 50% subset and assert the recovered sidecars + /// match the originals. Commitments come from the bid (here mocked via the same + /// `KzgCommitments` used to build the columns) since Gloas columns don't carry them. + #[track_caller] + fn test_reconstruct_data_columns_gloas(kzg: &Kzg, spec: &ChainSpec) { + let num_of_blobs = 2; + let (blobs, _proofs) = create_test_gloas_blobs::(num_of_blobs); + let blob_refs: Vec<_> = blobs.iter().collect(); + let column_sidecars = blobs_to_data_column_sidecars_gloas::( + &blob_refs, + Hash256::random(), + Slot::new(0), + kzg, + spec, + ) + .unwrap(); + + let commitments = + KzgCommitments::::new(vec![KzgCommitment::empty_for_testing(); num_of_blobs]) + .unwrap(); + + let subset = column_sidecars[..column_sidecars.len() / 2].to_vec(); + let reconstructed_columns = + reconstruct_data_columns(kzg, subset, &commitments, spec).unwrap(); for i in 0..E::number_of_columns() { assert_eq!(reconstructed_columns.get(i), column_sidecars.get(i), "{i}"); diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index d70fc1b3ec..804268a613 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -48,6 +48,7 @@ pub mod payload_attestation_verification; pub mod payload_bid_verification; pub mod payload_envelope_streamer; pub mod payload_envelope_verification; +pub mod pending_payload_cache; pub mod pending_payload_envelopes; pub mod persisted_beacon_chain; pub mod persisted_custody; diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 43c3337bc9..df1b005820 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -2043,6 +2043,12 @@ pub static DATA_AVAILABILITY_OVERFLOW_MEMORY_BLOCK_CACHE_SIZE: LazyLock> = LazyLock::new(|| { + try_create_int_gauge( + "pending_payload_cache_size", + "Number of entries in the pending payload availability cache.", + ) +}); pub static DATA_AVAILABILITY_RECONSTRUCTION_TIME: LazyLock> = LazyLock::new(|| { try_create_histogram( @@ -2150,6 +2156,10 @@ pub fn scrape_for_metrics(beacon_chain: &BeaconChain) { &DATA_AVAILABILITY_OVERFLOW_MEMORY_BLOCK_CACHE_SIZE, da_checker_metrics.block_cache_size, ); + set_gauge_by_usize( + &PENDING_PAYLOAD_CACHE_SIZE, + beacon_chain.pending_payload_cache.cache_size(), + ); if let Some((size, num_lookups)) = beacon_chain.pre_finalization_block_cache.metrics() { set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_CACHE_SIZE, size); diff --git a/beacon_node/beacon_chain/src/payload_envelope_verification/execution_pending_envelope.rs b/beacon_node/beacon_chain/src/payload_envelope_verification/execution_pending_envelope.rs index 4b8e7347cc..b678bdbaea 100644 --- a/beacon_node/beacon_chain/src/payload_envelope_verification/execution_pending_envelope.rs +++ b/beacon_node/beacon_chain/src/payload_envelope_verification/execution_pending_envelope.rs @@ -1,23 +1,22 @@ -use std::sync::Arc; - +use bls::Hash256; use slot_clock::SlotClock; use state_processing::{VerifySignatures, envelope_processing::verify_execution_payload_envelope}; -use types::EthSpec; +use std::sync::Arc; +use types::{EthSpec, SignedExecutionPayloadEnvelope}; use crate::{ BeaconChain, BeaconChainError, BeaconChainTypes, NotifyExecutionLayer, PayloadVerificationOutcome, block_verification::PayloadVerificationHandle, payload_envelope_verification::{ - EnvelopeError, EnvelopeImportData, MaybeAvailableEnvelope, - gossip_verified_envelope::GossipVerifiedEnvelope, load_snapshot_from_state_root, - payload_notifier::PayloadNotifier, + EnvelopeError, gossip_verified_envelope::GossipVerifiedEnvelope, + load_snapshot_from_state_root, payload_notifier::PayloadNotifier, }, }; pub struct ExecutionPendingEnvelope { - pub signed_envelope: MaybeAvailableEnvelope, - pub import_data: EnvelopeImportData, + pub signed_envelope: Arc>, + pub block_root: Hash256, pub payload_verification_handle: PayloadVerificationHandle, } @@ -29,7 +28,6 @@ impl GossipVerifiedEnvelope { ) -> Result, EnvelopeError> { let signed_envelope = self.signed_envelope; let envelope = &signed_envelope.message; - let payload = &envelope.payload; // Define a future that will verify the execution payload with an execution engine. // @@ -87,14 +85,8 @@ impl GossipVerifiedEnvelope { )?; Ok(ExecutionPendingEnvelope { - signed_envelope: MaybeAvailableEnvelope::AvailabilityPending { - block_hash: payload.block_hash, - envelope: signed_envelope, - }, - import_data: EnvelopeImportData { - block_root, - _phantom: Default::default(), - }, + signed_envelope, + block_root, payload_verification_handle, }) } diff --git a/beacon_node/beacon_chain/src/payload_envelope_verification/import.rs b/beacon_node/beacon_chain/src/payload_envelope_verification/import.rs index b40e8337fb..73ddb43273 100644 --- a/beacon_node/beacon_chain/src/payload_envelope_verification/import.rs +++ b/beacon_node/beacon_chain/src/payload_envelope_verification/import.rs @@ -9,13 +9,18 @@ use tracing::{debug, error, info, info_span, instrument, warn}; use types::{BlockImportSource, Hash256, SignedExecutionPayloadEnvelope}; use super::{ - AvailableEnvelope, AvailableExecutedEnvelope, EnvelopeError, EnvelopeImportData, - ExecutedEnvelope, gossip_verified_envelope::GossipVerifiedEnvelope, + AvailableEnvelope, AvailableExecutedEnvelope, EnvelopeError, + gossip_verified_envelope::GossipVerifiedEnvelope, }; use crate::{ - AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, - NotifyExecutionLayer, block_verification_types::AvailableBlockData, metrics, - payload_envelope_verification::ExecutionPendingEnvelope, validator_monitor::get_slot_delay_ms, + AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, + NotifyExecutionLayer, + block_verification_types::AvailableBlockData, + metrics, + payload_envelope_verification::{ + AvailabilityPendingExecutedEnvelope, ExecutionPendingEnvelope, + }, + validator_monitor::get_slot_delay_ms, }; const ENVELOPE_METRICS_CACHE_SLOT_LIMIT: u32 = 64; @@ -28,13 +33,13 @@ impl BeaconChain { /// /// Returns an `Err` if the given payload envelope was invalid, or an error was encountered during /// verification. - #[instrument(skip_all, fields(block_root = ?block_root, block_source = %block_source))] + #[instrument(skip_all, fields(block_root = ?block_root, envelope_source = %envelope_source))] pub async fn process_execution_payload_envelope( self: &Arc, block_root: Hash256, unverified_envelope: GossipVerifiedEnvelope, notify_execution_layer: NotifyExecutionLayer, - block_source: BlockImportSource, + envelope_source: BlockImportSource, publish_fn: impl FnOnce() -> Result<(), EnvelopeError>, ) -> Result { let block_slot = unverified_envelope.signed_envelope.slot(); @@ -50,7 +55,7 @@ impl BeaconChain { ); } - // TODO(gloas) insert the pre-executed envelope into some type of cache. + // TODO(gloas) insert the pre-executed envelope into some type of cache? let _full_timer = metrics::start_timer(&metrics::ENVELOPE_PROCESSING_TIMES); @@ -79,12 +84,11 @@ impl BeaconChain { let executed_envelope = chain .into_executed_payload_envelope(execution_pending) .await - .inspect_err(|_| { - // TODO(gloas) If the envelope fails execution for whatever reason (e.g. engine offline), - // and we keep it in the cache, then the node will NOT perform lookup and - // reprocess this block until the block is evicted from DA checker, causing the - // chain to get stuck temporarily if the block is canonical. Therefore we remove - // it from the cache if execution fails. + .map_err(|error| match error { + BlockError::ExecutionPayloadError(error) => { + EnvelopeError::ExecutionPayloadError(error) + } + error => EnvelopeError::ImportError(error), })?; // Record the time it took to wait for execution layer verification. @@ -94,15 +98,9 @@ impl BeaconChain { .set_time_executed(block_root, block_slot, timestamp); } - match executed_envelope { - ExecutedEnvelope::Available(envelope) => { - self.import_available_execution_payload_envelope(Box::new(envelope)) - .await - } - ExecutedEnvelope::AvailabilityPending() => Err(EnvelopeError::InternalError( - "Pending payload envelope not yet implemented".to_owned(), - )), - } + self.check_envelope_availability_and_import(executed_envelope) + .await + .map_err(EnvelopeError::ImportError) }; // Verify and import the payload envelope. @@ -112,7 +110,7 @@ impl BeaconChain { info!( ?block_root, %block_slot, - source = %block_source, + source = %envelope_source, "Execution payload envelope imported" ); @@ -138,6 +136,14 @@ impl BeaconChain { } Err(EnvelopeError::BeaconChainError(e)) } + Err(EnvelopeError::ImportError(BlockError::BeaconChainError(e))) => { + if matches!(e.as_ref(), BeaconChainError::TokioJoin(_)) { + debug!(error = ?e, "Envelope processing cancelled"); + } else { + warn!(error = ?e, "Execution payload envelope rejected"); + } + Err(EnvelopeError::ImportError(BlockError::BeaconChainError(e))) + } Err(other) => { warn!( reason = other.to_string(), @@ -148,6 +154,19 @@ impl BeaconChain { } } + #[instrument(skip_all)] + async fn check_envelope_availability_and_import( + self: &Arc, + envelope: AvailabilityPendingExecutedEnvelope, + ) -> Result { + let slot = envelope.envelope.slot(); + let availability = self + .pending_payload_cache + .put_executed_payload_envelope(envelope)?; + self.process_payload_envelope_availability(slot, availability, || Ok(())) + .await + } + /// Accepts a fully-verified payload envelope and awaits on its payload verification handle to /// get a fully `ExecutedEnvelope`. /// @@ -156,10 +175,10 @@ impl BeaconChain { async fn into_executed_payload_envelope( self: Arc, pending_envelope: ExecutionPendingEnvelope, - ) -> Result, EnvelopeError> { + ) -> Result, BlockError> { let ExecutionPendingEnvelope { signed_envelope, - import_data, + block_root, payload_verification_handle, } = pending_envelope; @@ -173,16 +192,13 @@ impl BeaconChain { .payload_verification_status .is_optimistic() { - return Err(EnvelopeError::OptimisticSyncNotSupported { - block_root: import_data.block_root, - }); + return Err(BlockError::OptimisticSyncNotSupported { block_root }); } - Ok(ExecutedEnvelope::new( + Ok(AvailabilityPendingExecutedEnvelope::new( signed_envelope, - import_data, + block_root, payload_verification_outcome, - self.spec.clone(), )) } @@ -190,18 +206,13 @@ impl BeaconChain { pub async fn import_available_execution_payload_envelope( self: &Arc, envelope: Box>, - ) -> Result { + ) -> Result { let AvailableExecutedEnvelope { envelope, - import_data, + block_root, payload_verification_outcome, } = *envelope; - let EnvelopeImportData { - block_root, - _phantom, - } = import_data; - let block_root = { let chain = self.clone(); self.spawn_blocking_handle( @@ -232,13 +243,13 @@ impl BeaconChain { signed_envelope: AvailableEnvelope, block_root: Hash256, payload_verification_status: PayloadVerificationStatus, - ) -> Result { + ) -> Result { // Everything in this initial section is on the hot path for processing the envelope. // Take an upgradable read lock on fork choice so we can check if this block has already // been imported. We don't want to repeat work importing a block that is already imported. let fork_choice_reader = self.canonical_head.fork_choice_upgradable_read_lock(); if !fork_choice_reader.contains_block(&block_root) { - return Err(EnvelopeError::BlockRootUnknown { block_root }); + return Err(BlockError::EnvelopeBlockRootUnknown(block_root)); } // TODO(gloas) add defensive check to see if payload envelope is already in fork choice @@ -253,7 +264,7 @@ impl BeaconChain { // node which can be eligible for head. fork_choice .on_valid_payload_envelope_received(block_root) - .map_err(|e| EnvelopeError::InternalError(format!("{e:?}")))?; + .map_err(|e| BlockError::InternalError(format!("{e:?}")))?; // TODO(gloas) emit SSE event if the payload became the new head payload diff --git a/beacon_node/beacon_chain/src/payload_envelope_verification/mod.rs b/beacon_node/beacon_chain/src/payload_envelope_verification/mod.rs index b153a3cd6a..a1e4e34eb6 100644 --- a/beacon_node/beacon_chain/src/payload_envelope_verification/mod.rs +++ b/beacon_node/beacon_chain/src/payload_envelope_verification/mod.rs @@ -18,14 +18,13 @@ //! //! ``` -use std::marker::PhantomData; +use state_processing::envelope_processing::EnvelopeProcessingError; use std::sync::Arc; - -use state_processing::{BlockProcessingError, envelope_processing::EnvelopeProcessingError}; use store::Error as DBError; +use strum::AsRefStr; use tracing::instrument; use types::{ - BeaconState, BeaconStateError, ChainSpec, DataColumnSidecarList, EthSpec, ExecutionBlockHash, + BeaconState, BeaconStateError, DataColumnSidecarList, EthSpec, ExecutionBlockHash, ExecutionPayloadEnvelope, Hash256, SignedExecutionPayloadEnvelope, Slot, }; @@ -41,39 +40,18 @@ mod payload_notifier; pub use execution_pending_envelope::ExecutionPendingEnvelope; -// TODO(gloas): could remove this type completely, or remove the generic -#[derive(PartialEq)] -pub struct EnvelopeImportData { - pub block_root: Hash256, - _phantom: PhantomData, -} - #[derive(Debug)] -#[allow(dead_code)] pub struct AvailableEnvelope { - execution_block_hash: ExecutionBlockHash, envelope: Arc>, - columns: DataColumnSidecarList, - /// Timestamp at which this envelope first became available (UNIX timestamp, time since 1970). - columns_available_timestamp: Option, - pub spec: Arc, + pub columns: DataColumnSidecarList, } impl AvailableEnvelope { pub fn new( - execution_block_hash: ExecutionBlockHash, envelope: Arc>, columns: DataColumnSidecarList, - columns_available_timestamp: Option, - spec: Arc, ) -> Self { - Self { - execution_block_hash, - envelope, - columns, - columns_available_timestamp, - spec, - } + Self { envelope, columns } } pub fn message(&self) -> &ExecutionPayloadEnvelope { @@ -94,14 +72,6 @@ impl AvailableEnvelope { } } -pub enum MaybeAvailableEnvelope { - Available(AvailableEnvelope), - AvailabilityPending { - block_hash: ExecutionBlockHash, - envelope: Arc>, - }, -} - /// This snapshot is to be used for verifying a payload envelope. #[derive(Debug, Clone)] pub struct EnvelopeProcessingSnapshot { @@ -111,46 +81,25 @@ pub struct EnvelopeProcessingSnapshot { pub beacon_block_root: Hash256, } -/// A payload envelope that has gone through processing checks and execution by an EL client. -/// This envelope hasn't necessarily completed data availability checks. -/// -/// -/// It contains 2 variants: -/// 1. `Available`: This envelope has been executed and also contains all data to consider it -/// fully available. -/// 2. `AvailabilityPending`: This envelope hasn't received all required blobs to consider it -/// fully available. -#[allow(dead_code)] -pub enum ExecutedEnvelope { - Available(AvailableExecutedEnvelope), - // TODO(gloas): check data column availability via DA checker - AvailabilityPending(), +/// A payload envelope that has completed all envelope processing checks, verification +/// by an EL client but does not have all requisite columns to get imported into +/// fork choice. +pub struct AvailabilityPendingExecutedEnvelope { + pub envelope: Arc>, + pub block_root: Hash256, + pub payload_verification_outcome: PayloadVerificationOutcome, } -impl ExecutedEnvelope { +impl AvailabilityPendingExecutedEnvelope { pub fn new( - envelope: MaybeAvailableEnvelope, - import_data: EnvelopeImportData, + envelope: Arc>, + block_root: Hash256, payload_verification_outcome: PayloadVerificationOutcome, - spec: Arc, ) -> Self { - match envelope { - MaybeAvailableEnvelope::Available(available_envelope) => { - Self::Available(AvailableExecutedEnvelope::new( - available_envelope, - import_data, - payload_verification_outcome, - )) - } - // TODO(gloas): check data column availability via DA checker - MaybeAvailableEnvelope::AvailabilityPending { - block_hash, - envelope, - } => Self::Available(AvailableExecutedEnvelope::new( - AvailableEnvelope::new(block_hash, envelope, vec![], None, spec), - import_data, - payload_verification_outcome, - )), + Self { + envelope, + block_root, + payload_verification_outcome, } } } @@ -159,25 +108,25 @@ impl ExecutedEnvelope { /// by an EL client **and** has all requisite blob data to be imported into fork choice. pub struct AvailableExecutedEnvelope { pub envelope: AvailableEnvelope, - pub import_data: EnvelopeImportData, + pub block_root: Hash256, pub payload_verification_outcome: PayloadVerificationOutcome, } impl AvailableExecutedEnvelope { pub fn new( envelope: AvailableEnvelope, - import_data: EnvelopeImportData, + block_root: Hash256, payload_verification_outcome: PayloadVerificationOutcome, ) -> Self { Self { envelope, - import_data, + block_root, payload_verification_outcome, } } } -#[derive(Debug)] +#[derive(Debug, AsRefStr)] pub enum EnvelopeError { /// The envelope's block root is unknown. BlockRootUnknown { block_root: Hash256 }, @@ -205,22 +154,16 @@ pub enum EnvelopeError { payload_slot: Slot, latest_finalized_slot: Slot, }, - /// Optimistic sync is not supported for Gloas payload envelopes. - OptimisticSyncNotSupported { block_root: Hash256 }, /// Some Beacon Chain Error BeaconChainError(Arc), /// Some Beacon State error BeaconStateError(BeaconStateError), - /// Some BlockProcessingError (for electra operations) - BlockProcessingError(BlockProcessingError), /// Some EnvelopeProcessingError EnvelopeProcessingError(EnvelopeProcessingError), /// Error verifying the execution payload ExecutionPayloadError(ExecutionPayloadError), - /// An error from block-level checks reused during envelope import - BlockError(BlockError), - /// Internal error - InternalError(String), + /// An error from importing the envelope. + ImportError(BlockError), } impl std::fmt::Display for EnvelopeError { @@ -253,13 +196,6 @@ impl From for EnvelopeError { } } -impl From for EnvelopeError { - fn from(e: BlockError) -> Self { - EnvelopeError::BlockError(e) - } -} - -/// Pull errors up from EnvelopeProcessingError to EnvelopeError impl From for EnvelopeError { fn from(e: EnvelopeProcessingError) -> Self { match e { diff --git a/beacon_node/beacon_chain/src/payload_envelope_verification/payload_notifier.rs b/beacon_node/beacon_chain/src/payload_envelope_verification/payload_notifier.rs index eb5e13b0cc..0bbe32525a 100644 --- a/beacon_node/beacon_chain/src/payload_envelope_verification/payload_notifier.rs +++ b/beacon_node/beacon_chain/src/payload_envelope_verification/payload_notifier.rs @@ -31,7 +31,8 @@ impl PayloadNotifier { match notify_execution_layer { NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => { - let new_payload_request = Self::build_new_payload_request(&envelope, &block)?; + let new_payload_request = Self::build_new_payload_request(&envelope, &block) + .map_err(EnvelopeError::ImportError)?; // TODO(gloas): check and test RLP block hash calculation post-Gloas if let Err(e) = new_payload_request.perform_optimistic_sync_verifications() { warn!( diff --git a/beacon_node/beacon_chain/src/pending_payload_cache/mod.rs b/beacon_node/beacon_chain/src/pending_payload_cache/mod.rs new file mode 100644 index 0000000000..2100a5fe9f --- /dev/null +++ b/beacon_node/beacon_chain/src/pending_payload_cache/mod.rs @@ -0,0 +1,781 @@ +//! This module builds out the data availability cache for Gloas. When a beacon block is received +//! over gossip/p2p we insert its bid into this cache, keyed by block root. As soon as the bid +//! is received we can begin using it to verify data columns. +//! +//! When a payload envelope is received and executed against the EL, it is inserted into this cache. +//! Once all required custody columns have been kzg verified and the envelope has been executed we can +//! import the envelope into fork choice and store it to disk. +//! +//! Note that the block must have arrived before the envelope or data columns can reach this cache. +//! Data columns require the bid (from the block) for verification. Columns that arrive before +//! the block are rejected with `BlockRootUnknown`. + +use crate::data_availability_checker::{AvailabilityCheckError, MissingCellsError}; +use crate::payload_envelope_verification::{ + AvailabilityPendingExecutedEnvelope, AvailableExecutedEnvelope, +}; +use crate::{BeaconChainTypes, CustodyContext, metrics}; +use kzg::Kzg; +use lru::LruCache; +use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::collections::HashMap; +use std::fmt; +use std::fmt::Debug; +use std::num::NonZeroUsize; +use std::sync::Arc; +use tracing::{Span, debug, error, instrument}; +use types::{ + ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, Epoch, EthSpec, Hash256, + PartialDataColumnSidecarRef, +}; + +mod pending_column; +mod pending_components; + +use crate::data_column_verification::{ + GossipVerifiedDataColumn, KzgVerifiedCustodyDataColumn, KzgVerifiedDataColumn, +}; +use crate::metrics::{ + KZG_DATA_COLUMN_RECONSTRUCTION_ATTEMPTS, KZG_DATA_COLUMN_RECONSTRUCTION_FAILURES, +}; +use crate::observed_data_sidecars::ObservationStrategy; +use pending_components::{PendingComponents, ReconstructColumnsDecision}; +use types::SignedExecutionPayloadBid; +use types::new_non_zero_usize; + +/// The LRU Cache stores `PendingComponents`, which store the block root, the execution payload bid, and its associated column data. +/// The execution payload bid stores the kzg commitments which we use to verify against incoming column data. +/// Setting this to 32 keeps memory usage reasonable. +/// +/// `PendingComponents` are now never removed from the cache manually and are only removed via LRU +/// eviction to prevent race conditions (#7961), so we expect this cache to be full all the time. +const AVAILABILITY_CACHE_CAPACITY: NonZeroUsize = new_non_zero_usize(32); + +/// This type is returned after adding a bid / column to the `DataAvailabilityChecker`. +/// +/// Indicates if the payloads data is fully `Available` or if we need more columns. +pub enum Availability { + MissingComponents(Hash256), + Available(Box>), +} + +impl Debug for Availability { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::MissingComponents(block_root) => { + write!(f, "MissingComponents({})", block_root) + } + Self::Available(envelope) => { + write!(f, "Available({:?})", envelope.block_root) + } + } + } +} + +pub type AvailabilityAndReconstructedColumns = (Availability, DataColumnSidecarList); + +#[derive(Debug)] +pub enum DataColumnReconstructionResult { + Success(AvailabilityAndReconstructedColumns), + NotStarted(&'static str), + RecoveredColumnsNotImported(&'static str), +} + +/// Cache to hold data columns for payloads pending data availability. +/// +/// In Gloas, beacon blocks can be immediately imported into fork choice. The execution payload +/// bid contains the payloads kzg commitments. This cache tracks data columns for payloads until all +/// required columns are received. +/// +/// Usually data becomes available on its slot within a second of receiving its first component +/// over gossip. However, data may never become available if a malicious proposer does not +/// publish its data, or there are network issues. Components are only removed via LRU eviction. +pub struct PendingPayloadCache { + /// Contains all the data we keep in memory, protected by an RwLock + availability_cache: RwLock>>, + kzg: Arc, + custody_context: Arc>, + spec: Arc, +} + +impl PendingPayloadCache { + pub fn new( + kzg: Arc, + custody_context: Arc>, + spec: Arc, + ) -> Result { + Ok(Self { + availability_cache: RwLock::new(LruCache::new(AVAILABILITY_CACHE_CAPACITY)), + kzg, + custody_context, + spec, + }) + } + + pub fn custody_context(&self) -> &Arc> { + &self.custody_context + } + + /// Returns all cached data columns for the given block root, if any. + #[instrument(skip_all, level = "trace")] + pub fn get_data_columns( + &self, + block_root: Hash256, + ) -> Option> { + self.peek_pending_components(&block_root, |components| { + components.map(|c| c.get_cached_data_columns()) + }) + } + + /// Returns the indices of cached data columns for the given block root. + #[instrument(skip_all, level = "trace")] + pub fn cached_data_column_indexes(&self, block_root: &Hash256) -> Option> { + self.peek_pending_components(block_root, |components| { + components.map(|components| components.get_cached_data_columns_indices()) + }) + } + + /// Return the cached Gloas payload bid for `block_root`, if present. + pub fn get_bid( + &self, + block_root: &Hash256, + ) -> Option>> { + self.peek_pending_components(block_root, |components| { + components.map(|components| components.bid.clone()) + }) + } + + /// Filter out cells that are already cached for the given column sidecar. + /// Returns the cells that still need KZG verification, or `None` if all cells are cached. + #[instrument(skip_all, level = "trace")] + pub fn missing_cells_for_column_sidecar<'a>( + &'_ self, + data_column: &'a DataColumnSidecar, + ) -> Result>, MissingCellsError> { + let block_root = data_column.block_root(); + let column_index = *data_column.index(); + + self.peek_pending_components(&block_root, |components| { + let Some(cached) = components.and_then(|c| c.verified_data_columns.get(&column_index)) + else { + return data_column.try_filter_to_partial_ref(|_, _, _| Ok(true)); + }; + + data_column.try_filter_to_partial_ref(|cell_idx, cell, proof| { + match cached.cell_matches(cell_idx, cell, proof) { + None => Ok(true), + Some(true) => Ok(false), + Some(false) => Err(MissingCellsError::MismatchesCachedColumn), + } + }) + }) + } + + /// Insert an executed payload envelope into the cache and performs an availability check + pub fn put_executed_payload_envelope( + &self, + executed_envelope: AvailabilityPendingExecutedEnvelope, + ) -> Result, AvailabilityCheckError> { + let epoch = executed_envelope.envelope.epoch(); + let beacon_block_root = executed_envelope.envelope.beacon_block_root(); + let bid = self + .get_bid(&beacon_block_root) + .ok_or(AvailabilityCheckError::MissingBid(beacon_block_root))?; + + let pending_components = + self.update_pending_components(beacon_block_root, bid, |pending_components| { + pending_components.insert_executed_payload_envelope(executed_envelope); + })?; + + let num_expected_columns = self + .custody_context + .num_of_data_columns_to_sample(epoch, &self.spec); + + pending_components.span.in_scope(|| { + debug!( + component = "executed envelope", + status = pending_components.status_str(num_expected_columns), + "Component added to data availability checker" + ); + }); + + self.check_availability(beacon_block_root, pending_components, num_expected_columns) + } + + /// Inserts a bid into the pending payload cache. + /// This will silently drop the bid if a bid for this block root already exists in the cache. + pub fn insert_bid(&self, block_root: Hash256, bid: Arc>) { + let mut write_lock = self.availability_cache.write(); + write_lock.get_or_insert_mut(block_root, || PendingComponents::new(block_root, bid)); + } + + /// Perform KZG verification on RPC custody columns and insert them into the cache. + /// After insertion check if the envelope becomes available. + #[instrument(skip_all, level = "trace")] + pub fn put_rpc_custody_columns( + &self, + block_root: Hash256, + custody_columns: DataColumnSidecarList, + ) -> Result, AvailabilityCheckError> { + let bid = self + .get_bid(&block_root) + .ok_or(AvailabilityCheckError::MissingBid(block_root))?; + let kzg_verified_columns = KzgVerifiedDataColumn::from_batch_with_scoring_and_commitments( + custody_columns, + bid.message.blob_kzg_commitments.as_ref(), + &self.kzg, + ) + .map_err(AvailabilityCheckError::InvalidColumn)?; + + let epoch = bid.message.slot.epoch(T::EthSpec::slots_per_epoch()); + let sampling_columns = self + .custody_context + .sampling_columns_for_epoch(epoch, &self.spec); + let verified_custody_columns = kzg_verified_columns + .into_iter() + .filter(|col| sampling_columns.contains(&col.index())) + .map(KzgVerifiedCustodyDataColumn::from_asserted_custody) + .collect::>(); + + self.put_kzg_verified_custody_data_columns(block_root, &verified_custody_columns) + } + + /// Perform KZG verification on gossip verified custody columns and insert them into the cache. + /// After insertion check if the envelope becomes available + #[instrument(skip_all, level = "trace")] + pub fn put_gossip_verified_data_columns( + &self, + block_root: Hash256, + data_columns: Vec>, + ) -> Result, AvailabilityCheckError> { + let bid = self + .get_bid(&block_root) + .ok_or(AvailabilityCheckError::MissingBid(block_root))?; + let epoch = bid.message.slot.epoch(T::EthSpec::slots_per_epoch()); + let sampling_columns = self + .custody_context + .sampling_columns_for_epoch(epoch, &self.spec); + let custody_columns = data_columns + .into_iter() + .filter(|col| sampling_columns.contains(&col.index())) + .map(|c| KzgVerifiedCustodyDataColumn::from_asserted_custody(c.into_inner())) + .collect::>(); + + self.put_kzg_verified_custody_data_columns(block_root, &custody_columns) + } + + /// Insert KZG verified columns into the cache. + /// After insertion check if the envelope becomes available. + pub fn put_kzg_verified_custody_data_columns( + &self, + block_root: Hash256, + kzg_verified_data_columns: &[KzgVerifiedCustodyDataColumn], + ) -> Result, AvailabilityCheckError> { + let bid = self + .get_bid(&block_root) + .ok_or(AvailabilityCheckError::MissingBid(block_root))?; + + let pending_components = + self.update_pending_components(block_root, bid.clone(), |pending_components| { + pending_components.merge_data_columns(kzg_verified_data_columns) + })?; + + let epoch = bid.message.slot.epoch(T::EthSpec::slots_per_epoch()); + + let num_expected_columns = self + .custody_context + .num_of_data_columns_to_sample(epoch, &self.spec); + + pending_components.span.in_scope(|| { + debug!( + component = "data_columns", + status = pending_components.status_str(num_expected_columns), + "Component added to data availability checker" + ); + }); + + self.check_availability(block_root, pending_components, num_expected_columns) + } + + #[instrument(skip_all, level = "debug")] + pub fn reconstruct_data_columns( + &self, + block_root: &Hash256, + ) -> Result, AvailabilityCheckError> { + let bid = self + .get_bid(block_root) + .ok_or(AvailabilityCheckError::MissingBid(*block_root))?; + + let verified_data_columns = match self.check_and_set_reconstruction_started(block_root) { + ReconstructColumnsDecision::Yes(verified_data_columns) => verified_data_columns, + ReconstructColumnsDecision::No(reason) => { + return Ok(DataColumnReconstructionResult::NotStarted(reason)); + } + }; + let existing_column_indices = verified_data_columns + .iter() + .map(|data_column| *data_column.index()) + .collect::>(); + + metrics::inc_counter(&KZG_DATA_COLUMN_RECONSTRUCTION_ATTEMPTS); + let timer = metrics::start_timer(&metrics::DATA_AVAILABILITY_RECONSTRUCTION_TIME); + + let all_data_columns = KzgVerifiedCustodyDataColumn::reconstruct_columns( + &self.kzg, + verified_data_columns, + bid.message.blob_kzg_commitments.as_ref(), + &self.spec, + ) + .map_err(|e| { + error!( + ?block_root, + error = ?e, + "Error reconstructing data columns" + ); + self.handle_reconstruction_failure(block_root); + metrics::inc_counter(&KZG_DATA_COLUMN_RECONSTRUCTION_FAILURES); + AvailabilityCheckError::ReconstructColumnsError(e) + })?; + + let slot = bid.message.slot; + let columns_to_sample = self + .custody_context() + .sampling_columns_for_epoch(slot.epoch(T::EthSpec::slots_per_epoch()), &self.spec); + + let data_columns_to_import_and_publish = all_data_columns + .into_iter() + .filter(|d| { + columns_to_sample.contains(&d.index()) + && !existing_column_indices.contains(&d.index()) + }) + .collect::>(); + + metrics::stop_timer(timer); + metrics::inc_counter_by( + &metrics::DATA_AVAILABILITY_RECONSTRUCTED_COLUMNS, + data_columns_to_import_and_publish.len() as u64, + ); + + debug!( + count = data_columns_to_import_and_publish.len(), + ?block_root, + %slot, + "Reconstructed columns" + ); + + self.put_kzg_verified_custody_data_columns(*block_root, &data_columns_to_import_and_publish) + .map(|availability| { + DataColumnReconstructionResult::Success(( + availability, + data_columns_to_import_and_publish + .into_iter() + .map(|d| d.clone_arc()) + .collect::>(), + )) + }) + } + + // ── Metrics ── + + /// Number of pending component entries in memory in the cache. + pub fn cache_size(&self) -> usize { + self.availability_cache.read().len() + } + + // ── Internal helpers ── + + fn check_availability( + &self, + block_root: Hash256, + pending_components: MappedRwLockReadGuard<'_, PendingComponents>, + num_expected_columns: usize, + ) -> Result, AvailabilityCheckError> { + if let Some(available_envelope) = pending_components.make_available(num_expected_columns)? { + // Explicitly drop read lock before acquiring write lock + drop(pending_components); + if let Some(components) = self.availability_cache.write().get_mut(&block_root) { + // Clean up span now that data is available + components.span = Span::none(); + } + + // We never remove the pending components manually to avoid race conditions. + // Components are only removed via LRU eviction as finality advances. + Ok(Availability::Available(Box::new(available_envelope))) + } else { + Ok(Availability::MissingComponents(block_root)) + } + } + + /// Gets or creates `PendingComponents` and applies the `update_fn` while holding the write lock. + /// + /// Once the update is complete, the write lock is downgraded and a read guard with a + /// reference of the updated `PendingComponents` is returned. + fn update_pending_components( + &self, + block_root: Hash256, + bid: Arc>, + update_fn: F, + ) -> Result>, AvailabilityCheckError> + where + F: FnOnce(&mut PendingComponents), + { + let mut write_lock = self.availability_cache.write(); + + { + let pending_components = write_lock + .get_or_insert_mut(block_root, || PendingComponents::new(block_root, bid)); + update_fn(pending_components) + } + + RwLockReadGuard::try_map(RwLockWriteGuard::downgrade(write_lock), |cache| { + cache.peek(&block_root) + }) + .map_err(|_| { + AvailabilityCheckError::Unexpected("pending components should exist".to_string()) + }) + } + + fn peek_pending_components>) -> R>( + &self, + block_root: &Hash256, + f: F, + ) -> R { + f(self.availability_cache.read().peek(block_root)) + } + + /// Check whether data column reconstruction should be attempted. + /// TODO(gloas): rethink reconstruction for the cell model + fn check_and_set_reconstruction_started( + &self, + block_root: &Hash256, + ) -> ReconstructColumnsDecision { + let mut write_lock = self.availability_cache.write(); + let Some(pending_components) = write_lock.get_mut(block_root) else { + return ReconstructColumnsDecision::No("block already imported"); + }; + + let epoch = pending_components.bid.epoch(); + + let total_column_count = T::EthSpec::number_of_columns(); + let sampling_column_count = self + .custody_context + .num_of_data_columns_to_sample(epoch, &self.spec); + + if pending_components.reconstruction_started { + return ReconstructColumnsDecision::No("already started"); + } + let received_column_count = pending_components.num_completed_columns(); + if received_column_count >= sampling_column_count { + return ReconstructColumnsDecision::No("all sampling columns received"); + } + if received_column_count < total_column_count / 2 { + return ReconstructColumnsDecision::No("not enough columns"); + } + + pending_components.reconstruction_started = true; + ReconstructColumnsDecision::Yes(pending_components.get_cached_data_columns()) + } + + /// This could mean some invalid data columns made it through to the `DataAvailabilityChecker`. + /// In this case, we remove all data columns in `PendingComponents`, reset reconstruction + /// status so that we can attempt to retrieve columns from peers again. + fn handle_reconstruction_failure(&self, block_root: &Hash256) { + if let Some(pending_components_mut) = self.availability_cache.write().get_mut(block_root) { + pending_components_mut.verified_data_columns = HashMap::new(); + pending_components_mut.reconstruction_started = false; + } + } + + /// Maintain the cache by removing entries older than the cutoff epoch. + pub fn do_maintenance(&self, cutoff_epoch: Epoch) -> Result<(), AvailabilityCheckError> { + let mut write_lock = self.availability_cache.write(); + let mut keys_to_remove = vec![]; + for (key, value) in write_lock.iter() { + if value.bid.epoch() < cutoff_epoch { + keys_to_remove.push(*key); + } + } + for key in keys_to_remove { + write_lock.pop(&key); + } + + Ok(()) + } +} + +#[cfg(test)] +mod data_availability_checker_tests { + use super::*; + + use crate::block_verification::PayloadVerificationOutcome; + use crate::custody_context::NodeCustodyType; + use crate::test_utils::{ + DiskHarnessType, NumBlobs, generate_data_column_indices_rand_order, + generate_rand_block_and_data_columns, get_kzg, + }; + use fork_choice::PayloadVerificationStatus; + use logging::create_test_tracing_subscriber; + use types::test_utils::test_unstructured; + use types::{ + ExecutionPayloadEnvelope, ExecutionPayloadGloas, ExecutionRequests, ForkName, + MinimalEthSpec, SignedExecutionPayloadEnvelope, + }; + + type E = MinimalEthSpec; + type T = DiskHarnessType; + + const NUM_BLOBS: usize = 1; + + /// Stand up a cache + a 1-blob Gloas block for the given custody type. The bid is registered + /// in the cache; `custody` is pre-filtered to the sampling subset. + fn setup(node_custody: NodeCustodyType) -> Setup { + setup_with(node_custody, NumBlobs::Number(NUM_BLOBS)) + } + + fn setup_zero_blob(node_custody: NodeCustodyType) -> Setup { + setup_with(node_custody, NumBlobs::Number(0)) + } + + fn setup_with(node_custody: NodeCustodyType, num_blobs: NumBlobs) -> Setup { + create_test_tracing_subscriber(); + let spec = Arc::new(ForkName::Gloas.make_genesis_spec(E::default_spec())); + let kzg = get_kzg(&spec); + let custody_context = Arc::new(CustodyContext::::new( + node_custody, + generate_data_column_indices_rand_order::(), + &spec, + )); + let cache = Arc::new( + PendingPayloadCache::::new(kzg, custody_context, spec.clone()) + .expect("create cache"), + ); + + let mut u = test_unstructured(); + let (block, columns) = + generate_rand_block_and_data_columns::(ForkName::Gloas, num_blobs, &mut u, &spec) + .expect("generate test block"); + let block_root = block.canonical_root(); + let bid = Arc::new( + block + .message() + .body() + .signed_execution_payload_bid() + .expect("Gloas block has bid") + .clone(), + ); + cache.insert_bid(block_root, bid.clone()); + + let epoch = bid.message.slot.epoch(E::slots_per_epoch()); + let sampling = cache + .custody_context() + .sampling_columns_for_epoch(epoch, &cache.spec); + let custody = columns + .into_iter() + .filter(|c| sampling.contains(c.index())) + .collect(); + + Setup { + cache, + block_root, + custody, + } + } + + struct Setup { + cache: Arc>, + block_root: Hash256, + custody: DataColumnSidecarList, + } + + impl Setup { + fn put_envelope(&self) -> Availability { + self.cache + .put_executed_payload_envelope(executed_envelope(self.block_root)) + .expect("put envelope") + } + + fn put_columns(&self, columns: DataColumnSidecarList) -> Availability { + self.cache + .put_rpc_custody_columns(self.block_root, columns) + .expect("put columns") + } + + fn reconstruct(&self) -> Result, AvailabilityCheckError> { + self.cache.reconstruct_data_columns(&self.block_root) + } + + fn cached_indexes(&self) -> Vec { + self.cache + .cached_data_column_indexes(&self.block_root) + .expect("entry") + } + } + + /// Hand-rolled executed envelope with bypassed verification; the cache only inspects + /// `beacon_block_root` and the verification outcome, never the signature or payload. + fn executed_envelope(block_root: Hash256) -> AvailabilityPendingExecutedEnvelope { + AvailabilityPendingExecutedEnvelope { + envelope: Arc::new(SignedExecutionPayloadEnvelope { + message: ExecutionPayloadEnvelope { + payload: ExecutionPayloadGloas::default(), + execution_requests: ExecutionRequests::default(), + builder_index: 0, + beacon_block_root: block_root, + parent_beacon_block_root: Hash256::random(), + }, + signature: bls::Signature::infinity().expect("infinity sig"), + }), + block_root, + payload_verification_outcome: PayloadVerificationOutcome { + payload_verification_status: PayloadVerificationStatus::Verified, + }, + } + } + + #[track_caller] + fn assert_missing(availability: Availability) { + assert!( + matches!(availability, Availability::MissingComponents(_)), + "expected MissingComponents, got {availability:?}", + ); + } + + #[track_caller] + fn assert_available(availability: Availability) -> Box> { + match availability { + Availability::Available(env) => env, + other => panic!("expected Available, got {other:?}"), + } + } + + // ─── Tier 1: real-path availability flows ─────────────────────────────── + + /// Envelope first → MissingComponents. Then all sampling columns → Available. + #[tokio::test] + async fn availability_arrives_envelope_first() { + let s = setup(NodeCustodyType::Fullnode); + assert_missing(s.put_envelope()); + let envelope = assert_available(s.put_columns(s.custody.clone())); + assert_eq!(envelope.block_root, s.block_root); + assert_eq!(envelope.envelope.columns.len(), s.custody.len()); + } + + /// Columns first → MissingComponents. Then envelope → Available. + #[tokio::test] + async fn availability_arrives_columns_first() { + let s = setup(NodeCustodyType::Fullnode); + assert_missing(s.put_columns(s.custody.clone())); + let envelope = assert_available(s.put_envelope()); + assert_eq!(envelope.block_root, s.block_root); + assert_eq!(envelope.envelope.columns.len(), s.custody.len()); + } + + /// N-1 columns + envelope is still MissingComponents; the Nth column flips to Available. + /// Guards the strict count comparison in `make_available`. + #[tokio::test] + async fn partial_columns_then_complete() { + let mut s = setup(NodeCustodyType::Fullnode); + assert!(s.custody.len() >= 2, "needs at least 2 sampling columns"); + let last = s.custody.pop().expect("non-empty custody"); + + s.put_envelope(); + assert_missing(s.put_columns(s.custody.clone())); + assert_available(s.put_columns(vec![last])); + } + + /// Zero-blob block + envelope → Available. Guards the `num_blobs_expected == 0` early-return + /// in `make_available`. + #[tokio::test] + async fn zero_blob_envelope_immediately_available() { + let s = setup_zero_blob(NodeCustodyType::Fullnode); + let envelope = assert_available(s.put_envelope()); + assert!(envelope.envelope.columns.is_empty()); + } + + /// Receiving the same column twice keeps a single cache entry. Guards `PendingColumn::insert` + /// staying only-if-empty under repeated arrivals. + #[tokio::test] + async fn dedups_repeated_column_inserts() { + let s = setup(NodeCustodyType::Fullnode); + let column = s.custody.first().cloned().expect("sampling column"); + let column_index = *column.index(); + s.put_columns(vec![column.clone()]); + s.put_columns(vec![column]); + + assert_eq!(s.cached_indexes(), vec![column_index]); + assert_eq!( + s.cache.get_data_columns(s.block_root).map(|c| c.len()), + Some(1), + ); + } + + // ─── Tier 2: reconstruction state machine ─────────────────────────────── + // + // Reconstruction only triggers when `total/2 ≤ received < sampling_count`. Fullnode's small + // sampling count never satisfies this, so these tests use `Supernode`. + + /// Fewer than `number_of_columns / 2` columns received → reconstruction is `NotStarted`. + #[tokio::test] + async fn reconstruction_below_threshold_is_not_started() { + let s = setup(NodeCustodyType::Supernode); + let half = E::number_of_columns() / 2; + s.put_columns(s.custody.iter().take(half - 1).cloned().collect()); + assert!(matches!( + s.reconstruct().expect("reconstruct call"), + DataColumnReconstructionResult::NotStarted("not enough columns") + )); + } + + /// All sampling columns received → reconstruction unnecessary, returns `NotStarted`. + #[tokio::test] + async fn reconstruction_already_complete_is_not_started() { + let s = setup(NodeCustodyType::Supernode); + s.put_columns(s.custody.clone()); + assert!(matches!( + s.reconstruct().expect("reconstruct call"), + DataColumnReconstructionResult::NotStarted("all sampling columns received") + )); + } + + /// Envelope + 50% of sampling columns → reconstruction recovers the rest, the entry flips + /// to `Available`, and the cache holds every sampling column. + #[tokio::test] + async fn reconstruction_success_fills_missing_columns() { + let s = setup(NodeCustodyType::Supernode); + s.put_envelope(); + let sampling_count = s.custody.len(); + let half = sampling_count / 2; + s.put_columns(s.custody.iter().take(half).cloned().collect()); + assert_eq!(s.cached_indexes().len(), half); + + let result = s.reconstruct().expect("reconstruction must succeed"); + let (availability, _recovered) = match result { + DataColumnReconstructionResult::Success(inner) => inner, + other => panic!("expected Success, got {other:?}"), + }; + assert_available(availability); + assert_eq!(s.cached_indexes().len(), sampling_count); + } + + // ─── Tier 3: invariants ───────────────────────────────────────────────── + + /// `get_data_columns` and `cached_data_column_indexes` must agree on which columns are + /// complete. Drift between these two would corrupt the DB on import. + #[tokio::test] + async fn cached_columns_match_completed_indexes() { + let mut s = setup(NodeCustodyType::Fullnode); + let last = s.custody.pop().expect("non-empty custody"); + + let assert_lengths_match = |s: &Setup| { + let indexes_len = s.cached_indexes().len(); + let sidecars_len = s.cache.get_data_columns(s.block_root).expect("entry").len(); + assert_eq!(indexes_len, sidecars_len); + }; + + s.put_columns(s.custody.clone()); + assert_lengths_match(&s); + + s.put_columns(vec![last]); + assert_lengths_match(&s); + } +} diff --git a/beacon_node/beacon_chain/src/pending_payload_cache/pending_column.rs b/beacon_node/beacon_chain/src/pending_payload_cache/pending_column.rs new file mode 100644 index 0000000000..890c17ba67 --- /dev/null +++ b/beacon_node/beacon_chain/src/pending_payload_cache/pending_column.rs @@ -0,0 +1,63 @@ +use kzg::KzgProof; +use ssz_types::VariableList; +use std::sync::Arc; +use types::{Cell, ColumnIndex, DataColumnSidecar, DataColumnSidecarGloas, EthSpec, Hash256, Slot}; + +#[derive(Clone)] +pub struct PendingColumn { + cells: Vec, KzgProof)>>, +} + +impl PendingColumn { + /// Allocate a `PendingColumn` whose `cells` vec has space for `blob_count` entries, all + /// initialised to `None`. Required so that `insert(idx, ...)` can write into `cells[idx]`. + pub fn new_with_capacity(blob_count: usize) -> Self { + Self { + cells: vec![None; blob_count], + } + } + + pub fn insert(&mut self, index: usize, cell: &Cell, proof: &KzgProof) { + if let Some(existing_cell) = self.cells.get_mut(index) + && existing_cell.is_none() + { + *existing_cell = Some((cell.clone(), *proof)); + } + } + + pub fn cell_matches(&self, index: usize, cell: &Cell, proof: &KzgProof) -> Option { + self.cells + .get(index)? + .as_ref() + .map(|(c, p)| c == cell && p == proof) + } + + /// Returns a full `DataColumnSidecar` if all cells are present, or `None` if any are missing. + pub fn to_full_sidecar( + &self, + index: ColumnIndex, + slot: Slot, + beacon_block_root: Hash256, + ) -> Option>> { + let mut column = Vec::with_capacity(self.cells.len()); + let mut kzg_proofs = Vec::with_capacity(self.cells.len()); + + for cell in self.cells.iter() { + let (cell, proof) = cell.as_ref()?; + // TODO(gloas): we likely want to go and arc all cells. This will help us from requiring a clone + // in PendingColumn::insert + column.push(cell.clone()); + kzg_proofs.push(*proof); + } + + // TODO(gloas): this hard-codes the Gloas sidecar variant. Pass the fork in once + // post-Gloas variants are introduced (or move construction to a fork-aware helper). + Some(Arc::new(DataColumnSidecar::Gloas(DataColumnSidecarGloas { + index, + column: VariableList::try_from(column).ok()?, + kzg_proofs: VariableList::try_from(kzg_proofs).ok()?, + slot, + beacon_block_root, + }))) + } +} diff --git a/beacon_node/beacon_chain/src/pending_payload_cache/pending_components.rs b/beacon_node/beacon_chain/src/pending_payload_cache/pending_components.rs new file mode 100644 index 0000000000..e7b9009577 --- /dev/null +++ b/beacon_node/beacon_chain/src/pending_payload_cache/pending_components.rs @@ -0,0 +1,180 @@ +use crate::data_availability_checker::AvailabilityCheckError; +use crate::data_column_verification::KzgVerifiedCustodyDataColumn; +use crate::payload_envelope_verification::AvailabilityPendingExecutedEnvelope; +use crate::payload_envelope_verification::AvailableEnvelope; +use crate::payload_envelope_verification::AvailableExecutedEnvelope; +use crate::pending_payload_cache::pending_column::PendingColumn; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::sync::Arc; +use tracing::{Span, debug, debug_span}; +use types::DataColumnSidecar; +use types::{ColumnIndex, EthSpec, Hash256, SignedExecutionPayloadBid}; + +/// This represents the components of a payload pending data availability. +/// +/// The columns are all gossip and kzg verified. +/// The payload is considered "available" when all required columns are received. +pub struct PendingComponents { + pub block_root: Hash256, + pub bid: Arc>, + /// a cached post executed payload envelope + pub envelope: Option>, + /// A column entry in this map may only have some cells filled in (i.e. a partial data column) + pub verified_data_columns: HashMap>, + pub reconstruction_started: bool, + pub(crate) span: Span, +} + +impl PendingComponents { + pub fn num_blobs_expected(&self) -> usize { + self.bid.message.blob_kzg_commitments.len() + } + + /// Returns columns that have all cells present. + pub fn get_cached_data_columns(&self) -> Vec>> { + let slot = self.bid.message.slot; + let block_root = self.block_root; + self.verified_data_columns + .iter() + .filter_map(|(col_idx, col)| col.to_full_sidecar(*col_idx, slot, block_root)) + .collect() + } + + /// Returns the indices of columns that have all cells present. + pub fn get_cached_data_columns_indices(&self) -> Vec { + let slot = self.bid.message.slot; + let block_root = self.block_root; + self.verified_data_columns + .iter() + .filter_map(|(col_idx, col)| { + col.to_full_sidecar(*col_idx, slot, block_root) + .map(|_| *col_idx) + }) + .collect() + } + + /// Merges a given set of data columns into the cache. + pub(crate) fn merge_data_columns( + &mut self, + kzg_verified_data_columns: &[KzgVerifiedCustodyDataColumn], + ) { + let num_blobs_expected = self.num_blobs_expected(); + for data_column in kzg_verified_data_columns { + let data_column = data_column.as_data_column(); + // The Vec-backed `PendingColumn` keys cells by index, so we have to allocate up to + // `num_blobs_expected` entries before inserting; otherwise `cells.get_mut(idx)` returns + // None and the insert is a no-op. + let col = self + .verified_data_columns + .entry(*data_column.index()) + .or_insert_with(|| PendingColumn::new_with_capacity(num_blobs_expected)); + for (cell_idx, (cell, proof)) in data_column + .column() + .iter() + .zip(data_column.kzg_proofs().iter()) + .enumerate() + { + col.insert(cell_idx, cell, proof); + } + } + } + + // TODO(gloas): merge partial columns + + /// Inserts an executed payload envelope into the cache. + pub fn insert_executed_payload_envelope( + &mut self, + envelope: AvailabilityPendingExecutedEnvelope, + ) { + self.envelope = Some(envelope); + } + + pub fn num_completed_columns(&self) -> usize { + self.get_cached_data_columns().len() + } + + /// Returns `Some` if the envelope and all required data columns have been received. + pub fn make_available( + &self, + num_expected_columns: usize, + ) -> Result>, AvailabilityCheckError> { + // Check if the payload has been received and executed + let Some(envelope) = &self.envelope else { + return Ok(None); + }; + + let AvailabilityPendingExecutedEnvelope { + envelope, + block_root, + payload_verification_outcome, + } = envelope; + + let columns = if self.num_blobs_expected() == 0 { + self.span.in_scope(|| { + debug!("Bid has no blobs, data is available"); + }); + vec![] + } else { + let columns = self.get_cached_data_columns(); + match columns.len().cmp(&num_expected_columns) { + Ordering::Greater => { + return Err(AvailabilityCheckError::Unexpected(format!( + "too many columns: got {} expected {num_expected_columns}", + columns.len() + ))); + } + Ordering::Equal => { + self.span.in_scope(|| { + debug!("All data columns received, data is available"); + }); + columns + } + Ordering::Less => { + // Not enough data columns received yet + return Ok(None); + } + } + }; + + let available_envelope = AvailableEnvelope::new(envelope.clone(), columns); + + Ok(Some(AvailableExecutedEnvelope { + envelope: available_envelope, + block_root: *block_root, + payload_verification_outcome: payload_verification_outcome.clone(), + })) + } + + /// Constructs a fresh `PendingComponents` with no envelope and no columns yet. + pub fn new(block_root: Hash256, bid: Arc>) -> Self { + let span = debug_span!(parent: None, "lh_pending_components", %block_root); + let _guard = span.clone().entered(); + Self { + block_root, + bid, + envelope: None, + verified_data_columns: HashMap::new(), + reconstruction_started: false, + span, + } + } + + pub fn status_str(&self, num_expected_columns: usize) -> String { + format!( + "envelope {}, data_columns {}/{}", + self.envelope.is_some(), + self.num_completed_columns(), + num_expected_columns + ) + } +} + +// This enum is only used internally within the crate in the reconstruction function to improve +// readability, so it's OK to not box the variant value, and it shouldn't impact memory much with +// the current usage, as it's deconstructed immediately. +#[allow(clippy::large_enum_variant)] +pub(crate) enum ReconstructColumnsDecision { + Yes(Vec>>), + No(&'static str), +} diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 4378da8405..8e9cc61208 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -2851,11 +2851,42 @@ where .await .expect("newPayload should succeed"); - // Store the envelope. + // Store the envelope and the data columns derived from the block. + // + // Production stores columns inside `import_available_execution_payload_envelope` after + // the cache is satisfied. The harness sidesteps that flow but must still persist columns + // or the `DataColumnMissing` invariant fires for any block with `num_expected_blobs > 0`. + let block = self + .chain + .store + .get_blinded_block(&block_root) + .expect("should read block from store") + .expect("block should exist in store"); + let mut ops = vec![]; + let block_with_full_payload = self + .chain + .store + .make_full_block(&block_root, block.clone()) + .expect("should reconstruct full block"); + let columns = + generate_data_column_sidecars_from_block(&block_with_full_payload, &self.spec); + if !columns.is_empty() + && let Some(store_op) = self.chain.get_blobs_or_columns_store_op( + block_root, + block.slot(), + AvailableBlockData::DataColumns(columns), + ) + { + ops.push(store_op); + } + ops.push(store::StoreOp::PutPayloadEnvelope( + block_root, + std::sync::Arc::new(signed_envelope), + )); self.chain .store - .put_payload_envelope(&block_root, &signed_envelope) - .expect("should store envelope"); + .do_atomically_with_block_and_blobs_cache(ops) + .expect("should persist envelope and columns"); // Update fork choice so it knows the payload was received. self.chain @@ -2876,11 +2907,10 @@ where block: Arc>, ) -> RangeSyncBlock { let block_root = block_root.unwrap_or_else(|| get_block_root(&block)); - let has_blobs = block - .message() - .body() - .blob_kzg_commitments() - .is_ok_and(|c| !c.is_empty()); + // For Gloas, kzg commitments live in the bid (`signed_execution_payload_bid`), so the + // body's `blob_kzg_commitments()` accessor returns Err. `num_expected_blobs` already + // handles both shapes. + let has_blobs = block.num_expected_blobs() > 0; if !has_blobs { return RangeSyncBlock::new( block, @@ -3782,7 +3812,26 @@ pub fn generate_rand_block_and_blobs( SignedBeaconBlock::Fulu(SignedBeaconBlockFulu { ref mut message, .. }) => add_blob_transactions!(message, FullPayloadFulu, num_blobs, u, fork_name), - // TODO(EIP-7732) Add `SignedBeaconBlock::Gloas` variant + SignedBeaconBlock::Gloas(SignedBeaconBlockGloas { + ref mut message, .. + }) => { + // For Gloas, commitments are in the bid, not directly in the body. + // BlobSidecars cannot be created for Gloas because there's no merkle proof + // from the block body to the commitments. Return early with empty blob_sidecars. + let num_blobs = match num_blobs { + NumBlobs::Random => u.int_in_range(DEFAULT_MIN_BLOBS..=DEFAULT_MAX_BLOBS)?, + NumBlobs::Number(n) => n, + NumBlobs::None => 0, + }; + let (bundle, _transactions) = + execution_layer::test_utils::generate_blobs::(num_blobs, fork_name).unwrap(); + message + .body + .signed_execution_payload_bid + .message + .blob_kzg_commitments = bundle.commitments.clone(); + return Ok((block, blob_sidecars)); + } _ => return Ok((block, blob_sidecars)), }; diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 6646fe0b1e..533ef61219 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -323,18 +323,34 @@ fn update_data_column_signed_header( ) { for old_custody_column_sidecar in data_columns.as_mut_slice() { let old_column_sidecar = old_custody_column_sidecar.as_data_column(); - let new_column_sidecar = Arc::new(DataColumnSidecar::Fulu(DataColumnSidecarFulu { - index: *old_column_sidecar.index(), - column: old_column_sidecar.column().clone(), - kzg_commitments: old_column_sidecar.kzg_commitments().unwrap().clone(), - kzg_proofs: old_column_sidecar.kzg_proofs().clone(), - signed_block_header: signed_block.signed_block_header(), - kzg_commitments_inclusion_proof: signed_block - .message() - .body() - .kzg_commitments_merkle_proof() - .unwrap(), - })); + let new_column_sidecar = match old_column_sidecar.as_ref() { + DataColumnSidecar::Fulu(_) => { + Arc::new(DataColumnSidecar::Fulu(DataColumnSidecarFulu { + index: *old_column_sidecar.index(), + column: old_column_sidecar.column().clone(), + kzg_commitments: old_column_sidecar.kzg_commitments().unwrap().clone(), + kzg_proofs: old_column_sidecar.kzg_proofs().clone(), + signed_block_header: signed_block.signed_block_header(), + kzg_commitments_inclusion_proof: signed_block + .message() + .body() + .kzg_commitments_merkle_proof() + .unwrap(), + })) + } + // Gloas columns reference the block by `beacon_block_root` instead of holding the + // block header inline, so updating the parent root just means re-keying the column to + // the new canonical root. + DataColumnSidecar::Gloas(g) => { + Arc::new(DataColumnSidecar::Gloas(types::DataColumnSidecarGloas { + index: g.index, + column: g.column.clone(), + kzg_proofs: g.kzg_proofs.clone(), + slot: g.slot, + beacon_block_root: signed_block.canonical_root(), + })) + } + }; *old_custody_column_sidecar = CustodyDataColumn::from_asserted_custody(new_column_sidecar); } } @@ -1150,8 +1166,13 @@ async fn block_gossip_verification() { ) .await .expect("should import valid gossip verified block"); + if let Some(data_sidecars) = blobs_opt { + verify_and_process_gossip_data_sidecars(&harness, data_sidecars).await; + } // Post-Gloas, store the execution payload envelope so that subsequent blocks can look up - // the parent envelope. + // the parent envelope. This must run after gossip column processing because marking the + // payload as received in fork choice causes the gossip column path's + // `is_block_data_imported` gate to reject otherwise-valid columns as duplicates. if let Some(ref envelope) = snapshot.execution_envelope { harness .chain @@ -1165,9 +1186,6 @@ async fn block_gossip_verification() { .on_valid_payload_envelope_received(snapshot.beacon_block_root) .expect("should update fork choice with envelope"); } - if let Some(data_sidecars) = blobs_opt { - verify_and_process_gossip_data_sidecars(&harness, data_sidecars).await; - } } // Recompute the head to ensure we cache the latest view of fork choice. @@ -2246,7 +2264,6 @@ async fn rpc_block_allows_construction_past_da_boundary() { // Now verify the block is past the DA boundary let da_boundary = harness .chain - .data_availability_checker .data_availability_boundary() .expect("DA boundary should be set"); assert!( diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index cd0e700109..29d0e38b93 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -11,7 +11,8 @@ use std::sync::Arc; use types::data::FixedBlobSidecarList; use types::{ BlobSidecar, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSidecarGloas, Domain, EthSpec, - MinimalEthSpec, PayloadAttestationData, PayloadAttestationMessage, SignedRoot, Slot, + MinimalEthSpec, PayloadAttestationData, PayloadAttestationMessage, SignedExecutionPayloadBid, + SignedRoot, Slot, }; type E = MinimalEthSpec; @@ -84,6 +85,15 @@ async fn data_column_sidecar_event_on_process_gossip_data_column() { let epoch = slot.epoch(E::slots_per_epoch()); random_sidecar.slot = slot; random_sidecar.index = harness.chain.sampling_columns_for_epoch(epoch)[0]; + + // For gloas, the bid must be known, e.g. in the pending payload cache + let mut bid = SignedExecutionPayloadBid::::empty(); + bid.message.slot = Slot::new(10); + harness + .chain + .pending_payload_cache + .insert_bid(random_sidecar.beacon_block_root, Arc::new(bid)); + DataColumnSidecar::Gloas(random_sidecar) } else { let mut random_sidecar = DataColumnSidecarFulu::arbitrary(&mut u).unwrap(); diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index cfdd54857a..0ff9f6841d 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -23,6 +23,7 @@ use beacon_chain::{ }, custody_context::NodeCustodyType, historical_blocks::HistoricalBlockError, + kzg_utils::reconstruct_blobs, migrate::MigratorConfig, }; use bls::{Keypair, Signature, SignatureBytes}; @@ -68,6 +69,43 @@ static KEYPAIRS: LazyLock> = type E = MinimalEthSpec; type TestHarness = BeaconChainHarness>; +/// Retrieve or reconstruct blobs for a given block root. This uses the block's epoch to determine +/// whether to retrieve blobs directly or reconstruct them from columns. +/// +/// Returns `None` for Gloas blocks (which have no blob sidecar representation). +fn get_or_reconstruct_blobs( + chain: &BeaconChain, + block_root: &Hash256, +) -> Result>, BeaconChainError> { + let Some(block) = chain.store.get_blinded_block(block_root)? else { + return Ok(None); + }; + + if block.fork_name_unchecked().gloas_enabled() { + return Ok(None); + } + + if chain.spec.is_peer_das_enabled_for_epoch(block.epoch()) { + let fork_name = chain.spec.fork_name_at_epoch(block.epoch()); + if let Some(columns) = chain.store.get_data_columns(block_root, fork_name)? { + let num_required_columns = T::EthSpec::number_of_columns() / 2; + if columns.len() >= num_required_columns { + reconstruct_blobs(&chain.kzg, columns, None, &block, &chain.spec) + .map(Some) + .map_err(BeaconChainError::FailedToReconstructBlobs) + } else { + Err(BeaconChainError::InsufficientColumnsToReconstructBlobs { + columns_found: columns.len(), + }) + } + } else { + Ok(None) + } + } else { + Ok(chain.get_blobs(block_root)?.blobs()) + } +} + fn get_store(db_path: &TempDir) -> Arc, BeaconNodeBackend>> { let store_config = StoreConfig { prune_payloads: false, @@ -2835,10 +2873,7 @@ async fn reproduction_unaligned_checkpoint_sync_pruned_payload() { .is_ok() ); - let wss_blobs_opt = harness - .chain - .get_or_reconstruct_blobs(&wss_block_root) - .unwrap(); + let wss_blobs_opt = get_or_reconstruct_blobs(&harness.chain, &wss_block_root).unwrap(); let wss_state = full_store .get_state(&wss_state_root, Some(checkpoint_slot), CACHE_STATE_IN_TESTS) @@ -2971,10 +3006,7 @@ async fn weak_subjectivity_sync_test( .state_root_at_slot(checkpoint_slot) .unwrap() .unwrap(); - let wss_blobs_opt = harness - .chain - .get_or_reconstruct_blobs(&wss_block_root) - .unwrap(); + let wss_blobs_opt = get_or_reconstruct_blobs(&harness.chain, &wss_block_root).unwrap(); let wss_state = full_store .get_state(&wss_state_root, Some(checkpoint_slot), CACHE_STATE_IN_TESTS) .unwrap() @@ -3063,6 +3095,29 @@ async fn weak_subjectivity_sync_test( let beacon_chain = Arc::new(beacon_chain); let wss_block_root = wss_block.canonical_root(); + + // For Gloas, blobs aren't a standalone shape — the WSS data is the column sidecar set, which + // `get_or_reconstruct_blobs` returns `None` for. Copy the WSS block's columns straight from + // the source store so that the destination has them after checkpoint sync, matching what + // network-driven WSS would produce in production. + if wss_block.fork_name_unchecked().gloas_enabled() + && let Ok(Some(source_columns)) = harness + .chain + .store + .get_data_columns(&wss_block_root, ForkName::Gloas) + && !source_columns.is_empty() + && let Some(store_op) = beacon_chain.get_blobs_or_columns_store_op( + wss_block_root, + wss_block.slot(), + beacon_chain::block_verification_types::AvailableBlockData::DataColumns(source_columns), + ) + { + beacon_chain + .store + .do_atomically_with_block_and_blobs_cache(vec![store_op]) + .unwrap(); + } + let store_wss_block = harness .chain .get_block(&wss_block_root) @@ -3070,9 +3125,7 @@ async fn weak_subjectivity_sync_test( .unwrap() .unwrap(); // This test may break in the future if we no longer store the full checkpoint data columns. - let store_wss_blobs_opt = beacon_chain - .get_or_reconstruct_blobs(&wss_block_root) - .unwrap(); + let store_wss_blobs_opt = get_or_reconstruct_blobs(&beacon_chain, &wss_block_root).unwrap(); assert_eq!(store_wss_block, wss_block); // TODO(fulu): Remove this condition once #6760 (PeerDAS checkpoint sync) is merged. @@ -3130,12 +3183,43 @@ async fn weak_subjectivity_sync_test( .await .unwrap(); - // Store the envelope and apply it to fork choice. + // Store the envelope, its columns, and apply to fork choice. if let Some(envelope) = &snapshot.execution_envelope { + // Persist data columns for Gloas blocks. This mirrors what production does in + // `import_available_execution_payload_envelope` and what the harness now does in + // `process_envelope` — the WSS forward-sync loop bypasses both, so do it directly. + let mut ops = vec![]; + let columns_block = beacon_chain + .store + .get_blinded_block(&block_root) + .unwrap() + .and_then(|b| beacon_chain.store.make_full_block(&block_root, b).ok()); + if let Some(full_block) = columns_block { + let columns = beacon_chain::test_utils::generate_data_column_sidecars_from_block( + &full_block, + &beacon_chain.spec, + ); + if !columns.is_empty() + && let Some(store_op) = beacon_chain.get_blobs_or_columns_store_op( + block_root, + full_block.slot(), + beacon_chain::block_verification_types::AvailableBlockData::DataColumns( + columns, + ), + ) + { + ops.push(store_op); + } + } + ops.push(store::StoreOp::PutPayloadEnvelope( + block_root, + std::sync::Arc::new(envelope.as_ref().clone()), + )); beacon_chain .store - .put_payload_envelope(&block_root, envelope) + .do_atomically_with_block_and_blobs_cache(ops) .unwrap(); + // Update fork choice so head selection accounts for Full payload status. beacon_chain .canonical_head diff --git a/beacon_node/http_api/src/beacon/execution_payload_envelope.rs b/beacon_node/http_api/src/beacon/execution_payload_envelope.rs index 65e1a83840..2e7fe693d6 100644 --- a/beacon_node/http_api/src/beacon/execution_payload_envelope.rs +++ b/beacon_node/http_api/src/beacon/execution_payload_envelope.rs @@ -7,6 +7,7 @@ use crate::version::{ execution_optimistic_finalized_beacon_response, }; use beacon_chain::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumn}; +use beacon_chain::payload_envelope_verification::EnvelopeError; use beacon_chain::{BeaconChain, BeaconChainTypes, NotifyExecutionLayer}; use bytes::Bytes; use eth2::types as api_types; @@ -148,7 +149,7 @@ pub async fn publish_execution_payload_envelope( PubsubMessage::ExecutionPayload(Box::new(envelope_for_gossip)), ) .map_err(|_| { - beacon_chain::payload_envelope_verification::EnvelopeError::BeaconChainError(Arc::new( + EnvelopeError::BeaconChainError(Arc::new( beacon_chain::BeaconChainError::UnableToPublish, )) }) @@ -272,7 +273,7 @@ fn build_gloas_data_columns( let index = *col.index(); match GossipVerifiedDataColumn::new_for_block_publishing(col, chain) { Ok(verified) => Some(verified), - Err(GossipDataColumnError::PriorKnownUnpublished) => None, + Err(GossipDataColumnError::PriorKnown { .. }) => None, Err(e) => { warn!( %slot, diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 644ade956a..e96c86b17f 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -246,7 +246,7 @@ pub async fn publish_block>( if let Err(e) = Box::pin(chain.process_gossip_data_columns(sampling_columns, publish_fn)).await { - let msg = format!("Invalid data column: {e}"); + let msg = format!("Invalid data column: {e:?}"); return if let BroadcastValidation::Gossip = validation_level { Err(warp_utils::reject::broadcast_without_import(msg)) } else { diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 184bfffc9a..b47f8e946a 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -382,6 +382,7 @@ pub async fn proposer_boost_re_org_weight_misprediction() { /// - `num_empty_votes`: percentage of comm of attestations for the parent block /// - `num_head_votes`: number of attestations for the head block /// - `should_re_org`: whether the proposer should build on the parent rather than the head +#[allow(clippy::large_stack_frames)] pub async fn proposer_boost_re_org_test( ReOrgTest { head_slot, diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index b09dc95db4..4b34d7bfc0 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -1,5 +1,5 @@ use beacon_chain::{ - AvailabilityProcessingStatus, BlockError, attestation_verification::Error as AttnError, + AvailabilityProcessingStatus, attestation_verification::Error as AttnError, light_client_finality_update_verification::Error as LightClientFinalityUpdateError, light_client_optimistic_update_verification::Error as LightClientOptimisticUpdateError, sync_committee_verification::Error as SyncCommitteeError, @@ -733,7 +733,7 @@ pub fn register_sync_committee_error(error: &SyncCommitteeError) { } pub(crate) fn register_process_result_metrics( - result: &std::result::Result, + result: &std::result::Result>, source: BlockSource, block_component: &'static str, ) { diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 57871a2009..d34668b138 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -698,15 +698,6 @@ impl NetworkBeaconProcessor { } Err(err) => { match err { - GossipDataColumnError::InvalidVariant => { - // TODO(gloas) we should probably penalize the peer here - debug!( - %slot, - %block_root, - %index, - "Invalid gossip data column variant." - ) - } GossipDataColumnError::PriorKnownUnpublished => { debug!( %slot, @@ -732,7 +723,27 @@ impl NetworkBeaconProcessor { column_sidecar, )); } - GossipDataColumnError::PubkeyCacheTimeout + GossipDataColumnError::BlockRootUnknown { + block_root: unknown_block_root, + .. + } => { + debug!( + action = "ignoring", + %unknown_block_root, + "Unknown block root for column" + ); + // TODO(gloas): wire this into proper lookup sync. Sending + // `UnknownBlockHashFromAttestation` here is a Fulu-shaped fallback that + // mixes column processing with the attestation lookup path and is not + // the right primitive for Gloas column lookups. + self.propagate_validation_result( + message_id, + peer_id, + MessageAcceptance::Ignore, + ); + } + GossipDataColumnError::InvalidVariant + | GossipDataColumnError::PubkeyCacheTimeout | GossipDataColumnError::BeaconChainError(_) => { crit!( error = ?err, @@ -743,6 +754,7 @@ impl NetworkBeaconProcessor { | GossipDataColumnError::UnknownValidator(_) | GossipDataColumnError::ProposerIndexMismatch { .. } | GossipDataColumnError::IsNotLaterThanParent { .. } + | GossipDataColumnError::BlockSlotMismatch { .. } | GossipDataColumnError::InvalidSubnetId { .. } | GossipDataColumnError::InvalidInclusionProof | GossipDataColumnError::InvalidKzgProof { .. } @@ -803,6 +815,19 @@ impl NetworkBeaconProcessor { MessageAcceptance::Ignore, ); } + GossipDataColumnError::InternalError(err) => { + error!( + error = ?err, + %block_root, + %index, + "Internal error while processing data columns" + ); + self.propagate_validation_result( + message_id, + peer_id, + MessageAcceptance::Ignore, + ); + } } } } @@ -904,14 +929,6 @@ impl NetworkBeaconProcessor { ) { match err { GossipPartialDataColumnError::GossipDataColumnError(err) => match err { - GossipDataColumnError::InvalidVariant => { - // TODO(gloas) we should probably penalize the peer here - debug!( - %block_root, - %index, - "Invalid gossip partial data column variant." - ) - } GossipDataColumnError::PriorKnownUnpublished => { debug!( %block_root, @@ -933,6 +950,24 @@ impl NetworkBeaconProcessor { slot, }); } + GossipDataColumnError::BlockRootUnknown { + block_root: unknown_block_root, + .. + } => { + debug!( + action = "requesting block", + %unknown_block_root, + "Unknown block root for partial column" + ); + // TODO(gloas): wire this into proper lookup sync. Sending + // `UnknownBlockHashFromAttestation` here is a Fulu-shaped fallback that + // mixes column processing with the attestation lookup path and is not + // the right primitive for Gloas column lookups. + self.send_sync_message(SyncMessage::UnknownBlockHashFromAttestation( + peer_id, + unknown_block_root, + )); + } GossipDataColumnError::PubkeyCacheTimeout | GossipDataColumnError::BeaconChainError(_) => { crit!( @@ -940,10 +975,12 @@ impl NetworkBeaconProcessor { "Internal error when verifying partial column sidecar" ) } - GossipDataColumnError::ProposalSignatureInvalid + GossipDataColumnError::InvalidVariant + | GossipDataColumnError::ProposalSignatureInvalid | GossipDataColumnError::UnknownValidator(_) | GossipDataColumnError::ProposerIndexMismatch { .. } | GossipDataColumnError::IsNotLaterThanParent { .. } + | GossipDataColumnError::BlockSlotMismatch { .. } | GossipDataColumnError::InvalidSubnetId { .. } | GossipDataColumnError::InvalidInclusionProof | GossipDataColumnError::InvalidKzgProof { .. } @@ -993,6 +1030,14 @@ impl NetworkBeaconProcessor { "gossip_partial_data_column_high", ); } + GossipDataColumnError::InternalError(err) => { + error!( + error = ?err, + %block_root, + %index, + "Internal error while handling partial data column verification" + ); + } }, GossipPartialDataColumnError::MissingHeader => { metrics::inc_counter( @@ -1052,7 +1097,7 @@ impl NetworkBeaconProcessor { "gossip_partial_data_column_low", ); } - GossipPartialDataColumnError::InternalError(_) => { + GossipPartialDataColumnError::InternalError(err) => { error!( error = ?err, %block_root, @@ -1323,6 +1368,7 @@ impl NetworkBeaconProcessor { let data_column_slot = verified_data_column.slot(); let data_column_index = verified_data_column.index(); + // TODO(gloas): implement partial messages if let DataColumnSidecar::Fulu(col) = verified_data_column.as_data_column() && self .chain @@ -1353,7 +1399,7 @@ impl NetworkBeaconProcessor { .await; register_process_result_metrics(&result, metrics::BlockSource::Gossip, "data_column"); - match &result { + match result { Ok(availability) => match availability { AvailabilityProcessingStatus::Imported(block_root) => { debug!( @@ -1366,6 +1412,14 @@ impl NetworkBeaconProcessor { &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, processing_start_time.elapsed().as_millis() as i64, ); + + // If a block is in the da_checker, sync maybe awaiting for an event when block is finally + // imported. A block can become imported both after processing a block or data column. If + // importing a block results in `Imported`, notify. Do not notify of data column errors. + self.send_sync_message(SyncMessage::GossipBlockProcessResult { + block_root, + imported: true, + }); } AvailabilityProcessingStatus::MissingComponents(slot, block_root) => { trace!( @@ -1375,7 +1429,7 @@ impl NetworkBeaconProcessor { "Processed data column, waiting for other components" ); - self.check_reconstruction_trigger(*slot, block_root).await; + self.check_reconstruction_trigger(slot, &block_root).await; } }, Err(BlockError::DuplicateFullyImported(_)) => { @@ -1399,16 +1453,6 @@ impl NetworkBeaconProcessor { ); } } - - // If a block is in the da_checker, sync maybe awaiting for an event when block is finally - // imported. A block can become imported both after processing a block or data column. If a - // importing a block results in `Imported`, notify. Do not notify of data column errors. - if matches!(result, Ok(AvailabilityProcessingStatus::Imported(_))) { - self.send_sync_message(SyncMessage::GossipBlockProcessResult { - block_root, - imported: true, - }); - } } /// Process a gossip-verified partial data column by merging it in the assembler @@ -1575,7 +1619,7 @@ impl NetworkBeaconProcessor { slot, process_fn: Box::pin(async move { cloned_self - .attempt_data_column_reconstruction(block_root) + .attempt_data_column_reconstruction(slot, block_root) .await; }), }, @@ -1827,7 +1871,10 @@ impl NetworkBeaconProcessor { return None; } // BlobNotRequired is unreachable. Only constructed in `process_gossip_blob` - Err(e @ BlockError::InternalError(_)) | Err(e @ BlockError::BlobNotRequired(_)) => { + Err(e @ BlockError::InternalError(_)) + | Err(e @ BlockError::BlobNotRequired(_)) + | Err(e @ BlockError::EnvelopeBlockRootUnknown(_)) + | Err(e @ BlockError::OptimisticSyncNotSupported { .. }) => { error!(error = %e, "Internal block gossip validation error"); return None; } @@ -3814,8 +3861,7 @@ impl NetworkBeaconProcessor { | EnvelopeError::UnknownValidator { .. } | EnvelopeError::IncorrectBlockProposer { .. } | EnvelopeError::ExecutionPayloadError(_) - | EnvelopeError::EnvelopeProcessingError(_) - | EnvelopeError::BlockError(_) => { + | EnvelopeError::EnvelopeProcessingError(_) => { self.propagate_validation_result( message_id, peer_id, @@ -3895,11 +3941,9 @@ impl NetworkBeaconProcessor { } EnvelopeError::PriorToFinalization { .. } - | EnvelopeError::OptimisticSyncNotSupported { .. } | EnvelopeError::BeaconChainError(_) | EnvelopeError::BeaconStateError(_) - | EnvelopeError::BlockProcessingError(_) - | EnvelopeError::InternalError(_) => { + | EnvelopeError::ImportError(_) => { self.propagate_validation_result( message_id, peer_id, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 6a3ccbcd65..7bf969db10 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -1014,6 +1014,7 @@ impl NetworkBeaconProcessor { } // Publish partial columns without eager send + // TODO(gloas): implement publish partial columns without eager send if let Some(assembler) = self.chain.data_availability_checker.partial_assembler() { let columns = assembler.get_partials_and_mark_as_local_fetched(block_root, &header); if !columns.is_empty() { @@ -1034,8 +1035,8 @@ impl NetworkBeaconProcessor { /// Attempts to reconstruct all data columns if the conditions checked in /// [`DataAvailabilityCheckerInner::check_and_set_reconstruction_started`] are satisfied. #[instrument(level = "debug", skip_all, fields(?block_root))] - async fn attempt_data_column_reconstruction(self: &Arc, block_root: Hash256) { - let result = self.chain.reconstruct_data_columns(block_root).await; + async fn attempt_data_column_reconstruction(self: &Arc, slot: Slot, block_root: Hash256) { + let result = self.chain.reconstruct_data_columns(slot, block_root).await; match result { Ok(Some((availability_processing_status, data_columns_to_publish))) => { diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 8f89b66948..988a68c9dd 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -731,6 +731,8 @@ impl NetworkBeaconProcessor { .map(|block| block.into_available_block()) .collect::>(); + // TODO(gloas) when implementing backfill sync for gloas + // we need a batch verify kzg function in the new da checker match self .chain .data_availability_checker diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index f13815f7b6..18d34b40b3 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -10,7 +10,7 @@ use crate::{ }; use beacon_chain::block_verification_types::LookupBlock; use beacon_chain::custody_context::NodeCustodyType; -use beacon_chain::data_column_verification::validate_data_column_sidecar_for_gossip_fulu; +use beacon_chain::data_column_verification::GossipVerifiedDataColumn; use beacon_chain::kzg_utils::blobs_to_data_column_sidecars; use beacon_chain::observed_data_sidecars::DoNotObserve; use beacon_chain::test_utils::{ @@ -1195,12 +1195,8 @@ async fn accept_processed_gossip_data_columns_without_import() { .map(|data_column| { let subnet_id = DataColumnSubnetId::from_column_index(*data_column.index(), &rig.chain.spec); - validate_data_column_sidecar_for_gossip_fulu::<_, DoNotObserve>( - data_column, - subnet_id, - &rig.chain, - ) - .expect("should be valid data column") + GossipVerifiedDataColumn::<_, DoNotObserve>::new(data_column, subnet_id, &rig.chain) + .expect("should be valid data column") }) .collect(); diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index f5c0fdb4e5..bb43396473 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -548,13 +548,19 @@ mod tests { #[test] fn no_blobs_into_responses() { + let spec = Arc::new(test_spec::()); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..4) .map(|_| { - generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut u) - .unwrap() - .0 - .into() + generate_rand_block_and_blobs::( + spec.fork_name_at_epoch(Epoch::new(0)), + NumBlobs::None, + &mut u, + ) + .unwrap() + .0 + .into() }) .collect::>>>(); @@ -565,7 +571,6 @@ mod tests { // Send blocks and complete terminate response info.add_blocks(blocks_req_id, blocks).unwrap(); - let spec = Arc::new(test_spec::()); let da_checker = Arc::new(test_da_checker(spec.clone(), NodeCustodyType::Fullnode)); // Assert response is finished and RpcBlocks can be constructed diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 734295ac1d..347b018a93 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -181,7 +181,9 @@ pub enum SyncMessage { result: BlockProcessingResult, }, - /// A block from gossip has completed processing, + /// A gossip-received component has completed processing and the block may now be imported. + /// In Fulu this is sent after block or blob processing. In Gloas this is also sent after + /// data column or payload envelope processing triggers availability. GossipBlockProcessResult { block_root: Hash256, imported: bool }, } @@ -905,9 +907,13 @@ impl SyncManager { }), ); } - // TODO(gloas) support gloas data column variant DataColumnSidecar::Gloas(_) => { - error!("Gloas variant not yet supported") + // TODO(gloas): proper lookup sync for Gloas. Routing into + // `handle_unknown_block_root` here mixes column processing with the + // single-block-lookup path; the Gloas column-arrives-before-block + // case wants its own queue/wakeup. + debug!(%block_root, "Received unknown block data column message"); + self.handle_unknown_block_root(peer_id, block_root); } } } diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index b1ba87c75d..465e23998b 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -1085,10 +1085,22 @@ impl SyncNetworkContext { block_root: Hash256, lookup_peers: Arc>>, ) -> Result { + let slot = self + .chain + .canonical_head + .fork_choice_read_lock() + .get_block(&block_root) + .map(|block| block.slot) + .or_else(|| self.chain.slot().ok()) + .ok_or_else(|| { + RpcRequestSendError::InternalError(format!( + "Unable to determine slot for block {block_root:?}" + )) + })?; + let custody_indexes_imported = self .chain - .data_availability_checker - .cached_data_column_indexes(&block_root) + .cached_data_column_indexes(&block_root, slot) .unwrap_or_default(); let current_epoch = self.chain.epoch().map_err(|e| { diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index d27c92c21a..c1b2793491 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -2087,8 +2087,7 @@ async fn too_many_processing_failures(depth: usize) { r.build_chain_and_trigger_last_block(depth).await; // Simulate that a peer always returns empty r.simulate( - SimulateConfig::new() - .with_process_result(|| BlockProcessingResult::Err(BlockError::BlockSlotLimitReached)), + SimulateConfig::new().with_process_result(|| BlockError::BlockSlotLimitReached.into()), ) .await; // We register multiple penalties, the lookup fails and sync does not progress @@ -2156,9 +2155,10 @@ async fn test_single_block_lookup_duplicate_response() { let mut r = TestRig::default(); r.build_chain_and_trigger_last_block(1).await; // Send a DuplicateFullyImported response, the lookup should complete successfully - r.simulate(SimulateConfig::new().with_process_result(|| { - BlockProcessingResult::Err(BlockError::DuplicateFullyImported(Hash256::ZERO)) - })) + r.simulate( + SimulateConfig::new() + .with_process_result(|| BlockError::DuplicateFullyImported(Hash256::ZERO).into()), + ) .await; // The block was not actually imported r.assert_head_slot(0); diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 593aa27915..a60859585c 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -1514,6 +1514,14 @@ where } } + /// Returns whether the execution payload for a block has been received. + /// + /// Returns `false` for unknown blocks and pre-Gloas nodes. + pub fn is_payload_received(&self, block_root: &Hash256) -> bool { + self.proto_array.is_payload_received(block_root) + && self.is_finalized_checkpoint_or_descendant(*block_root) + } + /// Returns whether the proposer should extend the execution payload chain of the given block. pub fn should_extend_payload(&self, block_root: &Hash256) -> Result> { let proposer_boost_root = self.fc_store.proposer_boost_root(); diff --git a/consensus/types/src/block/signed_beacon_block.rs b/consensus/types/src/block/signed_beacon_block.rs index 76bb9a09db..11ac17dece 100644 --- a/consensus/types/src/block/signed_beacon_block.rs +++ b/consensus/types/src/block/signed_beacon_block.rs @@ -351,6 +351,12 @@ impl> SignedBeaconBlock self.message() .body() .blob_kzg_commitments() + .or_else(|_| { + self.message() + .body() + .signed_execution_payload_bid() + .map(|bid| &bid.message.blob_kzg_commitments) + }) .map(|c| c.len()) .unwrap_or(0) } diff --git a/consensus/types/src/execution/signed_execution_payload_bid.rs b/consensus/types/src/execution/signed_execution_payload_bid.rs index 3d4f45a267..2ad6dcea1a 100644 --- a/consensus/types/src/execution/signed_execution_payload_bid.rs +++ b/consensus/types/src/execution/signed_execution_payload_bid.rs @@ -23,6 +23,14 @@ pub struct SignedExecutionPayloadBid { } impl SignedExecutionPayloadBid { + pub fn epoch(&self) -> crate::Epoch { + self.message.slot.epoch(E::slots_per_epoch()) + } + + pub fn slot(&self) -> crate::Slot { + self.message.slot + } + pub fn empty() -> Self { Self { message: ExecutionPayloadBid::default(), From fd0852a8e59be905824cebfb4f2fe028a920159b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 19 May 2026 11:35:31 +1000 Subject: [PATCH 12/18] Remove outdated SPRP hint (#9312) While working on this code in another branch I noticed we had this messy, complicated and incorrect code about SPRP (slots-per-restore-point), which is no longer a relevant concept since the introduction of hot state diffs. In the name of simplicity, I've removed any kind of hinting here in favour of a simple out of bounds error. The benefit of adding complex hinting code (which is not tested) to such a function is not worth it IMO. Users will work it out (or ask) if we just tell them their request is out of bounds. Co-Authored-By: Michael Sproul --- beacon_node/http_api/src/beacon/states.rs | 74 +++++++++-------------- 1 file changed, 27 insertions(+), 47 deletions(-) diff --git a/beacon_node/http_api/src/beacon/states.rs b/beacon_node/http_api/src/beacon/states.rs index 84ef3c1f26..52b05a807b 100644 --- a/beacon_node/http_api/src/beacon/states.rs +++ b/beacon_node/http_api/src/beacon/states.rs @@ -390,54 +390,34 @@ pub fn get_beacon_state_committees( if let Some(shuffling) = maybe_cached_shuffling { shuffling } else { - let possibly_built_cache = - match RelativeEpoch::from_epoch(current_epoch, epoch) { - Ok(relative_epoch) - if state.committee_cache_is_initialized( - relative_epoch, - ) => - { - state.committee_cache(relative_epoch).cloned() - } - _ => CommitteeCache::initialized( - state, - epoch, - &chain.spec, - ), + let possibly_built_cache = match RelativeEpoch::from_epoch( + current_epoch, + epoch, + ) { + Ok(relative_epoch) + if state.committee_cache_is_initialized( + relative_epoch, + ) => + { + state.committee_cache(relative_epoch).cloned() } - .map_err( - |e| match e { - BeaconStateError::EpochOutOfBounds => { - let max_sprp = - T::EthSpec::slots_per_historical_root() - as u64; - let first_subsequent_restore_point_slot = - ((epoch.start_slot( - T::EthSpec::slots_per_epoch(), - ) / max_sprp) - + 1) - * max_sprp; - if epoch < current_epoch { - warp_utils::reject::custom_bad_request( - format!( - "epoch out of bounds, \ - try state at slot {}", - first_subsequent_restore_point_slot, - ), - ) - } else { - warp_utils::reject::custom_bad_request( - "epoch out of bounds, \ - too far in future" - .into(), - ) - } - } - _ => warp_utils::reject::unhandled_error( - BeaconChainError::from(e), - ), - }, - )?; + _ => CommitteeCache::initialized( + state, + epoch, + &chain.spec, + ), + } + .map_err(|e| match e { + BeaconStateError::EpochOutOfBounds => { + warp_utils::reject::custom_bad_request(format!( + "epoch {} out of bounds for state at {}", + epoch, current_epoch + )) + } + _ => warp_utils::reject::unhandled_error( + BeaconChainError::from(e), + ), + })?; // Attempt to write to the beacon cache (only if the cache // size is not the default value). From 398efc3acca5c8d01befbbe09d35d24cbd04752c Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 18 May 2026 23:12:17 -0600 Subject: [PATCH 13/18] Use dedicated cache for HTTP API route (#9318) - PR https://github.com/sigp/lighthouse/pull/9305 wants to store PTCs in the committee cache. BUT the http API route wants to use the committee cache and insert historical committees (i.e. given state at epoch 1000, compute and store the committee for epoch 900). If we want a single cache to serve both use cases we need to: - Have entries in the committee cache that have no PTC: Makes reading PTCs from the cache not deterministic - Compute historical PTC: A bunch of complicated code that's useless Instead we can add a separate cache for the API, very simple one, that caches committees only. And have the one in the beacon chain compute and cache PTCs always. ### Performance impact Slightly additional memory cost for users of the `beacon/states/committees` route. Caching is almost equivalent, except for queries of recent committees that may already exist in the beacon chain's committee cache. ### AI disclousure This PR was written by hand 90%. Claude fixed some warp type issues Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> --- beacon_node/client/src/builder.rs | 5 ++ beacon_node/http_api/src/beacon/states.rs | 81 ++++++++++++----------- beacon_node/http_api/src/caches.rs | 43 ++++++++++++ beacon_node/http_api/src/lib.rs | 17 ++++- beacon_node/http_api/src/test_utils.rs | 7 +- beacon_node/src/config.rs | 3 + 6 files changed, 115 insertions(+), 41 deletions(-) create mode 100644 beacon_node/http_api/src/caches.rs diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 9dfb8304bc..f532ef716e 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -36,6 +36,7 @@ use rand::SeedableRng; use rand::rngs::{OsRng, StdRng}; use slasher::Slasher; use slasher_service::SlasherService; +use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -639,6 +640,10 @@ where network_globals: self.network_globals.clone(), beacon_processor_send: Some(beacon_processor_channels.beacon_processor_tx.clone()), sse_logging_components: runtime_context.sse_logging_components.clone(), + historical_committee_cache: Arc::new(http_api::HistoricalCommitteeCache::new( + NonZeroUsize::new(self.http_api_config.historical_committee_cache_size) + .unwrap_or(NonZeroUsize::MIN), + )), }); let exit = runtime_context.executor.exit(); diff --git a/beacon_node/http_api/src/beacon/states.rs b/beacon_node/http_api/src/beacon/states.rs index 52b05a807b..1b765aa227 100644 --- a/beacon_node/http_api/src/beacon/states.rs +++ b/beacon_node/http_api/src/beacon/states.rs @@ -1,4 +1,5 @@ use crate::StateId; +use crate::caches::{HistoricalCommitteeCache, HistoricalShufflingId}; use crate::task_spawner::{Priority, TaskSpawner}; use crate::utils::ResponseFilter; use crate::validator::pubkey_to_validator_index; @@ -13,7 +14,10 @@ use eth2::types::{ }; use ssz::Encode; use std::sync::Arc; -use types::{AttestationShufflingId, BeaconStateError, CommitteeCache, EthSpec, RelativeEpoch}; +use types::{ + AttestationShufflingId, BeaconStateError, CommitteeCache, EthSpec, RelativeEpoch, + RelativeEpochError, +}; use warp::filters::BoxedFilter; use warp::http::Response; use warp::hyper::Body; @@ -26,6 +30,8 @@ type BeaconStatesPath = BoxedFilter<( Arc>, )>; +type BeaconStatesCommitteesFilter = BoxedFilter<(Arc,)>; + // GET beacon/states/{state_id}/pending_consolidations pub fn get_beacon_state_pending_consolidations( beacon_states_path: BeaconStatesPath, @@ -337,17 +343,20 @@ pub fn get_beacon_state_sync_committees( // GET beacon/states/{state_id}/committees?slot,index,epoch pub fn get_beacon_state_committees( beacon_states_path: BeaconStatesPath, + beacon_states_committees_filter: BeaconStatesCommitteesFilter, ) -> ResponseFilter { beacon_states_path .clone() .and(warp::path("committees")) .and(warp::query::()) + .and(beacon_states_committees_filter) .and(warp::path::end()) .then( |state_id: StateId, task_spawner: TaskSpawner, chain: Arc>, - query: eth2::types::CommitteesQuery| { + query: eth2::types::CommitteesQuery, + historical_committee_cache: Arc| { task_spawner.blocking_json_task(Priority::P1, move || { let (data, execution_optimistic, finalized) = state_id .map_state_and_execution_optimistic_and_finalized( @@ -364,33 +373,33 @@ pub fn get_beacon_state_committees( let shuffling_id = if let Ok(Some(shuffling_decision_block)) = chain.block_root_at_slot(decision_slot, WhenSlotSkipped::Prev) { - Some(AttestationShufflingId { - shuffling_epoch: epoch, - shuffling_decision_block, - }) + Some(HistoricalShufflingId::ShufflingId( + AttestationShufflingId { + shuffling_epoch: epoch, + shuffling_decision_block, + }, + )) + } else if epoch < chain.head().finalized_checkpoint().epoch { + // Use the case for finalized epochs + Some(HistoricalShufflingId::FinalizedEpoch(epoch)) } else { None }; // Attempt to read from the chain cache if there exists a // shuffling_id - let maybe_cached_shuffling = if let Some(shuffling_id) = - shuffling_id.as_ref() - { - chain - .shuffling_cache - .try_write_for(std::time::Duration::from_secs(1)) - .and_then(|mut cache_write| cache_write.get(shuffling_id)) - .and_then(|cache_item| cache_item.wait().ok()) - } else { - None - }; + let maybe_cached_shuffling = + if let Some(shuffling_id) = shuffling_id.as_ref() { + historical_committee_cache.get(shuffling_id) + } else { + None + }; let committee_cache = if let Some(shuffling) = maybe_cached_shuffling { shuffling } else { - let possibly_built_cache = match RelativeEpoch::from_epoch( + let committee_cache = match RelativeEpoch::from_epoch( current_epoch, epoch, ) { @@ -401,11 +410,19 @@ pub fn get_beacon_state_committees( { state.committee_cache(relative_epoch).cloned() } - _ => CommitteeCache::initialized( - state, - epoch, - &chain.spec, - ), + Ok(_) | Err(RelativeEpochError::EpochTooLow { .. }) => { + CommitteeCache::initialized( + state, + epoch, + &chain.spec, + ) + } + Err(RelativeEpochError::EpochTooHigh { .. }) => { + Err(BeaconStateError::EpochOutOfBounds) + } + Err(RelativeEpochError::ArithError(e)) => { + Err(BeaconStateError::ArithError(e)) + } } .map_err(|e| match e { BeaconStateError::EpochOutOfBounds => { @@ -419,22 +436,12 @@ pub fn get_beacon_state_committees( ), })?; - // Attempt to write to the beacon cache (only if the cache - // size is not the default value). - if chain.config.shuffling_cache_size - != beacon_chain::shuffling_cache::DEFAULT_CACHE_SIZE - && let Some(shuffling_id) = shuffling_id - && let Some(mut cache_write) = chain - .shuffling_cache - .try_write_for(std::time::Duration::from_secs(1)) - { - cache_write.insert_committee_cache( - shuffling_id, - &possibly_built_cache, - ); + if let Some(shuffling_id) = shuffling_id { + historical_committee_cache + .insert(shuffling_id, committee_cache.clone()); } - possibly_built_cache + committee_cache }; // Use either the supplied slot or all slots in the epoch. diff --git a/beacon_node/http_api/src/caches.rs b/beacon_node/http_api/src/caches.rs new file mode 100644 index 0000000000..d92571594a --- /dev/null +++ b/beacon_node/http_api/src/caches.rs @@ -0,0 +1,43 @@ +use lru::LruCache; +use parking_lot::Mutex; +use std::num::NonZeroUsize; +use std::sync::Arc; +use types::{AttestationShufflingId, CommitteeCache, Epoch}; + +/// See `shuffling_cache::DEFAULT_CACHE_SIZE` for rationale +pub const DEFAULT_HISTORICAL_COMMITTEE_CACHE_SIZE: usize = 16; + +/// Indexes the `HistoricalCommitteeCache`. We can compute committees for very old epochs, and we +/// can't retrieve the decision root cheaply from a state. For those cases we allow the cache to +/// key those committees by finalized epoch. +#[derive(Eq, Hash, PartialEq)] +pub enum HistoricalShufflingId { + FinalizedEpoch(Epoch), + ShufflingId(AttestationShufflingId), +} + +/// Dedicated cache for attestation committees, used exclusively by the HTTP API. +/// +/// This may contain committees for finalized and unfinalized epochs. The name is slightly +/// missleading :) +pub struct HistoricalCommitteeCache { + committees: Mutex>>, +} + +impl HistoricalCommitteeCache { + pub fn new(size: NonZeroUsize) -> Self { + Self { + committees: Mutex::new(LruCache::new(size)), + } + } +} + +impl HistoricalCommitteeCache { + pub fn get(&self, id: &HistoricalShufflingId) -> Option> { + self.committees.lock().get(id).cloned() + } + + pub fn insert(&self, id: HistoricalShufflingId, cache: Arc) { + self.committees.lock().put(id, cache); + } +} diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index f31817c5ba..74bf1ccd76 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -12,6 +12,7 @@ mod beacon; mod block_id; mod build_block_contents; mod builder_states; +mod caches; mod custody; mod database; mod light_client; @@ -40,6 +41,8 @@ use crate::beacon::execution_payload_envelope::{ post_beacon_execution_payload_envelope_ssz, }; use crate::beacon::pool::*; +use crate::caches::DEFAULT_HISTORICAL_COMMITTEE_CACHE_SIZE; +pub use crate::caches::HistoricalCommitteeCache; use crate::light_client::{get_light_client_bootstrap, get_light_client_updates}; use crate::utils::{AnyVersionFilter, EthV1Filter}; use crate::validator::post_validator_liveness_epoch; @@ -132,6 +135,7 @@ pub struct Context { pub network_globals: Option>>, pub beacon_processor_send: Option>, pub sse_logging_components: Option, + pub historical_committee_cache: Arc, } /// Configuration for the HTTP server. @@ -148,6 +152,7 @@ pub struct Config { #[serde(with = "eth2::types::serde_status_code")] pub duplicate_block_status_code: StatusCode, pub target_peers: usize, + pub historical_committee_cache_size: usize, } impl Default for Config { @@ -163,6 +168,7 @@ impl Default for Config { enable_beacon_processor: true, duplicate_block_status_code: StatusCode::ACCEPTED, target_peers: 100, + historical_committee_cache_size: DEFAULT_HISTORICAL_COMMITTEE_CACHE_SIZE, } } } @@ -416,6 +422,11 @@ pub fn serve( }) .boxed(); + let historical_committee_cache = ctx.historical_committee_cache.clone(); + let beacon_states_committees_filter = warp::any() + .map(move || historical_committee_cache.clone()) + .boxed(); + // Create a `warp` filter that provides access to the network sender channel. let network_tx = ctx .network_senders @@ -628,8 +639,10 @@ pub fn serve( states::get_beacon_state_validators_id(beacon_states_path.clone()); // GET beacon/states/{state_id}/committees?slot,index,epoch - let get_beacon_state_committees = - states::get_beacon_state_committees(beacon_states_path.clone()); + let get_beacon_state_committees = states::get_beacon_state_committees( + beacon_states_path.clone(), + beacon_states_committees_filter, + ); // GET beacon/states/{state_id}/sync_committees?epoch let get_beacon_state_sync_committees = diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index 27e2a27d35..f27a04d17a 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -1,4 +1,4 @@ -use crate::{Config, Context}; +use crate::{Config, Context, caches::HistoricalCommitteeCache}; use beacon_chain::{ BeaconChain, BeaconChainTypes, custody_context::NodeCustodyType, @@ -22,10 +22,10 @@ use lighthouse_network::{ }; use network::{NetworkReceivers, NetworkSenders}; use sensitive_url::SensitiveUrl; -use std::future::Future; use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; +use std::{future::Future, num::NonZeroUsize}; use store::MemoryStore; use task_executor::test_utils::TestRuntime; use types::{ChainSpec, EthSpec}; @@ -293,6 +293,9 @@ pub async fn create_api_server_with_config( network_globals: Some(network_globals), beacon_processor_send: Some(beacon_processor_send), sse_logging_components: None, + historical_committee_cache: Arc::new(HistoricalCommitteeCache::new( + NonZeroUsize::new(http_config.historical_committee_cache_size).unwrap(), + )), }); let (listening_socket, server) = diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 8ba2c0f321..f10f9e3b45 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -215,6 +215,9 @@ pub fn get_config( if let Some(cache_size) = clap_utils::parse_optional(cli_args, "shuffling-cache-size")? { client_config.chain.shuffling_cache_size = cache_size; + // Mantain backwards compatibility with users customizing `shuffling_cache_size` to tweak + // the behaviour of the HTTP API route `beacon/states/committees` + client_config.http_api.historical_committee_cache_size = cache_size; } if let Some(batches) = clap_utils::parse_optional(cli_args, "blob-publication-batches")? { From 2c76ee5b6b03cdcd43563e89d1befa7f07f4cc75 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 20 May 2026 06:56:49 -0600 Subject: [PATCH 14/18] Gloas lookup sync boilerplate (#9322) Implements the boring boilerplate to send envelopes by root requests and process them. Pre-step to - https://github.com/sigp/lighthouse/pull/9155 Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> --- beacon_node/beacon_chain/src/beacon_chain.rs | 6 + beacon_node/beacon_processor/src/lib.rs | 10 ++ .../src/scheduler/work_queue.rs | 6 + .../src/service/api_types.rs | 2 + .../src/network_beacon_processor/mod.rs | 19 +++ .../network_beacon_processor/sync_methods.rs | 57 +++++++ beacon_node/network/src/router.rs | 35 ++++- .../network/src/sync/block_lookups/mod.rs | 2 + beacon_node/network/src/sync/manager.rs | 66 +++++++- .../network/src/sync/network_context.rs | 145 +++++++++++++++++- .../src/sync/network_context/requests.rs | 4 + .../requests/payload_envelopes_by_root.rs | 54 +++++++ 12 files changed, 398 insertions(+), 8 deletions(-) create mode 100644 beacon_node/network/src/sync/network_context/requests/payload_envelopes_by_root.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index af8cd477d6..f3f6cd299e 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -6339,6 +6339,12 @@ impl BeaconChain { .contains_block(root) } + pub fn envelope_is_known_to_fork_choice(&self, root: &Hash256) -> bool { + self.canonical_head + .fork_choice_read_lock() + .is_payload_received(root) + } + /// Determines the beacon proposer for the next slot. If that proposer is registered in the /// `execution_layer`, provide the `execution_layer` with the necessary information to produce /// `PayloadAttributes` for future calls to fork choice. diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 25944bcf8a..ce3851ea54 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -418,6 +418,7 @@ pub enum Work { process_fn: AsyncFn, }, RpcCustodyColumn(AsyncFn), + RpcEnvelope(AsyncFn), ColumnReconstruction(AsyncFn), IgnoredRpcBlock { process_fn: BlockingFn, @@ -485,6 +486,7 @@ pub enum WorkType { RpcBlock, RpcBlobs, RpcCustodyColumn, + RpcEnvelope, ColumnReconstruction, IgnoredRpcBlock, ChainSegment, @@ -548,6 +550,7 @@ impl Work { Work::RpcBlock { .. } => WorkType::RpcBlock, Work::RpcBlobs { .. } => WorkType::RpcBlobs, Work::RpcCustodyColumn { .. } => WorkType::RpcCustodyColumn, + Work::RpcEnvelope(_) => WorkType::RpcEnvelope, Work::ColumnReconstruction(_) => WorkType::ColumnReconstruction, Work::IgnoredRpcBlock { .. } => WorkType::IgnoredRpcBlock, Work::ChainSegment { .. } => WorkType::ChainSegment, @@ -825,6 +828,8 @@ impl BeaconProcessor { Some(item) } else if let Some(item) = work_queues.rpc_custody_column_queue.pop() { Some(item) + } else if let Some(item) = work_queues.rpc_envelope_queue.pop() { + Some(item) // Check delayed blocks before gossip blocks, the gossip blocks might rely // on the delayed ones. } else if let Some(item) = work_queues.delayed_block_queue.pop() { @@ -1192,6 +1197,9 @@ impl BeaconProcessor { work_queues.rpc_block_queue.push(work, work_id) } Work::RpcBlobs { .. } => work_queues.rpc_blob_queue.push(work, work_id), + Work::RpcEnvelope(_) => { + work_queues.rpc_envelope_queue.push(work, work_id) + } Work::RpcCustodyColumn { .. } => { work_queues.rpc_custody_column_queue.push(work, work_id) } @@ -1330,6 +1338,7 @@ impl BeaconProcessor { WorkType::RpcBlobs | WorkType::IgnoredRpcBlock => { work_queues.rpc_blob_queue.len() } + WorkType::RpcEnvelope => work_queues.rpc_envelope_queue.len(), WorkType::RpcCustodyColumn => work_queues.rpc_custody_column_queue.len(), WorkType::ColumnReconstruction => { work_queues.column_reconstruction_queue.len() @@ -1523,6 +1532,7 @@ impl BeaconProcessor { } | Work::RpcBlobs { process_fn } | Work::RpcCustodyColumn(process_fn) + | Work::RpcEnvelope(process_fn) | Work::ColumnReconstruction(process_fn) => task_spawner.spawn_async(process_fn), Work::IgnoredRpcBlock { process_fn } => task_spawner.spawn_blocking(process_fn), Work::GossipBlock(work) diff --git a/beacon_node/beacon_processor/src/scheduler/work_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_queue.rs index eb57b97df2..2fdc15182c 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_queue.rs @@ -120,6 +120,7 @@ pub struct BeaconProcessorQueueLengths { rpc_block_queue: usize, rpc_blob_queue: usize, rpc_custody_column_queue: usize, + rpc_envelope_queue: usize, column_reconstruction_queue: usize, chain_segment_queue: usize, backfill_chain_segment: usize, @@ -195,6 +196,8 @@ impl BeaconProcessorQueueLengths { // We don't request more than `PARENT_DEPTH_TOLERANCE` (32) lookups, so we can limit // this queue size. With 48 max blobs per block, each column sidecar list could be up to 12MB. rpc_custody_column_queue: 64, + // Bounded by `PARENT_DEPTH_TOLERANCE`; one envelope per Gloas block. + rpc_envelope_queue: 1024, column_reconstruction_queue: 1, chain_segment_queue: 64, backfill_chain_segment: 64, @@ -253,6 +256,7 @@ pub struct WorkQueues { pub rpc_block_queue: FifoQueue>, pub rpc_blob_queue: FifoQueue>, pub rpc_custody_column_queue: FifoQueue>, + pub rpc_envelope_queue: FifoQueue>, pub column_reconstruction_queue: LifoQueue>, pub chain_segment_queue: FifoQueue>, pub backfill_chain_segment: FifoQueue>, @@ -323,6 +327,7 @@ impl WorkQueues { let rpc_block_queue = FifoQueue::new(queue_lengths.rpc_block_queue); let rpc_blob_queue = FifoQueue::new(queue_lengths.rpc_blob_queue); let rpc_custody_column_queue = FifoQueue::new(queue_lengths.rpc_custody_column_queue); + let rpc_envelope_queue = FifoQueue::new(queue_lengths.rpc_envelope_queue); let column_reconstruction_queue = LifoQueue::new(queue_lengths.column_reconstruction_queue); let chain_segment_queue = FifoQueue::new(queue_lengths.chain_segment_queue); let backfill_chain_segment = FifoQueue::new(queue_lengths.backfill_chain_segment); @@ -391,6 +396,7 @@ impl WorkQueues { rpc_block_queue, rpc_blob_queue, rpc_custody_column_queue, + rpc_envelope_queue, chain_segment_queue, column_reconstruction_queue, backfill_chain_segment, diff --git a/beacon_node/lighthouse_network/src/service/api_types.rs b/beacon_node/lighthouse_network/src/service/api_types.rs index f598f59aee..2429b813e9 100644 --- a/beacon_node/lighthouse_network/src/service/api_types.rs +++ b/beacon_node/lighthouse_network/src/service/api_types.rs @@ -23,6 +23,8 @@ pub enum SyncRequestId { SingleBlock { id: SingleLookupReqId }, /// Request searching for a set of blobs given a hash. SingleBlob { id: SingleLookupReqId }, + /// Request searching for a payload envelope given a hash. + SinglePayloadEnvelope { id: SingleLookupReqId }, /// Request searching for a set of data columns given a hash and list of column indices. DataColumnsByRoot(DataColumnsByRootRequestId), /// Blocks by range request diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 7bf969db10..7817feb0bd 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -588,6 +588,25 @@ impl NetworkBeaconProcessor { }) } + /// Create a new `Work` event for an RPC-fetched payload envelope. `process_lookup_envelope` + /// reports the result back to sync. + pub fn send_lookup_envelope( + self: &Arc, + block_root: Hash256, + envelope: Arc>, + seen_timestamp: Duration, + process_type: BlockProcessType, + ) -> Result<(), Error> { + let s = self.clone(); + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::RpcEnvelope(Box::pin(async move { + s.process_lookup_envelope(block_root, envelope, seen_timestamp, process_type) + .await; + })), + }) + } + /// Create a new `Work` event for some custody columns. `process_rpc_custody_columns` reports /// the result back to sync. pub fn send_rpc_custody_columns( diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 988a68c9dd..e3ba6fb3c4 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -426,6 +426,63 @@ impl NetworkBeaconProcessor { }); } + /// Attempt to verify and import an execution payload envelope received via RPC. + #[instrument( + name = "lh_process_lookup_envelope", + parent = None, + level = "debug", + skip_all, + fields(?block_root), + )] + pub async fn process_lookup_envelope( + self: Arc>, + block_root: Hash256, + envelope: Arc>, + _seen_timestamp: Duration, + process_type: BlockProcessType, + ) { + debug!( + ?block_root, + slot = %envelope.slot(), + ?process_type, + "Processing RPC payload envelope" + ); + + // Gossip verification runs the same signature / slot / builder-index / block-hash checks + // independently of gossip propagation, so we can reuse it for RPC-fetched envelopes. + #[allow(clippy::result_large_err)] + let result = match self + .chain + .clone() + .verify_envelope_for_gossip(envelope.clone()) + .await + { + Ok(verified) => { + self.chain + .process_execution_payload_envelope( + block_root, + verified, + NotifyExecutionLayer::Yes, + BlockImportSource::Lookup, + || Ok(()), + ) + .await + } + Err(e) => Err(e), + }; + + // TODO(gloas): structured penalty classification arrives with the envelope lookup state + // machine; for now, fold the EnvelopeError into BlockError::InternalError so it flows + // through the existing `BlockProcessingResult::Err` path. + let result: Result = + result.map_err(|e| BlockError::InternalError(format!("envelope: {e}"))); + + self.send_sync_message(SyncMessage::BlockComponentProcessed { + process_type, + result: result.into(), + }); + } + pub fn process_historic_data_columns( &self, batch_id: CustodyBackfillBatchId, diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index a718997e0a..35939c6f39 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -26,6 +26,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, error, trace, warn}; use types::{ BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, PartialDataColumn, SignedBeaconBlock, + SignedExecutionPayloadEnvelope, }; /// Handles messages from the network and routes them to the appropriate service to be handled. @@ -348,10 +349,13 @@ impl Router { Response::DataColumnsByRange(data_column) => { self.on_data_columns_by_range_response(peer_id, app_request_id, data_column); } - // TODO(EIP-7732): implement outgoing payload envelopes by range and root - // responses once sync manager requests them. - Response::PayloadEnvelopesByRoot(_) | Response::PayloadEnvelopesByRange(_) => { - debug!("Requesting envelopes by root and by range not supported yet"); + Response::PayloadEnvelopesByRoot(envelope) => { + self.on_payload_envelopes_by_root_response(peer_id, app_request_id, envelope); + } + // TODO(EIP-7732): implement outgoing payload envelopes by range responses + // once sync manager requests them. + Response::PayloadEnvelopesByRange(_) => { + debug!("Requesting envelopes by range not supported yet"); } // Lighthouse currently only serves BlocksByHead and does not issue it as a client, // so receiving a response is unexpected. Drop it without crashing. @@ -821,6 +825,29 @@ impl Router { } } + /// Handle a `PayloadEnvelopesByRoot` response from the peer. + pub fn on_payload_envelopes_by_root_response( + &mut self, + peer_id: PeerId, + app_request_id: AppRequestId, + envelope: Option>>, + ) { + let sync_request_id = match app_request_id { + AppRequestId::Sync(id @ SyncRequestId::SinglePayloadEnvelope { .. }) => id, + other => { + crit!(request = ?other, %peer_id, "PayloadEnvelopesByRoot response on incorrect request"); + return; + } + }; + + self.send_to_sync(SyncMessage::RpcPayloadEnvelope { + sync_request_id, + peer_id, + envelope, + seen_timestamp: self.chain.slot_clock.now_duration().unwrap_or_default(), + }); + } + fn handle_beacon_processor_send_result( &mut self, result: Result<(), crate::network_beacon_processor::Error>, diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 3929f74aa0..f10610c751 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -559,6 +559,8 @@ impl BlockLookups { BlockProcessType::SingleCustodyColumn(id) => { self.on_processing_result_inner::>(id, result, cx) } + // TODO(gloas): route into the payload envelope lookup state machine. + BlockProcessType::SinglePayloadEnvelope(_) => Ok(LookupResult::Pending), }; self.on_lookup_result(process_type.id(), lookup_result, "processing_result", cx); } diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 347b018a93..14a38f0e72 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -73,7 +73,8 @@ use strum::IntoStaticStr; use tokio::sync::mpsc; use tracing::{debug, error, info, trace}; use types::{ - BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, Hash256, SignedBeaconBlock, Slot, + BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, Hash256, SignedBeaconBlock, + SignedExecutionPayloadEnvelope, Slot, }; /// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync @@ -132,6 +133,14 @@ pub enum SyncMessage { seen_timestamp: Duration, }, + /// A payload envelope has been received from the RPC. + RpcPayloadEnvelope { + sync_request_id: SyncRequestId, + peer_id: PeerId, + envelope: Option>>, + seen_timestamp: Duration, + }, + /// A block with an unknown parent has been received. UnknownParentBlock(PeerId, Arc>, Hash256), @@ -193,6 +202,7 @@ pub enum BlockProcessType { SingleBlock { id: Id }, SingleBlob { id: Id }, SingleCustodyColumn(Id), + SinglePayloadEnvelope(Id), } impl BlockProcessType { @@ -200,7 +210,8 @@ impl BlockProcessType { match self { BlockProcessType::SingleBlock { id } | BlockProcessType::SingleBlob { id } - | BlockProcessType::SingleCustodyColumn(id) => *id, + | BlockProcessType::SingleCustodyColumn(id) + | BlockProcessType::SinglePayloadEnvelope(id) => *id, } } } @@ -502,6 +513,9 @@ impl SyncManager { SyncRequestId::SingleBlob { id } => { self.on_single_blob_response(id, peer_id, RpcEvent::RPCError(error)) } + SyncRequestId::SinglePayloadEnvelope { id } => { + self.on_single_payload_envelope_response(id, peer_id, RpcEvent::RPCError(error)) + } SyncRequestId::DataColumnsByRoot(req_id) => { self.on_data_columns_by_root_response(req_id, peer_id, RpcEvent::RPCError(error)) } @@ -848,6 +862,17 @@ impl SyncManager { } => { self.rpc_data_column_received(sync_request_id, peer_id, data_column, seen_timestamp) } + SyncMessage::RpcPayloadEnvelope { + sync_request_id, + peer_id, + envelope, + seen_timestamp, + } => self.rpc_payload_envelope_received( + sync_request_id, + peer_id, + envelope, + seen_timestamp, + ), SyncMessage::UnknownParentBlock(peer_id, block, block_root) => { let block_slot = block.slot(); let parent_root = block.parent_root(); @@ -1209,6 +1234,27 @@ impl SyncManager { } } + // TODO(gloas): dispatch into block_lookups once the envelope lookup state machine lands. + fn rpc_payload_envelope_received( + &mut self, + sync_request_id: SyncRequestId, + peer_id: PeerId, + envelope: Option>>, + seen_timestamp: Duration, + ) { + match sync_request_id { + SyncRequestId::SinglePayloadEnvelope { id } => self + .on_single_payload_envelope_response( + id, + peer_id, + RpcEvent::from_chunk(envelope, seen_timestamp), + ), + _ => { + crit!(%peer_id, "bad request id for payload envelope"); + } + } + } + fn rpc_data_column_received( &mut self, sync_request_id: SyncRequestId, @@ -1237,6 +1283,22 @@ impl SyncManager { } } + fn on_single_payload_envelope_response( + &mut self, + id: SingleLookupReqId, + peer_id: PeerId, + envelope: RpcEvent>>, + ) { + if let Some(_resp) = self + .network + .on_single_payload_envelope_response(id, peer_id, envelope) + { + // TODO(gloas): dispatch into + // `block_lookups.on_download_response::>(...)` once + // the envelope lookup state machine lands. + } + } + fn on_single_blob_response( &mut self, id: SingleLookupReqId, diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 465e23998b..9d5ac40c0a 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -2,7 +2,10 @@ //! channel and stores a global RPC ID to perform requests. use self::custody::{ActiveCustodyRequest, Error as CustodyRequestError}; -pub use self::requests::{BlocksByRootSingleRequest, DataColumnsByRootSingleBlockRequest}; +pub use self::requests::{ + BlocksByRootSingleRequest, DataColumnsByRootSingleBlockRequest, + PayloadEnvelopesByRootSingleRequest, +}; use super::SyncMessage; use super::block_sidecar_coupling::RangeBlockComponentsRequest; use super::manager::BlockProcessType; @@ -37,6 +40,7 @@ pub use requests::LookupVerifyError; use requests::{ ActiveRequests, BlobsByRangeRequestItems, BlobsByRootRequestItems, BlocksByRangeRequestItems, BlocksByRootRequestItems, DataColumnsByRangeRequestItems, DataColumnsByRootRequestItems, + PayloadEnvelopesByRootRequestItems, }; #[cfg(test)] use slot_clock::SlotClock; @@ -52,7 +56,7 @@ use tracing::{Span, debug, debug_span, error, warn}; use types::data::FixedBlobSidecarList; use types::{ BlobSidecar, BlockImportSource, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, - ForkContext, Hash256, SignedBeaconBlock, Slot, + ForkContext, Hash256, SignedBeaconBlock, SignedExecutionPayloadEnvelope, Slot, }; pub mod custody; @@ -201,6 +205,9 @@ pub struct SyncNetworkContext { ActiveRequests>, /// A mapping of active BlobsByRoot requests, including both current slot and parent lookups. blobs_by_root_requests: ActiveRequests>, + /// A mapping of active PayloadEnvelopesByRoot requests + payload_envelopes_by_root_requests: + ActiveRequests>, /// A mapping of active DataColumnsByRoot requests data_columns_by_root_requests: ActiveRequests>, @@ -294,6 +301,7 @@ impl SyncNetworkContext { request_id: 1, blocks_by_root_requests: ActiveRequests::new("blocks_by_root"), blobs_by_root_requests: ActiveRequests::new("blobs_by_root"), + payload_envelopes_by_root_requests: ActiveRequests::new("payload_envelopes_by_root"), data_columns_by_root_requests: ActiveRequests::new("data_columns_by_root"), blocks_by_range_requests: ActiveRequests::new("blocks_by_range"), blobs_by_range_requests: ActiveRequests::new("blobs_by_range"), @@ -322,6 +330,7 @@ impl SyncNetworkContext { request_id: _, blocks_by_root_requests, blobs_by_root_requests, + payload_envelopes_by_root_requests, data_columns_by_root_requests, blocks_by_range_requests, blobs_by_range_requests, @@ -345,6 +354,10 @@ impl SyncNetworkContext { .active_requests_of_peer(peer_id) .into_iter() .map(|id| SyncRequestId::SingleBlob { id: *id }); + let payload_envelopes_by_root_ids = payload_envelopes_by_root_requests + .active_requests_of_peer(peer_id) + .into_iter() + .map(|id| SyncRequestId::SinglePayloadEnvelope { id: *id }); let data_column_by_root_ids = data_columns_by_root_requests .active_requests_of_peer(peer_id) .into_iter() @@ -363,6 +376,7 @@ impl SyncNetworkContext { .map(|req_id| SyncRequestId::DataColumnsByRange(*req_id)); blocks_by_root_ids .chain(blobs_by_root_ids) + .chain(payload_envelopes_by_root_ids) .chain(data_column_by_root_ids) .chain(blocks_by_range_ids) .chain(blobs_by_range_ids) @@ -419,6 +433,7 @@ impl SyncNetworkContext { request_id: _, blocks_by_root_requests, blobs_by_root_requests, + payload_envelopes_by_root_requests, data_columns_by_root_requests, blocks_by_range_requests, blobs_by_range_requests, @@ -441,6 +456,7 @@ impl SyncNetworkContext { for peer_id in blocks_by_root_requests .iter_request_peers() .chain(blobs_by_root_requests.iter_request_peers()) + .chain(payload_envelopes_by_root_requests.iter_request_peers()) .chain(data_columns_by_root_requests.iter_request_peers()) .chain(blocks_by_range_requests.iter_request_peers()) .chain(blobs_by_range_requests.iter_request_peers()) @@ -927,6 +943,81 @@ impl SyncNetworkContext { Ok(LookupRequestResult::RequestSent(id.req_id)) } + /// Request a payload envelope for a block root via PayloadEnvelopesByRoot RPC. + #[allow(dead_code)] + pub fn payload_lookup_request( + &mut self, + lookup_id: SingleLookupId, + lookup_peers: Arc>>, + block_root: Hash256, + ) -> Result { + // Skip the download if fork-choice already saw this envelope (e.g. imported via gossip + // before the lookup got here). + if self.chain.envelope_is_known_to_fork_choice(&block_root) { + return Ok(LookupRequestResult::NoRequestNeeded( + "envelope already known to fork-choice", + )); + } + + let active_request_count_by_peer = self.active_request_count_by_peer(); + let Some(peer_id) = lookup_peers + .read() + .iter() + .map(|peer| { + ( + active_request_count_by_peer.get(peer).copied().unwrap_or(0), + rand::random::(), + peer, + ) + }) + .min() + .map(|(_, _, peer)| *peer) + else { + return Ok(LookupRequestResult::Pending("no peers")); + }; + + let id = SingleLookupReqId { + lookup_id, + req_id: self.next_id(), + }; + + let request = PayloadEnvelopesByRootSingleRequest { block_root }; + + let network_request = RequestType::PayloadEnvelopesByRoot( + request + .clone() + .into_request(&self.fork_context) + .map_err(RpcRequestSendError::InternalError)?, + ); + self.network_send + .send(NetworkMessage::SendRequest { + peer_id, + request: network_request, + app_request_id: AppRequestId::Sync(SyncRequestId::SinglePayloadEnvelope { id }), + }) + .map_err(|_| RpcRequestSendError::InternalError("network send error".to_owned()))?; + + debug!( + method = "PayloadEnvelopesByRoot", + ?block_root, + peer = %peer_id, + %id, + "Sync RPC request sent" + ); + + self.payload_envelopes_by_root_requests.insert( + id, + peer_id, + // true = enforce that the peer returns a response. We only request a single envelope + // and the peer must have it. + true, + PayloadEnvelopesByRootRequestItems::new(request), + Span::none(), + ); + + Ok(LookupRequestResult::RequestSent(id.req_id)) + } + /// Request necessary blobs for `block_root`. Requests only the necessary blobs by checking: /// - If we have a downloaded but not yet processed block /// - If the da_checker has a pending block @@ -1476,6 +1567,27 @@ impl SyncNetworkContext { self.on_rpc_response_result(resp, peer_id) } + pub(crate) fn on_single_payload_envelope_response( + &mut self, + id: SingleLookupReqId, + peer_id: PeerId, + rpc_event: RpcEvent>>, + ) -> Option>>> { + let resp = self + .payload_envelopes_by_root_requests + .on_response(id, rpc_event); + let resp = resp.map(|res| { + res.and_then(|(mut envelopes, seen_timestamp)| { + match envelopes.pop() { + Some(envelope) => Ok((envelope, seen_timestamp)), + // Should never happen, we enforce at least 1 chunk. + None => Err(LookupVerifyError::NotEnoughResponsesReturned { actual: 0 }.into()), + } + }) + }); + self.on_rpc_response_result(resp, peer_id) + } + #[allow(clippy::type_complexity)] pub(crate) fn on_data_columns_by_root_response( &mut self, @@ -1652,6 +1764,35 @@ impl SyncNetworkContext { }) } + #[allow(dead_code)] + pub fn send_payload_for_processing( + &self, + block_root: Hash256, + envelope: Arc>, + seen_timestamp: Duration, + process_type: BlockProcessType, + ) -> Result<(), SendErrorProcessor> { + let beacon_processor = self + .beacon_processor_if_enabled() + .ok_or(SendErrorProcessor::ProcessorNotAvailable)?; + + debug!( + ?block_root, + ?process_type, + "Sending payload envelope for processing" + ); + + beacon_processor + .send_lookup_envelope(block_root, envelope, seen_timestamp, process_type) + .map_err(|e| { + error!( + error = ?e, + "Failed to send sync payload envelope to processor" + ); + SendErrorProcessor::SendError + }) + } + pub fn send_custody_columns_for_processing( &self, _id: Id, diff --git a/beacon_node/network/src/sync/network_context/requests.rs b/beacon_node/network/src/sync/network_context/requests.rs index ad60dffb45..8c091eca80 100644 --- a/beacon_node/network/src/sync/network_context/requests.rs +++ b/beacon_node/network/src/sync/network_context/requests.rs @@ -16,6 +16,9 @@ pub use data_columns_by_range::DataColumnsByRangeRequestItems; pub use data_columns_by_root::{ DataColumnsByRootRequestItems, DataColumnsByRootSingleBlockRequest, }; +pub use payload_envelopes_by_root::{ + PayloadEnvelopesByRootRequestItems, PayloadEnvelopesByRootSingleRequest, +}; use crate::metrics; @@ -27,6 +30,7 @@ mod blocks_by_range; mod blocks_by_root; mod data_columns_by_range; mod data_columns_by_root; +mod payload_envelopes_by_root; #[derive(Debug, PartialEq, Eq, IntoStaticStr)] pub enum LookupVerifyError { diff --git a/beacon_node/network/src/sync/network_context/requests/payload_envelopes_by_root.rs b/beacon_node/network/src/sync/network_context/requests/payload_envelopes_by_root.rs new file mode 100644 index 0000000000..a142d86e90 --- /dev/null +++ b/beacon_node/network/src/sync/network_context/requests/payload_envelopes_by_root.rs @@ -0,0 +1,54 @@ +use lighthouse_network::rpc::methods::PayloadEnvelopesByRootRequest; +use std::sync::Arc; +use types::{EthSpec, ForkContext, Hash256, SignedExecutionPayloadEnvelope}; + +use super::{ActiveRequestItems, LookupVerifyError}; + +#[derive(Debug, Clone)] +pub struct PayloadEnvelopesByRootSingleRequest { + pub block_root: Hash256, +} + +impl PayloadEnvelopesByRootSingleRequest { + pub fn into_request( + self, + fork_context: &ForkContext, + ) -> Result { + PayloadEnvelopesByRootRequest::new(vec![self.block_root], fork_context) + } +} + +pub struct PayloadEnvelopesByRootRequestItems { + request: PayloadEnvelopesByRootSingleRequest, + items: Vec>>, +} + +impl PayloadEnvelopesByRootRequestItems { + pub fn new(request: PayloadEnvelopesByRootSingleRequest) -> Self { + Self { + request, + items: vec![], + } + } +} + +impl ActiveRequestItems for PayloadEnvelopesByRootRequestItems { + type Item = Arc>; + + /// Append a response to the single chunk request. We expect exactly one envelope per + /// block root. Returns `true` when the single expected item has been received. + fn add(&mut self, envelope: Self::Item) -> Result { + let block_root = envelope.message.beacon_block_root; + if self.request.block_root != block_root { + return Err(LookupVerifyError::UnrequestedBlockRoot(block_root)); + } + + self.items.push(envelope); + // Always returns true, we expect a single envelope per block root + Ok(true) + } + + fn consume(&mut self) -> Vec { + std::mem::take(&mut self.items) + } +} From a9637c16502abb0215b9a76b7067207b6bc70d8c Mon Sep 17 00:00:00 2001 From: Daniel Knopik <107140945+dknopik@users.noreply.github.com> Date: Thu, 21 May 2026 05:25:02 +0200 Subject: [PATCH 15/18] Partial columns cleanup (#9321) #8314 left a few ugly potentially panicking location behind - all of them believed to be unreachable, but this PR fixes them regardless for good hygiene. Update to `ethereum_ssz 0.10.4` for two new helpers: `not_inplace` and `clone_zeroed`. Remove remaining `expect` and `todo!` in favour of these helpers and one new fallible (but practically infallible) method. Co-Authored-By: Daniel Knopik --- Cargo.lock | 4 ++-- .../src/data_column_verification.rs | 11 ++++++++- beacon_node/http_api/src/publish_blocks.rs | 12 ++++++---- .../lighthouse_network/src/types/partial.rs | 18 +++++--------- .../gossip_methods.rs | 24 +++++++++++-------- .../types/src/data/data_column_sidecar.rs | 17 ++++++------- 6 files changed, 47 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 078f699f3c..d42bcd8fc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3282,9 +3282,9 @@ dependencies = [ [[package]] name = "ethereum_ssz" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a4a4e4273b0135111fe9464e35465067766a8f664615b5a86338b73864407" +checksum = "e462875ad8693755ea8913d6e905715c76ea4836e2254e18c9cf0f7a8f8c2a13" dependencies = [ "alloy-primitives", "arbitrary", diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 71562b376b..45cd687b36 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -1220,7 +1220,16 @@ pub fn validate_partial_data_column_sidecar_for_gossip( header, }; } - Err(MissingCellsError::UnexpectedError(e)) => todo!("handle unexpected error {:?}", e), + Err(MissingCellsError::UnexpectedError(e)) => { + return PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipDataColumnError::InternalError(format!( + "An unexpected error occurred while validating partial data columns: {:?}", + e + )) + .into(), + header, + }; + } }; // We do not have to check block related data here, as we create the verifiable column from diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index e96c86b17f..ca4ab85524 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -524,11 +524,15 @@ pub(crate) fn publish_column_sidecars( if chain.config.enable_partial_columns && let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() { - let mut partial = fulu_data_col.to_partial(); - if let Some(header) = partial.sidecar.header.take() { - partial_header = Some(header); + match fulu_data_col.to_partial() { + Ok(mut partial) => { + if let Some(header) = partial.sidecar.header.take() { + partial_header = Some(header); + } + partial_columns.push(Arc::new(partial)); + } + Err(err) => crit!(?err, "Could not convert from full to partial"), } - partial_columns.push(Arc::new(partial)); } let subnet = DataColumnSubnetId::from_column_index(*data_col.index(), &chain.spec); diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index f25ce9ec36..26705b7106 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use tracing::{debug, error}; use types::core::{EthSpec, Hash256}; use types::data::{ - CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnPartsMetadata, + PartialDataColumn, PartialDataColumnHeader, PartialDataColumnPartsMetadata, PartialDataColumnSidecar, PartialDataColumnSidecarRef, }; @@ -32,12 +32,8 @@ impl OutgoingPartialColumn { header_sent_set: HeaderSentSet, ) -> Self { // For now, always request all cells - let mut requests = partial_column.sidecar.cells_present_bitmap.clone(); - for idx in 0..requests.len() { - requests - .set(idx, true) - .expect("Bound asserted via `len` above"); - } + let mut requests = partial_column.sidecar.cells_present_bitmap.clone_zeroed(); + requests.not_inplace(); let metadata = PartialDataColumnPartsMetadata:: { available: partial_column.sidecar.cells_present_bitmap.clone(), requests, @@ -45,10 +41,7 @@ impl OutgoingPartialColumn { .into(); let header_message = PartialDataColumnSidecarRef { - cells_present_bitmap: CellBitmap::::with_capacity( - partial_column.sidecar.cells_present_bitmap.len(), - ) - .expect("Taking length from bitmap with same bound"), + cells_present_bitmap: partial_column.sidecar.cells_present_bitmap.clone_zeroed(), column: vec![], kzg_proofs: vec![], header: Some(header).into(), @@ -210,7 +203,7 @@ impl Partial for OutgoingPartialColumn { let send = self .partial_column .sidecar - .filter(|idx| want.get(idx).expect("Bound checked above")) + .filter(|idx| want.get(idx).unwrap_or(false)) .map_err(|err| { error!(?err, "Unexpected error filtering sidecar"); PartialError::InvalidFormat @@ -262,6 +255,7 @@ mod tests { use fixed_bytes::FixedBytesExtended; use libp2p::identity::Keypair; use ssz_types::FixedVector; + use types::CellBitmap; use types::block::{BeaconBlockHeader, SignedBeaconBlockHeader}; use types::core::{MinimalEthSpec, Slot}; use types::data::PartialDataColumnHeader; diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index d34668b138..7a902649cb 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1381,16 +1381,20 @@ impl NetworkBeaconProcessor { &[&data_column_index.to_string()], ); - let mut column = col.to_partial(); - let header = column.sidecar.header.take(); - if let Some(header) = header { - self.send_network_message(NetworkMessage::PublishPartialColumns { - columns: vec![Arc::new(column)], - header: Arc::new(header), - }); - } else { - crit!("Converting from full to partial yielded headerless partial") - }; + match col.to_partial() { + Ok(mut column) => { + let header = column.sidecar.header.take(); + if let Some(header) = header { + self.send_network_message(NetworkMessage::PublishPartialColumns { + columns: vec![Arc::new(column)], + header: Arc::new(header), + }); + } else { + crit!("Converting from full to partial yielded headerless partial") + }; + } + Err(err) => crit!(?err, "Could not convert from full to partial"), + } } let result = self diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index 170aa99666..d15651730f 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -250,19 +250,16 @@ impl DataColumnSidecarFulu { } /// Convert this full data column into a verifiable partial data column. - pub fn to_partial(&self) -> PartialDataColumn { + /// Note: This is not expected to ever fail. + pub fn to_partial(&self) -> Result, PartialDataColumnSidecarError> { let cell_count = self.column.len(); - let mut bitmap = - CellBitmap::::with_capacity(cell_count).expect("our column has the same bound"); - for idx in 0..cell_count { - bitmap - .set(idx, true) - .expect("The correct size is initialized right above"); - } + let mut bitmap = CellBitmap::::with_capacity(cell_count) + .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?; + bitmap.not_inplace(); let block_root = self.block_root(); - PartialDataColumn { + Ok(PartialDataColumn { block_root, index: self.index, sidecar: PartialDataColumnSidecar { @@ -276,7 +273,7 @@ impl DataColumnSidecarFulu { }) .into(), }, - } + }) } } From 1caaa10fa86cfe9ad47cffc03f7de81b3e6642e6 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 21 May 2026 02:35:35 -0600 Subject: [PATCH 16/18] Drop unused EthSpec generic from Stores (#9281) Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 +- .../src/beacon_fork_choice_store.rs | 12 ++-- beacon_node/beacon_chain/src/builder.rs | 19 +++---- .../overflow_lru_cache.rs | 10 ++-- beacon_node/beacon_chain/src/migrate.rs | 4 +- .../payload_attestation_verification/tests.rs | 2 +- .../beacon_chain/src/persisted_custody.rs | 4 +- beacon_node/beacon_chain/src/test_utils.rs | 18 +++--- .../beacon_chain/tests/op_verification.rs | 2 +- .../beacon_chain/tests/prepare_payload.rs | 8 +-- .../beacon_chain/tests/schema_stability.rs | 2 +- beacon_node/beacon_chain/tests/store_tests.rs | 14 ++--- beacon_node/client/src/builder.rs | 15 +++-- beacon_node/http_api/src/test_utils.rs | 2 +- .../src/network_beacon_processor/mod.rs | 3 +- beacon_node/network/src/persisted_dht.rs | 13 ++--- .../network/src/subnet_service/tests/mod.rs | 7 +-- beacon_node/network/src/sync/tests/mod.rs | 2 +- beacon_node/src/lib.rs | 2 +- beacon_node/store/src/database/interface.rs | 13 ++--- .../store/src/database/leveldb_impl.rs | 13 ++--- beacon_node/store/src/database/redb_impl.rs | 15 ++--- beacon_node/store/src/forwards_iter.rs | 18 +++--- beacon_node/store/src/hot_cold_store.rs | 16 +++--- beacon_node/store/src/invariants.rs | 2 +- beacon_node/store/src/iter.rs | 56 ++++++++----------- beacon_node/store/src/lib.rs | 8 +-- beacon_node/store/src/memory_store.rs | 12 ++-- beacon_node/store/src/reconstruct.rs | 4 +- consensus/fork_choice/tests/tests.rs | 2 +- database_manager/src/lib.rs | 22 ++++---- 31 files changed, 141 insertions(+), 183 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index f3f6cd299e..2259e1d809 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -325,8 +325,8 @@ pub enum StateSkipConfig { } pub trait BeaconChainTypes: Send + Sync + 'static { - type HotStore: store::ItemStore; - type ColdStore: store::ItemStore; + type HotStore: store::ItemStore; + type ColdStore: store::ItemStore; type SlotClock: slot_clock::SlotClock; type EthSpec: types::EthSpec; } diff --git a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs index 95fde28f5b..133eaa2fc6 100644 --- a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs +++ b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs @@ -129,8 +129,8 @@ impl BalancesCache { /// Implements `fork_choice::ForkChoiceStore` in order to provide a persistent backing to the /// `fork_choice::ForkChoice` struct. #[derive(Debug, Educe)] -#[educe(PartialEq(bound(E: EthSpec, Hot: ItemStore, Cold: ItemStore)))] -pub struct BeaconForkChoiceStore, Cold: ItemStore> { +#[educe(PartialEq(bound(E: EthSpec, Hot: ItemStore, Cold: ItemStore)))] +pub struct BeaconForkChoiceStore { #[educe(PartialEq(ignore))] store: Arc>, balances_cache: BalancesCache, @@ -150,8 +150,8 @@ pub struct BeaconForkChoiceStore, Cold: ItemStore< impl BeaconForkChoiceStore where E: EthSpec, - Hot: ItemStore, - Cold: ItemStore, + Hot: ItemStore, + Cold: ItemStore, { /// Initialize `Self` from some `anchor` checkpoint which may or may not be the genesis state. /// @@ -267,8 +267,8 @@ where impl ForkChoiceStore for BeaconForkChoiceStore where E: EthSpec, - Hot: ItemStore, - Cold: ItemStore, + Hot: ItemStore, + Cold: ItemStore, { type Error = Error; diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index e668bef7c0..61c026e0a9 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -60,8 +60,8 @@ pub struct Witness( impl BeaconChainTypes for Witness where - THotStore: ItemStore + 'static, - TColdStore: ItemStore + 'static, + THotStore: ItemStore + 'static, + TColdStore: ItemStore + 'static, TSlotClock: SlotClock + 'static, E: EthSpec + 'static, { @@ -115,8 +115,8 @@ pub struct BeaconChainBuilder { impl BeaconChainBuilder> where - THotStore: ItemStore + 'static, - TColdStore: ItemStore + 'static, + THotStore: ItemStore + 'static, + TColdStore: ItemStore + 'static, TSlotClock: SlotClock + 'static, E: EthSpec + 'static, { @@ -1162,8 +1162,8 @@ where impl BeaconChainBuilder> where - THotStore: ItemStore + 'static, - TColdStore: ItemStore + 'static, + THotStore: ItemStore + 'static, + TColdStore: ItemStore + 'static, E: EthSpec + 'static, { /// Sets the `BeaconChain` slot clock to `TestingSlotClock`. @@ -1301,11 +1301,8 @@ mod test { let validator_count = 1; let genesis_time = 13_371_337; - let store: HotColdDB< - MinimalEthSpec, - MemoryStore, - MemoryStore, - > = HotColdDB::open_ephemeral(StoreConfig::default(), ChainSpec::minimal().into()).unwrap(); + let store: HotColdDB = + HotColdDB::open_ephemeral(StoreConfig::default(), ChainSpec::minimal().into()).unwrap(); let spec = MinimalEthSpec::default_spec(); let genesis_state = interop_genesis_state( diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 3034e196b9..8a80f835ab 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -802,7 +802,7 @@ mod test { fn get_store_with_spec( db_path: &TempDir, spec: Arc, - ) -> Arc, BeaconNodeBackend>> { + ) -> Arc> { let hot_path = db_path.path().join("hot_db"); let cold_path = db_path.path().join("cold_db"); let blobs_path = db_path.path().join("blobs_db"); @@ -860,8 +860,8 @@ mod test { ) where E: EthSpec, - Hot: ItemStore, - Cold: ItemStore, + Hot: ItemStore, + Cold: ItemStore, { let chain = &harness.chain; let head = chain.head_snapshot(); @@ -946,8 +946,8 @@ mod test { where E: EthSpec, T: BeaconChainTypes< - HotStore = BeaconNodeBackend, - ColdStore = BeaconNodeBackend, + HotStore = BeaconNodeBackend, + ColdStore = BeaconNodeBackend, EthSpec = E, >, { diff --git a/beacon_node/beacon_chain/src/migrate.rs b/beacon_node/beacon_chain/src/migrate.rs index 3c17c1ebba..9c70bcafa2 100644 --- a/beacon_node/beacon_chain/src/migrate.rs +++ b/beacon_node/beacon_chain/src/migrate.rs @@ -30,7 +30,7 @@ pub const DEFAULT_EPOCHS_PER_MIGRATION: u64 = 1; /// The background migrator runs a thread to perform pruning and migrate state from the hot /// to the cold database. -pub struct BackgroundMigrator, Cold: ItemStore> { +pub struct BackgroundMigrator { db: Arc>, /// Record of when the last migration ran, for enforcing `epochs_per_migration`. prev_migration: Arc>, @@ -135,7 +135,7 @@ pub struct FinalizationNotification { pub prev_migration: Arc>, } -impl, Cold: ItemStore> BackgroundMigrator { +impl BackgroundMigrator { /// Create a new `BackgroundMigrator` and spawn its thread if necessary. pub fn new(db: Arc>, config: MigratorConfig) -> Self { // Estimate last migration run from DB split slot. diff --git a/beacon_node/beacon_chain/src/payload_attestation_verification/tests.rs b/beacon_node/beacon_chain/src/payload_attestation_verification/tests.rs index 7faad98e55..c45df51ac8 100644 --- a/beacon_node/beacon_chain/src/payload_attestation_verification/tests.rs +++ b/beacon_node/beacon_chain/src/payload_attestation_verification/tests.rs @@ -43,7 +43,7 @@ struct TestContext { keypairs: Vec, spec: ChainSpec, genesis_block_root: Hash256, - store: Arc, store::MemoryStore>>, + store: Arc>, } impl TestContext { diff --git a/beacon_node/beacon_chain/src/persisted_custody.rs b/beacon_node/beacon_chain/src/persisted_custody.rs index ba221c67b5..cc7219fa90 100644 --- a/beacon_node/beacon_chain/src/persisted_custody.rs +++ b/beacon_node/beacon_chain/src/persisted_custody.rs @@ -9,7 +9,7 @@ pub const CUSTODY_DB_KEY: Hash256 = Hash256::ZERO; pub struct PersistedCustody(pub CustodyContextSsz); -pub fn load_custody_context, Cold: ItemStore>( +pub fn load_custody_context( store: Arc>, ) -> Option { let res: Result, _> = @@ -22,7 +22,7 @@ pub fn load_custody_context, Cold: ItemStore>( } /// Attempt to persist the custody context object to `self.store`. -pub fn persist_custody_context, Cold: ItemStore>( +pub fn persist_custody_context( store: Arc>, custody_context: CustodyContextSsz, ) -> Result<(), store::Error> { diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 8e9cc61208..c2ccad7d8c 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -124,8 +124,8 @@ pub fn get_kzg(spec: &ChainSpec) -> Arc { pub type BaseHarnessType = Witness; -pub type DiskHarnessType = BaseHarnessType, BeaconNodeBackend>; -pub type EphemeralHarnessType = BaseHarnessType, MemoryStore>; +pub type DiskHarnessType = BaseHarnessType; +pub type EphemeralHarnessType = BaseHarnessType; pub type BoxedMutator = Box< dyn FnOnce( @@ -334,7 +334,7 @@ impl Builder> { /// Manually restore from a given `MemoryStore`. pub fn resumed_ephemeral_store( mut self, - store: Arc, MemoryStore>>, + store: Arc>, ) -> Self { let mutator = move |builder: BeaconChainBuilder<_>| { builder @@ -350,7 +350,7 @@ impl Builder> { /// Disk store, start from genesis. pub fn fresh_disk_store( mut self, - store: Arc, BeaconNodeBackend>>, + store: Arc>, ) -> Self { let validator_keypairs = self .validator_keypairs @@ -384,7 +384,7 @@ impl Builder> { /// Disk store, resume. pub fn resumed_disk_store( mut self, - store: Arc, BeaconNodeBackend>>, + store: Arc>, ) -> Self { let mutator = move |builder: BeaconChainBuilder<_>| { builder @@ -399,8 +399,8 @@ impl Builder> { impl Builder> where E: EthSpec, - Hot: ItemStore, - Cold: ItemStore, + Hot: ItemStore, + Cold: ItemStore, { pub fn new(eth_spec_instance: E) -> Self { let runtime = TestRuntime::default(); @@ -760,8 +760,8 @@ pub type HarnessSyncContributions = Vec<( impl BeaconChainHarness> where E: EthSpec, - Hot: ItemStore, - Cold: ItemStore, + Hot: ItemStore, + Cold: ItemStore, { pub fn builder(eth_spec_instance: E) -> Builder> { create_test_tracing_subscriber(); diff --git a/beacon_node/beacon_chain/tests/op_verification.rs b/beacon_node/beacon_chain/tests/op_verification.rs index 2f97f10745..adc14541a9 100644 --- a/beacon_node/beacon_chain/tests/op_verification.rs +++ b/beacon_node/beacon_chain/tests/op_verification.rs @@ -27,7 +27,7 @@ static KEYPAIRS: LazyLock> = type E = MinimalEthSpec; type TestHarness = BeaconChainHarness>; -type HotColdDB = store::HotColdDB, BeaconNodeBackend>; +type HotColdDB = store::HotColdDB; fn get_store(db_path: &TempDir) -> Arc { let spec = Arc::new(test_spec::()); diff --git a/beacon_node/beacon_chain/tests/prepare_payload.rs b/beacon_node/beacon_chain/tests/prepare_payload.rs index 47dd1ef517..de8bfb3865 100644 --- a/beacon_node/beacon_chain/tests/prepare_payload.rs +++ b/beacon_node/beacon_chain/tests/prepare_payload.rs @@ -34,7 +34,7 @@ type TestHarness = BeaconChainHarness>; fn get_store( db_path: &TempDir, spec: Arc, -) -> Arc, BeaconNodeBackend>> { +) -> Arc> { let store_config = StoreConfig { prune_payloads: false, ..StoreConfig::default() @@ -46,7 +46,7 @@ fn get_store_generic( db_path: &TempDir, config: StoreConfig, spec: Arc, -) -> Arc, BeaconNodeBackend>> { +) -> Arc> { create_test_tracing_subscriber(); let hot_path = db_path.path().join("chain_db"); let cold_path = db_path.path().join("freezer_db"); @@ -64,7 +64,7 @@ fn get_store_generic( } fn get_harness( - store: Arc, BeaconNodeBackend>>, + store: Arc>, validator_count: usize, ) -> TestHarness { // Most tests expect to retain historic states, so we use this as the default. @@ -81,7 +81,7 @@ fn get_harness( } fn get_harness_generic( - store: Arc, BeaconNodeBackend>>, + store: Arc>, validator_count: usize, chain_config: ChainConfig, node_custody_type: NodeCustodyType, diff --git a/beacon_node/beacon_chain/tests/schema_stability.rs b/beacon_node/beacon_chain/tests/schema_stability.rs index 8200748ae6..899a40511d 100644 --- a/beacon_node/beacon_chain/tests/schema_stability.rs +++ b/beacon_node/beacon_chain/tests/schema_stability.rs @@ -20,7 +20,7 @@ use tempfile::{TempDir, tempdir}; use types::{ChainSpec, Hash256, MainnetEthSpec, Slot}; type E = MainnetEthSpec; -type Store = Arc, BeaconNodeBackend>>; +type Store = Arc>; type TestHarness = BeaconChainHarness>; const VALIDATOR_COUNT: usize = 32; diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 0ff9f6841d..7e50f4e5ac 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -106,7 +106,7 @@ fn get_or_reconstruct_blobs( } } -fn get_store(db_path: &TempDir) -> Arc, BeaconNodeBackend>> { +fn get_store(db_path: &TempDir) -> Arc> { let store_config = StoreConfig { prune_payloads: false, ..StoreConfig::default() @@ -118,7 +118,7 @@ fn get_store_generic( db_path: &TempDir, config: StoreConfig, spec: ChainSpec, -) -> Arc, BeaconNodeBackend>> { +) -> Arc> { create_test_tracing_subscriber(); let hot_path = db_path.path().join("chain_db"); let cold_path = db_path.path().join("freezer_db"); @@ -136,7 +136,7 @@ fn get_store_generic( } fn get_harness( - store: Arc, BeaconNodeBackend>>, + store: Arc>, validator_count: usize, ) -> TestHarness { // Most tests expect to retain historic states, so we use this as the default. @@ -153,7 +153,7 @@ fn get_harness( } fn get_harness_import_all_data_columns( - store: Arc, BeaconNodeBackend>>, + store: Arc>, validator_count: usize, ) -> TestHarness { // Most tests expect to retain historic states, so we use this as the default. @@ -171,7 +171,7 @@ fn get_harness_import_all_data_columns( } fn get_harness_generic( - store: Arc, BeaconNodeBackend>>, + store: Arc>, validator_count: usize, chain_config: ChainConfig, node_custody_type: NodeCustodyType, @@ -205,7 +205,7 @@ fn check_db_invariants(harness: &TestHarness) { } fn get_states_descendant_of_block( - store: &HotColdDB, BeaconNodeBackend>, + store: &HotColdDB, block_root: Hash256, ) -> Vec<(Hash256, Slot)> { let summaries = store.load_hot_state_summaries().unwrap(); @@ -5859,7 +5859,7 @@ async fn test_gloas_hot_state_hierarchy() { /// Check that the HotColdDB's split_slot is equal to the start slot of the last finalized epoch. fn check_split_slot( harness: &TestHarness, - store: Arc, BeaconNodeBackend>>, + store: Arc>, ) { let split_slot = store.get_split_slot(); assert_eq!( diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index f532ef716e..0a3c414632 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -98,8 +98,8 @@ impl where TSlotClock: SlotClock + Clone + 'static, E: EthSpec + 'static, - THotStore: ItemStore + 'static, - TColdStore: ItemStore + 'static, + THotStore: ItemStore + 'static, + TColdStore: ItemStore + 'static, { /// Instantiates a new, empty builder. /// @@ -815,8 +815,8 @@ impl where TSlotClock: SlotClock + Clone + 'static, E: EthSpec + 'static, - THotStore: ItemStore + 'static, - TColdStore: ItemStore + 'static, + THotStore: ItemStore + 'static, + TColdStore: ItemStore + 'static, { /// Consumes the internal `BeaconChainBuilder`, attaching the resulting `BeaconChain` to self. #[instrument(skip_all)] @@ -847,8 +847,7 @@ where } } -impl - ClientBuilder, BeaconNodeBackend>> +impl ClientBuilder> where TSlotClock: SlotClock + 'static, E: EthSpec + 'static, @@ -889,8 +888,8 @@ where impl ClientBuilder> where E: EthSpec + 'static, - THotStore: ItemStore + 'static, - TColdStore: ItemStore + 'static, + THotStore: ItemStore + 'static, + TColdStore: ItemStore + 'static, { /// Specifies that the slot clock should read the time from the computers system clock. pub fn system_time_slot_clock(mut self) -> Result { diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index f27a04d17a..467a5216b1 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -57,7 +57,7 @@ pub struct ApiServer> { type HarnessBuilder = Builder>; type Initializer = Box) -> HarnessBuilder>; -type Mutator = BoxedMutator, MemoryStore>; +type Mutator = BoxedMutator; impl InteractiveTester { pub async fn new(spec: Option, validator_count: usize) -> Self { diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 7817feb0bd..434f7ecc8b 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -1267,8 +1267,7 @@ use { }; #[cfg(test)] -pub(crate) type TestBeaconChainType = - Witness, MemoryStore>; +pub(crate) type TestBeaconChainType = Witness; #[cfg(test)] impl NetworkBeaconProcessor> { diff --git a/beacon_node/network/src/persisted_dht.rs b/beacon_node/network/src/persisted_dht.rs index 113b3cdd32..3672f97113 100644 --- a/beacon_node/network/src/persisted_dht.rs +++ b/beacon_node/network/src/persisted_dht.rs @@ -6,7 +6,7 @@ use types::{EthSpec, Hash256}; /// 32-byte key for accessing the `DhtEnrs`. All zero because `DhtEnrs` has its own column. pub const DHT_DB_KEY: Hash256 = Hash256::ZERO; -pub fn load_dht, Cold: ItemStore>( +pub fn load_dht( store: Arc>, ) -> Vec { // Load DHT from store @@ -20,7 +20,7 @@ pub fn load_dht, Cold: ItemStore>( } /// Attempt to persist the ENR's in the DHT to `self.store`. -pub fn persist_dht, Cold: ItemStore>( +pub fn persist_dht( store: Arc>, enrs: Vec, ) -> Result<(), store::Error> { @@ -28,7 +28,7 @@ pub fn persist_dht, Cold: ItemStore>( } /// Attempts to clear any DHT entries. -pub fn clear_dht, Cold: ItemStore>( +pub fn clear_dht( store: Arc>, ) -> Result<(), store::Error> { store.hot_db.delete::(&DHT_DB_KEY) @@ -75,11 +75,8 @@ mod tests { use types::{ChainSpec, MinimalEthSpec}; #[test] fn test_persisted_dht() { - let store: HotColdDB< - MinimalEthSpec, - MemoryStore, - MemoryStore, - > = HotColdDB::open_ephemeral(StoreConfig::default(), ChainSpec::minimal().into()).unwrap(); + let store: HotColdDB = + HotColdDB::open_ephemeral(StoreConfig::default(), ChainSpec::minimal().into()).unwrap(); let enrs = vec![Enr::from_str("enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8").unwrap()]; store .put_item(&DHT_DB_KEY, &PersistedDht { enrs: enrs.clone() }) diff --git a/beacon_node/network/src/subnet_service/tests/mod.rs b/beacon_node/network/src/subnet_service/tests/mod.rs index 619154d738..745934053a 100644 --- a/beacon_node/network/src/subnet_service/tests/mod.rs +++ b/beacon_node/network/src/subnet_service/tests/mod.rs @@ -25,12 +25,7 @@ const SLOT_DURATION_MILLIS: u64 = 400; const TEST_LOG_LEVEL: Option<&str> = None; -type TestBeaconChainType = Witness< - SystemTimeSlotClock, - MainnetEthSpec, - MemoryStore, - MemoryStore, ->; +type TestBeaconChainType = Witness; pub struct TestBeaconChain { chain: Arc>, diff --git a/beacon_node/network/src/sync/tests/mod.rs b/beacon_node/network/src/sync/tests/mod.rs index dd8c3ae432..4e185cc081 100644 --- a/beacon_node/network/src/sync/tests/mod.rs +++ b/beacon_node/network/src/sync/tests/mod.rs @@ -26,7 +26,7 @@ use types::{ForkName, Hash256, MinimalEthSpec as E, Slot}; mod lookups; mod range; -type T = Witness, MemoryStore>; +type T = Witness; /// This test utility enables integration testing of Lighthouse sync components. /// diff --git a/beacon_node/src/lib.rs b/beacon_node/src/lib.rs index e33da17e26..6400427f8c 100644 --- a/beacon_node/src/lib.rs +++ b/beacon_node/src/lib.rs @@ -20,7 +20,7 @@ use types::{ChainSpec, Epoch, EthSpec, ForkName}; /// A type-alias to the tighten the definition of a production-intended `Client`. pub type ProductionClient = - Client, BeaconNodeBackend>>; + Client>; /// The beacon node `Client` that is used in production. /// diff --git a/beacon_node/store/src/database/interface.rs b/beacon_node/store/src/database/interface.rs index 5646f1179c..7e0a09a3e9 100644 --- a/beacon_node/store/src/database/interface.rs +++ b/beacon_node/store/src/database/interface.rs @@ -6,18 +6,17 @@ use crate::{ColumnIter, ColumnKeyIter, DBColumn, Error, ItemStore, Key, KeyValue use crate::{KeyValueStoreOp, StoreConfig, config::DatabaseBackend}; use std::collections::HashSet; use std::path::Path; -use types::EthSpec; -pub enum BeaconNodeBackend { +pub enum BeaconNodeBackend { #[cfg(feature = "leveldb")] - LevelDb(leveldb_impl::LevelDB), + LevelDb(leveldb_impl::LevelDB), #[cfg(feature = "redb")] - Redb(redb_impl::Redb), + Redb(redb_impl::Redb), } -impl ItemStore for BeaconNodeBackend {} +impl ItemStore for BeaconNodeBackend {} -impl KeyValueStore for BeaconNodeBackend { +impl KeyValueStore for BeaconNodeBackend { fn get_bytes(&self, column: DBColumn, key: &[u8]) -> Result>, Error> { match self { #[cfg(feature = "leveldb")] @@ -183,7 +182,7 @@ impl KeyValueStore for BeaconNodeBackend { } } -impl BeaconNodeBackend { +impl BeaconNodeBackend { pub fn open(config: &StoreConfig, path: &Path) -> Result { metrics::inc_counter_vec(&metrics::DISK_DB_TYPE, &[&config.backend.to_string()]); match config.backend { diff --git a/beacon_node/store/src/database/leveldb_impl.rs b/beacon_node/store/src/database/leveldb_impl.rs index 6e01648263..0531eb900e 100644 --- a/beacon_node/store/src/database/leveldb_impl.rs +++ b/beacon_node/store/src/database/leveldb_impl.rs @@ -15,15 +15,13 @@ use leveldb::{ options::{Options, ReadOptions}, }; use std::collections::HashSet; -use std::marker::PhantomData; use std::path::Path; -use types::{EthSpec, Hash256}; +use types::Hash256; use super::interface::WriteOptions; -pub struct LevelDB { +pub struct LevelDB { db: Database, - _phantom: PhantomData, } impl From for leveldb::options::WriteOptions { @@ -34,7 +32,7 @@ impl From for leveldb::options::WriteOptions { } } -impl LevelDB { +impl LevelDB { pub fn open(path: &Path) -> Result { let mut options = Options::new(); @@ -42,10 +40,7 @@ impl LevelDB { let db = Database::open(path, options)?; - Ok(Self { - db, - _phantom: PhantomData, - }) + Ok(Self { db }) } pub fn read_options(&self) -> ReadOptions<'_, BytesKey> { diff --git a/beacon_node/store/src/database/redb_impl.rs b/beacon_node/store/src/database/redb_impl.rs index 4077326eca..dc39f22114 100644 --- a/beacon_node/store/src/database/redb_impl.rs +++ b/beacon_node/store/src/database/redb_impl.rs @@ -3,17 +3,15 @@ use crate::{DBColumn, Error, KeyValueStoreOp}; use parking_lot::RwLock; use redb::TableDefinition; use std::collections::HashSet; -use std::{borrow::BorrowMut, marker::PhantomData, path::Path}; +use std::{borrow::BorrowMut, path::Path}; use strum::IntoEnumIterator; -use types::EthSpec; use super::interface::WriteOptions; pub const DB_FILE_NAME: &str = "database.redb"; -pub struct Redb { +pub struct Redb { db: RwLock, - _phantom: PhantomData, } impl From for redb::Durability { @@ -26,19 +24,16 @@ impl From for redb::Durability { } } -impl Redb { +impl Redb { pub fn open(path: &Path) -> Result { let db_file = path.join(DB_FILE_NAME); let db = redb::Database::create(db_file)?; for column in DBColumn::iter() { - Redb::::create_table(&db, column.into())?; + Self::create_table(&db, column.into())?; } - Ok(Self { - db: db.into(), - _phantom: PhantomData, - }) + Ok(Self { db: db.into() }) } fn create_table(db: &redb::Database, table_name: &str) -> Result<(), Error> { diff --git a/beacon_node/store/src/forwards_iter.rs b/beacon_node/store/src/forwards_iter.rs index 255b7d8eac..ef4312f506 100644 --- a/beacon_node/store/src/forwards_iter.rs +++ b/beacon_node/store/src/forwards_iter.rs @@ -9,7 +9,7 @@ pub type HybridForwardsBlockRootsIterator<'a, E, Hot, Cold> = pub type HybridForwardsStateRootsIterator<'a, E, Hot, Cold> = HybridForwardsIterator<'a, E, Hot, Cold>; -impl, Cold: ItemStore> HotColdDB { +impl HotColdDB { fn simple_forwards_iterator( &self, column: DBColumn, @@ -116,7 +116,7 @@ impl, Cold: ItemStore> HotColdDB } /// Forwards root iterator that makes use of a slot -> root mapping in the freezer DB. -pub struct FrozenForwardsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { +pub struct FrozenForwardsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { inner: ColumnIter<'a, Vec>, column: DBColumn, next_slot: Slot, @@ -124,9 +124,7 @@ pub struct FrozenForwardsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemS _phantom: PhantomData<(E, Hot, Cold)>, } -impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> - FrozenForwardsIterator<'a, E, Hot, Cold> -{ +impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> FrozenForwardsIterator<'a, E, Hot, Cold> { /// `end_slot` is EXCLUSIVE here. pub fn new( store: &'a HotColdDB, @@ -148,7 +146,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> } } -impl, Cold: ItemStore> Iterator +impl Iterator for FrozenForwardsIterator<'_, E, Hot, Cold> { type Item = Result<(Hash256, Slot)>; @@ -199,7 +197,7 @@ impl Iterator for SimpleForwardsIterator { } /// Fusion of the above two approaches to forwards iteration. Fast and efficient. -pub enum HybridForwardsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { +pub enum HybridForwardsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { PreFinalization { iter: Box>, store: &'a HotColdDB, @@ -220,9 +218,7 @@ pub enum HybridForwardsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemSto Finished, } -impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> - HybridForwardsIterator<'a, E, Hot, Cold> -{ +impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> HybridForwardsIterator<'a, E, Hot, Cold> { /// Construct a new hybrid iterator. /// /// The `get_state` closure should return a beacon state and final block/state root to backtrack @@ -349,7 +345,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> } } -impl, Cold: ItemStore> Iterator +impl Iterator for HybridForwardsIterator<'_, E, Hot, Cold> { type Item = Result<(Hash256, Slot)>; diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index e9b9de76e6..a625a97004 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -49,7 +49,7 @@ use zstd::{Decoder, Encoder}; /// Stores vector fields like the `block_roots` and `state_roots` separately, and only stores /// intermittent "restore point" states pre-finalization. #[derive(Debug)] -pub struct HotColdDB, Cold: ItemStore> { +pub struct HotColdDB { /// The slot and state root at the point where the database is split between hot and cold. /// /// States with slots less than `split.slot` are in the cold DB, while states with slots @@ -217,11 +217,11 @@ pub enum HotColdDBError { Rollback, } -impl HotColdDB, MemoryStore> { +impl HotColdDB { pub fn open_ephemeral( config: StoreConfig, spec: Arc, - ) -> Result, MemoryStore>, Error> { + ) -> Result, Error> { config.verify::()?; let hierarchy = config.hierarchy_config.to_moduli()?; @@ -258,7 +258,7 @@ impl HotColdDB, MemoryStore> { } } -impl HotColdDB, BeaconNodeBackend> { +impl HotColdDB { /// Open a new or existing database, with the given paths to the hot and cold DBs. /// /// The `migrate_schema` function is passed in so that the parent `BeaconChain` can provide @@ -451,7 +451,7 @@ impl HotColdDB, BeaconNodeBackend> { } } -impl, Cold: ItemStore> HotColdDB { +impl HotColdDB { fn cold_storage_strategy(&self, slot: Slot) -> Result { // The start slot for the freezer HDiff is always 0 Ok(self.hierarchy.storage_strategy(slot, Slot::new(0))?) @@ -3575,7 +3575,7 @@ impl, Cold: ItemStore> HotColdDB /// This function previously did a combination of freezer migration alongside pruning. Now it is /// *just* responsible for copying relevant data to the freezer, while pruning is implemented /// in `prune_hot_db`. -pub fn migrate_database, Cold: ItemStore>( +pub fn migrate_database( store: Arc>, finalized_state_root: Hash256, finalized_block_root: Hash256, @@ -3786,7 +3786,7 @@ pub enum StateSummaryIteratorError { /// Return the ancestor state root of a state beyond SlotsPerHistoricalRoot using the roots iterator /// and the store -pub fn get_ancestor_state_root<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore>( +pub fn get_ancestor_state_root<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore>( store: &'a HotColdDB, from_state: &'a BeaconState, target_slot: Slot, @@ -3993,7 +3993,7 @@ impl StoreItem for HotStateSummary { impl HotStateSummary { /// Construct a new summary of the given state. - pub fn new, Cold: ItemStore>( + pub fn new( store: &HotColdDB, state_root: Hash256, state: &BeaconState, diff --git a/beacon_node/store/src/invariants.rs b/beacon_node/store/src/invariants.rs index d251fb8800..4ec72b82bd 100644 --- a/beacon_node/store/src/invariants.rs +++ b/beacon_node/store/src/invariants.rs @@ -242,7 +242,7 @@ pub enum InvariantViolation { ColdStateBaseSummaryMissing { slot: Slot, base_slot: Slot }, } -impl, Cold: ItemStore> HotColdDB { +impl HotColdDB { /// Run all database invariant checks. /// /// The `ctx` parameter provides data from the beacon chain layer (fork choice, state cache, diff --git a/beacon_node/store/src/iter.rs b/beacon_node/store/src/iter.rs index 0cb803d1ed..cf1ab86ffe 100644 --- a/beacon_node/store/src/iter.rs +++ b/beacon_node/store/src/iter.rs @@ -13,12 +13,12 @@ use types::{ /// /// It is assumed that all ancestors for this object are stored in the database. If this is not the /// case, the iterator will start returning `None` prior to genesis. -pub trait AncestorIter<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore, I: Iterator> { +pub trait AncestorIter<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore, I: Iterator> { /// Returns an iterator over the roots of the ancestors of `self`. fn try_iter_ancestor_roots(&self, store: &'a HotColdDB) -> Option; } -impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> +impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> AncestorIter<'a, E, Hot, Cold, BlockRootsIterator<'a, E, Hot, Cold>> for SignedBeaconBlock { /// Iterates across all available prior block roots of `self`, starting at the most recent and ending @@ -37,7 +37,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> } } -impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> +impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> AncestorIter<'a, E, Hot, Cold, StateRootsIterator<'a, E, Hot, Cold>> for BeaconState { /// Iterates across all available prior state roots of `self`, starting at the most recent and ending @@ -51,13 +51,11 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> } } -pub struct StateRootsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { +pub struct StateRootsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { inner: RootsIterator<'a, E, Hot, Cold>, } -impl, Cold: ItemStore> Clone - for StateRootsIterator<'_, E, Hot, Cold> -{ +impl Clone for StateRootsIterator<'_, E, Hot, Cold> { fn clone(&self) -> Self { Self { inner: self.inner.clone(), @@ -65,7 +63,7 @@ impl, Cold: ItemStore> Clone } } -impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> StateRootsIterator<'a, E, Hot, Cold> { +impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> StateRootsIterator<'a, E, Hot, Cold> { pub fn new(store: &'a HotColdDB, beacon_state: &'a BeaconState) -> Self { Self { inner: RootsIterator::new(store, beacon_state), @@ -79,7 +77,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> StateRootsIterator<' } } -impl, Cold: ItemStore> Iterator +impl Iterator for StateRootsIterator<'_, E, Hot, Cold> { type Item = Result<(Hash256, Slot), Error>; @@ -99,13 +97,11 @@ impl, Cold: ItemStore> Iterator /// exhausted. /// /// Returns `None` for roots prior to genesis or when there is an error reading from `Store`. -pub struct BlockRootsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { +pub struct BlockRootsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { inner: RootsIterator<'a, E, Hot, Cold>, } -impl, Cold: ItemStore> Clone - for BlockRootsIterator<'_, E, Hot, Cold> -{ +impl Clone for BlockRootsIterator<'_, E, Hot, Cold> { fn clone(&self) -> Self { Self { inner: self.inner.clone(), @@ -113,7 +109,7 @@ impl, Cold: ItemStore> Clone } } -impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> BlockRootsIterator<'a, E, Hot, Cold> { +impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> BlockRootsIterator<'a, E, Hot, Cold> { /// Create a new iterator over all block roots in the given `beacon_state` and prior states. pub fn new(store: &'a HotColdDB, beacon_state: &'a BeaconState) -> Self { Self { @@ -138,7 +134,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> BlockRootsIterator<' } } -impl, Cold: ItemStore> Iterator +impl Iterator for BlockRootsIterator<'_, E, Hot, Cold> { type Item = Result<(Hash256, Slot), Error>; @@ -151,13 +147,13 @@ impl, Cold: ItemStore> Iterator } /// Iterator over state and block roots that backtracks using the vectors from a `BeaconState`. -pub struct RootsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { +pub struct RootsIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { store: &'a HotColdDB, beacon_state: Cow<'a, BeaconState>, slot: Slot, } -impl, Cold: ItemStore> Clone for RootsIterator<'_, E, Hot, Cold> { +impl Clone for RootsIterator<'_, E, Hot, Cold> { fn clone(&self) -> Self { Self { store: self.store, @@ -167,7 +163,7 @@ impl, Cold: ItemStore> Clone for RootsIterator< } } -impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> RootsIterator<'a, E, Hot, Cold> { +impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> RootsIterator<'a, E, Hot, Cold> { pub fn new(store: &'a HotColdDB, beacon_state: &'a BeaconState) -> Self { Self { store, @@ -234,9 +230,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> RootsIterator<'a, E, } } -impl, Cold: ItemStore> Iterator - for RootsIterator<'_, E, Hot, Cold> -{ +impl Iterator for RootsIterator<'_, E, Hot, Cold> { /// (block_root, state_root, slot) type Item = Result<(Hash256, Hash256, Slot), Error>; @@ -246,15 +240,13 @@ impl, Cold: ItemStore> Iterator } /// Block iterator that uses the `parent_root` of each block to backtrack. -pub struct ParentRootBlockIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { +pub struct ParentRootBlockIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { store: &'a HotColdDB, next_block_root: Hash256, _phantom: PhantomData, } -impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> - ParentRootBlockIterator<'a, E, Hot, Cold> -{ +impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> ParentRootBlockIterator<'a, E, Hot, Cold> { pub fn new(store: &'a HotColdDB, start_block_root: Hash256) -> Self { Self { store, @@ -283,7 +275,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> } } -impl, Cold: ItemStore> Iterator +impl Iterator for ParentRootBlockIterator<'_, E, Hot, Cold> { type Item = Result<(Hash256, SignedBeaconBlock>), Error>; @@ -295,11 +287,11 @@ impl, Cold: ItemStore> Iterator #[derive(Clone)] /// Extends `BlockRootsIterator`, returning `SignedBeaconBlock` instances, instead of their roots. -pub struct BlockIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { +pub struct BlockIterator<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> { roots: BlockRootsIterator<'a, E, Hot, Cold>, } -impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> BlockIterator<'a, E, Hot, Cold> { +impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> BlockIterator<'a, E, Hot, Cold> { /// Create a new iterator over all blocks in the given `beacon_state` and prior states. pub fn new(store: &'a HotColdDB, beacon_state: &'a BeaconState) -> Self { Self { @@ -324,9 +316,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> BlockIterator<'a, E, } } -impl, Cold: ItemStore> Iterator - for BlockIterator<'_, E, Hot, Cold> -{ +impl Iterator for BlockIterator<'_, E, Hot, Cold> { type Item = Result>, Error>; fn next(&mut self) -> Option { @@ -338,7 +328,7 @@ impl, Cold: ItemStore> Iterator /// /// Return `Err(HistoryUnavailable)` in the case where no more backtrack states are available /// due to weak subjectivity sync. -fn next_historical_root_backtrack_state, Cold: ItemStore>( +fn next_historical_root_backtrack_state( store: &HotColdDB, current_state: &BeaconState, ) -> Result, Error> { @@ -386,7 +376,7 @@ mod test { harness.get_current_state() } - fn get_store() -> HotColdDB, MemoryStore> { + fn get_store() -> HotColdDB { let store = HotColdDB::open_ephemeral(Config::default(), Arc::new(E::default_spec())).unwrap(); // Init achor info so anchor slot is set. Use a random block as it is only used for the diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index bd8caa3ad5..56cdd18fbe 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -46,7 +46,7 @@ pub type ColumnKeyIter<'a, K> = Box> + 'a>; pub type RawEntryIter<'a> = Result, Vec), Error>> + 'a>, Error>; -pub trait KeyValueStore: Sync + Send + Sized + 'static { +pub trait KeyValueStore: Sync + Send + Sized + 'static { /// Retrieve some bytes in `column` with `key`. fn get_bytes(&self, column: DBColumn, key: &[u8]) -> Result>, Error>; @@ -177,7 +177,7 @@ pub enum KeyValueStoreOp { DeleteKey(DBColumn, Vec), } -pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'static { +pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'static { /// Store an item in `Self`. fn put(&self, key: &Hash256, item: &I) -> Result<(), Error> { let column = I::db_column(); @@ -493,7 +493,7 @@ mod tests { } } - fn test_impl(store: impl ItemStore) { + fn test_impl(store: impl ItemStore) { let key = Hash256::random(); let item = StorableThing { a: 1, b: 42 }; @@ -531,7 +531,7 @@ mod tests { #[test] fn exists() { - let store = MemoryStore::::open(); + let store = MemoryStore::open(); let key = Hash256::random(); let item = StorableThing { a: 1, b: 42 }; diff --git a/beacon_node/store/src/memory_store.rs b/beacon_node/store/src/memory_store.rs index 6baef61c9d..8be9278d90 100644 --- a/beacon_node/store/src/memory_store.rs +++ b/beacon_node/store/src/memory_store.rs @@ -4,28 +4,24 @@ use crate::{ }; use parking_lot::RwLock; use std::collections::{BTreeMap, HashSet}; -use std::marker::PhantomData; -use types::*; type DBMap = BTreeMap>; /// A thread-safe `BTreeMap` wrapper. -pub struct MemoryStore { +pub struct MemoryStore { db: RwLock, - _phantom: PhantomData, } -impl MemoryStore { +impl MemoryStore { /// Create a new, empty database. pub fn open() -> Self { Self { db: RwLock::new(BTreeMap::new()), - _phantom: PhantomData, } } } -impl KeyValueStore for MemoryStore { +impl KeyValueStore for MemoryStore { /// Get the value of some key from the database. Returns `None` if the key does not exist. fn get_bytes(&self, col: DBColumn, key: &[u8]) -> Result>, Error> { let column_key = BytesKey::from_vec(get_key_for_col(col, key)); @@ -148,4 +144,4 @@ impl KeyValueStore for MemoryStore { } } -impl ItemStore for MemoryStore {} +impl ItemStore for MemoryStore {} diff --git a/beacon_node/store/src/reconstruct.rs b/beacon_node/store/src/reconstruct.rs index 04a519af02..2fb40daa0d 100644 --- a/beacon_node/store/src/reconstruct.rs +++ b/beacon_node/store/src/reconstruct.rs @@ -15,8 +15,8 @@ use types::{EthSpec, Slot}; impl HotColdDB where E: EthSpec, - Hot: ItemStore, - Cold: ItemStore, + Hot: ItemStore, + Cold: ItemStore, { pub fn reconstruct_historic_states( self: &Arc, diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index 353893026b..848834b4d8 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -79,7 +79,7 @@ impl ForkChoiceTest { /// Get a value from the `ForkChoice` instantiation. fn get(&self, func: T) -> U where - T: Fn(&BeaconForkChoiceStore, MemoryStore>) -> U, + T: Fn(&BeaconForkChoiceStore) -> U, { func( self.harness diff --git a/database_manager/src/lib.rs b/database_manager/src/lib.rs index 608400fa7e..2509b500e0 100644 --- a/database_manager/src/lib.rs +++ b/database_manager/src/lib.rs @@ -55,7 +55,7 @@ pub fn display_db_version( let blobs_path = client_config.get_blobs_db_path(); let mut version = CURRENT_SCHEMA_VERSION; - HotColdDB::, BeaconNodeBackend>::open( + HotColdDB::::open( &hot_path, &cold_path, &blobs_path, @@ -143,13 +143,13 @@ pub fn inspect_db( let mut num_keys = 0; let sub_db = if inspect_config.freezer { - BeaconNodeBackend::::open(&client_config.store, &cold_path) + BeaconNodeBackend::open(&client_config.store, &cold_path) .map_err(|e| format!("Unable to open freezer DB: {e:?}"))? } else if inspect_config.blobs_db { - BeaconNodeBackend::::open(&client_config.store, &blobs_path) + BeaconNodeBackend::open(&client_config.store, &blobs_path) .map_err(|e| format!("Unable to open blobs DB: {e:?}"))? } else { - BeaconNodeBackend::::open(&client_config.store, &hot_path) + BeaconNodeBackend::open(&client_config.store, &hot_path) .map_err(|e| format!("Unable to open hot DB: {e:?}"))? }; @@ -264,17 +264,17 @@ pub fn compact_db( let (sub_db, db_name) = if compact_config.freezer { ( - BeaconNodeBackend::::open(&client_config.store, &cold_path)?, + BeaconNodeBackend::open(&client_config.store, &cold_path)?, "freezer_db", ) } else if compact_config.blobs_db { ( - BeaconNodeBackend::::open(&client_config.store, &blobs_path)?, + BeaconNodeBackend::open(&client_config.store, &blobs_path)?, "blobs_db", ) } else { ( - BeaconNodeBackend::::open(&client_config.store, &hot_path)?, + BeaconNodeBackend::open(&client_config.store, &hot_path)?, "hot_db", ) }; @@ -309,7 +309,7 @@ pub fn migrate_db( let mut from = CURRENT_SCHEMA_VERSION; let to = migrate_config.to; - let db = HotColdDB::, BeaconNodeBackend>::open( + let db = HotColdDB::::open( &hot_path, &cold_path, &blobs_path, @@ -339,7 +339,7 @@ pub fn prune_payloads( let cold_path = client_config.get_freezer_db_path(); let blobs_path = client_config.get_blobs_db_path(); - let db = HotColdDB::, BeaconNodeBackend>::open( + let db = HotColdDB::::open( &hot_path, &cold_path, &blobs_path, @@ -363,7 +363,7 @@ pub fn prune_blobs( let cold_path = client_config.get_freezer_db_path(); let blobs_path = client_config.get_blobs_db_path(); - let db = HotColdDB::, BeaconNodeBackend>::open( + let db = HotColdDB::::open( &hot_path, &cold_path, &blobs_path, @@ -398,7 +398,7 @@ pub fn prune_states( let cold_path = client_config.get_freezer_db_path(); let blobs_path = client_config.get_blobs_db_path(); - let db = HotColdDB::, BeaconNodeBackend>::open( + let db = HotColdDB::::open( &hot_path, &cold_path, &blobs_path, From b5d5644eebcb889b025c13b084c30ac1025adb59 Mon Sep 17 00:00:00 2001 From: Daniel Knopik <107140945+dknopik@users.noreply.github.com> Date: Thu, 21 May 2026 22:00:16 +0200 Subject: [PATCH 17/18] Add getBlobsV3 to `LIGHTHOUSE_CAPABILITIES` (#9330) Forgot to add `ENGINE_GET_BLOBS_V3` to `LIGHTHOUSE_CAPABILITIES`. Add `ENGINE_GET_BLOBS_V3` to `LIGHTHOUSE_CAPABILITIES`. Co-Authored-By: Daniel Knopik --- beacon_node/execution_layer/src/engine_api/http.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 110e155c77..7c63f78a22 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -94,6 +94,7 @@ pub static LIGHTHOUSE_CAPABILITIES: &[&str] = &[ ENGINE_GET_CLIENT_VERSION_V1, ENGINE_GET_BLOBS_V1, ENGINE_GET_BLOBS_V2, + ENGINE_GET_BLOBS_V3, ]; /// We opt to initialize the JsonClientVersionV1 rather than the ClientVersionV1 From 60abd4b5b985f5ef47baa799c43c085521e3e596 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Thu, 21 May 2026 23:21:20 -0700 Subject: [PATCH 18/18] Gloas alpha spec 8 (#9315) https://github.com/ethereum/consensus-specs/releases/tag/v1.7.0-alpha.8 Co-Authored-By: Eitan Seri-Levi Co-Authored-By: Michael Sproul --- beacon_node/beacon_chain/src/beacon_chain.rs | 30 ++- .../src/block_production/gloas.rs | 64 +++-- .../beacon_chain/src/execution_payload.rs | 20 +- .../gossip_verified_bid.rs | 98 +++++++- .../src/payload_bid_verification/mod.rs | 2 +- .../src/payload_bid_verification/tests.rs | 15 +- .../gossip_verified_proposer_preferences.rs | 47 +++- .../proposer_preferences_verification/mod.rs | 2 +- .../proposer_preference_cache.rs | 2 +- .../tests.rs | 6 +- .../tests/payload_invalidation.rs | 1 + beacon_node/execution_layer/src/engine_api.rs | 43 ++-- .../src/engine_api/json_structures.rs | 5 + beacon_node/execution_layer/src/lib.rs | 9 +- .../src/test_utils/mock_builder.rs | 7 +- .../src/test_utils/mock_execution_layer.rs | 7 +- beacon_node/http_api/tests/tests.rs | 2 +- .../gossip_methods.rs | 2 +- .../mainnet/config.yaml | 4 +- consensus/fork_choice/src/fork_choice.rs | 96 +++++--- consensus/proto_array/src/error.rs | 5 + .../src/fork_choice_test_definition.rs | 6 +- consensus/proto_array/src/proto_array.rs | 99 ++++++-- .../src/proto_array_fork_choice.rs | 30 +++ .../process_operations.rs | 44 ++-- .../state_processing/src/upgrade/gloas.rs | 94 ++++--- consensus/types/configs/mainnet.yaml | 4 +- .../types/src/builder/proposer_preferences.rs | 2 +- consensus/types/src/core/chain_spec.rs | 92 ++++++- consensus/types/src/state/beacon_state.rs | 4 +- testing/ef_tests/Makefile | 2 +- testing/ef_tests/src/cases/fork_choice.rs | 233 +++++++++++++++++- testing/ef_tests/src/handler.rs | 13 +- testing/ef_tests/tests/tests.rs | 6 + .../src/test_rig.rs | 8 +- .../src/proposer_preferences_service.rs | 2 +- 36 files changed, 863 insertions(+), 243 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 2259e1d809..db8f55a18a 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -96,8 +96,8 @@ use eth2::types::{ SseExtendedPayloadAttributes, SseHead, }; use execution_layer::{ - BlockProposalContents, BlockProposalContentsType, BuilderParams, ChainHealth, ExecutionLayer, - FailedCondition, PayloadAttributes, PayloadStatus, + BlockProposalContents, BlockProposalContentsType, BuilderParams, ChainHealth, + DEFAULT_GAS_LIMIT, ExecutionLayer, FailedCondition, PayloadAttributes, PayloadStatus, }; use fixed_bytes::FixedBytesExtended; use fork_choice::{ @@ -2185,12 +2185,20 @@ impl BeaconChain { // TODO(gloas) do we want to use a dedicated envelope cache instead? // Maybe the new gloas DA cache? (Or should the gloas DA cache use - // the envelopes_times_cache internally?) + // the envelopes_times_cache internally? + // The payload is considered present only if it was observed before + // the payload due deadline (PAYLOAD_DUE_BPS into the slot). + let payload_due = self.spec.get_payload_due(); let payload_present = self .envelope_times_cache .read() .cache - .contains_key(&beacon_block_root); + .get(&beacon_block_root) + .and_then(|entry| entry.timestamps.observed) + .is_some_and(|observed| { + let slot_start = self.slot_clock.start_of(request_slot); + slot_start.is_some_and(|start| observed.saturating_sub(start) < payload_due) + }); // TODO(EIP-7732): Check blob data availability. For now, default to true. let blob_data_available = true; @@ -6476,6 +6484,19 @@ impl BeaconChain { None }; + let target_gas_limit = if prepare_slot_fork.gloas_enabled() { + let proposer_gas_limit = execution_layer.get_proposer_gas_limit(proposer).await; + if proposer_gas_limit.is_none() { + warn!( + %proposer, + "No proposer gas limit configured, falling back to parent gas limit" + ); + } + proposer_gas_limit.or(Some(DEFAULT_GAS_LIMIT)) + } else { + None + }; + let payload_attributes = PayloadAttributes::new( self.slot_clock .start_of(prepare_slot) @@ -6486,6 +6507,7 @@ impl BeaconChain { withdrawals.map(Into::into), parent_beacon_block_root, slot_number, + target_gas_limit, ); execution_layer diff --git a/beacon_node/beacon_chain/src/block_production/gloas.rs b/beacon_node/beacon_chain/src/block_production/gloas.rs index 6510c20ba7..82dad6f6ad 100644 --- a/beacon_node/beacon_chain/src/block_production/gloas.rs +++ b/beacon_node/beacon_chain/src/block_production/gloas.rs @@ -2,11 +2,13 @@ use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::Arc; +use proto_array::PayloadStatus; + use bls::{PublicKeyBytes, Signature}; use execution_layer::{ - BlockProposalContentsGloas, BuilderParams, PayloadAttributes, PayloadParameters, + BlockProposalContentsGloas, BuilderParams, DEFAULT_GAS_LIMIT, PayloadAttributes, + PayloadParameters, }; -use fork_choice::PayloadStatus; use operation_pool::CompactAttestationRef; use ssz::Encode; use state_processing::common::{get_attesting_indices_from_state, get_indexed_payload_attestation}; @@ -150,8 +152,24 @@ impl BeaconChain { verification: ProduceBlockVerification, builder_boost_factor: Option, ) -> Result, BlockProductionError> { - // Extract the parent's execution requests from the envelope (if parent was full). - let parent_execution_requests = if parent_payload_status == PayloadStatus::Full { + let parent_root = if state.slot() > 0 { + *state + .get_block_root(state.slot() - 1) + .map_err(|_| BlockProductionError::UnableToGetBlockRootFromState)? + } else { + state.latest_block_header().canonical_root() + }; + + let should_build_on_full = self + .canonical_head + .fork_choice_read_lock() + .should_build_on_full(&parent_root, parent_payload_status) + .map_err(|e| { + BlockProductionError::BeaconChain(Box::new(BeaconChainError::ForkChoiceError(e))) + })?; + + // Extract the parent's execution requests from the envelope (if building on full). + let parent_execution_requests = if should_build_on_full { parent_envelope .as_ref() .map(|env| env.message.execution_requests.clone()) @@ -197,7 +215,7 @@ impl BeaconChain { .clone() .produce_execution_payload_bid( state, - parent_payload_status, + should_build_on_full, parent_envelope, produce_at_slot, BID_VALUE_SELF_BUILD, @@ -700,12 +718,12 @@ impl BeaconChain { /// data needed to construct the `ExecutionPayloadEnvelope` after the beacon block is /// created, plus the EL block value and `should_override_builder` flag used by the /// caller to compare against any cached p2p builder bid. - #[allow(clippy::type_complexity)] + #[allow(clippy::type_complexity, clippy::too_many_arguments)] #[instrument(level = "debug", skip_all)] pub async fn produce_execution_payload_bid( self: Arc, state: BeaconState, - parent_payload_status: PayloadStatus, + should_build_on_full: bool, parent_envelope: Option>>, produce_at_slot: Slot, bid_value: u64, @@ -751,20 +769,18 @@ impl BeaconChain { let parent_bid = state.latest_execution_payload_bid()?; - // TODO(gloas): need should_extend_payload check here as well let parent_block_slot = state.latest_block_header().slot; let parent_is_pre_gloas = !self .spec .fork_name_at_slot::(parent_block_slot) .gloas_enabled(); - let parent_block_hash = - if parent_payload_status == PayloadStatus::Full || parent_is_pre_gloas { - // Build on parent bid's payload. - parent_bid.block_hash - } else { - // Skip parent bid's payload. For genesis this is the EL genesis hash. - parent_bid.parent_block_hash - }; + let parent_block_hash = if should_build_on_full || parent_is_pre_gloas { + // Build on parent bid's payload. + parent_bid.block_hash + } else { + // Skip parent bid's payload. For genesis this is the EL genesis hash. + parent_bid.parent_block_hash + }; // TODO(gloas) this should be BlockProductionVersion::V4 // V3 is okay for now as long as we're not connected to a builder @@ -953,10 +969,7 @@ fn get_execution_payload_gloas( compute_timestamp_at_slot(state, state.slot(), spec).map_err(BeaconStateError::from)?; let random = *state.get_randao_mix(current_epoch)?; - // TODO(gloas): this gas limit calc is not necessarily right let parent_bid = state.latest_execution_payload_bid()?; - let latest_gas_limit = parent_bid.gas_limit; - let is_parent_block_full = parent_block_hash == parent_bid.block_hash; let withdrawals = if is_parent_block_full { @@ -992,7 +1005,6 @@ fn get_execution_payload_gloas( random, proposer_index, parent_block_hash, - latest_gas_limit, builder_params, withdrawals, parent_beacon_block_root, @@ -1020,7 +1032,6 @@ async fn prepare_execution_payload( random: Hash256, proposer_index: u64, parent_block_hash: ExecutionBlockHash, - parent_gas_limit: u64, builder_params: BuilderParams, withdrawals: Vec, parent_beacon_block_root: Hash256, @@ -1058,6 +1069,10 @@ where .get_suggested_fee_recipient(proposer_index) .await; let slot_number = Some(builder_params.slot.as_u64()); + let target_gas_limit = execution_layer + .get_proposer_gas_limit(proposer_index) + .await + .unwrap_or(DEFAULT_GAS_LIMIT); let payload_attributes = PayloadAttributes::new( timestamp, @@ -1066,13 +1081,12 @@ where Some(withdrawals), Some(parent_beacon_block_root), slot_number, + Some(target_gas_limit), ); - - let target_gas_limit = execution_layer.get_proposer_gas_limit(proposer_index).await; let payload_parameters = PayloadParameters { parent_hash: parent_block_hash, - parent_gas_limit, - proposer_gas_limit: target_gas_limit, + parent_gas_limit: None, + proposer_gas_limit: Some(target_gas_limit), payload_attributes: &payload_attributes, forkchoice_update_params: &forkchoice_update_params, current_fork: fork, diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index 16542eea2d..c8976fc6a8 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -342,7 +342,7 @@ pub fn get_execution_payload( Ok(join_handle) } -/// Prepares an execution payload for inclusion in a block. +/// Prepares an execution payload (pre-gloas) for inclusion in a block. /// /// ## Errors /// @@ -373,6 +373,13 @@ where { let spec = &chain.spec; let fork = spec.fork_name_at_slot::(builder_params.slot); + + if fork.gloas_enabled() { + return Err(BlockProductionError::InvalidBlockVariant( + "Called pre-gloas prepare_execution_payload on a gloas block".to_string(), + )); + } + let execution_layer = chain .execution_layer .as_ref() @@ -403,25 +410,20 @@ where .get_suggested_fee_recipient(proposer_index) .await; - let slot_number = if fork.gloas_enabled() { - Some(builder_params.slot.as_u64()) - } else { - None - }; - let payload_attributes = PayloadAttributes::new( timestamp, random, suggested_fee_recipient, withdrawals, parent_beacon_block_root, - slot_number, + None, + None, ); let target_gas_limit = execution_layer.get_proposer_gas_limit(proposer_index).await; let payload_parameters = PayloadParameters { parent_hash, - parent_gas_limit: latest_execution_payload_header_gas_limit, + parent_gas_limit: Some(latest_execution_payload_header_gas_limit), proposer_gas_limit: target_gas_limit, payload_attributes: &payload_attributes, forkchoice_update_params: &forkchoice_update_params, diff --git a/beacon_node/beacon_chain/src/payload_bid_verification/gossip_verified_bid.rs b/beacon_node/beacon_chain/src/payload_bid_verification/gossip_verified_bid.rs index 1f3f074598..354705b92c 100644 --- a/beacon_node/beacon_chain/src/payload_bid_verification/gossip_verified_bid.rs +++ b/beacon_node/beacon_chain/src/payload_bid_verification/gossip_verified_bid.rs @@ -43,9 +43,6 @@ pub(crate) fn verify_bid_consistency( if bid.fee_recipient != proposer_preferences.message.fee_recipient { return Err(PayloadBidError::InvalidFeeRecipient); } - if bid.gas_limit != proposer_preferences.message.gas_limit { - return Err(PayloadBidError::InvalidGasLimit); - } let max_blobs_per_block = spec.max_blobs_per_block(bid_slot.epoch(E::slots_per_epoch())) as usize; @@ -161,7 +158,23 @@ impl GossipVerifiedPayloadBid { }); } - // TODO(gloas) [IGNORE] bid.parent_block_hash is the block hash of a known execution payload in fork choice. + // TODO(gloas): [IGNORE] bid.parent_block_hash is the block hash of a known execution + // payload in fork choice. + + // TODO(gloas): This uses head state's bid gas_limit as parent_gas_limit, which is only + // correct when the bid's parent is the head. If the parent is an ancestor further back + // this check may be inaccurate. Fixing this requires storing + // gas_limit in fork choice or looking it up from the store by parent_block_hash. Taking the above + // TODO into consideration maybe should persist parent block hash and gas limit in fork choice? + if let Ok(parent_bid) = head_state.latest_execution_payload_bid() + && !is_gas_limit_target_compatible( + parent_bid.gas_limit, + signed_bid.message.gas_limit, + proposer_preferences.message.target_gas_limit, + )? + { + return Err(PayloadBidError::InvalidGasLimit); + } drop(fork_choice); @@ -263,8 +276,36 @@ impl BeaconChain { } } +/// Check if `gas_limit` is compatible with `target_gas_limit` under the +/// EIP-1559 transition rule from `parent_gas_limit`. +pub fn is_gas_limit_target_compatible( + parent_gas_limit: u64, + gas_limit: u64, + target_gas_limit: u64, +) -> Result { + let max_gas_limit_difference = (parent_gas_limit / 1024) + .max(1) + .checked_sub(1) + .ok_or(PayloadBidError::InvalidGasLimit)?; + let min_gas_limit = parent_gas_limit + .checked_sub(max_gas_limit_difference) + .ok_or(PayloadBidError::InvalidGasLimit)?; + let max_gas_limit = parent_gas_limit + .checked_add(max_gas_limit_difference) + .ok_or(PayloadBidError::InvalidGasLimit)?; + + if target_gas_limit >= min_gas_limit && target_gas_limit <= max_gas_limit { + Ok(gas_limit == target_gas_limit) + } else if target_gas_limit > max_gas_limit { + Ok(gas_limit == max_gas_limit) + } else { + Ok(gas_limit == min_gas_limit) + } +} + #[cfg(test)] mod tests { + use super::is_gas_limit_target_compatible; use bls::Signature; use kzg::KzgCommitment; use ssz_types::VariableList; @@ -288,11 +329,14 @@ mod tests { } } - fn make_preferences(fee_recipient: Address, gas_limit: u64) -> SignedProposerPreferences { + fn make_preferences( + fee_recipient: Address, + target_gas_limit: u64, + ) -> SignedProposerPreferences { SignedProposerPreferences { message: ProposerPreferences { fee_recipient, - gas_limit, + target_gas_limit, ..ProposerPreferences::default() }, signature: Signature::empty(), @@ -382,13 +426,41 @@ mod tests { } #[test] - fn test_gas_limit_mismatch() { - let (state, spec) = state_and_spec(); - let current_slot = Slot::new(10); - let bid = make_bid(current_slot, Address::ZERO, 30_000_000); - let prefs = make_preferences(Address::ZERO, 50_000_000); + fn test_is_gas_limit_target_compatible_increase_within_limit() { + assert!(is_gas_limit_target_compatible(60_000_000, 60_000_100, 60_000_100).unwrap()); + } - let result = verify_bid_consistency::(&bid, current_slot, &prefs, &state, &spec); - assert!(matches!(result, Err(PayloadBidError::InvalidGasLimit))); + #[test] + fn test_is_gas_limit_target_compatible_increase_exceeding_limit() { + // max_diff = 60_000_000 / 1024 - 1 = 58_592 + // max_gas_limit = 60_000_000 + 58_592 = 60_058_592 + assert!(is_gas_limit_target_compatible(60_000_000, 60_058_592, 100_000_000).unwrap()); + } + + #[test] + fn test_is_gas_limit_target_compatible_increase_exceeding_off_by_one() { + assert!(!is_gas_limit_target_compatible(60_000_000, 60_058_593, 100_000_000).unwrap()); + } + + #[test] + fn test_is_gas_limit_target_compatible_decrease_within_limit() { + assert!(is_gas_limit_target_compatible(60_000_000, 59_999_990, 59_999_990).unwrap()); + } + + #[test] + fn test_is_gas_limit_target_compatible_decrease_exceeding_limit() { + // min_gas_limit = 60_000_000 - 58_592 = 59_941_408 + assert!(is_gas_limit_target_compatible(60_000_000, 59_941_408, 30_000_000).unwrap()); + } + + #[test] + fn test_is_gas_limit_target_compatible_target_equals_parent() { + assert!(is_gas_limit_target_compatible(60_000_000, 60_000_000, 60_000_000).unwrap()); + } + + #[test] + fn test_is_gas_limit_target_compatible_parent_underflows() { + // parent=1023: max(1023/1024, 1) - 1 = max(0, 1) - 1 = 0, no change allowed + assert!(is_gas_limit_target_compatible(1023, 1023, 60_000_000).unwrap()); } } diff --git a/beacon_node/beacon_chain/src/payload_bid_verification/mod.rs b/beacon_node/beacon_chain/src/payload_bid_verification/mod.rs index 514695f5c0..a40fd14872 100644 --- a/beacon_node/beacon_chain/src/payload_bid_verification/mod.rs +++ b/beacon_node/beacon_chain/src/payload_bid_verification/mod.rs @@ -48,7 +48,7 @@ pub enum PayloadBidError { }, /// The bids fee recipient doesn't match the proposer preferences fee recipient. InvalidFeeRecipient, - /// The bids gas limit doesn't match the proposer preferences gas limit. + /// The bid's gas limit is not compatible with the proposer's target gas limit. InvalidGasLimit, /// The bids execution payment is non-zero ExecutionPaymentNonZero { execution_payment: u64 }, diff --git a/beacon_node/beacon_chain/src/payload_bid_verification/tests.rs b/beacon_node/beacon_chain/src/payload_bid_verification/tests.rs index c68e6d9d32..ccdf64d41d 100644 --- a/beacon_node/beacon_chain/src/payload_bid_verification/tests.rs +++ b/beacon_node/beacon_chain/src/payload_bid_verification/tests.rs @@ -101,6 +101,17 @@ impl TestContext { root: Hash256::ZERO, }; + // Set a non-zero gas_limit on latest_execution_payload_bid so the gas limit + // compatibility check doesn't reject all bids at genesis. + if let Ok(bid) = state.latest_execution_payload_bid_mut() { + bid.gas_limit = 30_000_000; + } + // Update body_root to reflect the modified bid (genesis block embeds it). + let genesis_body_root = genesis_block(&state, &spec) + .expect("should build genesis block") + .body_root(); + state.latest_block_header_mut().body_root = genesis_body_root; + let inactive_keypair = &keypairs[NUM_BUILDERS]; let inactive_creds = builder_withdrawal_credentials(&inactive_keypair.pk, &spec); let inactive_builder_index = state @@ -248,7 +259,7 @@ fn make_signed_preferences( proposal_slot: Slot, validator_index: u64, fee_recipient: Address, - gas_limit: u64, + target_gas_limit: u64, ) -> Arc { Arc::new(SignedProposerPreferences { message: ProposerPreferences { @@ -256,7 +267,7 @@ fn make_signed_preferences( proposal_slot, validator_index, fee_recipient, - gas_limit, + target_gas_limit, }, signature: Signature::empty(), }) diff --git a/beacon_node/beacon_chain/src/proposer_preferences_verification/gossip_verified_proposer_preferences.rs b/beacon_node/beacon_chain/src/proposer_preferences_verification/gossip_verified_proposer_preferences.rs index 4ba33fde72..586721d8c1 100644 --- a/beacon_node/beacon_chain/src/proposer_preferences_verification/gossip_verified_proposer_preferences.rs +++ b/beacon_node/beacon_chain/src/proposer_preferences_verification/gossip_verified_proposer_preferences.rs @@ -18,13 +18,16 @@ pub(crate) fn verify_preferences_consistency( preferences: &ProposerPreferences, current_slot: Slot, head_state: &BeaconState, + spec: &ChainSpec, ) -> Result<(), ProposerPreferencesError> { let proposal_slot = preferences.proposal_slot; let validator_index = preferences.validator_index; let current_epoch = current_slot.epoch(E::slots_per_epoch()); let proposal_epoch = proposal_slot.epoch(E::slots_per_epoch()); - if proposal_epoch < current_epoch || proposal_epoch > current_epoch.saturating_add(1u64) { + if proposal_epoch < current_epoch + || proposal_epoch > current_epoch.saturating_add(spec.min_seed_lookahead) + { return Err(ProposerPreferencesError::InvalidProposalEpoch { proposal_epoch }); } @@ -35,7 +38,7 @@ pub(crate) fn verify_preferences_consistency( }); } - if !head_state.is_valid_proposal_slot(preferences)? { + if !head_state.is_valid_proposal_slot(preferences, spec)? { return Err(ProposerPreferencesError::InvalidProposalSlot { validator_index, proposal_slot, @@ -83,7 +86,12 @@ impl GossipVerifiedProposerPreferences { }); } - verify_preferences_consistency(&signed_preferences.message, current_slot, head_state)?; + verify_preferences_consistency( + &signed_preferences.message, + current_slot, + head_state, + ctx.spec, + )?; // Verify signature proposer_preferences_signature_set( @@ -155,11 +163,13 @@ impl BeaconChain { #[cfg(test)] mod tests { use types::{ - Address, BeaconState, EthSpec, Hash256, MinimalEthSpec, ProposerPreferences, Slot, + Address, BeaconState, ChainSpec, EthSpec, Hash256, MinimalEthSpec, ProposerPreferences, + Slot, }; use super::verify_preferences_consistency; use crate::proposer_preferences_verification::ProposerPreferencesError; + use crate::test_utils::{fork_name_from_env, test_spec}; type E = MinimalEthSpec; @@ -169,20 +179,28 @@ mod tests { proposal_slot, validator_index, fee_recipient: Address::ZERO, - gas_limit: 30_000_000, + target_gas_limit: 30_000_000, } } fn state() -> BeaconState { - BeaconState::new(0, <_>::default(), &E::default_spec()) + let spec = spec(); + BeaconState::new(0, <_>::default(), &spec) + } + + fn spec() -> ChainSpec { + test_spec::() } #[test] fn test_invalid_epoch_too_old() { + if !fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let current_slot = Slot::new(2 * E::slots_per_epoch()); let prefs = make_preferences(Slot::new(3), 0); - let result = verify_preferences_consistency::(&prefs, current_slot, &state()); + let result = verify_preferences_consistency::(&prefs, current_slot, &state(), &spec()); assert!(matches!( result, Err(ProposerPreferencesError::InvalidProposalEpoch { .. }) @@ -191,10 +209,13 @@ mod tests { #[test] fn test_invalid_epoch_too_far_ahead() { + if !fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let current_slot = Slot::new(E::slots_per_epoch()); let prefs = make_preferences(Slot::new(3 * E::slots_per_epoch() + 1), 0); - let result = verify_preferences_consistency::(&prefs, current_slot, &state()); + let result = verify_preferences_consistency::(&prefs, current_slot, &state(), &spec()); assert!(matches!( result, Err(ProposerPreferencesError::InvalidProposalEpoch { .. }) @@ -203,10 +224,13 @@ mod tests { #[test] fn test_proposal_slot_already_passed() { + if !fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let current_slot = Slot::new(10); let prefs = make_preferences(Slot::new(9), 0); - let result = verify_preferences_consistency::(&prefs, current_slot, &state()); + let result = verify_preferences_consistency::(&prefs, current_slot, &state(), &spec()); assert!(matches!( result, Err(ProposerPreferencesError::ProposalSlotAlreadyPassed { .. }) @@ -215,10 +239,13 @@ mod tests { #[test] fn test_proposal_slot_equal_to_current() { + if !fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let current_slot = Slot::new(10); let prefs = make_preferences(Slot::new(10), 0); - let result = verify_preferences_consistency::(&prefs, current_slot, &state()); + let result = verify_preferences_consistency::(&prefs, current_slot, &state(), &spec()); assert!(matches!( result, Err(ProposerPreferencesError::ProposalSlotAlreadyPassed { .. }) diff --git a/beacon_node/beacon_chain/src/proposer_preferences_verification/mod.rs b/beacon_node/beacon_chain/src/proposer_preferences_verification/mod.rs index a2e96dfce1..6c79e56733 100644 --- a/beacon_node/beacon_chain/src/proposer_preferences_verification/mod.rs +++ b/beacon_node/beacon_chain/src/proposer_preferences_verification/mod.rs @@ -24,7 +24,7 @@ mod tests; #[derive(Debug)] pub enum ProposerPreferencesError { - /// The proposal slot is not in the current or next epoch. + /// The proposal slot is not within the proposer lookahead. InvalidProposalEpoch { proposal_epoch: Epoch }, /// The proposal slot has already passed. ProposalSlotAlreadyPassed { diff --git a/beacon_node/beacon_chain/src/proposer_preferences_verification/proposer_preference_cache.rs b/beacon_node/beacon_chain/src/proposer_preferences_verification/proposer_preference_cache.rs index 7bbdf34888..c423418fbc 100644 --- a/beacon_node/beacon_chain/src/proposer_preferences_verification/proposer_preference_cache.rs +++ b/beacon_node/beacon_chain/src/proposer_preferences_verification/proposer_preference_cache.rs @@ -87,7 +87,7 @@ mod tests { proposal_slot: slot, validator_index, fee_recipient: Address::ZERO, - gas_limit: 30_000_000, + target_gas_limit: 30_000_000, }, signature: Signature::empty(), }), diff --git a/beacon_node/beacon_chain/src/proposer_preferences_verification/tests.rs b/beacon_node/beacon_chain/src/proposer_preferences_verification/tests.rs index 468e08ff3b..53c1c4ded3 100644 --- a/beacon_node/beacon_chain/src/proposer_preferences_verification/tests.rs +++ b/beacon_node/beacon_chain/src/proposer_preferences_verification/tests.rs @@ -112,7 +112,7 @@ impl TestContext { let slot_in_epoch = slot.as_usize() % E::slots_per_epoch() as usize; let epoch = slot.epoch(E::slots_per_epoch()); let current_epoch = state.slot().epoch(E::slots_per_epoch()); - let index = if epoch == current_epoch.saturating_add(1u64) { + let index = if epoch == current_epoch.saturating_add(self.spec.min_seed_lookahead) { E::slots_per_epoch() as usize + slot_in_epoch } else { slot_in_epoch @@ -131,7 +131,7 @@ fn make_signed_preferences( proposal_slot, validator_index, fee_recipient: Address::ZERO, - gas_limit: 30_000_000, + target_gas_limit: 30_000_000, }, signature: Signature::empty(), }) @@ -271,7 +271,7 @@ fn same_validator_different_dependent_root_not_deduplicated() { validator_index: 42, dependent_root: Hash256::repeat_byte(0xaa), fee_recipient: Address::ZERO, - gas_limit: 30_000_000, + target_gas_limit: 30_000_000, }, signature: Signature::empty(), }), diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index be85fc2245..abf1fe48a6 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -1035,6 +1035,7 @@ async fn payload_preparation() { None, None, None, + None, ); assert_eq!(rig.previous_payload_attributes(), payload_attributes); } diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index acf5f2778b..7337a29c8f 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -178,6 +178,8 @@ pub struct PayloadAttributes { pub parent_beacon_block_root: Hash256, #[superstruct(only(V4), partial_getter(copy))] pub slot_number: u64, + #[superstruct(only(V4), partial_getter(copy))] + pub target_gas_limit: u64, } impl PayloadAttributes { @@ -188,19 +190,29 @@ impl PayloadAttributes { withdrawals: Option>, parent_beacon_block_root: Option, slot_number: Option, + target_gas_limit: Option, ) -> Self { - match (withdrawals, parent_beacon_block_root, slot_number) { - (Some(withdrawals), Some(parent_beacon_block_root), Some(slot_number)) => { - PayloadAttributes::V4(PayloadAttributesV4 { - timestamp, - prev_randao, - suggested_fee_recipient, - withdrawals, - parent_beacon_block_root, - slot_number, - }) - } - (Some(withdrawals), Some(parent_beacon_block_root), None) => { + match ( + withdrawals, + parent_beacon_block_root, + slot_number, + target_gas_limit, + ) { + ( + Some(withdrawals), + Some(parent_beacon_block_root), + Some(slot_number), + Some(target_gas_limit), + ) => PayloadAttributes::V4(PayloadAttributesV4 { + timestamp, + prev_randao, + suggested_fee_recipient, + withdrawals, + parent_beacon_block_root, + slot_number, + target_gas_limit, + }), + (Some(withdrawals), Some(parent_beacon_block_root), _, _) => { PayloadAttributes::V3(PayloadAttributesV3 { timestamp, prev_randao, @@ -209,13 +221,13 @@ impl PayloadAttributes { parent_beacon_block_root, }) } - (Some(withdrawals), None, _) => PayloadAttributes::V2(PayloadAttributesV2 { + (Some(withdrawals), None, _, _) => PayloadAttributes::V2(PayloadAttributesV2 { timestamp, prev_randao, suggested_fee_recipient, withdrawals, }), - (None, _, _) => PayloadAttributes::V1(PayloadAttributesV1 { + (None, _, _, _) => PayloadAttributes::V1(PayloadAttributesV1 { timestamp, prev_randao, suggested_fee_recipient, @@ -260,7 +272,7 @@ impl From for SsePayloadAttributes { withdrawals, parent_beacon_block_root, }), - // V4 maps to V3 for SSE (slot_number is not part of the SSE spec) + // V4 maps to V3 for SSE (slot_number/target_gas_limit are not part of the SSE spec) PayloadAttributes::V4(PayloadAttributesV4 { timestamp, prev_randao, @@ -268,6 +280,7 @@ impl From for SsePayloadAttributes { withdrawals, parent_beacon_block_root, slot_number: _, + target_gas_limit: _, }) => Self::V3(SsePayloadAttributesV3 { timestamp, prev_randao, diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 9d9391a1e1..fb516e3e16 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -777,6 +777,9 @@ pub struct JsonPayloadAttributes { #[superstruct(only(V4))] #[serde(with = "serde_utils::u64_hex_be")] pub slot_number: u64, + #[superstruct(only(V4))] + #[serde(with = "serde_utils::u64_hex_be")] + pub target_gas_limit: u64, } impl From for JsonPayloadAttributes { @@ -807,6 +810,7 @@ impl From for JsonPayloadAttributes { withdrawals: pa.withdrawals.into_iter().map(Into::into).collect(), parent_beacon_block_root: pa.parent_beacon_block_root, slot_number: pa.slot_number, + target_gas_limit: pa.target_gas_limit, }), } } @@ -840,6 +844,7 @@ impl From for PayloadAttributes { withdrawals: jpa.withdrawals.into_iter().map(Into::into).collect(), parent_beacon_block_root: jpa.parent_beacon_block_root, slot_number: jpa.slot_number, + target_gas_limit: jpa.target_gas_limit, }), } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index b2dabb7c01..b1b8b0deaa 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -73,6 +73,8 @@ pub const DEFAULT_EXECUTION_ENDPOINT: &str = "http://localhost:8551/"; /// Name for the default file used for the jwt secret. pub const DEFAULT_JWT_FILE: &str = "jwt.hex"; +pub const DEFAULT_GAS_LIMIT: u64 = 60_000_000; + /// A fee recipient address for use during block production. Only used as a very last resort if /// there is no address provided by the user. /// @@ -358,7 +360,10 @@ impl> BlockProposalContents { pub parent_hash: ExecutionBlockHash, - pub parent_gas_limit: u64, + // NOTE: The `parent_gas_limit` is a bit scuffed. We made it optional for Gloas because it + // isn't currently required, but it should possibly be made non-optional again if needed. + // Or we should superstruct this type. + pub parent_gas_limit: Option, pub proposer_gas_limit: Option, pub payload_attributes: &'a PayloadAttributes, pub forkchoice_update_params: &'a ForkchoiceUpdateParameters, @@ -2082,7 +2087,7 @@ fn verify_builder_bid( let payload_withdrawals_root = header.withdrawals_root().ok(); let expected_gas_limit = proposer_gas_limit - .and_then(|target_gas_limit| expected_gas_limit(parent_gas_limit, target_gas_limit, spec)); + .and_then(|target_gas_limit| expected_gas_limit(parent_gas_limit?, target_gas_limit, spec)); if header.parent_hash() != parent_hash { Err(Box::new(InvalidBuilderPayload::ParentHash { diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index d6243a7c4d..d456c9adc1 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -282,7 +282,7 @@ impl BidStuff for BuilderBid { #[derive(Clone)] pub struct PayloadParametersCloned { pub parent_hash: ExecutionBlockHash, - pub parent_gas_limit: u64, + pub parent_gas_limit: Option, pub proposer_gas_limit: Option, pub payload_attributes: PayloadAttributes, pub forkchoice_update_params: ForkchoiceUpdateParameters, @@ -903,6 +903,7 @@ impl MockBuilder { expected_withdrawals, None, None, + None, ), ForkName::Deneb | ForkName::Electra | ForkName::Fulu => PayloadAttributes::new( timestamp, @@ -911,6 +912,7 @@ impl MockBuilder { expected_withdrawals, Some(head_block_root), None, + None, ), ForkName::Gloas => PayloadAttributes::new( timestamp, @@ -919,6 +921,7 @@ impl MockBuilder { expected_withdrawals, Some(head_block_root), Some(slot.as_u64()), + None, // TODO(gloas): pass target_gas_limit ), ForkName::Base | ForkName::Altair => { return Err("invalid fork".to_string()); @@ -969,7 +972,7 @@ impl MockBuilder { let payload_parameters = PayloadParametersCloned { parent_hash: head_execution_hash, - parent_gas_limit: head_gas_limit, + parent_gas_limit: Some(head_gas_limit), proposer_gas_limit: Some(proposer_gas_limit), payload_attributes, forkchoice_update_params, diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 5b721bcab2..583808281f 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -105,6 +105,7 @@ impl MockExecutionLayer { None, None, None, + None, ); // Insert a proposer to ensure the fork choice updated command works. @@ -146,11 +147,12 @@ impl MockExecutionLayer { None, None, None, + None, ); let payload_parameters = PayloadParameters { parent_hash, - parent_gas_limit, + parent_gas_limit: Some(parent_gas_limit), proposer_gas_limit: None, payload_attributes: &payload_attributes, forkchoice_update_params: &forkchoice_update_params, @@ -199,11 +201,12 @@ impl MockExecutionLayer { None, None, None, + None, ); let payload_parameters = PayloadParameters { parent_hash, - parent_gas_limit, + parent_gas_limit: Some(parent_gas_limit), proposer_gas_limit: None, payload_attributes: &payload_attributes, forkchoice_update_params: &forkchoice_update_params, diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index a7fe34593a..3da0841a4e 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -2929,7 +2929,7 @@ impl ApiTester { proposal_slot, validator_index: validator_index as u64, fee_recipient: Address::repeat_byte(0xaa), - gas_limit: 30_000_000, + target_gas_limit: 30_000_000, }; let epoch = proposal_slot.epoch(E::slots_per_epoch()); diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 7a902649cb..3e8845f017 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -4058,7 +4058,6 @@ impl NetworkBeaconProcessor { PayloadBidError::BadSignature | PayloadBidError::InvalidBuilder { .. } | PayloadBidError::InvalidFeeRecipient - | PayloadBidError::InvalidGasLimit | PayloadBidError::ExecutionPaymentNonZero { .. } | PayloadBidError::InvalidBlobKzgCommitments { .. }, ) => { @@ -4076,6 +4075,7 @@ impl NetworkBeaconProcessor { | PayloadBidError::ParentBlockRootUnknown { .. } | PayloadBidError::ParentBlockRootNotCanonical { .. } | PayloadBidError::BuilderCantCoverBid { .. } + | PayloadBidError::InvalidGasLimit | PayloadBidError::BeaconStateError(_) | PayloadBidError::InternalError(_) | PayloadBidError::InvalidBidSlot { .. } diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml index 02bf37cb55..ced9679142 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml @@ -93,8 +93,8 @@ SYNC_MESSAGE_DUE_BPS: 3333 CONTRIBUTION_DUE_BPS: 6667 # Gloas -# 2**6 (= 64) epochs -MIN_BUILDER_WITHDRAWABILITY_DELAY: 64 +# 2**13 (= 8192) epochs +MIN_BUILDER_WITHDRAWABILITY_DELAY: 8192 # 2500 basis points, 25% of SLOT_DURATION_MS ATTESTATION_DUE_BPS_GLOAS: 2500 # 5000 basis points, 50% of SLOT_DURATION_MS diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index a60859585c..2de8ce7d81 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -1207,7 +1207,6 @@ where fn validate_on_payload_attestation( &self, indexed_payload_attestation: &IndexedPayloadAttestation, - is_from_block: AttestationFromBlock, ) -> Result<(), InvalidPayloadAttestation> { // This check is from `is_valid_indexed_payload_attestation`, but we do it immediately to // avoid wasting time on junk attestations. @@ -1233,25 +1232,6 @@ where }); } - // PTC votes can only change the vote for their assigned beacon block, return early otherwise - if block.slot != indexed_payload_attestation.data.slot { - return Ok(()); - } - - // Gossip payload attestations must be for the current slot. - // NOTE: signature is assumed to have been verified by caller. - // https://github.com/ethereum/consensus-specs/blob/master/specs/gloas/fork-choice.md - if matches!(is_from_block, AttestationFromBlock::False) - && indexed_payload_attestation.data.slot != self.fc_store.get_current_slot() - { - return Err( - InvalidPayloadAttestation::PayloadAttestationNotCurrentSlot { - attestation_slot: indexed_payload_attestation.data.slot, - current_slot: self.fc_store.get_current_slot(), - }, - ); - } - Ok(()) } @@ -1339,34 +1319,69 @@ where pub fn on_payload_attestation( &mut self, system_time_current_slot: Slot, - attestation: &IndexedPayloadAttestation, + payload_attestation: &IndexedPayloadAttestation, is_from_block: AttestationFromBlock, ptc: &[usize], ) -> Result<(), Error> { self.update_time(system_time_current_slot)?; - if attestation.data.beacon_block_root.is_zero() { + if payload_attestation.data.beacon_block_root.is_zero() { return Ok(()); } // TODO(gloas): Should ignore wrong-slot payload attestations at the caller, they could // have been processed at the correct slot when received on gossip, but then have the // wrong-slot by the time they make it to here (TOCTOU). - self.validate_on_payload_attestation(attestation, is_from_block)?; + // TODO(gloas): Consider inlining validate_on_payload_attestation here to look more like the spec. + self.validate_on_payload_attestation(payload_attestation)?; - // Resolve validator indices to PTC committee positions. - let ptc_indices: Vec = attestation - .attesting_indices - .iter() - .filter_map(|validator_index| ptc.iter().position(|&p| p == *validator_index as usize)) - .collect(); + // PTC votes can only change the vote for their assigned beacon block, return early otherwise. + let block = self + .proto_array + .get_block(&payload_attestation.data.beacon_block_root) + .ok_or(InvalidPayloadAttestation::UnknownHeadBlock { + beacon_block_root: payload_attestation.data.beacon_block_root, + })?; + if block.slot != payload_attestation.data.slot { + return Ok(()); + } + + // Gossip payload attestations must be for the current slot. + if matches!(is_from_block, AttestationFromBlock::False) + && payload_attestation.data.slot != self.fc_store.get_current_slot() + { + return Err( + InvalidPayloadAttestation::PayloadAttestationNotCurrentSlot { + attestation_slot: payload_attestation.data.slot, + current_slot: self.fc_store.get_current_slot(), + } + .into(), + ); + } + + // Resolve validator indices to all PTC committee positions. A validator may + // appear multiple times in the PTC committee. + let mut ptc_indices = vec![]; + let mut validators_found = 0; + for validator_index in payload_attestation.attesting_indices.iter() { + let mut found = false; + for (ptc_index, &ptc_validator_index) in ptc.iter().enumerate() { + if ptc_validator_index == *validator_index as usize { + ptc_indices.push(ptc_index); + found = true; + } + } + if found { + validators_found += 1; + } + } // Check that all the attesters are in the PTC - if ptc_indices.len() != attestation.attesting_indices.len() { + if validators_found != payload_attestation.attesting_indices.len() { return Err( InvalidPayloadAttestation::PayloadAttestationAttestersNotInPtc { - attesting_indices_len: attestation.attesting_indices.len(), - attesting_indices_in_ptc: ptc_indices.len(), + attesting_indices_len: payload_attestation.attesting_indices.len(), + attesting_indices_in_ptc: validators_found, } .into(), ); @@ -1374,10 +1389,10 @@ where for &ptc_index in &ptc_indices { self.proto_array.process_payload_attestation( - attestation.data.beacon_block_root, + payload_attestation.data.beacon_block_root, ptc_index, - attestation.data.payload_present, - attestation.data.blob_data_available, + payload_attestation.data.payload_present, + payload_attestation.data.blob_data_available, )?; } @@ -1522,6 +1537,17 @@ where && self.is_finalized_checkpoint_or_descendant(*block_root) } + /// Called by the proposer to decide whether to build on the full or empty parent. + pub fn should_build_on_full( + &self, + block_root: &Hash256, + parent_payload_status: PayloadStatus, + ) -> Result> { + self.proto_array + .should_build_on_full::(block_root, parent_payload_status) + .map_err(Error::ProtoArrayStringError) + } + /// Returns whether the proposer should extend the execution payload chain of the given block. pub fn should_extend_payload(&self, block_root: &Hash256) -> Result> { let proposer_boost_root = self.fc_store.proposer_boost_root(); diff --git a/consensus/proto_array/src/error.rs b/consensus/proto_array/src/error.rs index bb47af97d9..d185ed371c 100644 --- a/consensus/proto_array/src/error.rs +++ b/consensus/proto_array/src/error.rs @@ -1,3 +1,4 @@ +use crate::PayloadStatus; use safe_arith::ArithError; use types::{Checkpoint, Epoch, ExecutionBlockHash, Hash256, Slot}; @@ -62,6 +63,10 @@ pub enum Error { }, NoViableChildren, OnBlockRequiresProposerIndex, + InvalidPayloadStatus { + block_root: Hash256, + payload_status: PayloadStatus, + }, } impl From for Error { diff --git a/consensus/proto_array/src/fork_choice_test_definition.rs b/consensus/proto_array/src/fork_choice_test_definition.rs index d537f16bb2..43b76ec7cb 100644 --- a/consensus/proto_array/src/fork_choice_test_definition.rs +++ b/consensus/proto_array/src/fork_choice_test_definition.rs @@ -556,7 +556,11 @@ impl ForkChoiceTestDefinition { node_v29.payload_data_availability_votes = BitVector::from_bytes(smallvec::smallvec![fill; 64]) .expect("valid 512-bit bitvector"); - // Per spec, is_payload_timely/is_payload_data_available require + // Mark all PTC members as having participated. + node_v29.ptc_participation = + BitVector::from_bytes(smallvec::smallvec![0xFF; 64]) + .expect("valid 512-bit bitvector"); + // Per spec, payload_timeliness/payload_data_availability require // the payload to be in payload_states (payload_received). node_v29.payload_received = is_timely || is_data_available; } diff --git a/consensus/proto_array/src/proto_array.rs b/consensus/proto_array/src/proto_array.rs index 78f5026689..8ac8354f06 100644 --- a/consensus/proto_array/src/proto_array.rs +++ b/consensus/proto_array/src/proto_array.rs @@ -155,6 +155,10 @@ pub struct ProtoNode { /// Tiebreak derived as: `num_set_bits() > ptc_size / 2`. #[superstruct(only(V29))] pub payload_data_availability_votes: BitVector, + /// Tracks which PTC members have cast a vote. + /// Bit i set means PTC member i has submitted a payload attestation. + #[superstruct(only(V29))] + pub ptc_participation: BitVector, /// Whether the execution payload for this block has been received and validated locally. /// Maps to `root in store.payload_states` in the spec. #[superstruct(only(V29), partial_getter(copy))] @@ -193,31 +197,60 @@ impl ProtoNode { } } - pub fn is_payload_timely(&self) -> bool { + /// Checks if `timely` matches our view of payload timeliness. + /// Returns whether the execution payload for the node is considered `timely` + /// (or not `timely` when `timely` is `false`), taking into consideration local + /// availability and PTC votes. + pub fn payload_timeliness(&self, timely: bool) -> Result { let Ok(node) = self.as_v29() else { - return false; + return Err(Error::InvalidNodeVariant { + block_root: self.root(), + }); }; - // Equivalent to `if root not in store.payload_states` in the spec. + // Equivalent to `if not is_payload_verified(store, root)` in the spec. if !node.payload_received { - return false; + return Ok(!timely); } - node.payload_timeliness_votes.num_set_bits() > E::payload_timely_threshold() + let matching_votes = if timely { + node.payload_timeliness_votes.num_set_bits() + } else { + // We take into consideration only participating ptc votes. An unset bit + // in `payload_timeliness_votes` could be an absent vote or a no vote. + node.ptc_participation + .num_set_bits() + .saturating_sub(node.payload_timeliness_votes.num_set_bits()) + }; + Ok(matching_votes > E::payload_timely_threshold()) } - pub fn is_payload_data_available(&self) -> bool { + /// Checks if `available` matches our view of payload data availability. + /// Return whether the blob data for the node is considered `available` + /// (or not, when `available` is `False`), taking into consideration local + /// availability and PTC votes. + pub fn payload_data_availability(&self, available: bool) -> Result { let Ok(node) = self.as_v29() else { - return false; + return Err(Error::InvalidNodeVariant { + block_root: self.root(), + }); }; - // Equivalent to `if root not in store.payload_states` in the spec. + // Equivalent to `if not is_payload_verified(store, root)` in the spec. if !node.payload_received { - return false; + return Ok(!available); } - node.payload_data_availability_votes.num_set_bits() - > E::data_availability_timely_threshold() + let matching_votes = if available { + node.payload_data_availability_votes.num_set_bits() + } else { + // We take into consideration only participating ptc votes. An unset bit + // in `payload_data_availability_votes` could be an absent vote or a no vote. + node.ptc_participation + .num_set_bits() + .saturating_sub(node.payload_data_availability_votes.num_set_bits()) + }; + Ok(matching_votes > E::data_availability_timely_threshold()) } } @@ -605,6 +638,7 @@ impl ProtoArray { execution_payload_parent_hash, payload_timeliness_votes: BitVector::default(), payload_data_availability_votes: BitVector::default(), + ptc_participation: BitVector::default(), payload_received: false, proposer_index, // Spec: `record_block_timeliness` + `get_forkchoice_store`. @@ -1501,12 +1535,46 @@ impl ProtoArray { } } + /// Called by the proposer to decide whether to build on the full or empty + /// parent pending node. Returns false if the PTC has voted the data as unavailable. + pub fn should_build_on_full( + &self, + fc_node: &IndexedForkChoiceNode, + proto_node: &ProtoNode, + ) -> Result { + if fc_node.payload_status == PayloadStatus::Pending { + return Err(Error::InvalidPayloadStatus { + block_root: proto_node.root(), + payload_status: fc_node.payload_status, + }); + } + + if fc_node.payload_status == PayloadStatus::Empty { + return Ok(false); + } + // Check that false votes have not achieved an absolute majority. This allows the payload to be + // considered available when either a majority have voted true or not enough votes have + // been cast either way. + Ok(!proto_node.payload_data_availability::(false)?) + } + pub fn should_extend_payload( &self, fc_node: &IndexedForkChoiceNode, proto_node: &ProtoNode, proposer_boost_root: Hash256, ) -> Result { + let Ok(node) = proto_node.as_v29() else { + return Err(Error::InvalidNodeVariant { + block_root: fc_node.root, + }); + }; + + // Spec equivalent to `if not is_payload_verified(store, root): return False` + if !node.payload_received { + return Ok(false); + } + // Per spec: `proposer_root == Root()` is one of the `or` conditions that // makes `should_extend_payload` return True. if proposer_boost_root.is_zero() { @@ -1531,11 +1599,10 @@ impl ProtoArray { .ok_or(Error::InvalidNodeIndex(parent_index))? .root(); - Ok( - (proto_node.is_payload_timely::() && proto_node.is_payload_data_available::()) - || proposer_boost_parent_root != fc_node.root - || proposer_boost_node.is_parent_node_full(), - ) + Ok((proto_node.payload_timeliness::(true)? + && proto_node.payload_data_availability::(true)?) + || proposer_boost_parent_root != fc_node.root + || proposer_boost_node.is_parent_node_full()) } /// Update the tree with new finalization information. The tree is only actually pruned if both diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index 7abba8a1f6..96d2302266 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -640,6 +640,9 @@ impl ProtoArrayForkChoice { .map_err(|e| { format!("process_payload_attestation: data availability set failed: {e:?}") })?; + v29.ptc_participation + .set(ptc_index, true) + .map_err(|e| format!("process_payload_attestation: participation set failed: {e:?}"))?; Ok(()) } @@ -1006,6 +1009,33 @@ impl ProtoArrayForkChoice { }) } + /// Called by the proposer to decide whether to build on the full or empty + /// parent. Returns false if the PTC has voted the data as unavailable. + pub fn should_build_on_full( + &self, + block_root: &Hash256, + parent_payload_status: PayloadStatus, + ) -> Result { + let block_index = self + .proto_array + .indices + .get(block_root) + .ok_or_else(|| format!("Unknown block root: {block_root:?}"))?; + let proto_node = self + .proto_array + .nodes + .get(*block_index) + .ok_or_else(|| format!("Missing node at index: {block_index}"))?; + let fc_node = IndexedForkChoiceNode { + root: proto_node.root(), + proto_node_index: *block_index, + payload_status: parent_payload_status, + }; + self.proto_array + .should_build_on_full::(&fc_node, proto_node) + .map_err(|e| format!("{e:?}")) + } + /// Returns whether the proposer should extend the parent's execution payload chain. /// /// This checks timeliness, data availability, and proposer boost conditions per the spec. diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 422e0afe06..f88a325d4e 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -916,25 +916,24 @@ pub fn process_deposit_requests_post_gloas( /// Check if there is a pending deposit for a new validator with the given pubkey. // TODO(gloas): cache the deposit signature validation or remove this loop entirely if possible, // it is `O(n * m)` where `n` is max 8192 and `m` is max 128M. -fn is_pending_validator( - state: &BeaconState, +pub fn is_pending_validator<'a>( + pending_deposits: impl IntoIterator, pubkey: &PublicKeyBytes, spec: &ChainSpec, -) -> Result { - for deposit in state.pending_deposits()?.iter() { - if deposit.pubkey == *pubkey { - let deposit_data = DepositData { - pubkey: deposit.pubkey, - withdrawal_credentials: deposit.withdrawal_credentials, - amount: deposit.amount, - signature: deposit.signature.clone(), - }; - if is_valid_deposit_signature(&deposit_data, spec).is_ok() { - return Ok(true); - } - } - } - Ok(false) +) -> bool { + pending_deposits.into_iter().any(|deposit| { + deposit.pubkey == *pubkey + && is_valid_deposit_signature( + &DepositData { + pubkey: deposit.pubkey, + withdrawal_credentials: deposit.withdrawal_credentials, + amount: deposit.amount, + signature: deposit.signature.clone(), + }, + spec, + ) + .is_ok() + }) } pub fn process_deposit_request_post_gloas( @@ -964,7 +963,7 @@ pub fn process_deposit_request_post_gloas( if is_builder || (has_builder_prefix && !is_validator - && !is_pending_validator(state, &deposit_request.pubkey, spec)?) + && !is_pending_validator(state.pending_deposits()?, &deposit_request.pubkey, spec)) { // Apply builder deposits immediately apply_deposit_for_builder( @@ -1003,7 +1002,7 @@ pub fn apply_deposit_for_builder( signature: SignatureBytes, slot: Slot, spec: &ChainSpec, -) -> Result<(), BeaconStateError> { +) -> Result, BeaconStateError> { match builder_index_opt { None => { // Verify the deposit signature (proof of possession) which is not checked by the deposit contract @@ -1014,13 +1013,16 @@ pub fn apply_deposit_for_builder( signature, }; if is_valid_deposit_signature(&deposit_data, spec).is_ok() { - state.add_builder_to_registry( + let builder_index = state.add_builder_to_registry( pubkey, withdrawal_credentials, amount, slot, spec, )?; + Ok(Some(builder_index)) + } else { + Ok(None) } } Some(builder_index) => { @@ -1030,9 +1032,9 @@ pub fn apply_deposit_for_builder( .ok_or(BeaconStateError::UnknownBuilder(builder_index))? .balance .safe_add_assign(amount)?; + Ok(Some(builder_index)) } } - Ok(()) } // Make sure to build the pubkey cache before calling this function diff --git a/consensus/state_processing/src/upgrade/gloas.rs b/consensus/state_processing/src/upgrade/gloas.rs index 84cdbf22c2..c26547e304 100644 --- a/consensus/state_processing/src/upgrade/gloas.rs +++ b/consensus/state_processing/src/upgrade/gloas.rs @@ -1,18 +1,16 @@ -use crate::per_block_processing::{ - is_valid_deposit_signature, process_operations::apply_deposit_for_builder, -}; +use crate::per_block_processing::process_operations::apply_deposit_for_builder; +use crate::per_block_processing::process_operations::is_pending_validator; use milhouse::{List, Vector}; use safe_arith::SafeArith; use ssz_types::BitVector; use ssz_types::FixedVector; -use std::collections::HashSet; +use std::collections::HashMap; use std::mem; use tree_hash::TreeHash; use typenum::Unsigned; use types::{ BeaconState, BeaconStateError as Error, BeaconStateGloas, BuilderPendingPayment, ChainSpec, - DepositData, EthSpec, ExecutionPayloadBid, ExecutionRequests, Fork, - is_builder_withdrawal_credential, + EthSpec, ExecutionPayloadBid, ExecutionRequests, Fork, is_builder_withdrawal_credential, }; /// Transform a `Fulu` state into a `Gloas` state. @@ -80,6 +78,7 @@ pub fn upgrade_state_to_gloas( // Execution Bid latest_execution_payload_bid: ExecutionPayloadBid { block_hash: pre.latest_execution_payload_header.block_hash, + gas_limit: pre.latest_execution_payload_header.gas_limit, execution_requests_root: ExecutionRequests::::default().tree_hash_root(), ..Default::default() }, @@ -167,66 +166,57 @@ fn onboard_builders_from_pending_deposits( state: &mut BeaconState, spec: &ChainSpec, ) -> Result<(), Error> { - // Rather than tracking all `validator_pubkeys` in one place as the spec does, we keep a - // hashset for *just* the new validator pubkeys, and use the state's efficient - // `get_validator_index` function instead of an O(n) iteration over the full validator list. - let mut new_validator_pubkeys = HashSet::new(); - // Clone pending deposits to avoid borrow conflicts when mutating state. let current_pending_deposits = state.pending_deposits()?.clone(); let mut pending_deposits = List::empty(); + // TODO(gloas): introduce a global builder pubkey cache, see: + // https://github.com/sigp/lighthouse/issues/8783 + let mut builder_pubkey_to_index = state + .builders()? + .iter() + .enumerate() + .map(|(i, b)| (b.pubkey, i as u64)) + .collect::>(); + for deposit in ¤t_pending_deposits { // Deposits for existing validators stay in the pending queue. - if new_validator_pubkeys.contains(&deposit.pubkey) - || state.get_validator_index(&deposit.pubkey)?.is_some() - { + if state.get_validator_index(&deposit.pubkey)?.is_some() { pending_deposits.push(deposit.clone())?; continue; } - // Re-scan builder list each iteration because `apply_deposit_for_builder` may add - // new builders to the registry. - // TODO(gloas): this linear scan could be optimized, see: - // https://github.com/sigp/lighthouse/issues/8783 - let builder_index = state - .builders()? - .iter() - .position(|b| b.pubkey == deposit.pubkey); + if !builder_pubkey_to_index.contains_key(&deposit.pubkey) { + // Deposits without builder withdrawal credentials are for new validators. + if !is_builder_withdrawal_credential(deposit.withdrawal_credentials, spec) { + pending_deposits.push(deposit.clone())?; + continue; + } - let has_builder_credentials = - is_builder_withdrawal_credential(deposit.withdrawal_credentials, spec); - - if builder_index.is_some() || has_builder_credentials { - let builder_index_opt = builder_index.map(|i| i as u64); - apply_deposit_for_builder( - state, - builder_index_opt, - deposit.pubkey, - deposit.withdrawal_credentials, - deposit.amount, - deposit.signature.clone(), - deposit.slot, - spec, - )?; - continue; + // If there is a valid pending deposit for a new validator with this pubkey, + // keep this deposit in the pending queue to be applied to that validator later. + if is_pending_validator(&pending_deposits, &deposit.pubkey, spec) { + pending_deposits.push(deposit.clone())?; + continue; + } } - // If there is a pending deposit for a new validator that has a valid signature, - // track the pubkey so that subsequent builder deposits for the same pubkey stay - // in pending (applied to the validator later) rather than creating a builder. - // Deposits with invalid signatures are dropped since they would fail in - // apply_pending_deposit anyway. - let deposit_data = DepositData { - pubkey: deposit.pubkey, - withdrawal_credentials: deposit.withdrawal_credentials, - amount: deposit.amount, - signature: deposit.signature.clone(), - }; - if is_valid_deposit_signature(&deposit_data, spec).is_ok() { - new_validator_pubkeys.insert(deposit.pubkey); - pending_deposits.push(deposit.clone())?; + let builder_index = builder_pubkey_to_index.get(&deposit.pubkey).copied(); + + if let Some(new_builder_index) = apply_deposit_for_builder( + state, + builder_index, + deposit.pubkey, + deposit.withdrawal_credentials, + deposit.amount, + deposit.signature.clone(), + deposit.slot, + spec, + )? { + builder_pubkey_to_index + .entry(deposit.pubkey) + .or_insert(new_builder_index); } } diff --git a/consensus/types/configs/mainnet.yaml b/consensus/types/configs/mainnet.yaml index 25bf872a7a..743384bcc9 100644 --- a/consensus/types/configs/mainnet.yaml +++ b/consensus/types/configs/mainnet.yaml @@ -91,8 +91,8 @@ SYNC_MESSAGE_DUE_BPS: 3333 CONTRIBUTION_DUE_BPS: 6667 # Gloas -# 2**6 (= 64) epochs -MIN_BUILDER_WITHDRAWABILITY_DELAY: 64 +# 2**13 (= 8192) epochs +MIN_BUILDER_WITHDRAWABILITY_DELAY: 8192 # 2500 basis points, 25% of SLOT_DURATION_MS ATTESTATION_DUE_BPS_GLOAS: 2500 # 5000 basis points, 50% of SLOT_DURATION_MS diff --git a/consensus/types/src/builder/proposer_preferences.rs b/consensus/types/src/builder/proposer_preferences.rs index e3773e333d..4f27020105 100644 --- a/consensus/types/src/builder/proposer_preferences.rs +++ b/consensus/types/src/builder/proposer_preferences.rs @@ -16,7 +16,7 @@ pub struct ProposerPreferences { pub proposal_slot: Slot, pub validator_index: u64, pub fee_recipient: Address, - pub gas_limit: u64, + pub target_gas_limit: u64, } impl SignedRoot for ProposerPreferences {} diff --git a/consensus/types/src/core/chain_spec.rs b/consensus/types/src/core/chain_spec.rs index c54d032891..c42bb4b5b9 100644 --- a/consensus/types/src/core/chain_spec.rs +++ b/consensus/types/src/core/chain_spec.rs @@ -108,6 +108,7 @@ pub struct ChainSpec { pub proposer_reorg_cutoff_bps: u64, pub attestation_due_bps: u64, pub attestation_due_bps_gloas: u64, + pub payload_due_bps: u64, pub payload_attestation_due_bps: u64, pub aggregate_due_bps: u64, pub sync_message_due_bps: u64, @@ -118,6 +119,7 @@ pub struct ChainSpec { */ pub unaggregated_attestation_due: Duration, pub unaggregated_attestation_due_gloas: Duration, + pub payload_due: Duration, pub payload_attestation_due: Duration, pub aggregate_attestation_due: Duration, pub sync_message_due: Duration, @@ -894,6 +896,11 @@ impl ChainSpec { } } + /// Spec: `get_payload_due_ms`. + pub fn get_payload_due(&self) -> Duration { + self.payload_due + } + /// Spec: `get_payload_attestation_due_ms`. pub fn get_payload_attestation_due(&self) -> Duration { self.payload_attestation_due @@ -974,6 +981,9 @@ impl ChainSpec { self.unaggregated_attestation_due_gloas = self .compute_slot_component_duration(self.attestation_due_bps_gloas) .expect("invalid chain spec: cannot compute unaggregated_attestation_due_gloas"); + self.payload_due = self + .compute_slot_component_duration(self.payload_due_bps) + .expect("invalid chain spec: cannot compute payload_due"); self.payload_attestation_due = self .compute_slot_component_duration(self.payload_attestation_due_bps) .expect("invalid chain spec: cannot compute payload_attestation_due"); @@ -1108,6 +1118,7 @@ impl ChainSpec { proposer_reorg_cutoff_bps: 1667, attestation_due_bps: 3333, attestation_due_bps_gloas: 2500, + payload_due_bps: 7500, payload_attestation_due_bps: 7500, aggregate_due_bps: 6667, sync_message_due_bps: 3333, @@ -1118,6 +1129,7 @@ impl ChainSpec { */ unaggregated_attestation_due: Duration::from_millis(3999), unaggregated_attestation_due_gloas: Duration::from_millis(3000), + payload_due: Duration::from_millis(9000), payload_attestation_due: Duration::from_millis(9000), aggregate_attestation_due: Duration::from_millis(8000), sync_message_due: Duration::from_millis(3999), @@ -1270,7 +1282,7 @@ impl ChainSpec { gloas_fork_epoch: None, builder_payment_threshold_numerator: 6, builder_payment_threshold_denominator: 10, - min_builder_withdrawability_delay: Epoch::new(64), + min_builder_withdrawability_delay: Epoch::new(8192), churn_limit_quotient_gloas: option_wrapper(|| u64::checked_pow(2, 15)) .expect("calculation does not overflow"), consolidation_churn_limit_quotient: option_wrapper(|| u64::checked_pow(2, 16)) @@ -1440,6 +1452,7 @@ impl ChainSpec { */ unaggregated_attestation_due: Duration::from_millis(1999), unaggregated_attestation_due_gloas: Duration::from_millis(1500), + payload_due: Duration::from_millis(4500), payload_attestation_due: Duration::from_millis(4500), aggregate_attestation_due: Duration::from_millis(4000), sync_message_due: Duration::from_millis(1999), @@ -1531,6 +1544,7 @@ impl ChainSpec { proposer_reorg_cutoff_bps: 1667, attestation_due_bps: 3333, attestation_due_bps_gloas: 2500, + payload_due_bps: 7500, payload_attestation_due_bps: 7500, aggregate_due_bps: 6667, @@ -1540,6 +1554,7 @@ impl ChainSpec { */ unaggregated_attestation_due: Duration::from_millis(1666), unaggregated_attestation_due_gloas: Duration::from_millis(1250), + payload_due: Duration::from_millis(3750), payload_attestation_due: Duration::from_millis(3750), aggregate_attestation_due: Duration::from_millis(3333), sync_message_due: Duration::from_millis(1666), @@ -1693,7 +1708,7 @@ impl ChainSpec { gloas_fork_epoch: None, builder_payment_threshold_numerator: 6, builder_payment_threshold_denominator: 10, - min_builder_withdrawability_delay: Epoch::new(64), + min_builder_withdrawability_delay: Epoch::new(8192), churn_limit_quotient_gloas: option_wrapper(|| u64::checked_pow(2, 15)) .expect("calculation does not overflow"), consolidation_churn_limit_quotient: option_wrapper(|| u64::checked_pow(2, 16)) @@ -2136,6 +2151,9 @@ pub struct Config { #[serde(default = "default_attestation_due_bps_gloas")] #[serde(with = "serde_utils::quoted_u64")] attestation_due_bps_gloas: u64, + #[serde(default = "default_payload_due_bps")] + #[serde(with = "serde_utils::quoted_u64")] + payload_due_bps: u64, #[serde(default = "default_payload_attestation_due_bps")] #[serde(with = "serde_utils::quoted_u64")] payload_attestation_due_bps: u64, @@ -2379,6 +2397,10 @@ const fn default_attestation_due_bps_gloas() -> u64 { 2500 } +const fn default_payload_due_bps() -> u64 { + 7500 +} + const fn default_payload_attestation_due_bps() -> u64 { 7500 } @@ -2396,7 +2418,7 @@ const fn default_contribution_due_bps() -> u64 { } const fn default_min_builder_withdrawability_delay() -> u64 { - 64 + 8192 } const fn default_churn_limit_quotient_gloas() -> u64 { @@ -2656,6 +2678,7 @@ impl Config { proposer_reorg_cutoff_bps: spec.proposer_reorg_cutoff_bps, attestation_due_bps: spec.attestation_due_bps, attestation_due_bps_gloas: spec.attestation_due_bps_gloas, + payload_due_bps: spec.payload_due_bps, payload_attestation_due_bps: spec.payload_attestation_due_bps, aggregate_due_bps: spec.aggregate_due_bps, sync_message_due_bps: spec.sync_message_due_bps, @@ -2759,6 +2782,7 @@ impl Config { proposer_reorg_cutoff_bps, attestation_due_bps, attestation_due_bps_gloas, + payload_due_bps, payload_attestation_due_bps, aggregate_due_bps, sync_message_due_bps, @@ -2867,6 +2891,7 @@ impl Config { proposer_reorg_cutoff_bps, attestation_due_bps, attestation_due_bps_gloas, + payload_due_bps, payload_attestation_due_bps, aggregate_due_bps, sync_message_due_bps, @@ -3694,6 +3719,30 @@ mod yaml_tests { let custom_spec = custom_spec.compute_derived_values::(); let tiny_due = custom_spec.get_unaggregated_attestation_due(); assert_eq!(tiny_due, Duration::from_millis(1)); // 12000 * 1 / 10000 = 1.2 -> 1 + + // Test payload due (7500 bps = 75% of 12s = 9s) + let spec = ChainSpec::mainnet().compute_derived_values::(); + let payload_due = spec.get_payload_due(); + assert_eq!(payload_due, Duration::from_millis(9000)); // 12000 * 7500 / 10000 + + // Test payload attestation due (7500 bps = 75% of 12s = 9s) + let payload_att_due = spec.get_payload_attestation_due(); + assert_eq!(payload_att_due, Duration::from_millis(9000)); // 12000 * 7500 / 10000 + + // Test gloas attestation due (2500 bps = 25% of 12s = 3s) + assert_eq!( + spec.unaggregated_attestation_due_gloas, + Duration::from_millis(3000) + ); // 12000 * 2500 / 10000 + + // Test gloas with custom bps + let mut custom_spec = spec; + custom_spec.attestation_due_bps_gloas = 5000; + let custom_spec = custom_spec.compute_derived_values::(); + assert_eq!( + custom_spec.unaggregated_attestation_due_gloas, + Duration::from_millis(6000) + ); // 12000 * 5000 / 10000 } #[test] @@ -3715,6 +3764,19 @@ mod yaml_tests { Duration::from_millis(8000) ); + // Mainnet payload due: 12000ms slots, 7500 bps = 9000ms + assert_eq!(mainnet.get_payload_due(), Duration::from_millis(9000)); + assert_eq!( + mainnet.get_payload_attestation_due(), + Duration::from_millis(9000) + ); + + // Mainnet gloas: 12000ms slots, 2500 bps = 3000ms + assert_eq!( + mainnet.unaggregated_attestation_due_gloas, + Duration::from_millis(3000) + ); + // Minimal spec: 6000ms slots, 3333 bps = 1999ms, 6667 bps = 4000ms let minimal = ChainSpec::minimal(); assert_eq!( @@ -3730,6 +3792,18 @@ mod yaml_tests { minimal.get_contribution_message_due(), Duration::from_millis(4000) ); + // Minimal payload due: 6000ms slots, 7500 bps = 4500ms + assert_eq!(minimal.get_payload_due(), Duration::from_millis(4500)); + assert_eq!( + minimal.get_payload_attestation_due(), + Duration::from_millis(4500) + ); + + // Minimal gloas: 6000ms slots, 2500 bps = 1500ms + assert_eq!( + minimal.unaggregated_attestation_due_gloas, + Duration::from_millis(1500) + ); // Gnosis spec: 5000ms slots, 3333 bps = 1666ms, 6667 bps = 3333ms let gnosis = ChainSpec::gnosis(); @@ -3746,6 +3820,18 @@ mod yaml_tests { gnosis.get_contribution_message_due(), Duration::from_millis(3333) ); + // Gnosis payload due: 5000ms slots, 7500 bps = 3750ms + assert_eq!(gnosis.get_payload_due(), Duration::from_millis(3750)); + assert_eq!( + gnosis.get_payload_attestation_due(), + Duration::from_millis(3750) + ); + + // Gnosis gloas: 5000ms slots, 2500 bps = 1250ms + assert_eq!( + gnosis.unaggregated_attestation_due_gloas, + Duration::from_millis(1250) + ); } #[test] diff --git a/consensus/types/src/state/beacon_state.rs b/consensus/types/src/state/beacon_state.rs index 4d2c7533ca..027acfab7f 100644 --- a/consensus/types/src/state/beacon_state.rs +++ b/consensus/types/src/state/beacon_state.rs @@ -1333,6 +1333,7 @@ impl BeaconState { pub fn is_valid_proposal_slot( &self, preferences: &ProposerPreferences, + spec: &ChainSpec, ) -> Result { let current_epoch = self.current_epoch(); let proposal_epoch = preferences.proposal_slot.epoch(E::slots_per_epoch()); @@ -1341,8 +1342,7 @@ impl BeaconState { return Ok(false); } - let next_epoch = current_epoch.saturating_add(1u64); - if proposal_epoch > next_epoch { + if proposal_epoch > current_epoch.saturating_add(spec.min_seed_lookahead) { return Ok(false); } diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index 36f6684685..f566a89ded 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,6 +1,6 @@ # To download/extract nightly tests, run: # CONSENSUS_SPECS_TEST_VERSION=nightly make -CONSENSUS_SPECS_TEST_VERSION ?= v1.7.0-alpha.7 +CONSENSUS_SPECS_TEST_VERSION ?= v1.7.0-alpha.8 REPO_NAME := consensus-spec-tests OUTPUT_DIR := ./$(REPO_NAME) diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 8b0b74d256..69fce09505 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -1,6 +1,6 @@ use super::*; use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file}; -use ::fork_choice::{PayloadVerificationStatus, ProposerHeadError}; +use ::fork_choice::{AttestationFromBlock, PayloadVerificationStatus, ProposerHeadError}; use beacon_chain::beacon_proposer_cache::compute_proposer_duties_from_head; use beacon_chain::blob_verification::GossipBlobError; use beacon_chain::block_verification_types::LookupBlock; @@ -19,13 +19,16 @@ use beacon_chain::{ custody_context::NodeCustodyType, test_utils::{BeaconChainHarness, EphemeralHarnessType}, }; +use bls::AggregateSignature; use execution_layer::{ PayloadStatusV1, PayloadStatusV1Status, json_structures::JsonPayloadStatusV1Status, }; use serde::Deserialize; use ssz_derive::Decode; +use ssz_types::VariableList; use state_processing::VerifySignatures; use state_processing::envelope_processing::verify_execution_payload_envelope; +use state_processing::per_block_processing::is_valid_indexed_payload_attestation; use state_processing::state_advance::complete_state_advance; use std::future::Future; use std::sync::Arc; @@ -34,8 +37,8 @@ use types::{ Attestation, AttestationRef, AttesterSlashing, AttesterSlashingRef, BeaconBlock, BeaconState, BlobSidecar, BlobsList, BlockImportSource, Checkpoint, DataColumnSidecar, DataColumnSidecarList, DataColumnSubnetId, ExecutionBlockHash, Hash256, IndexedAttestation, - KzgProof, ProposerPreparationData, SignedBeaconBlock, SignedExecutionPayloadEnvelope, Slot, - Uint256, + IndexedPayloadAttestation, KzgProof, PayloadAttestationMessage, ProposerPreparationData, + SignedBeaconBlock, SignedExecutionPayloadEnvelope, Slot, Uint256, }; // When set to true, cache any states fetched from the db. @@ -63,6 +66,13 @@ pub struct ShouldOverrideFcu { result: bool, } +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PayloadVoteCheck { + block_root: Hash256, + votes: Vec>, +} + #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct Checks { @@ -78,6 +88,8 @@ pub struct Checks { get_proposer_head: Option, should_override_forkchoice_update: Option, head_payload_status: Option, + payload_timeliness_vote: Option, + payload_data_availability_vote: Option, } #[derive(Debug, Clone, Deserialize)] @@ -108,6 +120,7 @@ pub enum Step< TAttesterSlashing, TPowBlock, TExecutionPayload = String, + TPayloadAttestationMessage = String, > { Tick { tick: u64, @@ -146,6 +159,15 @@ pub enum Step< execution_payload: TExecutionPayload, valid: bool, }, + PayloadAttestationMessage { + payload_attestation_message: TPayloadAttestationMessage, + #[serde(default = "default_true")] + valid: bool, + }, +} + +fn default_true() -> bool { + true } #[derive(Debug, Clone, Deserialize)] @@ -170,6 +192,7 @@ pub struct ForkChoiceTest { AttesterSlashing, PowBlock, SignedExecutionPayloadEnvelope, + PayloadAttestationMessage, >, >, } @@ -184,8 +207,12 @@ impl LoadCase for ForkChoiceTest { .expect("path must be valid OsStr") .to_string(); let spec = &testing_spec::(fork_name); - let steps: Vec, String, String, String>> = - yaml_decode_file(&path.join("steps.yaml"))?; + + #[allow(clippy::type_complexity)] + let steps: Vec< + Step, String, String, String, String, String>, + > = yaml_decode_file(&path.join("steps.yaml"))?; + // Resolve the object names in `steps.yaml` into actual decoded block/attestation objects. let steps = steps .into_iter() @@ -301,6 +328,18 @@ impl LoadCase for ForkChoiceTest { valid, }) } + Step::PayloadAttestationMessage { + payload_attestation_message, + valid, + } => { + let msg: PayloadAttestationMessage = ssz_decode_file( + &path.join(format!("{payload_attestation_message}.ssz_snappy")), + )?; + Ok(Step::PayloadAttestationMessage { + payload_attestation_message: msg, + valid, + }) + } }) .collect::>()?; let anchor_state = ssz_decode_state(&path.join("anchor_state.ssz_snappy"), spec)?; @@ -381,6 +420,8 @@ impl Case for ForkChoiceTest { get_proposer_head, should_override_forkchoice_update: should_override_fcu, head_payload_status, + payload_timeliness_vote, + payload_data_availability_vote, } = checks.as_ref(); if let Some(expected_head) = head { @@ -431,6 +472,14 @@ impl Case for ForkChoiceTest { if let Some(expected_status) = head_payload_status { tester.check_head_payload_status(*expected_status)?; } + + if let Some(expected) = payload_timeliness_vote { + tester.check_payload_timeliness_vote(expected)?; + } + + if let Some(expected) = payload_data_availability_vote { + tester.check_payload_data_availability_vote(expected)?; + } } Step::MaybeValidBlockAndColumns { @@ -446,6 +495,13 @@ impl Case for ForkChoiceTest { } => { tester.process_execution_payload(execution_payload, *valid)?; } + Step::PayloadAttestationMessage { + payload_attestation_message, + valid, + } => { + tester + .process_payload_attestation_message(payload_attestation_message, *valid)?; + } } } @@ -1149,6 +1205,173 @@ impl Tester { expected_should_override_fcu.result, ) } + + pub fn process_payload_attestation_message( + &self, + msg: &PayloadAttestationMessage, + valid: bool, + ) -> Result<(), Error> { + let slot = msg.data.slot; + let block_root = msg.data.beacon_block_root; + + // Get the state at the block to compute the PTC and verify signature. + let store = &self.harness.chain.store; + let block = store + .get_blinded_block(&block_root) + .map_err(|e| Error::InternalError(format!("Failed to load block: {e:?}")))?; + + let state_opt = block.and_then(|block| { + store + .get_hot_state(&block.state_root(), CACHE_STATE_IN_TESTS) + .ok()? + }); + + // Build IndexedPayloadAttestation from the message. + let indexed = IndexedPayloadAttestation:: { + attesting_indices: VariableList::new(vec![msg.validator_index]).unwrap(), + data: msg.data.clone(), + signature: AggregateSignature::from(&msg.signature), + }; + + let result = if let Some(ref state) = state_opt { + is_valid_indexed_payload_attestation( + state, + &indexed, + VerifySignatures::True, + &self.spec, + ) + .map_err(|e| { + Error::InternalError(format!( + "payload attestation signature verification failed for validator {}: {:?}", + msg.validator_index, e + )) + }) + .and_then(|_| { + let ptc = state.get_ptc(slot, &self.spec).map_err(|e| { + Error::InternalError(format!( + "Could not compute PTC for block root {block_root:?} at slot {slot:?}: {e:?}" + )) + })?; + + self.harness + .chain + .canonical_head + .fork_choice_write_lock() + .on_payload_attestation( + self.harness.chain.slot().unwrap(), + &indexed, + AttestationFromBlock::False, + &ptc.0, + ) + .map_err(|e| { + Error::InternalError(format!( + "on_payload_attestation for validator {} failed: {:?}", + msg.validator_index, e + )) + }) + }) + } else { + Err(Error::InternalError(format!( + "Could not get state for block root {block_root:?} at slot {slot:?}" + ))) + }; + + if valid { + result?; + } else if result.is_ok() { + return Err(Error::DidntFail(format!( + "payload_attestation_message for validator {} should have failed", + msg.validator_index + ))); + } + + Ok(()) + } + + pub fn check_payload_timeliness_vote(&self, expected: &PayloadVoteCheck) -> Result<(), Error> { + let fc = self.harness.chain.canonical_head.fork_choice_read_lock(); + let proto_array = fc.proto_array().core_proto_array(); + + let node_index = proto_array + .indices + .get(&expected.block_root) + .ok_or_else(|| { + Error::InternalError(format!( + "Block root {:?} not found in proto array", + expected.block_root + )) + })?; + let node = proto_array + .nodes + .get(*node_index) + .ok_or_else(|| Error::InternalError(format!("Node index {} not found", node_index)))?; + let v29 = node + .as_v29() + .map_err(|_| Error::InternalError("Node is not V29".to_string()))?; + + let timeliness_votes = &v29.payload_timeliness_votes; + let participation = &v29.ptc_participation; + + for (i, expected_vote) in expected.votes.iter().enumerate() { + let actual = if !participation.get(i).unwrap() { + None // not yet voted + } else { + Some(timeliness_votes.get(i).unwrap()) + }; + if actual != *expected_vote { + return Err(Error::NotEqual(format!( + "payload_timeliness_vote[{}]: Got {:?} | Expected {:?}", + i, actual, expected_vote + ))); + } + } + + Ok(()) + } + + pub fn check_payload_data_availability_vote( + &self, + expected: &PayloadVoteCheck, + ) -> Result<(), Error> { + let fc = self.harness.chain.canonical_head.fork_choice_read_lock(); + let proto_array = fc.proto_array().core_proto_array(); + + let node_index = proto_array + .indices + .get(&expected.block_root) + .ok_or_else(|| { + Error::InternalError(format!( + "Block root {:?} not found in proto array", + expected.block_root + )) + })?; + let node = proto_array + .nodes + .get(*node_index) + .ok_or_else(|| Error::InternalError(format!("Node index {} not found", node_index)))?; + let v29 = node + .as_v29() + .map_err(|_| Error::InternalError("Node is not V29".to_string()))?; + + let availability_votes = &v29.payload_data_availability_votes; + let participation = &v29.ptc_participation; + + for (i, expected_vote) in expected.votes.iter().enumerate() { + let actual = if !participation.get(i).unwrap() { + None // not yet voted + } else { + Some(availability_votes.get(i).unwrap()) + }; + if actual != *expected_vote { + return Err(Error::NotEqual(format!( + "payload_data_availability_vote[{}]: Got {:?} | Expected {:?}", + i, actual, expected_vote + ))); + } + } + + Ok(()) + } } /// Checks that the `head` checkpoint from the beacon chain head matches the `fc` checkpoint gleaned diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index e380f51c0a..52cc5d57ae 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -715,10 +715,8 @@ impl Handler for ForkChoiceHandler { return false; } - // Deposit tests exist only for Electra and Fulu (not Gloas). - if self.handler_name == "deposit_with_reorg" - && (!fork_name.electra_enabled() || fork_name.gloas_enabled()) - { + // Deposit tests exist only for Electra and later. + if self.handler_name == "deposit_with_reorg" && !fork_name.electra_enabled() { return false; } @@ -727,10 +725,11 @@ impl Handler for ForkChoiceHandler { return false; } - // on_execution_payload_envelope and get_parent_payload_status tests exist only for - // Gloas and later. + // on_execution_payload_envelope, get_parent_payload_status, and + // on_payload_attestation_message tests exist only for Gloas and later. if (self.handler_name == "on_execution_payload_envelope" - || self.handler_name == "get_parent_payload_status") + || self.handler_name == "get_parent_payload_status" + || self.handler_name == "on_payload_attestation_message") && !fork_name.gloas_enabled() { return false; diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index ca383efdb0..0ff854bd21 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -1079,6 +1079,12 @@ fn fork_choice_get_parent_payload_status() { ForkChoiceHandler::::new("get_parent_payload_status").run(); } +#[test] +fn fork_choice_on_payload_attestation_message() { + ForkChoiceHandler::::new("on_payload_attestation_message").run(); + ForkChoiceHandler::::new("on_payload_attestation_message").run(); +} + #[test] fn optimistic_sync() { OptimisticSyncHandler::::default().run(); diff --git a/testing/execution_engine_integration/src/test_rig.rs b/testing/execution_engine_integration/src/test_rig.rs index ed6b5787b5..61f25e63a1 100644 --- a/testing/execution_engine_integration/src/test_rig.rs +++ b/testing/execution_engine_integration/src/test_rig.rs @@ -320,6 +320,7 @@ impl TestRig { Some(vec![]), None, None, + None, ), ) .await; @@ -366,11 +367,12 @@ impl TestRig { Some(vec![]), None, None, + None, ); let payload_parameters = PayloadParameters { parent_hash, - parent_gas_limit, + parent_gas_limit: Some(parent_gas_limit), proposer_gas_limit: None, payload_attributes: &payload_attributes, forkchoice_update_params: &forkchoice_update_params, @@ -527,11 +529,12 @@ impl TestRig { Some(vec![]), None, None, + None, ); let payload_parameters = PayloadParameters { parent_hash, - parent_gas_limit, + parent_gas_limit: Some(parent_gas_limit), proposer_gas_limit: None, payload_attributes: &payload_attributes, forkchoice_update_params: &forkchoice_update_params, @@ -588,6 +591,7 @@ impl TestRig { Some(vec![]), None, None, + None, ); let slot = Slot::new(42); let head_block_root = Hash256::repeat_byte(100); diff --git a/validator_client/validator_services/src/proposer_preferences_service.rs b/validator_client/validator_services/src/proposer_preferences_service.rs index fbefdf5d96..fc17a1bce6 100644 --- a/validator_client/validator_services/src/proposer_preferences_service.rs +++ b/validator_client/validator_services/src/proposer_preferences_service.rs @@ -136,7 +136,7 @@ impl ProposerPreferencesSer proposal_slot: duty.slot, validator_index: duty.validator_index, fee_recipient, - gas_limit: proposal_data.gas_limit, + target_gas_limit: proposal_data.gas_limit, }, )); }