diff --git a/.github/forbidden-files.txt b/.github/forbidden-files.txt index 8649fbb574..1c5e9acab9 100644 --- a/.github/forbidden-files.txt +++ b/.github/forbidden-files.txt @@ -12,4 +12,6 @@ beacon_node/http_api/src/block_rewards.rs common/eth2/src/lighthouse/attestation_performance.rs common/eth2/src/lighthouse/block_packing_efficiency.rs common/eth2/src/lighthouse/block_rewards.rs +common/test_random_derive/ consensus/types/src/execution/state_payload_status.rs +consensus/types/src/test_utils/test_random/ diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index c632042351..1d66bd30e7 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -85,8 +85,8 @@ jobs: while IFS= read -r file || [ -n "$file" ]; do # Skip comments and empty lines [[ "$file" =~ ^#.*$ || -z "$file" ]] && continue - if [ -f "$file" ]; then - echo "::error::Forbidden file exists: $file" + if [ -e "$file" ]; then + echo "::error::Forbidden file or directory exists: $file" status=1 fi done < .github/forbidden-files.txt @@ -414,10 +414,6 @@ jobs: cache: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Fetch libssl1.1 - run: wget https://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb - - name: Install libssl1.1 - run: sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb - name: Create Cargo config dir run: mkdir -p .cargo - name: Install custom Cargo config diff --git a/Cargo.lock b/Cargo.lock index e2b4860d72..277c9b06db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1224,6 +1224,8 @@ name = "beacon_chain" version = "0.2.0" dependencies = [ "alloy-primitives", + "arbitrary", + "beacon_chain", "bitvec", "bls", "criterion", @@ -1258,6 +1260,7 @@ dependencies = [ "parking_lot", "proto_array", "rand 0.9.2", + "rand_xorshift 0.4.0", "rayon", "safe_arith", "sensitive_url", @@ -1610,6 +1613,7 @@ dependencies = [ name = "builder_client" version = "0.1.0" dependencies = [ + "arbitrary", "bls", "context_deserialize", "eth2", @@ -1621,6 +1625,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "types", ] [[package]] @@ -2740,6 +2745,7 @@ dependencies = [ name = "doppelganger_service" version = "0.1.0" dependencies = [ + "arbitrary", "beacon_node_fallback", "bls", "environment", @@ -3116,6 +3122,7 @@ dependencies = [ name = "eth2" version = "0.1.0" dependencies = [ + "arbitrary", "bls", "context_deserialize", "educe", @@ -3132,7 +3139,6 @@ dependencies = [ "multiaddr", "pretty_reqwest_error", "proto_array", - "rand 0.9.2", "reqwest", "reqwest-eventsource", "sensitive_url", @@ -3140,7 +3146,6 @@ dependencies = [ "serde_json", "ssz_types", "superstruct", - "test_random_derive", "tokio", "types", "zeroize", @@ -3277,9 +3282,9 @@ dependencies = [ [[package]] name = "ethereum_ssz" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2128a84f7a3850d54ee343334e3392cca61f9f6aa9441eec481b9394b43c238b" +checksum = "368a4a4e4273b0135111fe9464e35465067766a8f664615b5a86338b73864407" dependencies = [ "alloy-primitives", "arbitrary", @@ -3294,9 +3299,9 @@ dependencies = [ [[package]] name = "ethereum_ssz_derive" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd596f91cff004fc8d02be44c21c0f9b93140a04b66027ae052f5f8e05b48eba" +checksum = "f2cd82c68120c89361e1a457245cf212f7d9f541bffaffed530c8f2d54a160b2" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -6053,6 +6058,7 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "anyhow", + "arbitrary", "async-channel 1.9.0", "beacon_chain", "beacon_processor", @@ -8429,7 +8435,7 @@ dependencies = [ "safe_arith", "smallvec", "ssz_types", - "test_random_derive", + "state_processing", "tokio", "tracing", "tree_hash", @@ -8714,14 +8720,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" -[[package]] -name = "test_random_derive" -version = "0.2.0" -dependencies = [ - "quote", - "syn 2.0.117", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -9362,12 +9360,12 @@ dependencies = [ "superstruct", "swap_or_not_shuffle", "tempfile", - "test_random_derive", "tokio", "tracing", "tree_hash", "tree_hash_derive", "typenum", + "types", "yaml_serde", ] diff --git a/Cargo.toml b/Cargo.toml index 1f58c322f1..71398530fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,6 @@ members = [ "common/system_health", "common/target_check", "common/task_executor", - "common/test_random_derive", "common/tracing_samplers", "common/validator_dir", "common/warp_utils", @@ -200,6 +199,7 @@ proto_array = { path = "consensus/proto_array" } quote = "1" r2d2 = "0.8" rand = "0.9.0" +rand_xorshift = "0.4.0" rayon = "1.7" regex = "1" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "stream", "rustls-tls"] } diff --git a/Makefile b/Makefile index 04973193ec..dd57bb038e 100644 --- a/Makefile +++ b/Makefile @@ -330,7 +330,7 @@ install-audit: cargo install --force cargo-audit audit-CI: - cargo audit --ignore RUSTSEC-2026-0049 --ignore RUSTSEC-2026-0098 --ignore RUSTSEC-2026-0099 --ignore RUSTSEC-2026-0104 + cargo audit --ignore RUSTSEC-2026-0049 --ignore RUSTSEC-2026-0098 --ignore RUSTSEC-2026-0099 --ignore RUSTSEC-2026-0104 --ignore RUSTSEC-2026-0118 --ignore RUSTSEC-2026-0119 # Runs cargo deny (check for banned crates, duplicate versions, and source restrictions) deny: install-deny deny-CI diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index a06db8934b..47ef4d7a03 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -16,9 +16,11 @@ participation_metrics = [] fork_from_env = [] portable = ["bls/supranational-portable"] test_backfill = [] +arbitrary = ["dep:arbitrary", "types/arbitrary"] [dependencies] alloy-primitives = { workspace = true } +arbitrary = { workspace = true, optional = true } bitvec = { workspace = true } bls = { workspace = true } educe = { workspace = true } @@ -74,11 +76,15 @@ types = { workspace = true } zstd = { workspace = true } [dev-dependencies] +arbitrary = { workspace = true } +beacon_chain = { path = ".", features = ["arbitrary"] } criterion = { workspace = true } maplit = { workspace = true } mockall = { workspace = true } mockall_double = { workspace = true } +rand_xorshift = { workspace = true } serde_json = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } [[bench]] name = "benches" diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 9d8b76aaed..f0fa9c7794 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -1041,8 +1041,6 @@ mod test { EphemeralHarnessType, NumBlobs, generate_data_column_indices_rand_order, generate_rand_block_and_data_columns, get_kzg, }; - use rand::SeedableRng; - use rand::prelude::StdRng; use slot_clock::{SlotClock, TestingSlotClock}; use std::collections::HashSet; use std::sync::Arc; @@ -1061,7 +1059,7 @@ mod test { fn should_exclude_rpc_columns_not_required_for_sampling() { // SETUP let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec())); - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let da_checker = new_da_checker(spec.clone()); let custody_context = &da_checker.custody_context; @@ -1093,9 +1091,10 @@ mod test { let (_, data_columns) = generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, - ); + ) + .unwrap(); let block_root = Hash256::random(); // Get 10 columns using the "latest" CGC (head) that block lookup would use. // The CGC change becomes effective after CUSTODY_CHANGE_DA_EFFECTIVE_DELAY_SECONDS, @@ -1147,7 +1146,7 @@ mod test { fn should_exclude_gossip_columns_not_required_for_sampling() { // SETUP let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec())); - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let da_checker = new_da_checker(spec.clone()); let custody_context = &da_checker.custody_context; @@ -1180,9 +1179,10 @@ mod test { let (_, data_columns) = generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, - ); + ) + .unwrap(); let block_root = Hash256::random(); // Get 10 columns using the "latest" CGC that gossip subscriptions would use. // The CGC change becomes effective after CUSTODY_CHANGE_DA_EFFECTIVE_DELAY_SECONDS, @@ -1230,7 +1230,7 @@ mod test { #[test] fn verify_kzg_for_range_sync_blocks_should_not_truncate_data_columns_fulu() { let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec())); - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let da_checker = new_da_checker(spec.clone()); // GIVEN multiple RPC blocks with data columns totalling more than 128 @@ -1239,9 +1239,10 @@ mod test { let (block, data_columns) = generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, - ); + ) + .unwrap(); let custody_columns = if index == 0 { // 128 valid data columns in the first block @@ -1293,7 +1294,7 @@ mod test { fn should_exclude_reconstructed_columns_not_required_for_sampling() { // SETUP let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec())); - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let da_checker = new_da_checker(spec.clone()); let custody_context = &da_checker.custody_context; @@ -1314,9 +1315,10 @@ mod test { let (block, data_columns) = generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, - ); + ) + .unwrap(); let block_root = Hash256::random(); // Add the block to the DA checker da_checker diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 8f1d4464e1..7d1bba2de9 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -1077,13 +1077,11 @@ mod pending_components_tests { use crate::PayloadVerificationOutcome; use crate::block_verification_types::BlockImportData; use crate::test_utils::{NumBlobs, generate_rand_block_and_blobs, test_spec}; + use arbitrary::Arbitrary; use fixed_bytes::FixedBytesExtended; use fork_choice::PayloadVerificationStatus; use kzg::KzgCommitment; - use rand::SeedableRng; - use rand::rngs::StdRng; use state_processing::ConsensusContext; - use types::test_utils::TestRandom; use types::{BeaconState, ForkName, MainnetEthSpec, SignedBeaconBlock, Slot}; type E = MainnetEthSpec; @@ -1096,10 +1094,10 @@ mod pending_components_tests { ); pub fn pre_setup() -> Setup { - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let spec = test_spec::(); let (block, blobs_vec) = - generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Random, &mut rng); + generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Random, &mut u).unwrap(); let max_len = spec.max_blobs_per_block(block.epoch()) as usize; let mut blobs: RuntimeFixedVector>>> = RuntimeFixedVector::default(max_len); @@ -1115,7 +1113,7 @@ mod pending_components_tests { for (index, blob) in blobs.iter().enumerate() { if let Some(invalid_blob) = blob { let mut blob_copy = invalid_blob.as_ref().clone(); - blob_copy.kzg_commitment = KzgCommitment::random_for_test(&mut rng); + blob_copy.kzg_commitment = KzgCommitment::arbitrary(&mut u).unwrap(); *invalid_blobs.get_mut(index).unwrap() = Some(Arc::new(blob_copy)); } } diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index 72080b92da..4d192cb5b9 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -582,20 +582,20 @@ mod tests { use tree_hash::TreeHash; use types::{ Attestation, AttestationBase, AttestationElectra, Fork, Hash256, SyncCommitteeMessage, - test_utils::{generate_deterministic_keypair, test_random_instance}, + test_utils::{generate_deterministic_keypair, test_arbitrary_instance}, }; type E = types::MainnetEthSpec; fn get_attestation_base(slot: Slot) -> Attestation { - let mut a: AttestationBase = test_random_instance(); + let mut a: AttestationBase = test_arbitrary_instance(); a.data.slot = slot; a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); Attestation::Base(a) } fn get_attestation_electra(slot: Slot) -> Attestation { - let mut a: AttestationElectra = test_random_instance(); + let mut a: AttestationElectra = test_arbitrary_instance(); a.data.slot = slot; a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); a.committee_bits = BitVector::new(); @@ -606,7 +606,7 @@ mod tests { } fn get_sync_contribution(slot: Slot) -> SyncCommitteeContribution { - let mut a: SyncCommitteeContribution = test_random_instance(); + let mut a: SyncCommitteeContribution = test_arbitrary_instance(); a.slot = slot; a.aggregation_bits = BitVector::new(); a diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index 7ecd581e85..8d4be693ac 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -474,12 +474,12 @@ where mod tests { use super::*; use fixed_bytes::FixedBytesExtended; - use types::{AttestationBase, Hash256, test_utils::test_random_instance}; + use types::{AttestationBase, Hash256, test_utils::test_arbitrary_instance}; type E = types::MainnetEthSpec; fn get_attestation(slot: Slot, beacon_block_root: u64) -> Attestation { - let a: AttestationBase = test_random_instance(); + let a: AttestationBase = test_arbitrary_instance(); let mut a = Attestation::Base(a); a.data_mut().slot = slot; a.data_mut().beacon_block_root = Hash256::from_low_u64_be(beacon_block_root); @@ -487,7 +487,7 @@ mod tests { } fn get_sync_contribution(slot: Slot, beacon_block_root: u64) -> SyncCommitteeContribution { - let mut a: SyncCommitteeContribution = test_random_instance(); + let mut a: SyncCommitteeContribution = test_arbitrary_instance(); a.slot = slot; a.beacon_block_root = Hash256::from_low_u64_be(beacon_block_root); a diff --git a/beacon_node/beacon_chain/src/payload_bid_verification/tests.rs b/beacon_node/beacon_chain/src/payload_bid_verification/tests.rs index bbd62b2a64..3d1a313e2e 100644 --- a/beacon_node/beacon_chain/src/payload_bid_verification/tests.rs +++ b/beacon_node/beacon_chain/src/payload_bid_verification/tests.rs @@ -257,11 +257,11 @@ fn make_signed_preferences( ) -> Arc { Arc::new(SignedProposerPreferences { message: ProposerPreferences { + dependent_root: Hash256::ZERO, proposal_slot, validator_index, fee_recipient, gas_limit, - ..ProposerPreferences::default() }, signature: Signature::empty(), }) diff --git a/beacon_node/beacon_chain/src/proposer_preferences_verification/gossip_verified_proposer_preferences.rs b/beacon_node/beacon_chain/src/proposer_preferences_verification/gossip_verified_proposer_preferences.rs index cc77453c49..4ba33fde72 100644 --- a/beacon_node/beacon_chain/src/proposer_preferences_verification/gossip_verified_proposer_preferences.rs +++ b/beacon_node/beacon_chain/src/proposer_preferences_verification/gossip_verified_proposer_preferences.rs @@ -154,7 +154,9 @@ impl BeaconChain { #[cfg(test)] mod tests { - use types::{Address, BeaconState, EthSpec, MinimalEthSpec, ProposerPreferences, Slot}; + use types::{ + Address, BeaconState, EthSpec, Hash256, MinimalEthSpec, ProposerPreferences, Slot, + }; use super::verify_preferences_consistency; use crate::proposer_preferences_verification::ProposerPreferencesError; @@ -163,7 +165,7 @@ mod tests { fn make_preferences(proposal_slot: Slot, validator_index: u64) -> ProposerPreferences { ProposerPreferences { - dependent_root: types::Hash256::ZERO, + dependent_root: Hash256::ZERO, proposal_slot, validator_index, fee_recipient: Address::ZERO, diff --git a/beacon_node/beacon_chain/src/proposer_preferences_verification/proposer_preference_cache.rs b/beacon_node/beacon_chain/src/proposer_preferences_verification/proposer_preference_cache.rs index 507e61dc10..7bbdf34888 100644 --- a/beacon_node/beacon_chain/src/proposer_preferences_verification/proposer_preference_cache.rs +++ b/beacon_node/beacon_chain/src/proposer_preferences_verification/proposer_preference_cache.rs @@ -70,20 +70,24 @@ mod tests { use std::sync::Arc; use bls::Signature; - use types::{Address, ProposerPreferences, SignedProposerPreferences, Slot}; + use types::{Address, Hash256, ProposerPreferences, SignedProposerPreferences, Slot}; use super::GossipVerifiedProposerPreferenceCache; use crate::proposer_preferences_verification::gossip_verified_proposer_preferences::GossipVerifiedProposerPreferences; - fn make_gossip_verified(slot: Slot, validator_index: u64) -> GossipVerifiedProposerPreferences { + fn make_gossip_verified( + slot: Slot, + validator_index: u64, + dependent_root: Hash256, + ) -> GossipVerifiedProposerPreferences { GossipVerifiedProposerPreferences { signed_preferences: Arc::new(SignedProposerPreferences { message: ProposerPreferences { + dependent_root, proposal_slot: slot, validator_index, fee_recipient: Address::ZERO, gas_limit: 30_000_000, - ..ProposerPreferences::default() }, signature: Signature::empty(), }), @@ -93,9 +97,10 @@ mod tests { #[test] fn prune_removes_old_retains_current() { let cache = GossipVerifiedProposerPreferenceCache::default(); + let root = Hash256::ZERO; for slot in [1, 2, 3, 7, 8, 9, 10] { - let verified = make_gossip_verified(Slot::new(slot), slot); + let verified = make_gossip_verified(Slot::new(slot), slot, root); cache.insert_seen_validator(&verified); cache.insert_preferences(verified); } @@ -104,11 +109,26 @@ mod tests { for slot in [1, 2, 3, 7] { assert!(cache.get_preferences(&Slot::new(slot)).is_none()); - assert!(!cache.get_seen_validator(&Slot::new(slot), types::Hash256::ZERO, slot)); + assert!(!cache.get_seen_validator(&Slot::new(slot), root, slot)); } for slot in [8, 9, 10] { assert!(cache.get_preferences(&Slot::new(slot)).is_some()); - assert!(cache.get_seen_validator(&Slot::new(slot), types::Hash256::ZERO, slot)); + assert!(cache.get_seen_validator(&Slot::new(slot), root, slot)); } } + + #[test] + fn different_dependent_roots_not_deduped() { + let cache = GossipVerifiedProposerPreferenceCache::default(); + let slot = Slot::new(5); + let root_a = Hash256::repeat_byte(0xaa); + let root_b = Hash256::repeat_byte(0xbb); + let validator_index = 42; + + let verified_a = make_gossip_verified(slot, validator_index, root_a); + cache.insert_seen_validator(&verified_a); + + assert!(cache.get_seen_validator(&slot, root_a, validator_index)); + assert!(!cache.get_seen_validator(&slot, root_b, validator_index)); + } } diff --git a/beacon_node/beacon_chain/src/proposer_preferences_verification/tests.rs b/beacon_node/beacon_chain/src/proposer_preferences_verification/tests.rs index ce2ea12bb5..468e08ff3b 100644 --- a/beacon_node/beacon_chain/src/proposer_preferences_verification/tests.rs +++ b/beacon_node/beacon_chain/src/proposer_preferences_verification/tests.rs @@ -127,11 +127,11 @@ fn make_signed_preferences( ) -> Arc { Arc::new(SignedProposerPreferences { message: ProposerPreferences { + dependent_root: Hash256::ZERO, proposal_slot, validator_index, fee_recipient: Address::ZERO, gas_limit: 30_000_000, - ..ProposerPreferences::default() }, signature: Signature::empty(), }) @@ -231,11 +231,10 @@ fn correct_proposer_bad_signature() { result, Err(ProposerPreferencesError::BadSignature) )); - assert!(!ctx.preferences_cache.get_seen_validator( - &slot, - types::Hash256::ZERO, - actual_proposer - )); + assert!( + !ctx.preferences_cache + .get_seen_validator(&slot, Hash256::ZERO, actual_proposer) + ); assert!(ctx.preferences_cache.get_preferences(&slot).is_none()); } diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index fb1a67a1eb..850c06696e 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -20,6 +20,8 @@ pub use crate::{ sync_committee_verification::Error as SyncCommitteeError, validator_monitor::{ValidatorMonitor, ValidatorMonitorConfig}, }; +#[cfg(feature = "arbitrary")] +use arbitrary::Arbitrary; use bls::get_withdrawal_credentials; use bls::{ AggregateSignature, Keypair, PublicKey, PublicKeyBytes, SecretKey, Signature, SignatureBytes, @@ -73,7 +75,6 @@ use typenum::U4294967296; use types::attestation::IndexedAttestationBase; use types::data::CustodyIndex; use types::execution::BlockProductionVersion; -use types::test_utils::TestRandom; pub use types::test_utils::generate_deterministic_keypairs; use types::*; @@ -96,7 +97,9 @@ pub const TEST_DATA_COLUMN_SIDECARS_GLOAS_SSZ: &[u8] = pub const DEFAULT_TARGET_AGGREGATORS: u64 = u64::MAX; // Minimum and maximum number of blobs to generate in each slot when using the `NumBlobs::Random` option (default). +#[cfg(feature = "arbitrary")] const DEFAULT_MIN_BLOBS: usize = 1; +#[cfg(feature = "arbitrary")] const DEFAULT_MAX_BLOBS: usize = 2; static KZG: LazyLock> = LazyLock::new(|| { @@ -3821,10 +3824,11 @@ pub enum NumBlobs { None, } +#[cfg(feature = "arbitrary")] macro_rules! add_blob_transactions { - ($message:expr, $payload_type:ty, $num_blobs:expr, $rng:expr, $fork_name:expr) => {{ + ($message:expr, $payload_type:ty, $num_blobs:expr, $u:expr, $fork_name:expr) => {{ let num_blobs = match $num_blobs { - NumBlobs::Random => $rng.random_range(DEFAULT_MIN_BLOBS..=DEFAULT_MAX_BLOBS), + NumBlobs::Random => $u.int_in_range(DEFAULT_MIN_BLOBS..=DEFAULT_MAX_BLOBS)?, NumBlobs::Number(n) => n, NumBlobs::None => 0, }; @@ -3841,28 +3845,30 @@ macro_rules! add_blob_transactions { }}; } +#[cfg(feature = "arbitrary")] +#[allow(clippy::type_complexity)] pub fn generate_rand_block_and_blobs( fork_name: ForkName, num_blobs: NumBlobs, - rng: &mut impl Rng, -) -> (SignedBeaconBlock>, Vec>) { - let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(rng)); + u: &mut arbitrary::Unstructured, +) -> arbitrary::Result<(SignedBeaconBlock>, Vec>)> { + let inner = map_fork_name!(fork_name, BeaconBlock, <_>::arbitrary(&mut *u)?); - let mut block = SignedBeaconBlock::from_block(inner, Signature::random_for_test(rng)); + let mut block = SignedBeaconBlock::from_block(inner, Signature::arbitrary(&mut *u)?); let mut blob_sidecars = vec![]; let bundle = match block { SignedBeaconBlock::Deneb(SignedBeaconBlockDeneb { ref mut message, .. - }) => add_blob_transactions!(message, FullPayloadDeneb, num_blobs, rng, fork_name), + }) => add_blob_transactions!(message, FullPayloadDeneb, num_blobs, u, fork_name), SignedBeaconBlock::Electra(SignedBeaconBlockElectra { ref mut message, .. - }) => add_blob_transactions!(message, FullPayloadElectra, num_blobs, rng, fork_name), + }) => add_blob_transactions!(message, FullPayloadElectra, num_blobs, u, fork_name), SignedBeaconBlock::Fulu(SignedBeaconBlockFulu { ref mut message, .. - }) => add_blob_transactions!(message, FullPayloadFulu, num_blobs, rng, fork_name), + }) => add_blob_transactions!(message, FullPayloadFulu, num_blobs, u, fork_name), // TODO(EIP-7732) Add `SignedBeaconBlock::Gloas` and `SignedBeaconBlock::Heze` variants - _ => return (block, blob_sidecars), + _ => return Ok((block, blob_sidecars)), }; let eth2::types::BlobsBundle { @@ -3887,21 +3893,23 @@ pub fn generate_rand_block_and_blobs( .unwrap(), }); } - (block, blob_sidecars) + Ok((block, blob_sidecars)) } +#[cfg(feature = "arbitrary")] +#[allow(clippy::type_complexity)] pub fn generate_rand_block_and_data_columns( fork_name: ForkName, num_blobs: NumBlobs, - rng: &mut impl Rng, + u: &mut arbitrary::Unstructured, spec: &ChainSpec, -) -> ( +) -> arbitrary::Result<( SignedBeaconBlock>, DataColumnSidecarList, -) { - let (block, _blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, rng); +)> { + let (block, _blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, u)?; let data_columns = generate_data_column_sidecars_from_block(&block, spec); - (block, data_columns) + Ok((block, data_columns)) } /// Generate data column sidecars from pre-computed cells and proofs. diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index e943514c4e..cd0e700109 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -1,3 +1,4 @@ +use arbitrary::Arbitrary; use beacon_chain::blob_verification::GossipVerifiedBlob; use beacon_chain::data_column_verification::GossipVerifiedDataColumn; use beacon_chain::test_utils::{ @@ -8,7 +9,6 @@ use rand::SeedableRng; use rand::rngs::StdRng; use std::sync::Arc; use types::data::FixedBlobSidecarList; -use types::test_utils::TestRandom; use types::{ BlobSidecar, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSidecarGloas, Domain, EthSpec, MinimalEthSpec, PayloadAttestationData, PayloadAttestationMessage, SignedRoot, Slot, @@ -74,19 +74,19 @@ async fn data_column_sidecar_event_on_process_gossip_data_column() { let mut data_column_event_receiver = event_handler.subscribe_data_column_sidecar(); // build and process a gossip verified data column - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let mut u = types::test_utils::test_unstructured(); let sidecar = { let slot = Slot::new(10); let fork_name = harness.spec.fork_name_at_slot::(slot); // DA checker only accepts sampling columns, so we need to create one with a sampling index. if fork_name.gloas_enabled() { - let mut random_sidecar = DataColumnSidecarGloas::random_for_test(&mut rng); + let mut random_sidecar = DataColumnSidecarGloas::arbitrary(&mut u).unwrap(); let epoch = slot.epoch(E::slots_per_epoch()); random_sidecar.slot = slot; random_sidecar.index = harness.chain.sampling_columns_for_epoch(epoch)[0]; DataColumnSidecar::Gloas(random_sidecar) } else { - let mut random_sidecar = DataColumnSidecarFulu::random_for_test(&mut rng); + let mut random_sidecar = DataColumnSidecarFulu::arbitrary(&mut u).unwrap(); let epoch = slot.epoch(E::slots_per_epoch()); random_sidecar.signed_block_header.message.slot = slot; random_sidecar.index = harness.chain.sampling_columns_for_epoch(epoch)[0]; diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 86adf50995..1576092c81 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -31,7 +31,9 @@ use fork_choice::PayloadStatus; use logging::create_test_tracing_subscriber; use maplit::hashset; use rand::Rng; +use rand::SeedableRng; use rand::rngs::StdRng; +use rand_xorshift::XorShiftRng; use slot_clock::{SlotClock, TestingSlotClock}; use ssz_types::VariableList; use state_processing::{BlockReplayer, state_advance::complete_state_advance}; @@ -50,7 +52,6 @@ use store::{ }; use tempfile::{TempDir, tempdir}; use tracing::info; -use types::test_utils::{SeedableRng, XorShiftRng}; use types::*; // Should ideally be divisible by 3. diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 347e092b9a..60d88d79ec 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -431,6 +431,7 @@ pub enum Work { Status(BlockingFn), BlocksByRangeRequest(AsyncFn), BlocksByRootsRequest(AsyncFn), + BlocksByHeadRequest(AsyncFn), PayloadEnvelopesByRangeRequest(AsyncFn), PayloadEnvelopesByRootRequest(AsyncFn), BlobsByRangeRequest(BlockingFn), @@ -492,6 +493,7 @@ pub enum WorkType { Status, BlocksByRangeRequest, BlocksByRootsRequest, + BlocksByHeadRequest, PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, BlobsByRangeRequest, @@ -555,6 +557,7 @@ impl Work { Work::Status(_) => WorkType::Status, Work::BlocksByRangeRequest(_) => WorkType::BlocksByRangeRequest, Work::BlocksByRootsRequest(_) => WorkType::BlocksByRootsRequest, + Work::BlocksByHeadRequest(_) => WorkType::BlocksByHeadRequest, Work::PayloadEnvelopesByRangeRequest(_) => WorkType::PayloadEnvelopesByRangeRequest, Work::PayloadEnvelopesByRootRequest(_) => WorkType::PayloadEnvelopesByRootRequest, Work::BlobsByRangeRequest(_) => WorkType::BlobsByRangeRequest, @@ -1006,6 +1009,8 @@ impl BeaconProcessor { Some(item) } else if let Some(item) = work_queues.block_broots_queue.pop() { Some(item) + } else if let Some(item) = work_queues.block_bhead_queue.pop() { + Some(item) } else if let Some(item) = work_queues.blob_brange_queue.pop() { Some(item) } else if let Some(item) = work_queues.blob_broots_queue.pop() { @@ -1215,6 +1220,9 @@ impl BeaconProcessor { Work::BlocksByRootsRequest { .. } => { work_queues.block_broots_queue.push(work, work_id) } + Work::BlocksByHeadRequest { .. } => { + work_queues.block_bhead_queue.push(work, work_id) + } Work::PayloadEnvelopesByRangeRequest { .. } => work_queues .payload_envelopes_brange_queue .push(work, work_id), @@ -1340,6 +1348,7 @@ impl BeaconProcessor { WorkType::Status => work_queues.status_queue.len(), WorkType::BlocksByRangeRequest => work_queues.block_brange_queue.len(), WorkType::BlocksByRootsRequest => work_queues.block_broots_queue.len(), + WorkType::BlocksByHeadRequest => work_queues.block_bhead_queue.len(), WorkType::PayloadEnvelopesByRangeRequest => { work_queues.payload_envelopes_brange_queue.len() } @@ -1543,6 +1552,7 @@ impl BeaconProcessor { } Work::BlocksByRangeRequest(work) | Work::BlocksByRootsRequest(work) + | Work::BlocksByHeadRequest(work) | Work::PayloadEnvelopesByRangeRequest(work) | Work::PayloadEnvelopesByRootRequest(work) => task_spawner.spawn_async(work), Work::ChainSegmentBackfill(process_fn) => { diff --git a/beacon_node/beacon_processor/src/scheduler/work_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_queue.rs index d1b002623c..23125e8e98 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_queue.rs @@ -132,6 +132,7 @@ pub struct BeaconProcessorQueueLengths { status_queue: usize, block_brange_queue: usize, block_broots_queue: usize, + block_bhead_queue: usize, blob_broots_queue: usize, blob_brange_queue: usize, dcbroots_queue: usize, @@ -208,6 +209,7 @@ impl BeaconProcessorQueueLengths { status_queue: 1024, block_brange_queue: 1024, block_broots_queue: 1024, + block_bhead_queue: 1024, blob_broots_queue: 1024, blob_brange_queue: 1024, dcbroots_queue: 1024, @@ -266,6 +268,7 @@ pub struct WorkQueues { pub status_queue: FifoQueue>, pub block_brange_queue: FifoQueue>, pub block_broots_queue: FifoQueue>, + pub block_bhead_queue: FifoQueue>, pub payload_envelopes_brange_queue: FifoQueue>, pub payload_envelopes_broots_queue: FifoQueue>, pub blob_broots_queue: FifoQueue>, @@ -338,6 +341,7 @@ impl WorkQueues { let status_queue = FifoQueue::new(queue_lengths.status_queue); let block_brange_queue = FifoQueue::new(queue_lengths.block_brange_queue); let block_broots_queue = FifoQueue::new(queue_lengths.block_broots_queue); + let block_bhead_queue = FifoQueue::new(queue_lengths.block_bhead_queue); let blob_broots_queue = FifoQueue::new(queue_lengths.blob_broots_queue); let blob_brange_queue = FifoQueue::new(queue_lengths.blob_brange_queue); let dcbroots_queue = FifoQueue::new(queue_lengths.dcbroots_queue); @@ -404,6 +408,7 @@ impl WorkQueues { status_queue, block_brange_queue, block_broots_queue, + block_bhead_queue, blob_broots_queue, blob_brange_queue, dcbroots_queue, diff --git a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs index 38306b3bb6..b1fa56af01 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs @@ -280,8 +280,8 @@ struct ReprocessQueue { queued_lc_updates: FnvHashMap, /// Light Client Updates per parent_root. awaiting_lc_updates_per_parent_root: HashMap>, - /// Column reconstruction per block root. - queued_column_reconstructions: HashMap, + /// Column reconstruction per block root. `None` means reconstruction was already dispatched. + queued_column_reconstructions: HashMap>, /// Queued backfill batches queued_backfill_batches: Vec, @@ -865,20 +865,20 @@ impl ReprocessQueue { && duration_from_current_slot >= reconstruction_deadline && current_slot == request.slot { - // If we are at least `reconstruction_deadline` seconds into the current slot, - // and the reconstruction request is for the current slot, process reconstruction immediately. reconstruction_delay = Duration::from_secs(0); } match self.queued_column_reconstructions.entry(request.block_root) { - Entry::Occupied(key) => { - self.column_reconstructions_delay_queue - .reset(key.get(), reconstruction_delay); + Entry::Occupied(entry) => { + if let Some(delay_key) = entry.get() { + self.column_reconstructions_delay_queue + .reset(delay_key, reconstruction_delay); + } } Entry::Vacant(vacant) => { let delay_key = self .column_reconstructions_delay_queue .insert(request, reconstruction_delay); - vacant.insert(delay_key); + vacant.insert(Some(delay_key)); } } } @@ -1039,7 +1039,9 @@ impl ReprocessQueue { } InboundEvent::ReadyColumnReconstruction(column_reconstruction) => { self.queued_column_reconstructions - .remove(&column_reconstruction.block_root); + .retain(|_, v| v.is_some()); + self.queued_column_reconstructions + .insert(column_reconstruction.block_root, None); if self .ready_work_tx .try_send(ReadyWork::ColumnReconstruction(column_reconstruction)) @@ -1398,7 +1400,10 @@ mod tests { queue.handle_message(InboundEvent::ReadyColumnReconstruction(reconstruction)); } - assert!(queue.queued_column_reconstructions.is_empty()); + assert_eq!( + queue.queued_column_reconstructions.get(&block_root), + Some(&None) + ); } /// Tests that column reconstruction queued after the deadline is triggered immediately diff --git a/beacon_node/builder_client/Cargo.toml b/beacon_node/builder_client/Cargo.toml index 09bf3f48b4..a329379160 100644 --- a/beacon_node/builder_client/Cargo.toml +++ b/beacon_node/builder_client/Cargo.toml @@ -16,5 +16,7 @@ serde = { workspace = true } serde_json = { workspace = true } [dev-dependencies] +arbitrary = { workspace = true } mockito = { workspace = true } tokio = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index 7dc0cbfc6d..bd064ca8bf 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -540,10 +540,10 @@ impl BuilderHttpClient { #[cfg(test)] mod tests { use super::*; + use arbitrary::Arbitrary; use bls::Signature; use eth2::types::MainnetEthSpec; use eth2::types::builder::{BuilderBid, BuilderBidFulu}; - use eth2::types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use mockito::{Matcher, Server, ServerGuard}; type E = MainnetEthSpec; @@ -689,12 +689,12 @@ mod tests { } fn fulu_signed_builder_bid() -> ForkVersionedResponse> { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); ForkVersionedResponse { version: ForkName::Fulu, metadata: EmptyMetadata {}, data: SignedBuilderBid { - message: BuilderBid::Fulu(BuilderBidFulu::random_for_test(rng)), + message: BuilderBid::Fulu(BuilderBidFulu::arbitrary(&mut u).unwrap()), signature: Signature::empty(), }, } diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 9a3fa258e4..403b76bc85 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -69,6 +69,13 @@ impl Block { } } + pub fn timestamp(&self) -> u64 { + match self { + Block::PoW(block) => block.timestamp, + Block::PoS(payload) => payload.timestamp(), + } + } + pub fn total_difficulty(&self) -> Option { match self { Block::PoW(block) => Some(block.total_difficulty), @@ -563,6 +570,23 @@ impl ExecutionBlockGenerator { self.insert_block(Block::PoS(payload))?; } + // Post-Gloas, the justified and finalized block hashes must be non-zero, since the + // CL always has a known parent_block_hash to reference. + if let Some(head_block) = self.blocks.get(&head_block_hash) + && self + .get_fork_at_timestamp(head_block.timestamp()) + .gloas_enabled() + { + assert!( + forkchoice_state.safe_block_hash != ExecutionBlockHash::zero(), + "post-Gloas safe_block_hash must not be zero" + ); + assert!( + forkchoice_state.finalized_block_hash != ExecutionBlockHash::zero(), + "post-Gloas finalized_block_hash must not be zero" + ); + } + let unknown_head_block_hash = !self.blocks.contains_key(&head_block_hash); let unknown_safe_block_hash = forkchoice_state.safe_block_hash != ExecutionBlockHash::zero() diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index ca44ed5def..d4c616d679 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1494,7 +1494,7 @@ pub fn serve( // POST beacon/pool/payload_attestations let post_beacon_pool_payload_attestations = post_beacon_pool_payload_attestations( &network_tx_filter, - optional_consensus_version_header_filter, + optional_consensus_version_header_filter.clone(), &beacon_pool_path, ); @@ -1539,6 +1539,22 @@ pub fn serve( }, ); + // POST validator/proposer_preferences (JSON) + let post_validator_proposer_preferences = post_validator_proposer_preferences( + eth_v1.clone(), + task_spawner_filter.clone(), + chain_filter.clone(), + network_tx_filter.clone(), + ); + + // POST validator/proposer_preferences (SSZ) + let post_validator_proposer_preferences_ssz = post_validator_proposer_preferences_ssz( + eth_v1.clone(), + task_spawner_filter.clone(), + chain_filter.clone(), + network_tx_filter.clone(), + ); + // POST beacon/execution_payload_envelope let post_beacon_execution_payload_envelope = post_beacon_execution_payload_envelope( eth_v1.clone(), @@ -3529,7 +3545,8 @@ pub fn serve( .uor(post_beacon_blinded_blocks_ssz) .uor(post_beacon_blinded_blocks_v2_ssz) .uor(post_beacon_execution_payload_envelope_ssz) - .uor(post_beacon_pool_payload_attestations_ssz), + .uor(post_beacon_pool_payload_attestations_ssz) + .uor(post_validator_proposer_preferences_ssz), ) .uor(post_beacon_blocks) .uor(post_beacon_blinded_blocks) @@ -3542,6 +3559,7 @@ pub fn serve( .uor(post_beacon_pool_sync_committees) .uor(post_beacon_pool_payload_attestations) .uor(post_beacon_pool_bls_to_execution_changes) + .uor(post_validator_proposer_preferences) .uor(post_beacon_execution_payload_envelope) .uor(post_beacon_state_validators) .uor(post_beacon_state_validator_balances) diff --git a/beacon_node/http_api/src/validator/mod.rs b/beacon_node/http_api/src/validator/mod.rs index 27fe5de6e7..77df94bc36 100644 --- a/beacon_node/http_api/src/validator/mod.rs +++ b/beacon_node/http_api/src/validator/mod.rs @@ -9,8 +9,11 @@ use crate::utils::{ use crate::version::{V1, V2, V3, unsupported_version_rejection}; use crate::{StateId, attester_duties, proposer_duties, ptc_duties, sync_committees}; use beacon_chain::attestation_verification::VerifiedAttestation; +use beacon_chain::proposer_preferences_verification::ProposerPreferencesError; use beacon_chain::{AttestationError, BeaconChain, BeaconChainError, BeaconChainTypes}; use bls::PublicKeyBytes; +use bytes::Bytes; +use eth2::CONSENSUS_VERSION_HEADER; use eth2::types::{ Accept, BeaconCommitteeSubscription, EndpointVersion, Failure, GenericResponse, StandardLivenessResponseData, StateId as CoreStateId, ValidatorAggregateAttestationQuery, @@ -20,14 +23,15 @@ use lighthouse_network::PubsubMessage; use network::{NetworkMessage, ValidatorSubscriptionMessage}; use reqwest::StatusCode; use slot_clock::SlotClock; +use ssz::Decode; use std::sync::Arc; use tokio::sync::mpsc::{Sender, UnboundedSender}; use tokio::sync::oneshot; use tracing::{debug, error, info, warn}; use types::{ - BeaconState, Epoch, EthSpec, ProposerPreparationData, SignedAggregateAndProof, - SignedContributionAndProof, SignedValidatorRegistrationData, Slot, SyncContributionData, - ValidatorSubscription, + BeaconState, Epoch, EthSpec, ForkName, ProposerPreparationData, SignedAggregateAndProof, + SignedContributionAndProof, SignedProposerPreferences, SignedValidatorRegistrationData, Slot, + SyncContributionData, ValidatorSubscription, }; use warp::{Filter, Rejection, Reply}; use warp_utils::reject::convert_rejection; @@ -329,8 +333,12 @@ pub fn get_validator_payload_attestation_data( let payload_attestation_data = chain .produce_payload_attestation_data(slot) .map_err(|e| match e { - BeaconChainError::InvalidSlot(_) - | BeaconChainError::NoBlockForSlot(_) => { + BeaconChainError::NoBlockForSlot(_) => { + warp_utils::reject::block_not_found(format!( + "No block received for slot {slot}" + )) + } + BeaconChainError::InvalidSlot(_) => { warp_utils::reject::custom_bad_request(format!( "Unable to produce payload attestation data: {e:?}" )) @@ -1144,3 +1152,117 @@ pub fn get_validator_duties_proposer( ) .boxed() } + +/// POST validator/proposer_preferences (JSON) +pub fn post_validator_proposer_preferences( + eth_v1: EthV1Filter, + task_spawner_filter: TaskSpawnerFilter, + chain_filter: ChainFilter, + network_tx_filter: NetworkTxFilter, +) -> ResponseFilter { + eth_v1 + .and(warp::path("validator")) + .and(warp::path("proposer_preferences")) + .and(warp::path::end()) + .and(warp_utils::json::json()) + .and(warp::header::(CONSENSUS_VERSION_HEADER)) + .and(task_spawner_filter) + .and(chain_filter) + .and(network_tx_filter) + .then( + |preferences: Vec, + _fork_name: ForkName, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>| { + task_spawner.blocking_response_task(Priority::P0, move || { + publish_proposer_preferences(&chain, &network_tx, preferences)?; + Ok(warp::reply()) + }) + }, + ) + .boxed() +} + +/// POST validator/proposer_preferences (SSZ) +pub fn post_validator_proposer_preferences_ssz( + eth_v1: EthV1Filter, + task_spawner_filter: TaskSpawnerFilter, + chain_filter: ChainFilter, + network_tx_filter: NetworkTxFilter, +) -> ResponseFilter { + eth_v1 + .and(warp::path("validator")) + .and(warp::path("proposer_preferences")) + .and(warp::path::end()) + .and(warp::body::bytes()) + .and(warp::header::(CONSENSUS_VERSION_HEADER)) + .and(task_spawner_filter) + .and(chain_filter) + .and(network_tx_filter) + .then( + |body_bytes: Bytes, + _fork_name: ForkName, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>| { + task_spawner.blocking_response_task(Priority::P0, move || { + let preferences = Vec::::from_ssz_bytes(&body_bytes) + .map_err(|e| { + warp_utils::reject::custom_bad_request(format!("invalid SSZ: {e:?}")) + })?; + publish_proposer_preferences(&chain, &network_tx, preferences)?; + Ok(warp::reply()) + }) + }, + ) + .boxed() +} + +fn publish_proposer_preferences( + chain: &BeaconChain, + network_tx: &UnboundedSender>, + preferences_list: Vec, +) -> Result<(), warp::Rejection> { + let mut failures = vec![]; + let mut num_already_known = 0; + + for (index, preferences) in preferences_list.into_iter().enumerate() { + let validator_index = preferences.message.validator_index; + match chain.verify_proposer_preferences_for_gossip(Arc::new(preferences)) { + Ok(verified) => { + crate::utils::publish_pubsub_message( + network_tx, + PubsubMessage::ProposerPreferences(verified.signed_preferences), + )?; + } + Err(ProposerPreferencesError::AlreadySeen { .. }) => { + num_already_known += 1; + } + Err(e) => { + error!( + error = ?e, + %validator_index, + "Failure verifying proposer preferences for gossip" + ); + failures.push(Failure::new(index, format!("{e:?}"))); + } + } + } + + if num_already_known > 0 { + debug!( + count = num_already_known, + "Some proposer preferences already known" + ); + } + + if failures.is_empty() { + Ok(()) + } else { + Err(warp_utils::reject::indexed_bad_request( + "error processing proposer preferences".to_string(), + failures, + )) + } +} diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index ce57b71366..5b068297c0 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -48,9 +48,10 @@ use tokio::time::Duration; use tree_hash::TreeHash; use types::ApplicationDomain; use types::{ - Domain, EthSpec, ExecutionBlockHash, Hash256, MainnetEthSpec, RelativeEpoch, SelectionProof, - SignedExecutionPayloadEnvelope, SignedRoot, SingleAttestation, Slot, - attestation::AttestationBase, consts::gloas::BUILDER_INDEX_SELF_BUILD, + Address, Domain, EthSpec, ExecutionBlockHash, Hash256, MainnetEthSpec, ProposerPreferences, + RelativeEpoch, SelectionProof, SignedExecutionPayloadEnvelope, SignedProposerPreferences, + SignedRoot, SingleAttestation, Slot, attestation::AttestationBase, + consts::gloas::BUILDER_INDEX_SELF_BUILD, }; type E = MainnetEthSpec; @@ -2898,6 +2899,162 @@ impl ApiTester { self } + fn make_valid_signed_proposer_preferences( + &self, + slot_offset: usize, + ) -> SignedProposerPreferences { + let head = self.chain.head_snapshot(); + let head_slot = head.beacon_block.slot(); + let head_state = &head.beacon_state; + let genesis_validators_root = self.chain.genesis_validators_root; + + let proposer_lookahead = head_state + .proposer_lookahead() + .expect("should get proposer_lookahead"); + + // Pick a future slot in the next epoch to ensure it's always valid. + // The lookahead covers 2 epochs: index = epoch_offset * slots_per_epoch + slot_in_epoch. + let slots_per_epoch = E::slots_per_epoch() as usize; + let next_epoch = head_slot.epoch(E::slots_per_epoch()) + 1; + let next_epoch_start = next_epoch.start_slot(E::slots_per_epoch()); + let proposal_slot = next_epoch_start + Slot::new((slot_offset % slots_per_epoch) as u64); + + let lookahead_index = slots_per_epoch + (slot_offset % slots_per_epoch); + let validator_index = *proposer_lookahead + .get(lookahead_index) + .expect("slot index should be in lookahead") as usize; + + let preferences = ProposerPreferences { + dependent_root: Hash256::ZERO, + proposal_slot, + validator_index: validator_index as u64, + fee_recipient: Address::repeat_byte(0xaa), + gas_limit: 30_000_000, + }; + + let epoch = proposal_slot.epoch(E::slots_per_epoch()); + let fork = head_state.fork(); + let domain = self.chain.spec.get_domain( + epoch, + Domain::ProposerPreferences, + &fork, + genesis_validators_root, + ); + let signing_root = preferences.signing_root(domain); + let sk = &self.validator_keypairs()[validator_index].sk; + let signature = sk.sign(signing_root); + + SignedProposerPreferences { + message: preferences, + signature, + } + } + + // Each sub-test uses a unique slot_offset (1-5) because the gossip cache deduplicates on + // (slot, dependent_root, validator_index). Reusing an offset from an earlier test would hit + // "already seen" instead of testing the intended condition. + pub async fn test_post_validator_proposer_preferences_valid(mut self) -> Self { + let signed = self.make_valid_signed_proposer_preferences(1); + let fork_name = self + .chain + .spec + .fork_name_at_slot::(signed.message.proposal_slot); + + self.client + .post_validator_proposer_preferences(&[signed], fork_name) + .await + .unwrap(); + + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "valid proposer preferences should be sent to network" + ); + + self + } + + pub async fn test_post_validator_proposer_preferences_valid_ssz(mut self) -> Self { + let signed = self.make_valid_signed_proposer_preferences(2); + let fork_name = self + .chain + .spec + .fork_name_at_slot::(signed.message.proposal_slot); + + self.client + .post_validator_proposer_preferences_ssz(&vec![signed], fork_name) + .await + .unwrap(); + + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "valid proposer preferences (SSZ) should be sent to network" + ); + + self + } + + pub async fn test_post_validator_proposer_preferences_invalid_sig(self) -> Self { + let mut signed = self.make_valid_signed_proposer_preferences(3); + signed.signature = Signature::empty(); + let fork_name = self + .chain + .spec + .fork_name_at_slot::(signed.message.proposal_slot); + + let result = self + .client + .post_validator_proposer_preferences(&[signed], fork_name) + .await; + + assert!(result.is_err(), "invalid signature should be rejected"); + + self + } + + pub async fn test_post_validator_proposer_preferences_invalid_sig_ssz(self) -> Self { + let mut signed = self.make_valid_signed_proposer_preferences(4); + signed.signature = Signature::empty(); + let fork_name = self + .chain + .spec + .fork_name_at_slot::(signed.message.proposal_slot); + + let result = self + .client + .post_validator_proposer_preferences_ssz(&vec![signed], fork_name) + .await; + + assert!( + result.is_err(), + "invalid signature should be rejected via SSZ route" + ); + + self + } + + pub async fn test_post_validator_proposer_preferences_duplicate(mut self) -> Self { + let signed = self.make_valid_signed_proposer_preferences(5); + let fork_name = self + .chain + .spec + .fork_name_at_slot::(signed.message.proposal_slot); + + // First submission should succeed. + self.client + .post_validator_proposer_preferences(std::slice::from_ref(&signed), fork_name) + .await + .unwrap(); + self.network_rx.network_recv.recv().await; + + // Second submission of the same preferences should return 200 (already known, not an error). + self.client + .post_validator_proposer_preferences(&[signed], fork_name) + .await + .unwrap(); + + self + } + pub async fn test_get_config_fork_schedule(self) -> Self { let result = self.client.get_config_fork_schedule().await.unwrap().data; @@ -4663,7 +4820,8 @@ impl ApiTester { .client .get_validator_payload_attestation_data(slot) .await - .unwrap(); + .unwrap() + .expect("expected payload attestation data for slot with block"); assert_eq!(response.version(), Some(fork_name)); @@ -4679,7 +4837,8 @@ impl ApiTester { .client .get_validator_payload_attestation_data_ssz(slot) .await - .unwrap(); + .unwrap() + .expect("expected SSZ payload attestation data for slot with block"); assert_eq!(ssz_result, expected); @@ -4750,6 +4909,7 @@ impl ApiTester { .get_validator_payload_attestation_data(slot) .await .unwrap() + .expect("expected payload attestation data for slot with block") .into_data(); assert_eq!(pa_data.beacon_block_root, block_root); @@ -4782,6 +4942,26 @@ impl ApiTester { self } + pub async fn test_get_validator_payload_attestation_data_no_block(self) -> Self { + // Advance the slot clock without producing a block + self.harness.advance_slot(); + let slot = self.chain.slot().unwrap(); + + // Should return None when no block exists for the slot + let result = self + .client + .get_validator_payload_attestation_data(slot) + .await + .unwrap(); + + assert!( + result.is_none(), + "expected None for empty slot, got: {result:?}" + ); + + self + } + #[allow(clippy::await_holding_lock)] // This is a test, so it should be fine. pub async fn test_get_validator_aggregate_attestation_v1(self) -> Self { let attestation = self @@ -8469,6 +8649,17 @@ async fn get_validator_payload_attestation_data_pre_gloas() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_payload_attestation_data_no_block() { + if !fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } + ApiTester::new_with_hard_forks() + .await + .test_get_validator_payload_attestation_data_no_block() + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn payload_attestation_present_after_envelope_publish() { ApiTester::new_with_hard_forks() @@ -9260,3 +9451,22 @@ async fn get_validator_blocks_v3_http_api_path() { .get_validator_blocks_v3_path_graffiti_policy() .await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn post_validator_proposer_preferences() { + if !fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } + ApiTester::new_with_hard_forks() + .await + .test_post_validator_proposer_preferences_valid() + .await + .test_post_validator_proposer_preferences_valid_ssz() + .await + .test_post_validator_proposer_preferences_invalid_sig() + .await + .test_post_validator_proposer_preferences_invalid_sig_ssz() + .await + .test_post_validator_proposer_preferences_duplicate() + .await; +} diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index d7285c5c8e..6b5144fa6f 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -589,6 +589,7 @@ impl PeerManager { Protocol::Ping => PeerAction::MidToleranceError, Protocol::BlocksByRange => PeerAction::MidToleranceError, Protocol::BlocksByRoot => PeerAction::MidToleranceError, + Protocol::BlocksByHead => PeerAction::MidToleranceError, Protocol::BlobsByRange => PeerAction::MidToleranceError, Protocol::PayloadEnvelopesByRange => PeerAction::MidToleranceError, Protocol::PayloadEnvelopesByRoot => PeerAction::MidToleranceError, @@ -617,6 +618,7 @@ impl PeerManager { Protocol::Ping => PeerAction::Fatal, Protocol::BlocksByRange => return, Protocol::BlocksByRoot => return, + Protocol::BlocksByHead => return, Protocol::PayloadEnvelopesByRange => return, Protocol::PayloadEnvelopesByRoot => return, Protocol::BlobsByRange => return, @@ -642,6 +644,7 @@ impl PeerManager { Protocol::Ping => PeerAction::LowToleranceError, Protocol::BlocksByRange => PeerAction::MidToleranceError, Protocol::BlocksByRoot => PeerAction::MidToleranceError, + Protocol::BlocksByHead => PeerAction::MidToleranceError, Protocol::PayloadEnvelopesByRange => PeerAction::MidToleranceError, Protocol::PayloadEnvelopesByRoot => PeerAction::MidToleranceError, Protocol::BlobsByRange => PeerAction::MidToleranceError, diff --git a/beacon_node/lighthouse_network/src/rpc/codec.rs b/beacon_node/lighthouse_network/src/rpc/codec.rs index f65ff776aa..75d7266fb6 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec.rs @@ -18,7 +18,7 @@ use tokio_util::codec::{Decoder, Encoder}; use types::SignedExecutionPayloadEnvelope; use types::{ BlobSidecar, ChainSpec, DataColumnSidecar, DataColumnsByRootIdentifier, EthSpec, ForkContext, - ForkName, Hash256, LightClientBootstrap, LightClientFinalityUpdate, + ForkName, ForkVersionDecode, Hash256, LightClientBootstrap, LightClientFinalityUpdate, LightClientOptimisticUpdate, LightClientUpdate, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, SignedBeaconBlockFulu, @@ -77,6 +77,7 @@ impl SSZSnappyInboundCodec { }, RpcSuccessResponse::BlocksByRange(res) => res.as_ssz_bytes(), RpcSuccessResponse::BlocksByRoot(res) => res.as_ssz_bytes(), + RpcSuccessResponse::BlocksByHead(res) => res.as_ssz_bytes(), RpcSuccessResponse::PayloadEnvelopesByRange(res) => res.as_ssz_bytes(), RpcSuccessResponse::PayloadEnvelopesByRoot(res) => res.as_ssz_bytes(), RpcSuccessResponse::BlobsByRange(res) => res.as_ssz_bytes(), @@ -359,6 +360,7 @@ impl Encoder> for SSZSnappyOutboundCodec { BlocksByRootRequest::V1(req) => req.block_roots.as_ssz_bytes(), BlocksByRootRequest::V2(req) => req.block_roots.as_ssz_bytes(), }, + RequestType::BlocksByHead(req) => req.as_ssz_bytes(), RequestType::PayloadEnvelopesByRange(req) => req.as_ssz_bytes(), RequestType::PayloadEnvelopesByRoot(req) => req.beacon_block_roots.as_ssz_bytes(), RequestType::BlobsByRange(req) => req.as_ssz_bytes(), @@ -553,6 +555,9 @@ fn handle_rpc_request( )?, }), ))), + SupportedProtocol::BlocksByHeadV1 => Ok(Some(RequestType::BlocksByHead( + BlocksByHeadRequest::from_ssz_bytes(decoded_buffer)?, + ))), SupportedProtocol::PayloadEnvelopesByRangeV1 => { Ok(Some(RequestType::PayloadEnvelopesByRange( PayloadEnvelopesByRangeRequest::from_ssz_bytes(decoded_buffer)?, @@ -949,6 +954,18 @@ fn handle_rpc_response( ), )), }, + SupportedProtocol::BlocksByHeadV1 => match fork_name { + Some(fork_name) => Ok(Some(RpcSuccessResponse::BlocksByHead(Arc::new( + SignedBeaconBlock::from_ssz_bytes_by_fork(decoded_buffer, fork_name)?, + )))), + None => Err(RPCError::ErrorResponse( + RpcErrorResponse::InvalidRequest, + format!( + "No context bytes provided for {:?} response", + versioned_protocol + ), + )), + }, } } @@ -1327,6 +1344,9 @@ mod tests { RequestType::BlocksByRoot(bbroot) => { assert_eq!(decoded, RequestType::BlocksByRoot(bbroot)) } + RequestType::BlocksByHead(bbhead) => { + assert_eq!(decoded, RequestType::BlocksByHead(bbhead)) + } RequestType::BlobsByRange(blbrange) => { assert_eq!(decoded, RequestType::BlobsByRange(blbrange)) } @@ -1903,6 +1923,31 @@ mod tests { ); } + // BlocksByHead is introduced in Fulu but the response is just `SignedBeaconBlock`, + // so the codec must accept blocks of any fork variant — the chain a Fulu peer walks + // back may straddle the Fulu boundary and include pre-Fulu canonical blocks. + #[test] + fn test_blocks_by_head_decodes_all_forks() { + let chain_spec = spec_with_all_forks_enabled(); + for (block, fork) in [ + (empty_base_block(&chain_spec), ForkName::Base), + (altair_block(&chain_spec), ForkName::Altair), + (bellatrix_block_small(&chain_spec), ForkName::Bellatrix), + ] { + let block_arc = Arc::new(block); + assert_eq!( + encode_then_decode_response( + SupportedProtocol::BlocksByHeadV1, + RpcResponse::Success(RpcSuccessResponse::BlocksByHead(block_arc.clone())), + fork, + &chain_spec, + ), + Ok(Some(RpcSuccessResponse::BlocksByHead(block_arc))), + "BlocksByHeadV1 must round-trip a {fork} block" + ); + } + } + // Test RPCResponse encoding/decoding for V2 messages #[test] fn test_context_bytes_v2() { @@ -2099,6 +2144,10 @@ mod tests { RequestType::BlobsByRange(blbrange_request()), RequestType::DataColumnsByRange(dcbrange_request()), RequestType::MetaData(MetadataRequest::new_v2()), + RequestType::BlocksByHead(BlocksByHeadRequest { + beacon_root: Hash256::zero(), + count: 32, + }), ]; for req in requests.iter() { for fork_name in ForkName::list_all() { diff --git a/beacon_node/lighthouse_network/src/rpc/config.rs b/beacon_node/lighthouse_network/src/rpc/config.rs index 9e1c6541ec..59f0b8e9a2 100644 --- a/beacon_node/lighthouse_network/src/rpc/config.rs +++ b/beacon_node/lighthouse_network/src/rpc/config.rs @@ -89,6 +89,7 @@ pub struct RateLimiterConfig { pub(super) goodbye_quota: Quota, pub(super) blocks_by_range_quota: Quota, pub(super) blocks_by_root_quota: Quota, + pub(super) blocks_by_head_quota: Quota, pub(super) payload_envelopes_by_range_quota: Quota, pub(super) payload_envelopes_by_root_quota: Quota, pub(super) blobs_by_range_quota: Quota, @@ -113,6 +114,8 @@ impl RateLimiterConfig { Quota::n_every(NonZeroU64::new(128).unwrap(), 10); pub const DEFAULT_BLOCKS_BY_ROOT_QUOTA: Quota = Quota::n_every(NonZeroU64::new(128).unwrap(), 10); + pub const DEFAULT_BLOCKS_BY_HEAD_QUOTA: Quota = + Quota::n_every(NonZeroU64::new(128).unwrap(), 10); pub const DEFAULT_PAYLOAD_ENVELOPES_BY_RANGE_QUOTA: Quota = Quota::n_every(NonZeroU64::new(128).unwrap(), 10); pub const DEFAULT_PAYLOAD_ENVELOPES_BY_ROOT_QUOTA: Quota = @@ -143,6 +146,7 @@ impl Default for RateLimiterConfig { goodbye_quota: Self::DEFAULT_GOODBYE_QUOTA, blocks_by_range_quota: Self::DEFAULT_BLOCKS_BY_RANGE_QUOTA, blocks_by_root_quota: Self::DEFAULT_BLOCKS_BY_ROOT_QUOTA, + blocks_by_head_quota: Self::DEFAULT_BLOCKS_BY_HEAD_QUOTA, payload_envelopes_by_range_quota: Self::DEFAULT_PAYLOAD_ENVELOPES_BY_RANGE_QUOTA, payload_envelopes_by_root_quota: Self::DEFAULT_PAYLOAD_ENVELOPES_BY_ROOT_QUOTA, blobs_by_range_quota: Self::DEFAULT_BLOBS_BY_RANGE_QUOTA, @@ -177,6 +181,7 @@ impl Debug for RateLimiterConfig { .field("goodbye", fmt_q!(&self.goodbye_quota)) .field("blocks_by_range", fmt_q!(&self.blocks_by_range_quota)) .field("blocks_by_root", fmt_q!(&self.blocks_by_root_quota)) + .field("blocks_by_head", fmt_q!(&self.blocks_by_head_quota)) .field( "payload_envelopes_by_range", fmt_q!(&self.payload_envelopes_by_range_quota), @@ -213,6 +218,7 @@ impl FromStr for RateLimiterConfig { let mut goodbye_quota = None; let mut blocks_by_range_quota = None; let mut blocks_by_root_quota = None; + let mut blocks_by_head_quota = None; let mut payload_envelopes_by_range_quota = None; let mut payload_envelopes_by_root_quota = None; let mut blobs_by_range_quota = None; @@ -232,6 +238,7 @@ impl FromStr for RateLimiterConfig { Protocol::Goodbye => goodbye_quota = goodbye_quota.or(quota), Protocol::BlocksByRange => blocks_by_range_quota = blocks_by_range_quota.or(quota), Protocol::BlocksByRoot => blocks_by_root_quota = blocks_by_root_quota.or(quota), + Protocol::BlocksByHead => blocks_by_head_quota = blocks_by_head_quota.or(quota), Protocol::PayloadEnvelopesByRange => { payload_envelopes_by_range_quota = payload_envelopes_by_range_quota.or(quota) } @@ -274,6 +281,8 @@ impl FromStr for RateLimiterConfig { .unwrap_or(Self::DEFAULT_BLOCKS_BY_RANGE_QUOTA), blocks_by_root_quota: blocks_by_root_quota .unwrap_or(Self::DEFAULT_BLOCKS_BY_ROOT_QUOTA), + blocks_by_head_quota: blocks_by_head_quota + .unwrap_or(Self::DEFAULT_BLOCKS_BY_HEAD_QUOTA), payload_envelopes_by_range_quota: payload_envelopes_by_range_quota .unwrap_or(Self::DEFAULT_PAYLOAD_ENVELOPES_BY_RANGE_QUOTA), payload_envelopes_by_root_quota: payload_envelopes_by_root_quota diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index baabf48683..f3f294d913 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -488,6 +488,18 @@ impl From for OldBlocksByRangeRequest { } } +/// Request a contiguous range of beacon blocks by walking the parent chain of `beacon_root`. +/// +/// New in Fulu (see consensus-specs PR 5181). The responder walks the parent chain of +/// `beacon_root` (inclusive) and emits up to `count` blocks in descending slot order. +#[derive(Encode, Decode, Clone, Debug, PartialEq)] +pub struct BlocksByHeadRequest { + /// The block root to start the parent walk from (inclusive). + pub beacon_root: Hash256, + /// The maximum number of blocks to return. + pub count: u64, +} + /// Request a number of beacon block bodies from a peer. #[superstruct(variants(V1, V2), variant_attributes(derive(Clone, Debug, PartialEq)))] #[derive(Clone, Debug, PartialEq)] @@ -622,6 +634,9 @@ pub enum RpcSuccessResponse { /// A response to a get BLOCKS_BY_ROOT request. BlocksByRoot(Arc>), + /// A response to a get BEACON_BLOCKS_BY_HEAD request. + BlocksByHead(Arc>), + /// A response to a get EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE request. A None response signifies /// the end of the batch. PayloadEnvelopesByRange(Arc>), @@ -669,6 +684,9 @@ pub enum ResponseTermination { /// Blocks by root stream termination. BlocksByRoot, + /// Blocks by head stream termination. + BlocksByHead, + /// Execution payload envelopes by range stream termination. PayloadEnvelopesByRange, @@ -696,6 +714,7 @@ impl ResponseTermination { match self { ResponseTermination::BlocksByRange => Protocol::BlocksByRange, ResponseTermination::BlocksByRoot => Protocol::BlocksByRoot, + ResponseTermination::BlocksByHead => Protocol::BlocksByHead, ResponseTermination::PayloadEnvelopesByRange => Protocol::PayloadEnvelopesByRange, ResponseTermination::PayloadEnvelopesByRoot => Protocol::PayloadEnvelopesByRoot, ResponseTermination::BlobsByRange => Protocol::BlobsByRange, @@ -793,6 +812,7 @@ impl RpcSuccessResponse { RpcSuccessResponse::Status(_) => Protocol::Status, RpcSuccessResponse::BlocksByRange(_) => Protocol::BlocksByRange, RpcSuccessResponse::BlocksByRoot(_) => Protocol::BlocksByRoot, + RpcSuccessResponse::BlocksByHead(_) => Protocol::BlocksByHead, RpcSuccessResponse::PayloadEnvelopesByRange(_) => Protocol::PayloadEnvelopesByRange, RpcSuccessResponse::PayloadEnvelopesByRoot(_) => Protocol::PayloadEnvelopesByRoot, RpcSuccessResponse::BlobsByRange(_) => Protocol::BlobsByRange, @@ -812,7 +832,9 @@ impl RpcSuccessResponse { pub fn slot(&self) -> Option { match self { - Self::BlocksByRange(r) | Self::BlocksByRoot(r) => Some(r.slot()), + Self::BlocksByRange(r) | Self::BlocksByRoot(r) | Self::BlocksByHead(r) => { + Some(r.slot()) + } Self::PayloadEnvelopesByRoot(r) | Self::PayloadEnvelopesByRange(r) => Some(r.slot()), Self::BlobsByRange(r) | Self::BlobsByRoot(r) => Some(r.slot()), Self::DataColumnsByRange(r) | Self::DataColumnsByRoot(r) => Some(r.slot()), @@ -864,6 +886,9 @@ impl std::fmt::Display for RpcSuccessResponse { RpcSuccessResponse::BlocksByRoot(block) => { write!(f, "BlocksByRoot: Block slot: {}", block.slot()) } + RpcSuccessResponse::BlocksByHead(block) => { + write!(f, "BlocksByHead: Block slot: {}", block.slot()) + } RpcSuccessResponse::PayloadEnvelopesByRange(envelope) => { write!( f, @@ -975,6 +1000,16 @@ impl std::fmt::Display for OldBlocksByRangeRequest { } } +impl std::fmt::Display for BlocksByHeadRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "BlocksByHead: beacon_root: {}, count: {}", + self.beacon_root, self.count + ) + } +} + impl std::fmt::Display for BlobsByRootRequest { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 9b451d0877..cf986b6a30 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -263,6 +263,9 @@ pub enum Protocol { /// The `BlocksByRoot` protocol name. #[strum(serialize = "beacon_blocks_by_root")] BlocksByRoot, + /// The `BlocksByHead` protocol name. + #[strum(serialize = "beacon_blocks_by_head")] + BlocksByHead, /// The `BlobsByRange` protocol name. #[strum(serialize = "blob_sidecars_by_range")] BlobsByRange, @@ -307,6 +310,7 @@ impl Protocol { Protocol::Goodbye => None, Protocol::BlocksByRange => Some(ResponseTermination::BlocksByRange), Protocol::BlocksByRoot => Some(ResponseTermination::BlocksByRoot), + Protocol::BlocksByHead => Some(ResponseTermination::BlocksByHead), Protocol::PayloadEnvelopesByRange => Some(ResponseTermination::PayloadEnvelopesByRange), Protocol::PayloadEnvelopesByRoot => Some(ResponseTermination::PayloadEnvelopesByRoot), Protocol::BlobsByRange => Some(ResponseTermination::BlobsByRange), @@ -339,6 +343,7 @@ pub enum SupportedProtocol { BlocksByRangeV2, BlocksByRootV1, BlocksByRootV2, + BlocksByHeadV1, PayloadEnvelopesByRangeV1, PayloadEnvelopesByRootV1, BlobsByRangeV1, @@ -367,6 +372,7 @@ impl SupportedProtocol { SupportedProtocol::PayloadEnvelopesByRootV1 => "1", SupportedProtocol::BlocksByRootV1 => "1", SupportedProtocol::BlocksByRootV2 => "2", + SupportedProtocol::BlocksByHeadV1 => "1", SupportedProtocol::BlobsByRangeV1 => "1", SupportedProtocol::BlobsByRootV1 => "1", SupportedProtocol::DataColumnsByRootV1 => "1", @@ -391,6 +397,7 @@ impl SupportedProtocol { SupportedProtocol::BlocksByRangeV2 => Protocol::BlocksByRange, SupportedProtocol::BlocksByRootV1 => Protocol::BlocksByRoot, SupportedProtocol::BlocksByRootV2 => Protocol::BlocksByRoot, + SupportedProtocol::BlocksByHeadV1 => Protocol::BlocksByHead, SupportedProtocol::PayloadEnvelopesByRangeV1 => Protocol::PayloadEnvelopesByRange, SupportedProtocol::PayloadEnvelopesByRootV1 => Protocol::PayloadEnvelopesByRoot, SupportedProtocol::BlobsByRangeV1 => Protocol::BlobsByRange, @@ -459,6 +466,13 @@ impl SupportedProtocol { ), ]); } + // BeaconBlocksByHead is new in Fulu (consensus-specs PR 5181). + if fork_context.fork_exists(ForkName::Fulu) { + supported.push(ProtocolId::new( + SupportedProtocol::BlocksByHeadV1, + Encoding::SSZSnappy, + )); + } supported } } @@ -565,6 +579,10 @@ impl ProtocolId { ::ssz_fixed_len(), ), Protocol::BlocksByRoot => RpcLimits::new(0, spec.max_blocks_by_root_request), + Protocol::BlocksByHead => RpcLimits::new( + ::ssz_fixed_len(), + ::ssz_fixed_len(), + ), Protocol::PayloadEnvelopesByRange => RpcLimits::new( ::ssz_fixed_len(), ::ssz_fixed_len(), @@ -610,6 +628,7 @@ impl ProtocolId { Protocol::Goodbye => RpcLimits::new(0, 0), // Goodbye request has no response Protocol::BlocksByRange => rpc_block_limits_by_fork(fork_context.current_fork_name()), Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork_name()), + Protocol::BlocksByHead => rpc_block_limits_by_fork(fork_context.current_fork_name()), Protocol::PayloadEnvelopesByRange => rpc_payload_limits(), Protocol::PayloadEnvelopesByRoot => rpc_payload_limits(), Protocol::BlobsByRange => rpc_blob_limits::(), @@ -649,6 +668,7 @@ impl ProtocolId { match self.versioned_protocol { SupportedProtocol::BlocksByRangeV2 | SupportedProtocol::BlocksByRootV2 + | SupportedProtocol::BlocksByHeadV1 | SupportedProtocol::PayloadEnvelopesByRangeV1 | SupportedProtocol::PayloadEnvelopesByRootV1 | SupportedProtocol::BlobsByRangeV1 @@ -802,6 +822,7 @@ pub enum RequestType { Goodbye(GoodbyeReason), BlocksByRange(OldBlocksByRangeRequest), BlocksByRoot(BlocksByRootRequest), + BlocksByHead(BlocksByHeadRequest), PayloadEnvelopesByRange(PayloadEnvelopesByRangeRequest), PayloadEnvelopesByRoot(PayloadEnvelopesByRootRequest), BlobsByRange(BlobsByRangeRequest), @@ -827,6 +848,7 @@ impl RequestType { RequestType::Goodbye(_) => 0, RequestType::BlocksByRange(req) => *req.count(), RequestType::BlocksByRoot(req) => req.block_roots().len() as u64, + RequestType::BlocksByHead(req) => req.count, RequestType::PayloadEnvelopesByRange(req) => req.count, RequestType::PayloadEnvelopesByRoot(req) => req.beacon_block_roots.len() as u64, RequestType::BlobsByRange(req) => req.max_blobs_requested(digest_epoch, spec), @@ -858,6 +880,7 @@ impl RequestType { BlocksByRootRequest::V1(_) => SupportedProtocol::BlocksByRootV1, BlocksByRootRequest::V2(_) => SupportedProtocol::BlocksByRootV2, }, + RequestType::BlocksByHead(_) => SupportedProtocol::BlocksByHeadV1, RequestType::PayloadEnvelopesByRange(_) => SupportedProtocol::PayloadEnvelopesByRangeV1, RequestType::PayloadEnvelopesByRoot(_) => SupportedProtocol::PayloadEnvelopesByRootV1, RequestType::BlobsByRange(_) => SupportedProtocol::BlobsByRangeV1, @@ -891,6 +914,7 @@ impl RequestType { // variants that have `multiple_responses()` can have values. RequestType::BlocksByRange(_) => ResponseTermination::BlocksByRange, RequestType::BlocksByRoot(_) => ResponseTermination::BlocksByRoot, + RequestType::BlocksByHead(_) => ResponseTermination::BlocksByHead, RequestType::PayloadEnvelopesByRange(_) => ResponseTermination::PayloadEnvelopesByRange, RequestType::PayloadEnvelopesByRoot(_) => ResponseTermination::PayloadEnvelopesByRoot, RequestType::BlobsByRange(_) => ResponseTermination::BlobsByRange, @@ -927,6 +951,10 @@ impl RequestType { ProtocolId::new(SupportedProtocol::BlocksByRootV2, Encoding::SSZSnappy), ProtocolId::new(SupportedProtocol::BlocksByRootV1, Encoding::SSZSnappy), ], + RequestType::BlocksByHead(_) => vec![ProtocolId::new( + SupportedProtocol::BlocksByHeadV1, + Encoding::SSZSnappy, + )], RequestType::PayloadEnvelopesByRange(_) => vec![ProtocolId::new( SupportedProtocol::PayloadEnvelopesByRangeV1, Encoding::SSZSnappy, @@ -985,6 +1013,7 @@ impl RequestType { RequestType::Goodbye(_) => false, RequestType::BlocksByRange(_) => false, RequestType::BlocksByRoot(_) => false, + RequestType::BlocksByHead(_) => false, RequestType::BlobsByRange(_) => false, RequestType::PayloadEnvelopesByRange(_) => false, RequestType::PayloadEnvelopesByRoot(_) => false, @@ -1098,6 +1127,7 @@ impl std::fmt::Display for RequestType { RequestType::Goodbye(reason) => write!(f, "Goodbye: {}", reason), RequestType::BlocksByRange(req) => write!(f, "Blocks by range: {}", req), RequestType::BlocksByRoot(req) => write!(f, "Blocks by root: {:?}", req), + RequestType::BlocksByHead(req) => write!(f, "Blocks by head: {}", req), RequestType::PayloadEnvelopesByRange(req) => { write!(f, "Payload envelopes by range: {:?}", req) } @@ -1172,6 +1202,8 @@ mod tests { fork_context.fork_exists(ForkName::Gloas) } + BlocksByHeadV1 => fork_context.fork_exists(ForkName::Fulu), + // Light client protocols are not in currently_supported() LightClientBootstrapV1 | LightClientOptimisticUpdateV1 diff --git a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs index ebdca386d8..a5c98a4d30 100644 --- a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs @@ -105,6 +105,8 @@ pub struct RPCRateLimiter { bbrange_rl: Limiter, /// BlocksByRoot rate limiter. bbroots_rl: Limiter, + /// BlocksByHead rate limiter. + bbhead_rl: Limiter, /// BlobsByRange rate limiter. blbrange_rl: Limiter, /// BlobsByRoot rate limiter. @@ -152,6 +154,8 @@ pub struct RPCRateLimiterBuilder { bbrange_quota: Option, /// Quota for the BlocksByRoot protocol. bbroots_quota: Option, + /// Quota for the BlocksByHead protocol. + bbhead_quota: Option, /// Quota for the ExecutionPayloadEnvelopesByRange protocol. perange_quota: Option, /// Quota for the ExecutionPayloadEnvelopesByRoot protocol. @@ -185,6 +189,7 @@ impl RPCRateLimiterBuilder { Protocol::Goodbye => self.goodbye_quota = q, Protocol::BlocksByRange => self.bbrange_quota = q, Protocol::BlocksByRoot => self.bbroots_quota = q, + Protocol::BlocksByHead => self.bbhead_quota = q, Protocol::PayloadEnvelopesByRange => self.perange_quota = q, Protocol::PayloadEnvelopesByRoot => self.peroots_quota = q, Protocol::BlobsByRange => self.blbrange_quota = q, @@ -211,6 +216,9 @@ impl RPCRateLimiterBuilder { let bbrange_quota = self .bbrange_quota .ok_or("BlocksByRange quota not specified")?; + let bbhead_quota = self + .bbhead_quota + .ok_or("BlocksByHead quota not specified")?; let perange_quota = self .perange_quota .ok_or("PayloadEnvelopesByRange quota not specified")?; @@ -252,6 +260,7 @@ impl RPCRateLimiterBuilder { let goodbye_rl = Limiter::from_quota(goodbye_quota)?; let bbroots_rl = Limiter::from_quota(bbroots_quota)?; let bbrange_rl = Limiter::from_quota(bbrange_quota)?; + let bbhead_rl = Limiter::from_quota(bbhead_quota)?; let envrange_rl = Limiter::from_quota(perange_quota)?; let envroots_rl = Limiter::from_quota(peroots_quota)?; let blbrange_rl = Limiter::from_quota(blbrange_quota)?; @@ -277,6 +286,7 @@ impl RPCRateLimiterBuilder { goodbye_rl, bbroots_rl, bbrange_rl, + bbhead_rl, envrange_rl, envroots_rl, blbrange_rl, @@ -332,6 +342,7 @@ impl RPCRateLimiter { goodbye_quota, blocks_by_range_quota, blocks_by_root_quota, + blocks_by_head_quota, payload_envelopes_by_range_quota, payload_envelopes_by_root_quota, blobs_by_range_quota, @@ -351,6 +362,7 @@ impl RPCRateLimiter { .set_quota(Protocol::Goodbye, goodbye_quota) .set_quota(Protocol::BlocksByRange, blocks_by_range_quota) .set_quota(Protocol::BlocksByRoot, blocks_by_root_quota) + .set_quota(Protocol::BlocksByHead, blocks_by_head_quota) .set_quota( Protocol::PayloadEnvelopesByRange, payload_envelopes_by_range_quota, @@ -406,6 +418,7 @@ impl RPCRateLimiter { Protocol::Goodbye => &mut self.goodbye_rl, Protocol::BlocksByRange => &mut self.bbrange_rl, Protocol::BlocksByRoot => &mut self.bbroots_rl, + Protocol::BlocksByHead => &mut self.bbhead_rl, Protocol::PayloadEnvelopesByRange => &mut self.envrange_rl, Protocol::PayloadEnvelopesByRoot => &mut self.envroots_rl, Protocol::BlobsByRange => &mut self.blbrange_rl, @@ -432,6 +445,7 @@ impl RPCRateLimiter { status_rl, bbrange_rl, bbroots_rl, + bbhead_rl, envrange_rl, envroots_rl, blbrange_rl, @@ -451,6 +465,7 @@ impl RPCRateLimiter { status_rl.prune(time_since_start); bbrange_rl.prune(time_since_start); bbroots_rl.prune(time_since_start); + bbhead_rl.prune(time_since_start); envrange_rl.prune(time_since_start); envroots_rl.prune(time_since_start); blbrange_rl.prune(time_since_start); diff --git a/beacon_node/lighthouse_network/src/service/api_types.rs b/beacon_node/lighthouse_network/src/service/api_types.rs index 486a443857..f598f59aee 100644 --- a/beacon_node/lighthouse_network/src/service/api_types.rs +++ b/beacon_node/lighthouse_network/src/service/api_types.rs @@ -161,6 +161,9 @@ pub enum Response { DataColumnsByRange(Option>>), /// A response to a get BLOCKS_BY_ROOT request. BlocksByRoot(Option>>), + /// A response to a get BEACON_BLOCKS_BY_HEAD request. A None response signals the end of the + /// batch. + BlocksByHead(Option>>), /// A response to a get `EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT` request. PayloadEnvelopesByRoot(Option>>), /// A response to a get `EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE` request. @@ -186,6 +189,10 @@ impl std::convert::From> for RpcResponse { Some(b) => RpcResponse::Success(RpcSuccessResponse::BlocksByRoot(b)), None => RpcResponse::StreamTermination(ResponseTermination::BlocksByRoot), }, + Response::BlocksByHead(r) => match r { + Some(b) => RpcResponse::Success(RpcSuccessResponse::BlocksByHead(b)), + None => RpcResponse::StreamTermination(ResponseTermination::BlocksByHead), + }, Response::BlocksByRange(r) => match r { Some(b) => RpcResponse::Success(RpcSuccessResponse::BlocksByRange(b)), None => RpcResponse::StreamTermination(ResponseTermination::BlocksByRange), diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index f0c1567cb0..41d937e324 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1691,6 +1691,14 @@ impl Network { request_type, }) } + RequestType::BlocksByHead(_) => { + metrics::inc_counter_vec(&metrics::TOTAL_RPC_REQUESTS, &["blocks_by_head"]); + Some(NetworkEvent::RequestReceived { + peer_id, + inbound_request_id, + request_type, + }) + } RequestType::PayloadEnvelopesByRange(_) => { metrics::inc_counter_vec( &metrics::TOTAL_RPC_REQUESTS, @@ -1827,6 +1835,9 @@ impl Network { RpcSuccessResponse::BlocksByRoot(resp) => { self.build_response(id, peer_id, Response::BlocksByRoot(Some(resp))) } + RpcSuccessResponse::BlocksByHead(resp) => { + self.build_response(id, peer_id, Response::BlocksByHead(Some(resp))) + } RpcSuccessResponse::PayloadEnvelopesByRange(resp) => self.build_response( id, peer_id, @@ -1871,6 +1882,7 @@ impl Network { let response = match termination { ResponseTermination::BlocksByRange => Response::BlocksByRange(None), ResponseTermination::BlocksByRoot => Response::BlocksByRoot(None), + ResponseTermination::BlocksByHead => Response::BlocksByHead(None), ResponseTermination::PayloadEnvelopesByRange => { Response::PayloadEnvelopesByRange(None) } diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 79c75d233a..76b3af1006 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -52,7 +52,7 @@ pub enum PubsubMessage { /// Gossipsub message providing notification of a signed execution payload bid. ExecutionPayloadBid(Box>), /// Gossipsub message providing notification of signed proposer preferences. - ProposerPreferences(Box), + ProposerPreferences(Arc), /// Gossipsub message providing notification of a light client finality update. LightClientFinalityUpdate(Box>), /// Gossipsub message providing notification of a light client optimistic update. @@ -416,7 +416,7 @@ impl PubsubMessage { GossipKind::ProposerPreferences => { let proposer_preferences = SignedProposerPreferences::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?; - Ok(PubsubMessage::ProposerPreferences(Box::new( + Ok(PubsubMessage::ProposerPreferences(Arc::new( proposer_preferences, ))) } diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index 319ea2b149..607f231a66 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -49,6 +49,8 @@ typenum = { workspace = true } types = { workspace = true } [dev-dependencies] +arbitrary = { workspace = true } +beacon_chain = { workspace = true, features = ["arbitrary"] } bls = { workspace = true } eth2 = { workspace = true } eth2_network_config = { workspace = true } @@ -62,3 +64,4 @@ rand_08 = { package = "rand", version = "0.8.5" } rand_chacha = "0.9.0" rand_chacha_03 = { package = "rand_chacha", version = "0.3.1" } serde_json = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 3c6f60b83c..7b5c8785bf 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -14,8 +14,8 @@ use beacon_processor::{ }; use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, - LightClientUpdatesByRangeRequest, PayloadEnvelopesByRangeRequest, + BlobsByRangeRequest, BlobsByRootRequest, BlocksByHeadRequest, DataColumnsByRangeRequest, + DataColumnsByRootRequest, LightClientUpdatesByRangeRequest, PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, }; use lighthouse_network::service::api_types::CustodyBackfillBatchId; @@ -550,15 +550,11 @@ impl NetworkBeaconProcessor { self: &Arc, message_id: MessageId, peer_id: PeerId, - proposer_preferences: Box, + proposer_preferences: Arc, ) -> Result<(), Error> { let processor = self.clone(); let process_fn = move || { - processor.process_gossip_proposer_preferences( - message_id, - peer_id, - Arc::new(*proposer_preferences), - ) + processor.process_gossip_proposer_preferences(message_id, peer_id, proposer_preferences) }; self.try_send(BeaconWorkEvent { @@ -727,6 +723,26 @@ impl NetworkBeaconProcessor { }) } + /// Create a new work event to process `BlocksByHeadRequest`s from the RPC network. + pub fn send_blocks_by_head_request( + self: &Arc, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + request: BlocksByHeadRequest, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = async move { + processor + .handle_blocks_by_head_request(peer_id, inbound_request_id, request) + .await; + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::BlocksByHeadRequest(Box::pin(process_fn)), + }) + } + /// Create a new work event to process `BlocksByRootRequest`s from the RPC network. pub fn send_blocks_by_roots_request( self: &Arc, diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index 8b31b67acb..37a6f3779a 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -7,8 +7,8 @@ use beacon_chain::payload_envelope_streamer::EnvelopeRequestSource; use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessStatus, WhenSlotSkipped}; use itertools::{Itertools, process_results}; use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, - PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, + BlobsByRangeRequest, BlobsByRootRequest, BlocksByHeadRequest, DataColumnsByRangeRequest, + DataColumnsByRootRequest, PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, }; use lighthouse_network::rpc::*; use lighthouse_network::{PeerId, ReportSource, Response, SyncInfo}; @@ -256,6 +256,266 @@ impl NetworkBeaconProcessor { Ok(()) } + /// Handle a `BeaconBlocksByHead` request from the peer. + /// + /// Walks the parent chain of `request.beacon_root` (inclusive) and emits up to + /// `min(request.count, MAX_REQUEST_BLOCKS_DENEB)` blocks in descending slot order. + /// See consensus-specs PR 5181. + #[instrument( + name = "lh_handle_blocks_by_head_request", + parent = None, + level = "debug", + skip_all, + fields(peer_id = %peer_id, client = tracing::field::Empty) + )] + pub async fn handle_blocks_by_head_request( + self: Arc, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + request: BlocksByHeadRequest, + ) { + let client = self.network_globals.client(&peer_id); + Span::current().record("client", field::display(client.kind)); + + self.terminate_response_stream( + peer_id, + inbound_request_id, + self.clone() + .handle_blocks_by_head_request_inner(peer_id, inbound_request_id, request) + .await, + Response::BlocksByHead, + ); + } + + async fn handle_blocks_by_head_request_inner( + self: Arc, + peer_id: PeerId, + inbound_request_id: InboundRequestId, + request: BlocksByHeadRequest, + ) -> Result<(), (RpcErrorResponse, &'static str)> { + let spec = &self.chain.spec; + // Cap the response at MAX_REQUEST_BLOCKS_DENEB regardless of what the peer asked for, + // matching the spec. + let max_request_blocks = spec.max_request_blocks(types::ForkName::Deneb) as u64; + let cap = request.count.min(max_request_blocks); + let beacon_root = request.beacon_root; + + debug!( + %peer_id, + beacon_root = ?beacon_root, + count = request.count, + cap, + "Received BlocksByHead Request" + ); + + if cap == 0 { + return Ok(()); + } + + // Walk the parent chain on a blocking thread because `get_blinded_block` hits the store + // synchronously and we may walk up to MAX_REQUEST_BLOCKS_DENEB ancestors. + let network_beacon_processor = self.clone(); + let block_roots = self + .executor + .spawn_blocking_handle( + move || network_beacon_processor.get_block_roots_ancestor_of_head(beacon_root, cap), + "get_block_roots_ancestor_of_head", + ) + .ok_or((RpcErrorResponse::ServerError, "shutting down"))? + .await + .map_err(|_| (RpcErrorResponse::ServerError, "tokio join"))??; + + let requested_blocks = block_roots.len(); + + let log_results = |peer_id, blocks_sent| { + debug!( + %peer_id, + requested = requested_blocks, + returned = blocks_sent, + "BlocksByHead outgoing response processed" + ); + }; + + let mut block_stream = match self.chain.get_blocks(block_roots) { + Ok(block_stream) => block_stream, + Err(e) => { + error!(error = ?e, "Error getting block stream"); + return Err((RpcErrorResponse::ServerError, "Iterator error")); + } + }; + + // Fetching blocks is async because it may have to hit the execution layer for payloads. + let mut blocks_sent = 0; + while let Some((root, result)) = block_stream.next().await { + match result.as_ref() { + Ok(Some(block)) => { + blocks_sent += 1; + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + inbound_request_id, + response: Response::BlocksByHead(Some(block.clone())), + }); + } + Ok(None) => { + error!( + %peer_id, + request_root = ?root, + "Block in the chain is not in the store" + ); + log_results(peer_id, blocks_sent); + return Err((RpcErrorResponse::ServerError, "Database inconsistency")); + } + Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => { + debug!( + block_root = ?root, + reason = "execution layer not synced", + "Failed to fetch execution payload for blocks by head request" + ); + log_results(peer_id, blocks_sent); + return Err(( + RpcErrorResponse::ResourceUnavailable, + "Execution layer not synced", + )); + } + Err(e) => { + if matches!( + e, + BeaconChainError::ExecutionLayerErrorPayloadReconstruction(_block_hash, boxed_error) + if matches!(**boxed_error, execution_layer::Error::EngineError(_)) + ) { + warn!( + info = "this may occur occasionally when the EE is busy", + block_root = ?root, + error = ?e, + "Error rebuilding payload for peer" + ); + } else { + error!( + block_root = ?root, + error = ?e, + "Error fetching block for peer" + ); + } + log_results(peer_id, blocks_sent); + return Err((RpcErrorResponse::ServerError, "Failed fetching blocks")); + } + } + } + + log_results(peer_id, blocks_sent); + Ok(()) + } + + /// Walks the parent chain of `head_root` (inclusive) and returns up to `count` block roots + /// in descending slot order. Synchronous so it can be run on a blocking thread. + /// + /// Two regimes are handled: + /// 1. Above finalization → fork-choice's in-memory proto-array supplies the roots + /// (zero DB reads). + /// 2. At or below finalization → the freezer DB's `BeaconBlockRoots` column (the + /// canonical slot→root index for finalized blocks, populated for + /// `[oldest_block_slot, split.slot)` with skip slots reusing the prior block's + /// root) supplies the roots. The head state is never consulted: its 8192-slot + /// `block_roots` bucket would silently truncate deep walks and is the wrong + /// source of truth for canonical history below finalization. + /// + /// Returns `ResourceUnavailable` if `head_root` is not known to the node. + fn get_block_roots_ancestor_of_head( + &self, + head_root: Hash256, + count: u64, + ) -> Result, (RpcErrorResponse, &'static str)> { + if count == 0 { + return Ok(vec![]); + } + + // 1. Walk ancestors in proto-array (in-memory, zero DB reads). Track the + // deepest slot we collected — that's where the freezer walk picks up. + let mut roots: Vec = Vec::with_capacity(count as usize); + let mut deepest_slot: Option = None; + { + let fork_choice = self.chain.canonical_head.fork_choice_read_lock(); + for (root, slot) in fork_choice + .proto_array() + .iter_block_roots(&head_root) + .take(count as usize) + { + roots.push(root); + deepest_slot = Some(slot); + } + } + + let store = &self.chain.store; + + // 2. Fallback: `head_root` is at or below finalization (proto-array doesn't + // track it). Look up its slot in the store, then verify it is the canonical + // block at that slot via the freezer index — a non-canonical hot-DB block at + // slot < split.slot can shadow the finalized chain. If the freezer + // disagrees (or doesn't have that slot), serve just the single block we + // found, satisfying the spec's "MUST return at least one block if you have + // it" clause. + let mut current_slot = if let Some(slot) = deepest_slot { + slot + } else { + let block = self + .chain + .get_blinded_block(&head_root) + .map_err(|e| { + error!(error = ?e, "Error reading blinded block for BlocksByHead beacon_root"); + (RpcErrorResponse::ServerError, "Database error") + })? + .ok_or((RpcErrorResponse::ResourceUnavailable, "Unknown beacon_root"))?; + let block_slot = block.slot(); + roots.push(head_root); + + match store.get_cold_block_root(block_slot) { + Ok(Some(r)) if r == head_root => {} // canonical, OK to walk back + Ok(_) => return Ok(roots), + Err(e) => { + error!(error = ?e, "Error reading freezer block_root for BlocksByHead"); + return Err((RpcErrorResponse::ServerError, "Database error")); + } + } + + block_slot + }; + + if (roots.len() as u64) >= count { + return Ok(roots); + } + + // 3. Spillover via the freezer DB's `BeaconBlockRoots` index (the canonical + // slot→root mapping for finalized blocks). Skip slots reuse the prior + // block's root; dedup on insert. + let oldest_block_slot = store.get_oldest_block_slot(); + let mut last_root = roots.last().copied(); + while (roots.len() as u64) < count && current_slot > oldest_block_slot { + current_slot = match current_slot.as_u64().checked_sub(1) { + Some(s) => Slot::from(s), + None => break, + }; + match store.get_cold_block_root(current_slot) { + Ok(Some(root)) => { + if Some(root) != last_root { + roots.push(root); + last_root = Some(root); + } + } + Ok(None) => { + // Hole in the freezer index (e.g. before `oldest_block_slot` on a + // checkpoint-synced node). Stop walking. + break; + } + Err(e) => { + error!(error = ?e, "Error walking freezer block_roots"); + return Err((RpcErrorResponse::ServerError, "Database error")); + } + } + } + + Ok(roots) + } + /// Handle a `ExecutionPayloadEnvelopesByRoot` request from the peer. #[instrument( name = "lh_handle_payload_envelopes_by_root_request", diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index c4e7f8f8d1..f13815f7b6 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -24,8 +24,8 @@ use itertools::Itertools; use libp2p::gossipsub::MessageAcceptance; use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, MetaDataV3, - PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, + BlobsByRangeRequest, BlobsByRootRequest, BlocksByHeadRequest, DataColumnsByRangeRequest, + MetaDataV3, PayloadEnvelopesByRangeRequest, PayloadEnvelopesByRootRequest, }; use lighthouse_network::{ Client, MessageId, NetworkConfig, NetworkGlobals, PeerId, Response, @@ -501,6 +501,16 @@ impl TestRig { .unwrap(); } + pub fn enqueue_blocks_by_head_request(&self, beacon_root: Hash256, count: u64) { + self.network_beacon_processor + .send_blocks_by_head_request( + PeerId::random(), + InboundRequestId::new_unchecked(42, 24), + BlocksByHeadRequest { beacon_root, count }, + ) + .unwrap(); + } + pub fn enqueue_blobs_by_root_request(&self, blob_ids: RuntimeVariableList) { self.network_beacon_processor .send_blobs_by_roots_request( @@ -2346,3 +2356,153 @@ async fn test_payload_envelopes_by_range_no_duplicates_with_skip_slots() { // 1. Gossip envelope arrives before its block → queued via UnknownBlockForEnvelope // 2. Block imported → envelope released and processed successfully // 3. Timeout path → envelope released and re-verified + +/// Drain `network_rx` collecting `Response::BlocksByHead(Some(_))` block roots until the +/// stream terminator (`None`) arrives. Panics on any other message type so tests fail +/// loudly if an error response sneaks in. +async fn drain_blocks_by_head_response(rig: &mut TestRig) -> Vec { + let mut roots = Vec::new(); + while let Some(msg) = rig.network_rx.recv().await { + match msg { + NetworkMessage::SendResponse { + response: Response::BlocksByHead(Some(block)), + .. + } => roots.push(block.canonical_root()), + NetworkMessage::SendResponse { + response: Response::BlocksByHead(None), + .. + } => return roots, + other => panic!("unexpected message: {:?}", other), + } + } + roots +} + +// `BlocksByHead` request that crosses the finalized boundary: proto-array supplies +// the unfinalized head + ancestors down to the finalized root, then the freezer's +// `BeaconBlockRoots` index supplies the rest. Verifies the spillover path +// `get_block_roots_ancestor_of_head` takes when count > proto-array depth. +#[tokio::test] +async fn test_blocks_by_head_spillover_into_freezer() { + // Long enough for finalization + state migration to populate the freezer. + let mut rig = TestRig::new(SLOTS_PER_EPOCH * 4).await; + + // Sanity-check the precondition: finalization advanced past genesis and the split + // slot is non-zero, so the freezer's `BeaconBlockRoots` column has entries. + assert!( + rig.chain + .canonical_head + .cached_head() + .finalized_checkpoint() + .epoch + > Epoch::new(0), + "test precondition: chain must have finalized past epoch 0", + ); + assert!( + rig.chain.store.get_split_slot() > Slot::new(0), + "test precondition: state migration must have populated the freezer", + ); + + let head_slot = rig.chain.canonical_head.cached_head().head_slot(); + let head_root = rig.chain.canonical_head.cached_head().head_block_root(); + + // Walk all the way back to slot 1: exercises both proto-array (above finalization) + // and freezer (at/below finalization). + let count = head_slot.as_u64(); + rig.enqueue_blocks_by_head_request(head_root, count); + let actual = drain_blocks_by_head_response(&mut rig).await; + + // Build the canonical descending root list independently. The harness has no skip + // slots so every slot in [1, head_slot] has a unique block, but we still dedup + // defensively to mirror the function under test. + let mut expected: Vec = Vec::new(); + let mut last: Option = None; + for offset in 0..count { + let slot = Slot::new(head_slot.as_u64() - offset); + if let Some(root) = rig + .chain + .block_root_at_slot(slot, WhenSlotSkipped::Prev) + .unwrap() + && Some(root) != last + { + expected.push(root); + last = Some(root); + } + } + + assert_eq!( + actual, expected, + "BlocksByHead must serve the full canonical parent chain across the finalized boundary", + ); + assert_eq!(actual.first(), Some(&head_root), "first root must be head"); +} + +// `BlocksByHead` with `beacon_root` set to a finalized block root (case-2 fallback in +// `get_block_roots_ancestor_of_head`): proto-array doesn't track it, so we +// `get_blinded_block` for its slot, verify canonicity via the freezer index, and walk +// back from there. +#[tokio::test] +async fn test_blocks_by_head_finalized_root() { + let mut rig = TestRig::new(SLOTS_PER_EPOCH * 4).await; + + let finalized_root = rig + .chain + .canonical_head + .cached_head() + .finalized_checkpoint() + .root; + let finalized_slot = rig + .chain + .get_blinded_block(&finalized_root) + .unwrap() + .expect("finalized block exists in store") + .slot(); + assert!( + finalized_slot > Slot::new(0), + "test precondition: finalized block must not be genesis", + ); + + let count = 8u64.min(finalized_slot.as_u64()); + rig.enqueue_blocks_by_head_request(finalized_root, count); + let actual = drain_blocks_by_head_response(&mut rig).await; + + let mut expected: Vec = Vec::new(); + let mut last: Option = None; + for offset in 0..count { + let slot = Slot::new(finalized_slot.as_u64() - offset); + if let Some(root) = rig + .chain + .block_root_at_slot(slot, WhenSlotSkipped::Prev) + .unwrap() + && Some(root) != last + { + expected.push(root); + last = Some(root); + } + } + + assert_eq!(actual, expected); + assert_eq!( + actual.first(), + Some(&finalized_root), + "first root must be the requested finalized root", + ); +} + +// `BlocksByHead` for a `beacon_root` we don't have. Spec says we MUST return an error +// (we map this to `ResourceUnavailable`). +#[tokio::test] +async fn test_blocks_by_head_unknown_root() { + let mut rig = TestRig::new(SLOTS_PER_EPOCH).await; + rig.enqueue_blocks_by_head_request(Hash256::repeat_byte(0xab), 4); + + match rig.network_rx.recv().await.expect("a network message") { + NetworkMessage::SendErrorResponse { error, .. } => { + assert_matches!( + error, + lighthouse_network::rpc::RpcErrorResponse::ResourceUnavailable + ); + } + other => panic!("expected SendErrorResponse, got {:?}", other), + } +} diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 4b2ef84f1d..d5ff0db37e 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -243,6 +243,13 @@ impl Router { request, ), ), + RequestType::BlocksByHead(request) => self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_blocks_by_head_request( + peer_id, + inbound_request_id, + request, + ), + ), RequestType::PayloadEnvelopesByRoot(request) => self .handle_beacon_processor_send_result( self.network_beacon_processor @@ -346,6 +353,11 @@ impl Router { Response::PayloadEnvelopesByRoot(_) | Response::PayloadEnvelopesByRange(_) => { debug!("Requesting envelopes by root and by range not supported yet"); } + // Lighthouse currently only serves BlocksByHead and does not issue it as a client, + // so receiving a response is unexpected. Drop it without crashing. + Response::BlocksByHead(_) => { + debug!("BlocksByHead response received but not requested by lighthouse"); + } // Light client responses should not be received Response::LightClientBootstrap(_) | Response::LightClientOptimisticUpdate(_) diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index 98cf3e0a1f..f5c0fdb4e5 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -501,10 +501,9 @@ mod tests { DataColumnsByRangeRequestId, DataColumnsByRangeRequester, Id, RangeRequestId, }, }; - use rand::SeedableRng; use std::{collections::HashMap, sync::Arc}; use tracing::Span; - use types::{Epoch, ForkName, MinimalEthSpec as E, SignedBeaconBlock, test_utils::XorShiftRng}; + use types::{Epoch, ForkName, MinimalEthSpec as E, SignedBeaconBlock}; fn components_id() -> ComponentsByRangeRequestId { ComponentsByRangeRequestId { @@ -549,10 +548,11 @@ mod tests { #[test] fn no_blobs_into_responses() { - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..4) .map(|_| { - generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng) + generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut u) + .unwrap() .0 .into() }) @@ -574,11 +574,12 @@ mod tests { #[test] fn empty_blobs_into_responses() { - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..4) .map(|_| { // Always generate some blobs. - generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Number(3), &mut rng) + generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Number(3), &mut u) + .unwrap() .0 .into() }) @@ -619,15 +620,16 @@ mod tests { .custody_context() .sampling_columns_for_epoch(Epoch::new(0), &spec) .to_vec(); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..4) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); @@ -729,15 +731,16 @@ mod tests { Span::none(), ); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..4) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); @@ -787,15 +790,16 @@ mod tests { .custody_context() .sampling_columns_for_epoch(Epoch::new(0), &spec) .to_vec(); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..2) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); @@ -884,15 +888,16 @@ mod tests { .custody_context() .sampling_columns_for_epoch(Epoch::new(0), &spec) .to_vec(); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..2) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); @@ -999,15 +1004,16 @@ mod tests { .custody_context() .sampling_columns_for_epoch(Epoch::new(0), &spec) .to_vec(); - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let blocks = (0..1) .map(|_| { generate_rand_block_and_data_columns::( ForkName::Fulu, NumBlobs::Number(1), - &mut rng, + &mut u, &spec, ) + .unwrap() }) .collect::>(); diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index a26996ec5e..d27c92c21a 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -38,7 +38,6 @@ use tracing::info; use types::{ BlobSidecar, BlockImportSource, ColumnIndex, DataColumnSidecar, EthSpec, ForkContext, ForkName, Hash256, MinimalEthSpec as E, SignedBeaconBlock, Slot, - test_utils::{SeedableRng, XorShiftRng}, }; const D: Duration = Duration::new(0, 0); @@ -279,7 +278,6 @@ impl TestRig { // deterministic seed let rng_08 = ::from_seed([0u8; 32]); - let rng = ChaCha20Rng::from_seed([0u8; 32]); init_tracing(); @@ -291,7 +289,7 @@ impl TestRig { sync_rx, sync_rx_queue: vec![], rng_08, - rng, + unstructured: types::test_utils::test_unstructured(), network_globals: beacon_processor.network_globals.clone(), sync_manager: SyncManager::new( chain, @@ -1492,8 +1490,7 @@ impl TestRig { num_blobs: NumBlobs, ) -> (SignedBeaconBlock, Vec>) { let fork_name = self.fork_name; - let rng = &mut self.rng; - generate_rand_block_and_blobs::(fork_name, num_blobs, rng) + generate_rand_block_and_blobs::(fork_name, num_blobs, &mut self.unstructured).unwrap() } pub fn send_sync_message(&mut self, sync_message: SyncMessage) { @@ -1829,16 +1826,17 @@ impl TestRig { } #[test] -fn stable_rng() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let (block, _) = generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng); +fn stable_arbitrary() { + let mut u = types::test_utils::test_unstructured(); + let (block, _) = + generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut u).unwrap(); assert_eq!( block.canonical_root(), Hash256::from_slice( - &hex::decode("adfd2e9e7a7976e8ccaed6eaf0257ed36a5b476732fee63ff44966602fd099ec") + &hex::decode("7348573d99ca404b502e2be790593203a1d899f9cf04f42ec9c5b4975803e3c5") .unwrap() ), - "rng produces a consistent value" + "arbitrary produces a consistent value" ); } diff --git a/beacon_node/network/src/sync/tests/mod.rs b/beacon_node/network/src/sync/tests/mod.rs index 8ffe24dda5..dd8c3ae432 100644 --- a/beacon_node/network/src/sync/tests/mod.rs +++ b/beacon_node/network/src/sync/tests/mod.rs @@ -11,7 +11,6 @@ use beacon_processor::WorkEvent; use lighthouse_network::rpc::RequestType; use lighthouse_network::service::api_types::{AppRequestId, Id}; use lighthouse_network::{NetworkGlobals, PeerId}; -use rand_chacha::ChaCha20Rng; use slot_clock::ManualSlotClock; use std::collections::{HashMap, HashSet}; use std::fs::OpenOptions; @@ -72,9 +71,8 @@ struct TestRig { network_globals: Arc>, /// Beacon chain harness harness: BeaconChainHarness>, - /// `rng` for generating test blocks and blobs. rng_08: rand_chacha_03::ChaCha20Rng, - rng: ChaCha20Rng, + unstructured: arbitrary::Unstructured<'static>, fork_name: ForkName, /// Blocks that will be used in the test but may not be known to `harness` yet. network_blocks_by_root: HashMap>, diff --git a/beacon_node/store/src/reconstruct.rs b/beacon_node/store/src/reconstruct.rs index 7aca692ef9..04a519af02 100644 --- a/beacon_node/store/src/reconstruct.rs +++ b/beacon_node/store/src/reconstruct.rs @@ -1,7 +1,8 @@ //! Implementation of historic state reconstruction (given complete block history). +use crate::forwards_iter::FrozenForwardsIterator; use crate::hot_cold_store::{HotColdDB, HotColdDBError}; use crate::metrics; -use crate::{Error, ItemStore}; +use crate::{DBColumn, Error, ItemStore}; use itertools::{Itertools, process_results}; use state_processing::{ BlockSignatureStrategy, ConsensusContext, VerifyBlockRoot, per_block_processing, @@ -9,7 +10,7 @@ use state_processing::{ }; use std::sync::Arc; use tracing::{debug, info}; -use types::EthSpec; +use types::{EthSpec, Slot}; impl HotColdDB where @@ -35,13 +36,6 @@ where }); } - debug!( - start_slot = %anchor.state_lower_limit, - "Starting state reconstruction batch" - ); - - let _t = metrics::start_timer(&metrics::STORE_BEACON_RECONSTRUCTION_TIME); - // Iterate blocks from the state lower limit to the upper limit. let split = self.get_split_info(); let lower_limit_slot = anchor.state_lower_limit; @@ -56,20 +50,86 @@ where // If `num_blocks` is not specified iterate all blocks. Add 1 so that we end on an epoch // boundary when `num_blocks` is a multiple of an epoch boundary. We want to be *inclusive* // of the state at slot `lower_limit_slot + num_blocks`. - let block_root_iter = self - .forwards_block_roots_iterator_until(lower_limit_slot, upper_limit_slot - 1, || { - Err(Error::StateShouldNotBeRequired(upper_limit_slot - 1)) - })? - .take(num_blocks.map_or(usize::MAX, |n| n + 1)); + let to_slot = num_blocks + .map(|n| std::cmp::min(lower_limit_slot + n as u64 + 1, upper_limit_slot)) + .unwrap_or(upper_limit_slot); + + let on_commit = |slot: Slot| -> Result<(), Error> { + info!( + %slot, + remaining = %(upper_limit_slot - 1 - slot), + "State reconstruction in progress" + ); + + // Update anchor. + let old_anchor = anchor.clone(); + let reconstruction_complete = slot + 1 == upper_limit_slot; + + if reconstruction_complete { + // The two limits have met in the middle! We're done! + let new_anchor = old_anchor.as_archive_anchor(); + self.compare_and_set_anchor_info_with_write(old_anchor, new_anchor)?; + } else { + // The lower limit has been raised, store it. + anchor.state_lower_limit = slot; + self.compare_and_set_anchor_info_with_write(old_anchor, anchor.clone())?; + } + + Ok(()) + }; + + self.reconstruct_historic_states_on_range(lower_limit_slot, to_slot, on_commit)?; + + // Check that the split point wasn't mutated during the state reconstruction process. + // It shouldn't have been, due to the serialization of requests through the store migrator, + // so this is just a paranoid check. + let latest_split = self.get_split_info(); + if split != latest_split { + return Err(Error::SplitPointModified(latest_split.slot, split.slot)); + } + + Ok(()) + } + + /// Reconstruct historic states for the slot range `(with_state_at_slot, to_slot)`. + /// + /// Loads the state at `with_state_at_slot` and replays blocks up to and including slot + /// `to_slot - 1`, writing all intermediate states to the freezer DB. + /// + /// The `BeaconBlockRoots` column must be populated for the range before this is called. + /// + /// `on_commit(slot)` is invoked after each atomic commit (whenever the hierarchy says to + /// commit, plus once at the final slot) so callers can update anchor metadata or log + /// progress. + pub fn reconstruct_historic_states_on_range( + self: &Arc, + with_state_at_slot: Slot, + to_slot: Slot, + mut on_commit: impl FnMut(Slot) -> Result<(), Error>, + ) -> Result<(), Error> { + debug!( + from_slot = %(with_state_at_slot + 1), + %to_slot, + "Starting state reconstruction batch" + ); + + let _t = metrics::start_timer(&metrics::STORE_BEACON_RECONSTRUCTION_TIME); + + // Iterate from `with_state_at_slot` so `tuple_windows` gives us the predecessor block + // root at each step for skip detection. + let block_root_iter = FrozenForwardsIterator::new( + self, + DBColumn::BeaconBlockRoots, + with_state_at_slot, + to_slot, + )?; // The state to be advanced. - let mut state = self.load_cold_state_by_slot(lower_limit_slot)?; - + let mut state = self.load_cold_state_by_slot(with_state_at_slot)?; state.build_caches(&self.spec)?; process_results(block_root_iter, |iter| -> Result<(), Error> { let mut io_batch = vec![]; - let mut prev_state_root = None; for ((prev_block_root, _), (block_root, slot)) in iter.tuple_windows() { @@ -114,32 +174,16 @@ where // Stage state for storage in freezer DB. self.store_cold_state(&state_root, &state, &mut io_batch)?; - let batch_complete = - num_blocks.is_some_and(|n_blocks| slot == lower_limit_slot + n_blocks as u64); - let reconstruction_complete = slot + 1 == upper_limit_slot; + let batch_complete = slot + 1 == to_slot; // Commit the I/O batch if: // // - The diff/snapshot for this slot is required for future slots, or - // - The reconstruction batch is complete (we are about to return), or - // - Reconstruction is complete. - if self.hierarchy.should_commit_immediately(slot)? - || batch_complete - || reconstruction_complete - { - info!( - %slot, - remaining = %(upper_limit_slot - 1 - slot), - "State reconstruction in progress" - ); - + // - The reconstruction batch is complete (we are about to return). + if self.hierarchy.should_commit_immediately(slot)? || batch_complete { self.cold_db.do_atomically(std::mem::take(&mut io_batch))?; - // Update anchor. - let old_anchor = anchor.clone(); - - if reconstruction_complete { - // The two limits have met in the middle! We're done! + if batch_complete { // Perform one last integrity check on the state reached. let computed_state_root = state.update_tree_hash_cache()?; if computed_state_root != state_root { @@ -149,23 +193,15 @@ where computed: computed_state_root, }); } - - let new_anchor = old_anchor.as_archive_anchor(); - self.compare_and_set_anchor_info_with_write(old_anchor, new_anchor)?; - - return Ok(()); - } else { - // The lower limit has been raised, store it. - anchor.state_lower_limit = slot; - - self.compare_and_set_anchor_info_with_write(old_anchor, anchor.clone())?; } + on_commit(slot)?; + // If this is the end of the batch, return Ok. The caller will run another // batch when there is idle capacity. if batch_complete { debug!( - start_slot = %lower_limit_slot, + start_slot = %(with_state_at_slot + 1), end_slot = %slot, "Finished state reconstruction batch" ); @@ -174,19 +210,10 @@ where } } - // Should always reach the `upper_limit_slot` or the end of the batch and return early - // above. + // Should always reach `to_slot` or the end of the batch and return early above. Err(Error::StateReconstructionLogicError) })??; - // Check that the split point wasn't mutated during the state reconstruction process. - // It shouldn't have been, due to the serialization of requests through the store migrator, - // so this is just a paranoid check. - let latest_split = self.get_split_info(); - if split != latest_split { - return Err(Error::SplitPointModified(latest_split.slot, split.slot)); - } - Ok(()) } } diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index 974508492a..5e015f2713 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -38,6 +38,6 @@ types = { workspace = true } zeroize = { workspace = true, optional = true } [dev-dependencies] -rand = { workspace = true } -test_random_derive = { path = "../../common/test_random_derive" } +arbitrary = { workspace = true } tokio = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index d084679b76..f1dd0224c1 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -46,7 +46,7 @@ use ssz::{Decode, Encode}; use std::fmt; use std::future::Future; use std::time::Duration; -use types::{PayloadAttestationData, PayloadAttestationMessage}; +use types::{PayloadAttestationData, PayloadAttestationMessage, SignedProposerPreferences}; pub const V1: EndpointVersion = EndpointVersion(1); pub const V2: EndpointVersion = EndpointVersion(2); @@ -1875,6 +1875,46 @@ impl BeaconNodeHttpClient { Ok(()) } + /// `POST validator/proposer_preferences` + pub async fn post_validator_proposer_preferences( + &self, + signed_preferences: &[SignedProposerPreferences], + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V1)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("proposer_preferences"); + + self.post_generic_with_consensus_version(path, &signed_preferences, None, fork_name) + .await?; + + Ok(()) + } + + /// `POST validator/proposer_preferences` (SSZ) + pub async fn post_validator_proposer_preferences_ssz( + &self, + signed_preferences: &Vec, + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V1)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("proposer_preferences"); + + let ssz_body = signed_preferences.as_ssz_bytes(); + + self.post_generic_with_consensus_version_and_ssz_body(path, ssz_body, None, fork_name) + .await?; + + Ok(()) + } + /// `POST beacon/rewards/sync_committee` pub async fn post_beacon_rewards_sync_committee( &self, @@ -3016,10 +3056,11 @@ impl BeaconNodeHttpClient { } /// `GET validator/payload_attestation_data/{slot}` + /// Returns `None` if no block has been received for the requested slot (404). pub async fn get_validator_payload_attestation_data( &self, slot: Slot, - ) -> Result, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -3028,16 +3069,23 @@ impl BeaconNodeHttpClient { .push("payload_attestation_data") .push(&slot.to_string()); - self.get_with_timeout(path, self.timeouts.payload_attestation) + let opt_response = self + .get_response(path, |b| b.timeout(self.timeouts.payload_attestation)) .await - .map(BeaconResponse::ForkVersioned) + .optional()?; + + match opt_response { + Some(response) => Ok(Some(BeaconResponse::ForkVersioned(response.json().await?))), + None => Ok(None), + } } /// `GET validator/payload_attestation_data/{slot}` in SSZ format + /// Returns `None` if no block has been received for the requested slot (404). pub async fn get_validator_payload_attestation_data_ssz( &self, slot: Slot, - ) -> Result { + ) -> Result, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -3050,9 +3098,9 @@ impl BeaconNodeHttpClient { .get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.payload_attestation) .await?; - let response_bytes = opt_response.ok_or(Error::StatusCode(StatusCode::NOT_FOUND))?; - - PayloadAttestationData::from_ssz_bytes(&response_bytes).map_err(Error::InvalidSsz) + opt_response + .map(|bytes| PayloadAttestationData::from_ssz_bytes(&bytes).map_err(Error::InvalidSsz)) + .transpose() } /// `GET v1/validator/aggregate_attestation?slot,attestation_data_root` diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 303974d522..da394fe23b 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -26,11 +26,6 @@ use std::sync::Arc; use std::time::Duration; use superstruct::superstruct; -#[cfg(test)] -use test_random_derive::TestRandom; -#[cfg(test)] -use types::test_utils::TestRandom; - // TODO(mac): Temporary module and re-export hack to expose old `consensus/types` via `eth2/types`. pub use crate::beacon_response::*; pub mod beacon_response { @@ -2390,7 +2385,7 @@ pub enum ContentType { Ssz, } -#[cfg_attr(test, derive(TestRandom))] +#[cfg_attr(test, derive(arbitrary::Arbitrary))] #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode, Decode)] #[serde(bound = "E: EthSpec")] pub struct BlobsBundle { @@ -2496,7 +2491,7 @@ pub struct BlobWrapper { mod test { use std::fmt::Debug; - use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use arbitrary::Arbitrary; use super::*; @@ -2524,13 +2519,16 @@ mod test { assert_eq!(request, deserialized_request); }; - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); for fork_name in ForkName::list_all() { - let signed_beacon_block = - map_fork_name!(fork_name, SignedBeaconBlock, <_>::random_for_test(rng)); + let signed_beacon_block = map_fork_name!( + fork_name, + SignedBeaconBlock, + <_>::arbitrary(&mut u).unwrap() + ); let request = if fork_name.deneb_enabled() && !fork_name.gloas_enabled() { - let kzg_proofs = KzgProofs::::random_for_test(rng); - let blobs = BlobsList::::random_for_test(rng); + let kzg_proofs = KzgProofs::::arbitrary(&mut u).unwrap(); + let blobs = BlobsList::::arbitrary(&mut u).unwrap(); let block_contents = SignedBlockContents { signed_block: Arc::new(signed_beacon_block), kzg_proofs, @@ -2558,12 +2556,15 @@ mod test { }; let mut fork_name = ForkName::Deneb; - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); loop { - let signed_beacon_block = - map_fork_name!(fork_name, SignedBeaconBlock, <_>::random_for_test(rng)); - let kzg_proofs = KzgProofs::::random_for_test(rng); - let blobs = BlobsList::::random_for_test(rng); + let signed_beacon_block = map_fork_name!( + fork_name, + SignedBeaconBlock, + <_>::arbitrary(&mut u).unwrap() + ); + let kzg_proofs = KzgProofs::::arbitrary(&mut u).unwrap(); + let blobs = BlobsList::::arbitrary(&mut u).unwrap(); let block_contents = SignedBlockContents { signed_block: Arc::new(signed_beacon_block), kzg_proofs, @@ -2581,27 +2582,30 @@ mod test { #[test] fn test_execution_payload_execution_payload_deserialize_by_fork() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let payloads = [ ExecutionPayload::Bellatrix( - ExecutionPayloadBellatrix::::random_for_test(rng), + ExecutionPayloadBellatrix::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Capella( + ExecutionPayloadCapella::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Deneb( + ExecutionPayloadDeneb::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Electra( + ExecutionPayloadElectra::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Fulu( + ExecutionPayloadFulu::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Gloas( + ExecutionPayloadGloas::::arbitrary(&mut u).unwrap(), + ), + ExecutionPayload::Heze( + ExecutionPayloadHeze::::arbitrary(&mut u).unwrap(), ), - ExecutionPayload::Capella(ExecutionPayloadCapella::::random_for_test( - rng, - )), - ExecutionPayload::Deneb(ExecutionPayloadDeneb::::random_for_test( - rng, - )), - ExecutionPayload::Electra(ExecutionPayloadElectra::::random_for_test( - rng, - )), - ExecutionPayload::Heze(ExecutionPayloadHeze::::random_for_test(rng)), - ExecutionPayload::Fulu(ExecutionPayloadFulu::::random_for_test(rng)), - ExecutionPayload::Gloas(ExecutionPayloadGloas::::random_for_test( - rng, - )), - ExecutionPayload::Heze(ExecutionPayloadHeze::::random_for_test(rng)), ]; let merged_forks = &ForkName::list_all()[2..]; assert_eq!( @@ -2620,70 +2624,54 @@ mod test { #[test] fn test_execution_payload_and_blobs_deserialize_by_fork() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let payloads = [ { - let execution_payload = - ExecutionPayload::Deneb( - ExecutionPayloadDeneb::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); + let execution_payload = ExecutionPayload::Deneb( + ExecutionPayloadDeneb::::arbitrary(&mut u).unwrap(), + ); + let blobs_bundle = BlobsBundle::::arbitrary(&mut u).unwrap(); ExecutionPayloadAndBlobs { execution_payload, blobs_bundle, } }, { - let execution_payload = - ExecutionPayload::Electra( - ExecutionPayloadElectra::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); + let execution_payload = ExecutionPayload::Electra( + ExecutionPayloadElectra::::arbitrary(&mut u).unwrap(), + ); + let blobs_bundle = BlobsBundle::::arbitrary(&mut u).unwrap(); ExecutionPayloadAndBlobs { execution_payload, blobs_bundle, } }, { - let execution_payload = - ExecutionPayload::Heze( - ExecutionPayloadHeze::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); + let execution_payload = ExecutionPayload::Fulu( + ExecutionPayloadFulu::::arbitrary(&mut u).unwrap(), + ); + let blobs_bundle = BlobsBundle::::arbitrary(&mut u).unwrap(); ExecutionPayloadAndBlobs { execution_payload, blobs_bundle, } }, { - let execution_payload = - ExecutionPayload::Fulu( - ExecutionPayloadFulu::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); + let execution_payload = ExecutionPayload::Gloas( + ExecutionPayloadGloas::::arbitrary(&mut u).unwrap(), + ); + let blobs_bundle = BlobsBundle::::arbitrary(&mut u).unwrap(); ExecutionPayloadAndBlobs { execution_payload, blobs_bundle, } }, { - let execution_payload = - ExecutionPayload::Gloas( - ExecutionPayloadGloas::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); - ExecutionPayloadAndBlobs { - execution_payload, - blobs_bundle, - } - }, - { - let execution_payload = - ExecutionPayload::Heze( - ExecutionPayloadHeze::::random_for_test(rng), - ); - let blobs_bundle = BlobsBundle::random_for_test(rng); + let execution_payload = ExecutionPayload::Heze( + ExecutionPayloadHeze::::arbitrary(&mut u).unwrap(), + ); + let blobs_bundle = BlobsBundle::::arbitrary(&mut u).unwrap(); ExecutionPayloadAndBlobs { execution_payload, blobs_bundle, diff --git a/common/test_random_derive/Cargo.toml b/common/test_random_derive/Cargo.toml deleted file mode 100644 index b38d5ef63a..0000000000 --- a/common/test_random_derive/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "test_random_derive" -version = "0.2.0" -authors = ["thojest "] -edition = { workspace = true } -description = "Procedural derive macros for implementation of TestRandom trait" - -[lib] -proc-macro = true - -[dependencies] -quote = { workspace = true } -syn = { workspace = true } diff --git a/common/test_random_derive/src/lib.rs b/common/test_random_derive/src/lib.rs deleted file mode 100644 index bf57d79aaa..0000000000 --- a/common/test_random_derive/src/lib.rs +++ /dev/null @@ -1,59 +0,0 @@ -use proc_macro::TokenStream; -use quote::quote; -use syn::{DeriveInput, parse_macro_input}; - -/// Returns true if some field has an attribute declaring it should be generated from default (not -/// randomized). -/// -/// The field attribute is: `#[test_random(default)]` -fn should_use_default(field: &syn::Field) -> bool { - field.attrs.iter().any(|attr| { - attr.path().is_ident("test_random") - && matches!(&attr.meta, syn::Meta::List(list) if list.tokens.to_string().replace(' ', "") == "default") - }) -} - -#[proc_macro_derive(TestRandom, attributes(test_random))] -pub fn test_random_derive(input: TokenStream) -> TokenStream { - let derived_input = parse_macro_input!(input as DeriveInput); - let name = &derived_input.ident; - let (impl_generics, ty_generics, where_clause) = &derived_input.generics.split_for_impl(); - - let syn::Data::Struct(struct_data) = &derived_input.data else { - panic!("test_random_derive only supports structs."); - }; - - // Build quotes for fields that should be generated and those that should be built from - // `Default`. - let mut quotes = vec![]; - for field in &struct_data.fields { - match &field.ident { - Some(ident) => { - if should_use_default(field) { - quotes.push(quote! { - #ident: <_>::default(), - }); - } else { - quotes.push(quote! { - #ident: <_>::random_for_test(rng), - }); - } - } - _ => panic!("test_random_derive only supports named struct fields."), - }; - } - - let output = quote! { - impl #impl_generics TestRandom for #name #ty_generics #where_clause { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - Self { - #( - #quotes - )* - } - } - } - }; - - output.into() -} diff --git a/common/warp_utils/src/reject.rs b/common/warp_utils/src/reject.rs index c478870950..b88fd79b23 100644 --- a/common/warp_utils/src/reject.rs +++ b/common/warp_utils/src/reject.rs @@ -110,6 +110,17 @@ pub fn not_synced(msg: String) -> warp::reject::Rejection { warp::reject::custom(NotSynced(msg)) } +/// A 404 Not Found response for when no block has been received for the +/// requested slot. +#[derive(Debug)] +pub struct BlockNotFound(pub String); + +impl Reject for BlockNotFound {} + +pub fn block_not_found(msg: String) -> warp::reject::Rejection { + warp::reject::custom(BlockNotFound(msg)) +} + #[derive(Debug)] pub struct InvalidAuthorization(pub String); @@ -199,6 +210,9 @@ pub async fn handle_rejection(err: warp::Rejection) -> Result() { code = StatusCode::SERVICE_UNAVAILABLE; message = format!("SERVICE_UNAVAILABLE: beacon node is syncing: {}", e.0); + } else if let Some(e) = err.find::() { + code = StatusCode::NOT_FOUND; + message = format!("NOT_FOUND: {}", e.0); } else if let Some(e) = err.find::() { code = StatusCode::FORBIDDEN; message = format!("FORBIDDEN: Invalid auth token: {}", e.0); diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 11e99aade0..3586b2f1eb 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -578,9 +578,9 @@ where // For Gloas blocks, `execution_status` is Irrelevant (no embedded payload). // If the payload envelope was received (Full), use the bid's block_hash as the // execution chain head. Otherwise fall back to the parent hash (Pending) or None. - // TODO(gloas): this is a bit messy, and we probably need a similar treatment for - // justified/finalized - // Can fix as part of: https://github.com/sigp/lighthouse/issues/8957 + // For justified/finalized hashes we always use the bid's parent_block_hash, since the + // payload from the justified/finalized block is not itself justified/finalized due to + // being applied immediately prior to the next block. let head_hash = self.get_block(&head_root).and_then(|b| { b.execution_status .block_hash() @@ -593,12 +593,16 @@ where }); let justified_root = self.justified_checkpoint().root; let finalized_root = self.finalized_checkpoint().root; - let justified_hash = self - .get_block(&justified_root) - .and_then(|b| b.execution_status.block_hash()); - let finalized_hash = self - .get_block(&finalized_root) - .and_then(|b| b.execution_status.block_hash()); + let justified_hash = self.get_block(&justified_root).and_then(|b| { + b.execution_status + .block_hash() + .or(b.execution_payload_parent_hash) + }); + let finalized_hash = self.get_block(&finalized_root).and_then(|b| { + b.execution_status + .block_hash() + .or(b.execution_payload_parent_hash) + }); self.forkchoice_update_parameters = ForkchoiceUpdateParameters { head_root, head_hash, diff --git a/consensus/state_processing/Cargo.toml b/consensus/state_processing/Cargo.toml index ae0af03231..72d0e17d99 100644 --- a/consensus/state_processing/Cargo.toml +++ b/consensus/state_processing/Cargo.toml @@ -37,7 +37,6 @@ rayon = { workspace = true } safe_arith = { workspace = true } smallvec = { workspace = true } ssz_types = { workspace = true } -test_random_derive = { path = "../../common/test_random_derive" } tracing = { workspace = true } tree_hash = { workspace = true } typenum = { workspace = true } @@ -45,4 +44,5 @@ types = { workspace = true } [dev-dependencies] beacon_chain = { workspace = true } +state_processing = { path = ".", features = ["arbitrary"] } tokio = { workspace = true } diff --git a/consensus/state_processing/src/verify_operation.rs b/consensus/state_processing/src/verify_operation.rs index 1e9c3d5fe3..8e67c3da43 100644 --- a/consensus/state_processing/src/verify_operation.rs +++ b/consensus/state_processing/src/verify_operation.rs @@ -14,11 +14,10 @@ use smallvec::{SmallVec, smallvec}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::marker::PhantomData; -use test_random_derive::TestRandom; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingOnDisk, AttesterSlashingRefOnDisk, BeaconState, ChainSpec, Epoch, EthSpec, Fork, ForkVersion, ProposerSlashing, - SignedBlsToExecutionChange, SignedVoluntaryExit, test_utils::TestRandom, + SignedBlsToExecutionChange, SignedVoluntaryExit, }; const MAX_FORKS_VERIFIED_AGAINST: usize = 2; @@ -138,7 +137,7 @@ struct SigVerifiedOpDecode { /// /// We need to store multiple `ForkVersion`s because attester slashings contain two indexed /// attestations which may be signed using different versions. -#[derive(Debug, PartialEq, Eq, Clone, Hash, Encode, Decode, TestRandom)] +#[derive(Debug, PartialEq, Eq, Clone, Hash, Encode, Decode)] #[cfg_attr(feature = "arbitrary", derive(Arbitrary))] pub struct VerifiedAgainst { fork_versions: SmallVec<[ForkVersion; MAX_FORKS_VERIFIED_AGAINST]>, @@ -423,20 +422,21 @@ impl TransformPersist for SignedBlsToExecutionChange { #[cfg(all(test, not(debug_assertions)))] mod test { use super::*; - use types::{ - MainnetEthSpec, - test_utils::{SeedableRng, TestRandom, XorShiftRng}, - }; + use types::MainnetEthSpec; type E = MainnetEthSpec; - fn roundtrip_test() { + fn roundtrip_test<'a, T>() + where + T: arbitrary::Arbitrary<'a> + TransformPersist + PartialEq + std::fmt::Debug, + { let runs = 10; - let mut rng = XorShiftRng::seed_from_u64(0xff0af5a356af1123); + let mut u = types::test_utils::test_unstructured(); for _ in 0..runs { - let op = T::random_for_test(&mut rng); - let verified_against = VerifiedAgainst::random_for_test(&mut rng); + let op = T::arbitrary(&mut u).expect("arbitrary op"); + let verified_against = + VerifiedAgainst::arbitrary(&mut u).expect("arbitrary verified_against"); let verified_op = SigVerifiedOp { op, diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 4aae4b7f39..9ee827c7b9 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -45,7 +45,7 @@ metastruct = "0.1.0" milhouse = { workspace = true } parking_lot = { workspace = true } rand = { workspace = true } -rand_xorshift = "0.4.0" +rand_xorshift = { workspace = true } rayon = { workspace = true } regex = { workspace = true } rpds = { workspace = true } @@ -58,7 +58,6 @@ ssz_types = { workspace = true } superstruct = { workspace = true } swap_or_not_shuffle = { workspace = true } tempfile = { workspace = true } -test_random_derive = { path = "../../common/test_random_derive" } tracing = { workspace = true } tree_hash = { workspace = true } tree_hash_derive = { workspace = true } @@ -71,6 +70,7 @@ criterion = { workspace = true } paste = { workspace = true } state_processing = { workspace = true } tokio = { workspace = true } +types = { path = ".", features = ["arbitrary"] } [lints.clippy] module_inception = "allow" diff --git a/consensus/types/src/attestation/aggregate_and_proof.rs b/consensus/types/src/attestation/aggregate_and_proof.rs index 4c6e775e56..76e33faf88 100644 --- a/consensus/types/src/attestation/aggregate_and_proof.rs +++ b/consensus/types/src/attestation/aggregate_and_proof.rs @@ -3,7 +3,6 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -12,7 +11,6 @@ use crate::{ }, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; #[superstruct( @@ -26,7 +24,6 @@ use crate::{ Deserialize, Encode, Decode, - TestRandom, TreeHash, ), context_deserialize(ForkName), diff --git a/consensus/types/src/attestation/attestation.rs b/consensus/types/src/attestation/attestation.rs index 28059efee6..4cfb7a4d24 100644 --- a/consensus/types/src/attestation/attestation.rs +++ b/consensus/types/src/attestation/attestation.rs @@ -10,7 +10,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{BitList, BitVector}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -20,7 +19,6 @@ use crate::{ }, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot, Slot, SlotData}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; #[derive(Debug, PartialEq, Clone)] @@ -49,7 +47,6 @@ impl From for Error { Deserialize, Decode, Encode, - TestRandom, Educe, TreeHash, ), @@ -614,7 +611,7 @@ impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for Vec> */ #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive(Debug, Clone, Serialize, Deserialize, Decode, Encode, TestRandom, TreeHash, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, Decode, Encode, TreeHash, PartialEq)] #[context_deserialize(ForkName)] pub struct SingleAttestation { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/attestation/attestation_data.rs b/consensus/types/src/attestation/attestation_data.rs index f3fceb9b70..2d88bce2b9 100644 --- a/consensus/types/src/attestation/attestation_data.rs +++ b/consensus/types/src/attestation/attestation_data.rs @@ -1,14 +1,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ attestation::Checkpoint, core::{Hash256, SignedRoot, Slot, SlotData}, fork::ForkName, - test_utils::TestRandom, }; /// The data upon which an attestation is based. @@ -16,18 +14,7 @@ use crate::{ /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[derive( - Debug, - Clone, - PartialEq, - Eq, - Serialize, - Deserialize, - Hash, - Encode, - Decode, - TreeHash, - TestRandom, - Default, + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Encode, Decode, TreeHash, Default, )] #[context_deserialize(ForkName)] pub struct AttestationData { diff --git a/consensus/types/src/attestation/checkpoint.rs b/consensus/types/src/attestation/checkpoint.rs index f5a95f0ad9..09f8f06e6e 100644 --- a/consensus/types/src/attestation/checkpoint.rs +++ b/consensus/types/src/attestation/checkpoint.rs @@ -1,13 +1,11 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Epoch, Hash256}, fork::ForkName, - test_utils::TestRandom, }; /// Casper FFG checkpoint, used in attestations. @@ -27,7 +25,6 @@ use crate::{ Encode, Decode, TreeHash, - TestRandom, )] #[context_deserialize(ForkName)] pub struct Checkpoint { diff --git a/consensus/types/src/attestation/indexed_attestation.rs b/consensus/types/src/attestation/indexed_attestation.rs index 272b015d90..ae15f474f3 100644 --- a/consensus/types/src/attestation/indexed_attestation.rs +++ b/consensus/types/src/attestation/indexed_attestation.rs @@ -11,10 +11,9 @@ use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName, test_utils::TestRandom}; +use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName}; /// Details an attestation that can be slashable. /// @@ -31,7 +30,6 @@ use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName, test_ut Deserialize, Decode, Encode, - TestRandom, Educe, TreeHash, ), @@ -212,10 +210,8 @@ impl Hash for IndexedAttestation { #[cfg(test)] mod tests { use super::*; - use crate::{ - core::{Epoch, MainnetEthSpec}, - test_utils::{SeedableRng, XorShiftRng}, - }; + use crate::core::{Epoch, MainnetEthSpec}; + use arbitrary::Arbitrary; #[test] pub fn test_is_double_vote_true() { @@ -278,9 +274,9 @@ mod tests { target_epoch: u64, source_epoch: u64, ) -> IndexedAttestation { - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let mut indexed_vote = - IndexedAttestation::Base(IndexedAttestationBase::random_for_test(&mut rng)); + IndexedAttestation::Base(IndexedAttestationBase::arbitrary(&mut u).unwrap()); indexed_vote.data_mut().source.epoch = Epoch::new(source_epoch); indexed_vote.data_mut().target.epoch = Epoch::new(target_epoch); diff --git a/consensus/types/src/attestation/indexed_payload_attestation.rs b/consensus/types/src/attestation/indexed_payload_attestation.rs index bb2087e330..67fdf77bdf 100644 --- a/consensus/types/src/attestation/indexed_payload_attestation.rs +++ b/consensus/types/src/attestation/indexed_payload_attestation.rs @@ -1,14 +1,12 @@ -use crate::test_utils::TestRandom; use crate::{EthSpec, ForkName, PayloadAttestationData}; use bls::AggregateSignature; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(TestRandom, TreeHash, Debug, Clone, PartialEq, Encode, Decode, Serialize, Deserialize)] +#[derive(TreeHash, Debug, Clone, PartialEq, Encode, Decode, Serialize, Deserialize)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[serde(bound = "E: EthSpec", deny_unknown_fields)] #[cfg_attr(feature = "arbitrary", arbitrary(bound = "E: EthSpec"))] diff --git a/consensus/types/src/attestation/participation_flags.rs b/consensus/types/src/attestation/participation_flags.rs index 66831abfac..a88ea0d3f7 100644 --- a/consensus/types/src/attestation/participation_flags.rs +++ b/consensus/types/src/attestation/participation_flags.rs @@ -1,15 +1,11 @@ use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use ssz::{Decode, DecodeError, Encode}; -use test_random_derive::TestRandom; use tree_hash::{PackedEncoding, TreeHash, TreeHashType}; -use crate::{ - core::{Hash256, consts::altair::NUM_FLAG_INDICES}, - test_utils::TestRandom, -}; +use crate::core::{Hash256, consts::altair::NUM_FLAG_INDICES}; -#[derive(Debug, Default, Clone, Copy, PartialEq, Deserialize, Serialize, TestRandom)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Deserialize, Serialize)] #[serde(transparent)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub struct ParticipationFlags { diff --git a/consensus/types/src/attestation/payload_attestation.rs b/consensus/types/src/attestation/payload_attestation.rs index 115a5ec4d6..d5e76f941b 100644 --- a/consensus/types/src/attestation/payload_attestation.rs +++ b/consensus/types/src/attestation/payload_attestation.rs @@ -1,5 +1,4 @@ use crate::attestation::payload_attestation_data::PayloadAttestationData; -use crate::test_utils::TestRandom; use crate::{EthSpec, ForkName}; use bls::AggregateSignature; use context_deserialize::context_deserialize; @@ -7,10 +6,9 @@ use educe::Educe; use serde::{Deserialize, Serialize}; use ssz::BitVector; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(TestRandom, TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] +#[derive(TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[serde(bound = "E: EthSpec", deny_unknown_fields)] #[cfg_attr(feature = "arbitrary", arbitrary(bound = "E: EthSpec"))] diff --git a/consensus/types/src/attestation/payload_attestation_data.rs b/consensus/types/src/attestation/payload_attestation_data.rs index 58d36fd01d..198d380c14 100644 --- a/consensus/types/src/attestation/payload_attestation_data.rs +++ b/consensus/types/src/attestation/payload_attestation_data.rs @@ -1,14 +1,10 @@ -use crate::test_utils::TestRandom; use crate::{ForkName, Hash256, SignedRoot, Slot}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive( - TestRandom, TreeHash, Debug, Clone, PartialEq, Eq, Encode, Decode, Serialize, Deserialize, Hash, -)] +#[derive(TreeHash, Debug, Clone, PartialEq, Eq, Encode, Decode, Serialize, Deserialize, Hash)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[context_deserialize(ForkName)] pub struct PayloadAttestationData { diff --git a/consensus/types/src/attestation/payload_attestation_message.rs b/consensus/types/src/attestation/payload_attestation_message.rs index 82e2137b09..7be022efd3 100644 --- a/consensus/types/src/attestation/payload_attestation_message.rs +++ b/consensus/types/src/attestation/payload_attestation_message.rs @@ -1,14 +1,12 @@ use crate::ForkName; use crate::attestation::payload_attestation_data::PayloadAttestationData; -use crate::test_utils::TestRandom; use bls::Signature; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(TestRandom, TreeHash, Debug, Clone, PartialEq, Encode, Decode, Serialize, Deserialize)] +#[derive(TreeHash, Debug, Clone, PartialEq, Encode, Decode, Serialize, Deserialize)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[context_deserialize(ForkName)] pub struct PayloadAttestationMessage { diff --git a/consensus/types/src/attestation/pending_attestation.rs b/consensus/types/src/attestation/pending_attestation.rs index 84353ac118..79a77b47cb 100644 --- a/consensus/types/src/attestation/pending_attestation.rs +++ b/consensus/types/src/attestation/pending_attestation.rs @@ -2,10 +2,9 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::BitList; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName, test_utils::TestRandom}; +use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName}; /// An attestation that has been included in the state but not yet fully processed. /// @@ -15,7 +14,7 @@ use crate::{attestation::AttestationData, core::EthSpec, fork::ForkName, test_ut derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct PendingAttestation { pub aggregation_bits: BitList, diff --git a/consensus/types/src/attestation/signed_aggregate_and_proof.rs b/consensus/types/src/attestation/signed_aggregate_and_proof.rs index 48c3f4c567..f9db76e9d2 100644 --- a/consensus/types/src/attestation/signed_aggregate_and_proof.rs +++ b/consensus/types/src/attestation/signed_aggregate_and_proof.rs @@ -3,7 +3,6 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -13,7 +12,6 @@ use crate::{ }, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; /// A Validators signed aggregate proof to publish on the `beacon_aggregate_and_proof` @@ -31,7 +29,6 @@ use crate::{ Deserialize, Encode, Decode, - TestRandom, TreeHash, ), context_deserialize(ForkName), diff --git a/consensus/types/src/block/beacon_block.rs b/consensus/types/src/block/beacon_block.rs index b2181d9ff8..38861ccf3b 100644 --- a/consensus/types/src/block/beacon_block.rs +++ b/consensus/types/src/block/beacon_block.rs @@ -9,7 +9,6 @@ use ssz::{Decode, DecodeError}; use ssz_derive::{Decode, Encode}; use ssz_types::{BitList, BitVector, FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use typenum::Unsigned; @@ -34,7 +33,6 @@ use crate::{ slashing::{AttesterSlashingBase, ProposerSlashing}, state::BeaconStateError, sync_committee::SyncAggregate, - test_utils::TestRandom, }; /// A block of the `BeaconChain`. @@ -49,7 +47,6 @@ use crate::{ Encode, Decode, TreeHash, - TestRandom, Educe, ), educe(PartialEq, Hash(bound(E: EthSpec, Payload: AbstractExecPayload))), @@ -1004,10 +1001,8 @@ impl fmt::Display for BlockImportSource { #[cfg(test)] mod tests { use super::*; - use crate::{ - core::MainnetEthSpec, - test_utils::{SeedableRng, XorShiftRng, test_ssz_tree_hash_pair_with}, - }; + use crate::{core::MainnetEthSpec, test_utils::test_ssz_tree_hash_pair_with}; + use arbitrary::Arbitrary; use ssz::Encode; type BeaconBlock = super::BeaconBlock; @@ -1016,16 +1011,10 @@ mod tests { #[test] fn roundtrip_base_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Base.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockBase { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyBase::random_for_test(rng), - }; + let inner_block = BeaconBlockBase::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Base(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1035,16 +1024,10 @@ mod tests { #[test] fn roundtrip_altair_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Altair.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockAltair { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyAltair::random_for_test(rng), - }; + let inner_block = BeaconBlockAltair::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Altair(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1054,16 +1037,10 @@ mod tests { #[test] fn roundtrip_capella_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Capella.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockCapella { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyCapella::random_for_test(rng), - }; + let inner_block = BeaconBlockCapella::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Capella(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1073,16 +1050,10 @@ mod tests { #[test] fn roundtrip_deneb_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Deneb.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockDeneb { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyDeneb::random_for_test(rng), - }; + let inner_block = BeaconBlockDeneb::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Deneb(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1092,17 +1063,10 @@ mod tests { #[test] fn roundtrip_electra_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Electra.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockElectra { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyElectra::random_for_test(rng), - }; - + let inner_block = BeaconBlockElectra::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Electra(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1112,17 +1076,10 @@ mod tests { #[test] fn roundtrip_fulu_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Fulu.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockFulu { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyFulu::random_for_test(rng), - }; - + let inner_block = BeaconBlockFulu::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Fulu(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1132,17 +1089,10 @@ mod tests { #[test] fn roundtrip_gloas_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Gloas.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockGloas { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyGloas::random_for_test(rng), - }; - + let inner_block = BeaconBlockGloas::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Gloas(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1152,17 +1102,10 @@ mod tests { #[test] fn roundtrip_heze_block() { - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let spec = &ForkName::Heze.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockHeze { - slot: Slot::random_for_test(rng), - proposer_index: u64::random_for_test(rng), - parent_root: Hash256::random_for_test(rng), - state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyHeze::random_for_test(rng), - }; - + let inner_block = BeaconBlockHeze::arbitrary(&mut u).unwrap(); let block = BeaconBlock::Heze(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { @@ -1175,7 +1118,7 @@ mod tests { type E = MainnetEthSpec; let mut spec = E::default_spec(); - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = crate::test_utils::test_unstructured(); let altair_fork_epoch = spec.altair_fork_epoch.unwrap(); @@ -1208,7 +1151,7 @@ mod tests { { let good_base_block = BeaconBlock::Base(BeaconBlockBase { slot: base_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have a base block with a slot higher than the fork epoch. let bad_base_block = { @@ -1230,7 +1173,7 @@ mod tests { { let good_altair_block = BeaconBlock::Altair(BeaconBlockAltair { slot: altair_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have an Altair block with a epoch lower than the fork epoch. let bad_altair_block = { @@ -1252,7 +1195,7 @@ mod tests { { let good_block = BeaconBlock::Capella(BeaconBlockCapella { slot: capella_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have an Capella block with a epoch lower than the fork epoch. let bad_block = { @@ -1274,7 +1217,7 @@ mod tests { { let good_block = BeaconBlock::Deneb(BeaconBlockDeneb { slot: deneb_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have a Deneb block with a epoch lower than the fork epoch. let bad_block = { @@ -1296,7 +1239,7 @@ mod tests { { let good_block = BeaconBlock::Electra(BeaconBlockElectra { slot: electra_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have an Electra block with a epoch lower than the fork epoch. let bad_block = { @@ -1318,7 +1261,7 @@ mod tests { { let good_block = BeaconBlock::Fulu(BeaconBlockFulu { slot: fulu_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); assert_eq!( @@ -1332,7 +1275,7 @@ mod tests { { let good_block = BeaconBlock::Gloas(BeaconBlockGloas { slot: gloas_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have a Fulu block with a epoch lower than the fork epoch. let _bad_block = { @@ -1357,7 +1300,7 @@ mod tests { { let good_block = BeaconBlock::Heze(BeaconBlockHeze { slot: heze_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); let _bad_block = { let mut bad = good_block.clone(); diff --git a/consensus/types/src/block/beacon_block_body.rs b/consensus/types/src/block/beacon_block_body.rs index 93c82c5cfc..2a86dfd919 100644 --- a/consensus/types/src/block/beacon_block_body.rs +++ b/consensus/types/src/block/beacon_block_body.rs @@ -9,7 +9,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -38,7 +37,6 @@ use crate::{ }, state::BeaconStateError, sync_committee::SyncAggregate, - test_utils::TestRandom, }; /// The number of leaves (including padding) on the `BeaconBlockBody` Merkle tree. @@ -65,7 +63,6 @@ pub const BLOB_KZG_COMMITMENTS_INDEX: usize = 11; Encode, Decode, TreeHash, - TestRandom, Educe, ), educe(PartialEq, Hash(bound(E: EthSpec, Payload: AbstractExecPayload))), diff --git a/consensus/types/src/block/beacon_block_header.rs b/consensus/types/src/block/beacon_block_header.rs index 06e1023d91..3d5b02d6b6 100644 --- a/consensus/types/src/block/beacon_block_header.rs +++ b/consensus/types/src/block/beacon_block_header.rs @@ -2,7 +2,6 @@ use bls::SecretKey; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -10,16 +9,13 @@ use crate::{ block::SignedBeaconBlockHeader, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot, Slot}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; /// A header of a `BeaconBlock`. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct BeaconBlockHeader { pub slot: Slot, diff --git a/consensus/types/src/block/signed_beacon_block.rs b/consensus/types/src/block/signed_beacon_block.rs index a3bf50ea78..367658b544 100644 --- a/consensus/types/src/block/signed_beacon_block.rs +++ b/consensus/types/src/block/signed_beacon_block.rs @@ -8,7 +8,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tracing::instrument; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -33,7 +32,6 @@ use crate::{ fork::{Fork, ForkName, ForkVersionDecode, InconsistentFork, map_fork_name}, kzg_ext::format_kzg_commitments, state::BeaconStateError, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] @@ -77,7 +75,6 @@ impl From for Hash256 { Decode, TreeHash, Educe, - TestRandom ), educe(PartialEq, Hash(bound(E: EthSpec))), serde(bound = "E: EthSpec, Payload: AbstractExecPayload"), diff --git a/consensus/types/src/block/signed_beacon_block_header.rs b/consensus/types/src/block/signed_beacon_block_header.rs index 2fcd8a705f..6e81850a3f 100644 --- a/consensus/types/src/block/signed_beacon_block_header.rs +++ b/consensus/types/src/block/signed_beacon_block_header.rs @@ -2,23 +2,19 @@ use bls::{PublicKey, Signature}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ block::BeaconBlockHeader, core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; /// A signed header of a `BeaconBlock`. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SignedBeaconBlockHeader { pub message: BeaconBlockHeader, diff --git a/consensus/types/src/builder/builder.rs b/consensus/types/src/builder/builder.rs index 2bd50f42cc..18961c5969 100644 --- a/consensus/types/src/builder/builder.rs +++ b/consensus/types/src/builder/builder.rs @@ -1,18 +1,14 @@ -use crate::test_utils::TestRandom; use crate::{Address, Epoch, ForkName}; use bls::PublicKeyBytes; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; pub type BuilderIndex = u64; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash, -)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct Builder { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/builder/builder_bid.rs b/consensus/types/src/builder/builder_bid.rs index 6c2544c7dc..3fbcf36cfd 100644 --- a/consensus/types/src/builder/builder_bid.rs +++ b/consensus/types/src/builder/builder_bid.rs @@ -5,7 +5,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz::Decode; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -18,7 +17,6 @@ use crate::{ }, fork::{ForkName, ForkVersionDecode}, kzg_ext::KzgCommitments, - test_utils::TestRandom, }; #[superstruct( @@ -33,9 +31,13 @@ use crate::{ TreeHash, Decode, Clone, - TestRandom ), - serde(bound = "E: EthSpec", deny_unknown_fields) + serde(bound = "E: EthSpec", deny_unknown_fields), + cfg_attr( + feature = "arbitrary", + derive(arbitrary::Arbitrary), + arbitrary(bound = "E: EthSpec"), + ), ), map_ref_into(ExecutionPayloadHeaderRef), map_ref_mut_into(ExecutionPayloadHeaderRefMut) diff --git a/consensus/types/src/builder/builder_pending_payment.rs b/consensus/types/src/builder/builder_pending_payment.rs index 0f1b68ad97..61c76dfc15 100644 --- a/consensus/types/src/builder/builder_pending_payment.rs +++ b/consensus/types/src/builder/builder_pending_payment.rs @@ -1,24 +1,11 @@ -use crate::test_utils::TestRandom; use crate::{BuilderPendingWithdrawal, ForkName}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[derive( - Debug, - PartialEq, - Eq, - Hash, - Clone, - Default, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, + Debug, PartialEq, Eq, Hash, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, )] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/builder/builder_pending_withdrawal.rs b/consensus/types/src/builder/builder_pending_withdrawal.rs index dbbb029a5d..4b1003a28b 100644 --- a/consensus/types/src/builder/builder_pending_withdrawal.rs +++ b/consensus/types/src/builder/builder_pending_withdrawal.rs @@ -1,24 +1,11 @@ -use crate::test_utils::TestRandom; use crate::{Address, ForkName}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[derive( - Debug, - PartialEq, - Eq, - Hash, - Clone, - Default, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, + Debug, PartialEq, Eq, Hash, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, )] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/builder/proposer_preferences.rs b/consensus/types/src/builder/proposer_preferences.rs index 38f1b36be3..e3773e333d 100644 --- a/consensus/types/src/builder/proposer_preferences.rs +++ b/consensus/types/src/builder/proposer_preferences.rs @@ -1,16 +1,12 @@ -use crate::test_utils::TestRandom; use crate::{Address, ForkName, Hash256, SignedRoot, Slot}; use bls::Signature; use context_deserialize::context_deserialize; use educe::Educe; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive( - Default, Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe, TestRandom, -)] +#[derive(Default, Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[educe(PartialEq, Hash)] #[context_deserialize(ForkName)] @@ -25,7 +21,7 @@ pub struct ProposerPreferences { impl SignedRoot for ProposerPreferences {} -#[derive(TestRandom, TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] +#[derive(TreeHash, Debug, Clone, Encode, Decode, Serialize, Deserialize, Educe)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[educe(PartialEq, Hash)] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/consolidation/consolidation_request.rs b/consensus/types/src/consolidation/consolidation_request.rs index 3f09517a90..b24d0bee66 100644 --- a/consensus/types/src/consolidation/consolidation_request.rs +++ b/consensus/types/src/consolidation/consolidation_request.rs @@ -3,19 +3,15 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Address, SignedRoot}, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct ConsolidationRequest { pub source_address: Address, diff --git a/consensus/types/src/consolidation/pending_consolidation.rs b/consensus/types/src/consolidation/pending_consolidation.rs index fcd76e43b6..df71316f07 100644 --- a/consensus/types/src/consolidation/pending_consolidation.rs +++ b/consensus/types/src/consolidation/pending_consolidation.rs @@ -1,15 +1,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{fork::ForkName, test_utils::TestRandom}; +use crate::fork::ForkName; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct PendingConsolidation { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/core/enr_fork_id.rs b/consensus/types/src/core/enr_fork_id.rs index c3b400cd13..f4ad072175 100644 --- a/consensus/types/src/core/enr_fork_id.rs +++ b/consensus/types/src/core/enr_fork_id.rs @@ -1,18 +1,15 @@ use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Epoch, test_utils::TestRandom}; +use crate::core::Epoch; /// Specifies a fork which allows nodes to identify each other on the network. This fork is used in /// a nodes local ENR. /// /// Spec v0.11 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct EnrForkId { /// Fork digest of the current fork computed from [`ChainSpec::compute_fork_digest`]. #[serde(with = "serde_utils::bytes_4_hex")] diff --git a/consensus/types/src/core/execution_block_hash.rs b/consensus/types/src/core/execution_block_hash.rs index 71e63727ee..41e00115c6 100644 --- a/consensus/types/src/core/execution_block_hash.rs +++ b/consensus/types/src/core/execution_block_hash.rs @@ -1,14 +1,10 @@ use std::fmt; use fixed_bytes::FixedBytesExtended; -use rand::RngCore; use serde::{Deserialize, Serialize}; use ssz::{Decode, DecodeError, Encode}; -use crate::{ - core::{Hash256, Hash256Ext}, - test_utils::TestRandom, -}; +use crate::core::{Hash256, Hash256Ext}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[derive(Default, Clone, Copy, Serialize, Deserialize, Eq, PartialEq, Hash)] @@ -95,12 +91,6 @@ impl tree_hash::TreeHash for ExecutionBlockHash { } } -impl TestRandom for ExecutionBlockHash { - fn random_for_test(rng: &mut impl RngCore) -> Self { - Self(Hash256::random_for_test(rng)) - } -} - impl std::str::FromStr for ExecutionBlockHash { type Err = String; diff --git a/consensus/types/src/core/graffiti.rs b/consensus/types/src/core/graffiti.rs index d0e0e1b1a8..02b805a2a8 100644 --- a/consensus/types/src/core/graffiti.rs +++ b/consensus/types/src/core/graffiti.rs @@ -1,13 +1,10 @@ use std::{fmt, str::FromStr}; -use rand::RngCore; use regex::bytes::Regex; use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error}; use ssz::{Decode, DecodeError, Encode}; use tree_hash::{PackedEncoding, TreeHash}; -use crate::{core::Hash256, test_utils::TestRandom}; - pub const GRAFFITI_BYTES_LEN: usize = 32; /// The 32-byte `graffiti` field on a beacon block. @@ -180,9 +177,3 @@ impl TreeHash for Graffiti { self.0.tree_hash_root() } } - -impl TestRandom for Graffiti { - fn random_for_test(rng: &mut impl RngCore) -> Self { - Self::from(Hash256::random_for_test(rng).0) - } -} diff --git a/consensus/types/src/core/signing_data.rs b/consensus/types/src/core/signing_data.rs index 907f03fac7..e698b4fdbe 100644 --- a/consensus/types/src/core/signing_data.rs +++ b/consensus/types/src/core/signing_data.rs @@ -1,14 +1,13 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; -use crate::{core::Hash256, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Hash256, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SigningData { pub object_root: Hash256, diff --git a/consensus/types/src/core/slot_epoch.rs b/consensus/types/src/core/slot_epoch.rs index 837391546c..177161a2ab 100644 --- a/consensus/types/src/core/slot_epoch.rs +++ b/consensus/types/src/core/slot_epoch.rs @@ -12,15 +12,11 @@ use std::{fmt, hash::Hash}; -use rand::RngCore; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use ssz::{Decode, DecodeError, Encode}; -use crate::{ - core::{ChainSpec, SignedRoot}, - test_utils::TestRandom, -}; +use crate::core::{ChainSpec, SignedRoot}; #[cfg(feature = "saturating-arith")] use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssign}; diff --git a/consensus/types/src/core/slot_epoch_macros.rs b/consensus/types/src/core/slot_epoch_macros.rs index 1b0c3bcfc1..09e0f1d120 100644 --- a/consensus/types/src/core/slot_epoch_macros.rs +++ b/consensus/types/src/core/slot_epoch_macros.rs @@ -293,12 +293,6 @@ macro_rules! impl_ssz { } impl SignedRoot for $type {} - - impl TestRandom for $type { - fn random_for_test(rng: &mut impl RngCore) -> Self { - $type::from(u64::random_for_test(rng)) - } - } }; } diff --git a/consensus/types/src/data/blob_sidecar.rs b/consensus/types/src/data/blob_sidecar.rs index 70b95615e5..4020278d64 100644 --- a/consensus/types/src/data/blob_sidecar.rs +++ b/consensus/types/src/data/blob_sidecar.rs @@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, RuntimeFixedVector, RuntimeVariableList, VariableList}; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -25,7 +24,6 @@ use crate::{ fork::ForkName, kzg_ext::KzgProofs, state::BeaconStateError, - test_utils::TestRandom, }; /// Container of the data that identifies an individual blob. @@ -55,7 +53,7 @@ impl Ord for BlobIdentifier { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Educe)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, Educe)] #[context_deserialize(ForkName)] #[serde(bound = "E: EthSpec")] #[educe(PartialEq, Eq, Hash(bound(E: EthSpec)))] diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index d734280183..5da73374fa 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -12,7 +12,6 @@ use ssz_derive::{Decode, Encode}; use ssz_types::Error as SszError; use ssz_types::{FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -26,7 +25,6 @@ use crate::{ fork::ForkName, kzg_ext::{KzgCommitments, KzgError}, state::BeaconStateError, - test_utils::TestRandom, }; pub type ColumnIndex = u64; @@ -53,7 +51,6 @@ pub type DataColumnSidecarList = Vec>>; Deserialize, Decode, Encode, - TestRandom, Educe, TreeHash, ), diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index df65be1ae3..c0e713b4b8 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -5,7 +5,6 @@ use crate::{ execution::AbstractExecPayload, kzg_ext::KzgCommitments, state::BeaconStateError, - test_utils::TestRandom, }; use educe::Educe; use kzg::KzgProof; @@ -14,7 +13,6 @@ use ssz::BitList; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, ListEncodedOption, VariableList}; use std::fmt::Display; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -134,7 +132,7 @@ impl PartialDataColumnSidecar { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, Encode, Decode, TreeHash, TestRandom, Educe)] +#[derive(Debug, Clone, Encode, Decode, TreeHash, Educe)] #[educe(PartialEq, Eq, Hash(bound = "E: EthSpec"))] pub struct PartialDataColumnHeader { pub kzg_commitments: KzgCommitments, diff --git a/consensus/types/src/deposit/deposit.rs b/consensus/types/src/deposit/deposit.rs index 0b08bd6509..22dbdfbb71 100644 --- a/consensus/types/src/deposit/deposit.rs +++ b/consensus/types/src/deposit/deposit.rs @@ -2,11 +2,10 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use typenum::U33; -use crate::{core::Hash256, deposit::DepositData, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Hash256, deposit::DepositData, fork::ForkName}; pub const DEPOSIT_TREE_DEPTH: usize = 32; @@ -14,9 +13,7 @@ pub const DEPOSIT_TREE_DEPTH: usize = 32; /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct Deposit { pub proof: FixedVector, diff --git a/consensus/types/src/deposit/deposit_data.rs b/consensus/types/src/deposit/deposit_data.rs index 51697f5d1a..bd39643ebd 100644 --- a/consensus/types/src/deposit/deposit_data.rs +++ b/consensus/types/src/deposit/deposit_data.rs @@ -2,23 +2,19 @@ use bls::{PublicKeyBytes, SecretKey, SignatureBytes}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, Hash256, SignedRoot}, deposit::DepositMessage, fork::ForkName, - test_utils::TestRandom, }; /// The data supplied by the user to the deposit contract. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct DepositData { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/deposit/deposit_message.rs b/consensus/types/src/deposit/deposit_message.rs index 4495a5c023..9cb282e2d9 100644 --- a/consensus/types/src/deposit/deposit_message.rs +++ b/consensus/types/src/deposit/deposit_message.rs @@ -2,20 +2,18 @@ use bls::PublicKeyBytes; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Hash256, SignedRoot}, fork::ForkName, - test_utils::TestRandom, }; /// The data supplied by the user to the deposit contract. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct DepositMessage { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/deposit/deposit_request.rs b/consensus/types/src/deposit/deposit_request.rs index 8d3c6e88ba..b17450a851 100644 --- a/consensus/types/src/deposit/deposit_request.rs +++ b/consensus/types/src/deposit/deposit_request.rs @@ -3,15 +3,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Hash256, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Hash256, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct DepositRequest { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/deposit/deposit_tree_snapshot.rs b/consensus/types/src/deposit/deposit_tree_snapshot.rs index 24f41397a0..979f266d1b 100644 --- a/consensus/types/src/deposit/deposit_tree_snapshot.rs +++ b/consensus/types/src/deposit/deposit_tree_snapshot.rs @@ -3,11 +3,10 @@ use fixed_bytes::FixedBytesExtended; use int_to_bytes::int_to_bytes32; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; -use crate::{core::Hash256, deposit::DEPOSIT_TREE_DEPTH, test_utils::TestRandom}; +use crate::{core::Hash256, deposit::DEPOSIT_TREE_DEPTH}; -#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq, TestRandom)] +#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq)] pub struct FinalizedExecutionBlock { pub deposit_root: Hash256, pub deposit_count: u64, @@ -26,7 +25,8 @@ impl From<&DepositTreeSnapshot> for FinalizedExecutionBlock { } } -#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq, TestRandom)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Encode, Decode, Deserialize, Serialize, Clone, Debug, PartialEq)] pub struct DepositTreeSnapshot { pub finalized: Vec, pub deposit_root: Hash256, diff --git a/consensus/types/src/deposit/pending_deposit.rs b/consensus/types/src/deposit/pending_deposit.rs index 4c039af39c..ed0f866ecc 100644 --- a/consensus/types/src/deposit/pending_deposit.rs +++ b/consensus/types/src/deposit/pending_deposit.rs @@ -2,19 +2,15 @@ use bls::{PublicKeyBytes, SignatureBytes}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Hash256, Slot}, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct PendingDeposit { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/execution/bls_to_execution_change.rs b/consensus/types/src/execution/bls_to_execution_change.rs index de14f1b4c5..48a089bc63 100644 --- a/consensus/types/src/execution/bls_to_execution_change.rs +++ b/consensus/types/src/execution/bls_to_execution_change.rs @@ -2,20 +2,16 @@ use bls::{PublicKeyBytes, SecretKey}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Address, ChainSpec, Domain, Hash256, SignedRoot}, execution::SignedBlsToExecutionChange, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct BlsToExecutionChange { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/execution/eth1_data.rs b/consensus/types/src/execution/eth1_data.rs index 89a4e634a6..f2a00ca87b 100644 --- a/consensus/types/src/execution/eth1_data.rs +++ b/consensus/types/src/execution/eth1_data.rs @@ -1,28 +1,16 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Hash256, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Hash256, fork::ForkName}; /// Contains data obtained from the Eth1 chain. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[derive( - Debug, - PartialEq, - Clone, - Default, - Eq, - Hash, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, + Debug, PartialEq, Clone, Default, Eq, Hash, Serialize, Deserialize, Encode, Decode, TreeHash, )] #[context_deserialize(ForkName)] pub struct Eth1Data { diff --git a/consensus/types/src/execution/execution_payload.rs b/consensus/types/src/execution/execution_payload.rs index be228485c5..f5509ee06d 100644 --- a/consensus/types/src/execution/execution_payload.rs +++ b/consensus/types/src/execution/execution_payload.rs @@ -6,14 +6,12 @@ use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Address, EthSpec, ExecutionBlockHash, Hash256, Slot}, fork::{ForkName, ForkVersionDecode}, state::BeaconStateError, - test_utils::TestRandom, withdrawal::Withdrawals, }; @@ -35,7 +33,6 @@ pub type Transactions = VariableList< Encode, Decode, TreeHash, - TestRandom, Educe, ), context_deserialize(ForkName), diff --git a/consensus/types/src/execution/execution_payload_bid.rs b/consensus/types/src/execution/execution_payload_bid.rs index da2071680b..fb4e7e7f31 100644 --- a/consensus/types/src/execution/execution_payload_bid.rs +++ b/consensus/types/src/execution/execution_payload_bid.rs @@ -1,6 +1,5 @@ use crate::kzg_ext::KzgCommitments; use crate::state::BeaconStateError; -use crate::test_utils::TestRandom; use crate::{Address, EthSpec, ExecutionBlockHash, ForkName, Hash256, SignedRoot, Slot}; use context_deserialize::context_deserialize; use educe::Educe; @@ -8,7 +7,6 @@ use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::BitVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[superstruct( @@ -23,7 +21,6 @@ use tree_hash_derive::TreeHash; Encode, Decode, TreeHash, - TestRandom, Educe, ), educe(PartialEq, Hash(bound(E: EthSpec))), diff --git a/consensus/types/src/execution/execution_payload_envelope.rs b/consensus/types/src/execution/execution_payload_envelope.rs index a6d123bd21..87a0ea7a63 100644 --- a/consensus/types/src/execution/execution_payload_envelope.rs +++ b/consensus/types/src/execution/execution_payload_envelope.rs @@ -1,5 +1,4 @@ use crate::execution::{ExecutionPayloadGloas, ExecutionRequests}; -use crate::test_utils::TestRandom; use crate::{EthSpec, ForkName, Hash256, SignedRoot, Slot}; use context_deserialize::context_deserialize; use educe::Educe; @@ -7,10 +6,14 @@ use fixed_bytes::FixedBytesExtended; use serde::{Deserialize, Serialize}; use ssz::{BYTES_PER_LENGTH_OFFSET, Encode as SszEncode}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(Debug, Clone, Serialize, Encode, Decode, Deserialize, TestRandom, TreeHash, Educe)] +#[cfg_attr( + feature = "arbitrary", + derive(arbitrary::Arbitrary), + arbitrary(bound = "E: EthSpec") +)] +#[derive(Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe)] #[educe(PartialEq, Hash(bound(E: EthSpec)))] #[context_deserialize(ForkName)] #[serde(bound = "E: EthSpec")] diff --git a/consensus/types/src/execution/execution_payload_header.rs b/consensus/types/src/execution/execution_payload_header.rs index 8f46f6d323..110644d949 100644 --- a/consensus/types/src/execution/execution_payload_header.rs +++ b/consensus/types/src/execution/execution_payload_header.rs @@ -6,7 +6,6 @@ use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -20,7 +19,6 @@ use crate::{ fork::ForkName, map_execution_payload_ref_into_execution_payload_header, state::BeaconStateError, - test_utils::TestRandom, }; #[superstruct( @@ -35,7 +33,6 @@ use crate::{ Encode, Decode, TreeHash, - TestRandom, Educe, ), educe(PartialEq, Hash(bound(E: EthSpec))), diff --git a/consensus/types/src/execution/execution_requests.rs b/consensus/types/src/execution/execution_requests.rs index 92d717778e..218b7edc17 100644 --- a/consensus/types/src/execution/execution_requests.rs +++ b/consensus/types/src/execution/execution_requests.rs @@ -6,7 +6,6 @@ use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -14,7 +13,6 @@ use crate::{ core::{EthSpec, Hash256}, deposit::DepositRequest, fork::ForkName, - test_utils::TestRandom, withdrawal::WithdrawalRequest, }; @@ -30,9 +28,7 @@ pub type ConsolidationRequests = derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive( - Debug, Educe, Default, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, Educe, Default, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[educe(PartialEq, Eq, Hash(bound(E: EthSpec)))] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/execution/inclusion_list.rs b/consensus/types/src/execution/inclusion_list.rs index 0b61d89090..8b865dc415 100644 --- a/consensus/types/src/execution/inclusion_list.rs +++ b/consensus/types/src/execution/inclusion_list.rs @@ -1,16 +1,15 @@ -use crate::test_utils::TestRandom; use crate::{EthSpec, Hash256, SignedRoot, Slot, Transactions}; use bls::{PublicKeyBytes, Signature}; use educe::Educe; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; pub type InclusionListCommittee = FixedVector::InclusionListCommitteeSize>; -#[derive(Debug, Clone, Serialize, Deserialize, Encode, Educe, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Educe, Decode, TreeHash)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec"))] #[serde(bound = "E: EthSpec")] #[educe(PartialEq, Eq, Hash(bound(E: EthSpec)))] pub struct InclusionList { diff --git a/consensus/types/src/execution/payload.rs b/consensus/types/src/execution/payload.rs index 8b5b5fadaa..5e8d3b14cb 100644 --- a/consensus/types/src/execution/payload.rs +++ b/consensus/types/src/execution/payload.rs @@ -6,7 +6,6 @@ use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; use std::{borrow::Cow, fmt::Debug, hash::Hash}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -22,7 +21,6 @@ use crate::{ fork::ForkName, map_execution_payload_into_blinded_payload, map_execution_payload_into_full_payload, state::BeaconStateError, - test_utils::TestRandom, }; #[derive(Debug, PartialEq)] @@ -71,7 +69,6 @@ pub trait OwnedExecPayload: + DeserializeOwned + Encode + Decode - + TestRandom + for<'a> arbitrary::Arbitrary<'a> + 'static { @@ -84,7 +81,6 @@ impl OwnedExecPayload for P where + DeserializeOwned + Encode + Decode - + TestRandom + for<'a> arbitrary::Arbitrary<'a> + 'static { @@ -93,19 +89,12 @@ impl OwnedExecPayload for P where /// `ExecPayload` functionality the requires ownership. #[cfg(not(feature = "arbitrary"))] pub trait OwnedExecPayload: - ExecPayload + Default + Serialize + DeserializeOwned + Encode + Decode + TestRandom + 'static + ExecPayload + Default + Serialize + DeserializeOwned + Encode + Decode + 'static { } #[cfg(not(feature = "arbitrary"))] impl OwnedExecPayload for P where - P: ExecPayload - + Default - + Serialize - + DeserializeOwned - + Encode - + Decode - + TestRandom - + 'static + P: ExecPayload + Default + Serialize + DeserializeOwned + Encode + Decode + 'static { } @@ -173,7 +162,6 @@ pub trait AbstractExecPayload: Deserialize, Encode, Decode, - TestRandom, TreeHash, Educe, ), @@ -547,7 +535,6 @@ impl TryFrom> for FullPayload { Deserialize, Encode, Decode, - TestRandom, TreeHash, Educe, ), diff --git a/consensus/types/src/execution/signed_bls_to_execution_change.rs b/consensus/types/src/execution/signed_bls_to_execution_change.rs index 535960fb3f..0ed7de5350 100644 --- a/consensus/types/src/execution/signed_bls_to_execution_change.rs +++ b/consensus/types/src/execution/signed_bls_to_execution_change.rs @@ -2,15 +2,12 @@ use bls::Signature; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{execution::BlsToExecutionChange, fork::ForkName, test_utils::TestRandom}; +use crate::{execution::BlsToExecutionChange, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SignedBlsToExecutionChange { pub message: BlsToExecutionChange, diff --git a/consensus/types/src/execution/signed_execution_payload_bid.rs b/consensus/types/src/execution/signed_execution_payload_bid.rs index fe9b03d7ed..89620f266b 100644 --- a/consensus/types/src/execution/signed_execution_payload_bid.rs +++ b/consensus/types/src/execution/signed_execution_payload_bid.rs @@ -1,6 +1,5 @@ use crate::execution::{ExecutionPayloadBidGloas, ExecutionPayloadBidHeze, ExecutionPayloadBidRef}; use crate::state::BeaconStateError; -use crate::test_utils::TestRandom; use crate::{EthSpec, ForkName}; use bls::Signature; use context_deserialize::{ContextDeserialize, context_deserialize}; @@ -8,7 +7,6 @@ use educe::Educe; use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[superstruct( @@ -21,7 +19,6 @@ use tree_hash_derive::TreeHash; Deserialize, Encode, Decode, - TestRandom, TreeHash, Educe, ), diff --git a/consensus/types/src/execution/signed_execution_payload_envelope.rs b/consensus/types/src/execution/signed_execution_payload_envelope.rs index 522c8b3f54..316a580476 100644 --- a/consensus/types/src/execution/signed_execution_payload_envelope.rs +++ b/consensus/types/src/execution/signed_execution_payload_envelope.rs @@ -1,4 +1,3 @@ -use crate::test_utils::TestRandom; use crate::{ BeaconState, BeaconStateError, ChainSpec, Domain, Epoch, EthSpec, ExecutionBlockHash, ExecutionPayloadEnvelope, Fork, ForkName, Hash256, SignedRoot, Slot, @@ -10,10 +9,14 @@ use educe::Educe; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -#[derive(Debug, Clone, Serialize, Encode, Decode, Deserialize, TestRandom, TreeHash, Educe)] +#[cfg_attr( + feature = "arbitrary", + derive(arbitrary::Arbitrary), + arbitrary(bound = "E: EthSpec") +)] +#[derive(Debug, Clone, Serialize, Encode, Decode, Deserialize, TreeHash, Educe)] #[educe(PartialEq, Hash(bound(E: EthSpec)))] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/exit/signed_voluntary_exit.rs b/consensus/types/src/exit/signed_voluntary_exit.rs index b49401a721..072541e766 100644 --- a/consensus/types/src/exit/signed_voluntary_exit.rs +++ b/consensus/types/src/exit/signed_voluntary_exit.rs @@ -2,18 +2,15 @@ use bls::Signature; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{exit::VoluntaryExit, fork::ForkName, test_utils::TestRandom}; +use crate::{exit::VoluntaryExit, fork::ForkName}; /// An exit voluntarily submitted a validator who wishes to withdraw. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SignedVoluntaryExit { pub message: VoluntaryExit, diff --git a/consensus/types/src/exit/voluntary_exit.rs b/consensus/types/src/exit/voluntary_exit.rs index 30c6a97c4d..fac0a4ad0b 100644 --- a/consensus/types/src/exit/voluntary_exit.rs +++ b/consensus/types/src/exit/voluntary_exit.rs @@ -2,23 +2,19 @@ use bls::SecretKey; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, Domain, Epoch, Hash256, SignedRoot}, exit::SignedVoluntaryExit, fork::ForkName, - test_utils::TestRandom, }; /// An exit voluntarily submitted a validator who wishes to withdraw. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct VoluntaryExit { /// Earliest epoch when voluntary exit can be processed. diff --git a/consensus/types/src/fork/fork.rs b/consensus/types/src/fork/fork.rs index 371b11e05c..675d61cc52 100644 --- a/consensus/types/src/fork/fork.rs +++ b/consensus/types/src/fork/fork.rs @@ -1,27 +1,16 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Epoch, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Epoch, fork::ForkName}; /// Specifies a fork of the `BeaconChain`, to prevent replay attacks. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[derive( - Debug, - Clone, - Copy, - PartialEq, - Default, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, + Debug, Clone, Copy, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, )] #[context_deserialize(ForkName)] pub struct Fork { diff --git a/consensus/types/src/fork/fork_data.rs b/consensus/types/src/fork/fork_data.rs index 1b9c8bad9f..5f98132f62 100644 --- a/consensus/types/src/fork/fork_data.rs +++ b/consensus/types/src/fork/fork_data.rs @@ -1,22 +1,18 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Hash256, SignedRoot}, fork::ForkName, - test_utils::TestRandom, }; /// Specifies a fork of the `BeaconChain`, to prevent replay attacks. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct ForkData { #[serde(with = "serde_utils::bytes_4_hex")] diff --git a/consensus/types/src/light_client/light_client_bootstrap.rs b/consensus/types/src/light_client/light_client_bootstrap.rs index a920aaaa33..c1847e2ba0 100644 --- a/consensus/types/src/light_client/light_client_bootstrap.rs +++ b/consensus/types/src/light_client/light_client_bootstrap.rs @@ -7,7 +7,6 @@ use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -21,7 +20,6 @@ use crate::{ }, state::BeaconState, sync_committee::SyncCommittee, - test_utils::TestRandom, }; /// A LightClientBootstrap is the initializer we send over to light_client nodes @@ -29,17 +27,7 @@ use crate::{ #[superstruct( variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( - derive( - Debug, - Clone, - Serialize, - Deserialize, - Educe, - Decode, - Encode, - TestRandom, - TreeHash, - ), + derive(Debug, Clone, Serialize, Deserialize, Educe, Decode, Encode, TreeHash,), educe(PartialEq), serde(bound = "E: EthSpec", deny_unknown_fields), cfg_attr( diff --git a/consensus/types/src/light_client/light_client_finality_update.rs b/consensus/types/src/light_client/light_client_finality_update.rs index 50a38b8d90..a173fa0ee6 100644 --- a/consensus/types/src/light_client/light_client_finality_update.rs +++ b/consensus/types/src/light_client/light_client_finality_update.rs @@ -6,7 +6,6 @@ use ssz_derive::Decode; use ssz_derive::Encode; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -19,23 +18,12 @@ use crate::{ LightClientHeaderElectra, LightClientHeaderFulu, }, sync_committee::SyncAggregate, - test_utils::TestRandom, }; #[superstruct( variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( - derive( - Debug, - Clone, - Serialize, - Deserialize, - Educe, - Decode, - Encode, - TestRandom, - TreeHash, - ), + derive(Debug, Clone, Serialize, Deserialize, Educe, Decode, Encode, TreeHash,), educe(PartialEq), serde(bound = "E: EthSpec", deny_unknown_fields), cfg_attr( diff --git a/consensus/types/src/light_client/light_client_header.rs b/consensus/types/src/light_client/light_client_header.rs index 25191e1f66..84b6cab0ac 100644 --- a/consensus/types/src/light_client/light_client_header.rs +++ b/consensus/types/src/light_client/light_client_header.rs @@ -7,7 +7,6 @@ use ssz::Decode; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -19,23 +18,12 @@ use crate::{ }, fork::ForkName, light_client::{ExecutionPayloadProofLen, LightClientError, consts::EXECUTION_PAYLOAD_INDEX}, - test_utils::TestRandom, }; #[superstruct( variants(Altair, Capella, Deneb, Electra, Fulu,), variant_attributes( - derive( - Debug, - Clone, - Serialize, - Deserialize, - Educe, - Decode, - Encode, - TestRandom, - TreeHash, - ), + derive(Debug, Clone, Serialize, Deserialize, Educe, Decode, Encode, TreeHash,), educe(PartialEq), serde(bound = "E: EthSpec", deny_unknown_fields), cfg_attr( diff --git a/consensus/types/src/light_client/light_client_optimistic_update.rs b/consensus/types/src/light_client/light_client_optimistic_update.rs index 1459759f81..f4f7370071 100644 --- a/consensus/types/src/light_client/light_client_optimistic_update.rs +++ b/consensus/types/src/light_client/light_client_optimistic_update.rs @@ -4,7 +4,6 @@ use serde::{Deserialize, Deserializer, Serialize}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash::Hash256; use tree_hash_derive::TreeHash; @@ -17,7 +16,6 @@ use crate::{ LightClientHeaderDeneb, LightClientHeaderElectra, LightClientHeaderFulu, }, sync_committee::SyncAggregate, - test_utils::TestRandom, }; /// A LightClientOptimisticUpdate is the update we send on each slot, @@ -25,17 +23,7 @@ use crate::{ #[superstruct( variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( - derive( - Debug, - Clone, - Serialize, - Deserialize, - Educe, - Decode, - Encode, - TestRandom, - TreeHash, - ), + derive(Debug, Clone, Serialize, Deserialize, Educe, Decode, Encode, TreeHash,), educe(PartialEq), serde(bound = "E: EthSpec", deny_unknown_fields), cfg_attr( diff --git a/consensus/types/src/light_client/light_client_update.rs b/consensus/types/src/light_client/light_client_update.rs index 2b2b1c1896..753ab10e7f 100644 --- a/consensus/types/src/light_client/light_client_update.rs +++ b/consensus/types/src/light_client/light_client_update.rs @@ -10,7 +10,6 @@ use ssz_derive::Decode; use ssz_derive::Encode; use ssz_types::FixedVector; use superstruct::superstruct; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use typenum::{U4, U5, U6, U7}; @@ -23,7 +22,6 @@ use crate::{ LightClientHeaderDeneb, LightClientHeaderElectra, LightClientHeaderFulu, }, sync_committee::{SyncAggregate, SyncCommittee}, - test_utils::TestRandom, }; pub type FinalizedRootProofLen = U6; @@ -47,17 +45,7 @@ type NextSyncCommitteeBranchElectra = FixedVector AttesterSlashing { } } -impl TestRandom for AttesterSlashing { - fn random_for_test(rng: &mut impl RngCore) -> Self { - if rng.random_bool(0.5) { - AttesterSlashing::Base(AttesterSlashingBase::random_for_test(rng)) - } else { - AttesterSlashing::Electra(AttesterSlashingElectra::random_for_test(rng)) - } - } -} - impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for Vec> { fn context_deserialize(deserializer: D, context: ForkName) -> Result where diff --git a/consensus/types/src/slashing/proposer_slashing.rs b/consensus/types/src/slashing/proposer_slashing.rs index 697bd1a9aa..b5ffbc562c 100644 --- a/consensus/types/src/slashing/proposer_slashing.rs +++ b/consensus/types/src/slashing/proposer_slashing.rs @@ -1,18 +1,15 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{block::SignedBeaconBlockHeader, fork::ForkName, test_utils::TestRandom}; +use crate::{block::SignedBeaconBlockHeader, fork::ForkName}; /// Two conflicting proposals from the same proposer (validator). /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct ProposerSlashing { pub signed_header_1: SignedBeaconBlockHeader, diff --git a/consensus/types/src/state/beacon_state.rs b/consensus/types/src/state/beacon_state.rs index d31f8ed372..4b8fec928b 100644 --- a/consensus/types/src/state/beacon_state.rs +++ b/consensus/types/src/state/beacon_state.rs @@ -17,7 +17,6 @@ use ssz_types::{BitVector, FixedVector}; use std::collections::BTreeMap; use superstruct::superstruct; use swap_or_not_shuffle::compute_shuffled_index; -use test_random_derive::TestRandom; use tracing::instrument; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -52,7 +51,6 @@ use crate::{ get_active_validator_indices, }, sync_committee::{SyncCommittee, SyncDuty}, - test_utils::TestRandom, validator::Validator, withdrawal::PendingPartialWithdrawal, }; @@ -291,7 +289,6 @@ impl From for Hash256 { Encode, Decode, TreeHash, - TestRandom, CompareFields, ), serde(bound = "E: EthSpec", deny_unknown_fields), @@ -471,21 +468,21 @@ where // History #[metastruct(exclude_from(tree_lists))] pub latest_block_header: BeaconBlockHeader, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[compare_fields(as_iter)] pub block_roots: Vector, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[compare_fields(as_iter)] pub state_roots: Vector, // Frozen in Capella, replaced by historical_summaries - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[compare_fields(as_iter)] pub historical_roots: List, // Ethereum 1.0 chain data #[metastruct(exclude_from(tree_lists))] pub eth1_data: Eth1Data, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub eth1_data_votes: List, #[superstruct(getter(copy))] #[metastruct(exclude_from(tree_lists))] @@ -494,42 +491,42 @@ where // Registry #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub validators: Validators, #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub balances: List, // Randomness - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub randao_mixes: Vector, // Slashings - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[serde(with = "ssz_types::serde_utils::quoted_u64_fixed_vec")] pub slashings: Vector, // Attestations (genesis fork only) #[superstruct(only(Base))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub previous_epoch_attestations: List, E::MaxPendingAttestations>, #[superstruct(only(Base))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub current_epoch_attestations: List, E::MaxPendingAttestations>, // Participation (Altair and later) #[compare_fields(as_iter)] #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas, Heze))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[compare_fields(as_iter)] pub previous_epoch_participation: List, #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas, Heze))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub current_epoch_participation: List, // Finality - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude_from(tree_lists))] pub justification_bits: BitVector, #[superstruct(getter(copy))] @@ -545,7 +542,7 @@ where // Inactivity #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas, Heze))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub inactivity_scores: List, // Light-client sync committees @@ -587,7 +584,7 @@ where )] #[metastruct(exclude_from(tree_lists))] pub latest_execution_payload_header: ExecutionPayloadHeaderFulu, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas, Heze))] #[metastruct(exclude_from(tree_lists))] pub latest_block_hash: ExecutionBlockHash, @@ -601,7 +598,7 @@ where pub next_withdrawal_validator_index: u64, // Deep history valid from Capella onwards. #[superstruct(only(Capella, Deneb, Electra, Fulu, Gloas, Heze))] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub historical_summaries: List, // Electra @@ -628,28 +625,28 @@ where #[metastruct(exclude_from(tree_lists))] pub earliest_consolidation_epoch: Epoch, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Electra, Fulu, Gloas, Heze))] pub pending_deposits: List, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Electra, Fulu, Gloas, Heze))] pub pending_partial_withdrawals: List, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Electra, Fulu, Gloas, Heze))] pub pending_consolidations: List, // Fulu #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Fulu, Gloas, Heze))] #[serde(with = "ssz_types::serde_utils::quoted_u64_fixed_vec")] pub proposer_lookahead: Vector, // Gloas #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas, Heze))] pub builders: List, @@ -658,18 +655,18 @@ where #[superstruct(only(Gloas, Heze), partial_getter(copy))] pub next_withdrawal_builder_index: BuilderIndex, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas, Heze))] #[metastruct(exclude_from(tree_lists))] pub execution_payload_availability: BitVector, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas, Heze))] pub builder_pending_payments: Vector, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas, Heze))] pub builder_pending_withdrawals: List, @@ -688,12 +685,12 @@ where pub latest_execution_payload_bid: ExecutionPayloadBidHeze, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas, Heze))] pub payload_expected_withdrawals: List, #[compare_fields(as_iter)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[superstruct(only(Gloas, Heze))] pub ptc_window: Vector, E::PtcWindowLength>, @@ -701,44 +698,44 @@ where #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub total_active_balance: Option<(Epoch, u64)>, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub committee_caches: [Arc; CACHED_EPOCHS], #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub progressive_balances_cache: ProgressiveBalancesCache, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub pubkey_cache: PubkeyCache, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub exit_cache: ExitCache, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub slashings_cache: SlashingsCache, /// Epoch cache of values that are useful for block processing that are static over an epoch. #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] #[metastruct(exclude)] pub epoch_cache: EpochCache, } diff --git a/consensus/types/src/state/historical_batch.rs b/consensus/types/src/state/historical_batch.rs index 0167d64f62..6e6e31eceb 100644 --- a/consensus/types/src/state/historical_batch.rs +++ b/consensus/types/src/state/historical_batch.rs @@ -2,13 +2,11 @@ use context_deserialize::context_deserialize; use milhouse::Vector; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{EthSpec, Hash256}, fork::ForkName, - test_utils::TestRandom, }; /// Historical block and state roots. @@ -19,12 +17,12 @@ use crate::{ derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct HistoricalBatch { - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub block_roots: Vector, - #[test_random(default)] + #[cfg_attr(feature = "arbitrary", arbitrary(default))] pub state_roots: Vector, } diff --git a/consensus/types/src/state/historical_summary.rs b/consensus/types/src/state/historical_summary.rs index f520e46483..80c65316c9 100644 --- a/consensus/types/src/state/historical_summary.rs +++ b/consensus/types/src/state/historical_summary.rs @@ -2,7 +2,6 @@ use compare_fields::CompareFields; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -10,7 +9,6 @@ use crate::{ core::{EthSpec, Hash256}, fork::ForkName, state::BeaconState, - test_utils::TestRandom, }; /// `HistoricalSummary` matches the components of the phase0 `HistoricalBatch` @@ -28,7 +26,6 @@ use crate::{ Encode, Decode, TreeHash, - TestRandom, CompareFields, Clone, Copy, diff --git a/consensus/types/src/sync_committee/contribution_and_proof.rs b/consensus/types/src/sync_committee/contribution_and_proof.rs index 2a344b89de..2b0a1c63f0 100644 --- a/consensus/types/src/sync_committee/contribution_and_proof.rs +++ b/consensus/types/src/sync_committee/contribution_and_proof.rs @@ -2,14 +2,12 @@ use bls::{SecretKey, Signature}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, sync_committee::{SyncCommitteeContribution, SyncSelectionProof}, - test_utils::TestRandom, }; /// A Validators aggregate sync committee contribution and selection proof. @@ -18,7 +16,7 @@ use crate::{ derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] pub struct ContributionAndProof { diff --git a/consensus/types/src/sync_committee/signed_contribution_and_proof.rs b/consensus/types/src/sync_committee/signed_contribution_and_proof.rs index 0027003b9f..c788b01b13 100644 --- a/consensus/types/src/sync_committee/signed_contribution_and_proof.rs +++ b/consensus/types/src/sync_committee/signed_contribution_and_proof.rs @@ -2,14 +2,12 @@ use bls::{SecretKey, Signature}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot}, fork::{Fork, ForkName}, sync_committee::{ContributionAndProof, SyncCommitteeContribution, SyncSelectionProof}, - test_utils::TestRandom, }; /// A Validators signed contribution proof to publish on the `sync_committee_contribution_and_proof` @@ -19,7 +17,7 @@ use crate::{ derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] pub struct SignedContributionAndProof { diff --git a/consensus/types/src/sync_committee/sync_aggregate.rs b/consensus/types/src/sync_committee/sync_aggregate.rs index e5848aa22c..263faf1286 100644 --- a/consensus/types/src/sync_committee/sync_aggregate.rs +++ b/consensus/types/src/sync_committee/sync_aggregate.rs @@ -5,14 +5,12 @@ use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::BitVector; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{EthSpec, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT}, fork::ForkName, sync_committee::SyncCommitteeContribution, - test_utils::TestRandom, }; #[derive(Debug, PartialEq)] @@ -32,7 +30,7 @@ impl From for Error { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Educe)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, Educe)] #[educe(PartialEq, Hash(bound(E: EthSpec)))] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] diff --git a/consensus/types/src/sync_committee/sync_aggregator_selection_data.rs b/consensus/types/src/sync_committee/sync_aggregator_selection_data.rs index e905ca036b..c828e874e0 100644 --- a/consensus/types/src/sync_committee/sync_aggregator_selection_data.rs +++ b/consensus/types/src/sync_committee/sync_aggregator_selection_data.rs @@ -1,19 +1,15 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{SignedRoot, Slot}, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Clone, Serialize, Deserialize, Hash, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Hash, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SyncAggregatorSelectionData { pub slot: Slot, diff --git a/consensus/types/src/sync_committee/sync_committee.rs b/consensus/types/src/sync_committee/sync_committee.rs index 5448411800..413258f77d 100644 --- a/consensus/types/src/sync_committee/sync_committee.rs +++ b/consensus/types/src/sync_committee/sync_committee.rs @@ -6,10 +6,9 @@ use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::FixedVector; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::EthSpec, fork::ForkName, sync_committee::SyncSubnetId, test_utils::TestRandom}; +use crate::{core::EthSpec, fork::ForkName, sync_committee::SyncSubnetId}; #[derive(Debug, PartialEq)] pub enum Error { @@ -32,7 +31,7 @@ impl From for Error { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] pub struct SyncCommittee { diff --git a/consensus/types/src/sync_committee/sync_committee_contribution.rs b/consensus/types/src/sync_committee/sync_committee_contribution.rs index 09376fbe5c..c646d0b7e3 100644 --- a/consensus/types/src/sync_committee/sync_committee_contribution.rs +++ b/consensus/types/src/sync_committee/sync_committee_contribution.rs @@ -3,14 +3,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::BitVector; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{EthSpec, Hash256, SignedRoot, Slot, SlotData}, fork::ForkName, sync_committee::SyncCommitteeMessage, - test_utils::TestRandom, }; #[derive(Debug, PartialEq)] @@ -26,7 +24,7 @@ pub enum Error { derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(bound = "E: EthSpec")] #[context_deserialize(ForkName)] pub struct SyncCommitteeContribution { @@ -79,7 +77,7 @@ impl SyncCommitteeContribution { impl SignedRoot for Hash256 {} /// This is not in the spec, but useful for determining uniqueness of sync committee contributions -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct SyncContributionData { pub slot: Slot, pub beacon_block_root: Hash256, diff --git a/consensus/types/src/sync_committee/sync_committee_message.rs b/consensus/types/src/sync_committee/sync_committee_message.rs index ed42555c43..87291c59c4 100644 --- a/consensus/types/src/sync_committee/sync_committee_message.rs +++ b/consensus/types/src/sync_committee/sync_committee_message.rs @@ -2,18 +2,16 @@ use bls::{SecretKey, Signature}; use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot, Slot, SlotData}, fork::{Fork, ForkName}, - test_utils::TestRandom, }; /// The data upon which a `SyncCommitteeContribution` is based. #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct SyncCommitteeMessage { pub slot: Slot, diff --git a/consensus/types/src/test_utils/generate_random_block_and_blobs.rs b/consensus/types/src/test_utils/generate_random_block_and_blobs.rs index 2a38b5be1f..c511fd72e7 100644 --- a/consensus/types/src/test_utils/generate_random_block_and_blobs.rs +++ b/consensus/types/src/test_utils/generate_random_block_and_blobs.rs @@ -1,6 +1,5 @@ -use bls::Signature; +use arbitrary::Arbitrary; use kzg::{KzgCommitment, KzgProof}; -use rand::Rng; use crate::{ block::{BeaconBlock, SignedBeaconBlock}, @@ -9,22 +8,22 @@ use crate::{ execution::FullPayload, fork::{ForkName, map_fork_name}, kzg_ext::{KzgCommitments, KzgProofs}, - test_utils::TestRandom, }; type BlobsBundle = (KzgCommitments, KzgProofs, BlobsList); +#[allow(clippy::type_complexity)] pub fn generate_rand_block_and_blobs( fork_name: ForkName, num_blobs: usize, - rng: &mut impl Rng, -) -> (SignedBeaconBlock>, Vec>) { - let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(rng)); - let mut block = SignedBeaconBlock::from_block(inner, Signature::random_for_test(rng)); + u: &mut arbitrary::Unstructured, +) -> arbitrary::Result<(SignedBeaconBlock>, Vec>)> { + let inner = map_fork_name!(fork_name, BeaconBlock, <_>::arbitrary(u)?); + let mut block = SignedBeaconBlock::from_block(inner, bls::Signature::arbitrary(u)?); let mut blob_sidecars = vec![]; if block.fork_name_unchecked() < ForkName::Deneb { - return (block, blob_sidecars); + return Ok((block, blob_sidecars)); } let (commitments, proofs, blobs) = generate_blobs::(num_blobs).unwrap(); @@ -50,7 +49,7 @@ pub fn generate_rand_block_and_blobs( .unwrap(), }); } - (block, blob_sidecars) + Ok((block, blob_sidecars)) } pub fn generate_blobs(n_blobs: usize) -> Result, String> { @@ -74,13 +73,13 @@ pub fn generate_blobs(n_blobs: usize) -> Result, Stri #[cfg(test)] mod test { use super::*; - use rand::rng; use ssz_types::FixedVector; #[test] fn test_verify_blob_inclusion_proof() { + let mut u = crate::test_utils::test_unstructured(); let (_block, blobs) = - generate_rand_block_and_blobs::(ForkName::Deneb, 2, &mut rng()); + generate_rand_block_and_blobs::(ForkName::Deneb, 2, &mut u).unwrap(); for blob in blobs { assert!(blob.verify_blob_sidecar_inclusion_proof()); } @@ -88,8 +87,9 @@ mod test { #[test] fn test_verify_blob_inclusion_proof_from_existing_proof() { + let mut u = crate::test_utils::test_unstructured(); let (block, mut blob_sidecars) = - generate_rand_block_and_blobs::(ForkName::Deneb, 1, &mut rng()); + generate_rand_block_and_blobs::(ForkName::Deneb, 1, &mut u).unwrap(); let BlobSidecar { index, blob, @@ -105,11 +105,12 @@ mod test { #[test] fn test_verify_blob_inclusion_proof_invalid() { + let mut u = crate::test_utils::test_unstructured(); let (_block, blobs) = - generate_rand_block_and_blobs::(ForkName::Deneb, 1, &mut rng()); + generate_rand_block_and_blobs::(ForkName::Deneb, 1, &mut u).unwrap(); for mut blob in blobs { - blob.kzg_commitment_inclusion_proof = FixedVector::random_for_test(&mut rng()); + blob.kzg_commitment_inclusion_proof = FixedVector::arbitrary(&mut u).unwrap(); assert!(!blob.verify_blob_sidecar_inclusion_proof()); } } diff --git a/consensus/types/src/test_utils/macros.rs b/consensus/types/src/test_utils/macros.rs index 662527f5a4..09afd27ae3 100644 --- a/consensus/types/src/test_utils/macros.rs +++ b/consensus/types/src/test_utils/macros.rs @@ -14,10 +14,8 @@ macro_rules! ssz_tests { #[test] pub fn test_ssz_round_trip() { use ssz::{Decode, ssz_encode}; - use $crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = <$type>::random_for_test(&mut rng); + let original: $type = $crate::test_utils::test_arbitrary_instance(); let bytes = ssz_encode(&original); let decoded = <$type>::from_ssz_bytes(&bytes).unwrap(); @@ -33,10 +31,8 @@ macro_rules! tree_hash_tests { #[test] pub fn test_tree_hash_root() { use tree_hash::TreeHash; - use $crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = <$type>::random_for_test(&mut rng); + let original: $type = $crate::test_utils::test_arbitrary_instance(); // Tree hashing should not panic. original.tree_hash_root(); diff --git a/consensus/types/src/test_utils/mod.rs b/consensus/types/src/test_utils/mod.rs index c4409b4392..5cf728be66 100644 --- a/consensus/types/src/test_utils/mod.rs +++ b/consensus/types/src/test_utils/mod.rs @@ -5,15 +5,36 @@ mod macros; mod generate_deterministic_keypairs; #[cfg(test)] mod generate_random_block_and_blobs; -mod test_random; pub use generate_deterministic_keypairs::generate_deterministic_keypair; pub use generate_deterministic_keypairs::generate_deterministic_keypairs; pub use generate_deterministic_keypairs::load_keypairs_from_yaml; -pub use test_random::{TestRandom, test_random_instance}; -pub use rand::{RngCore, SeedableRng}; -pub use rand_xorshift::XorShiftRng; +/// Deterministic 256 KiB seed. +#[cfg(feature = "arbitrary")] +static SEED: std::sync::LazyLock> = std::sync::LazyLock::new(|| { + use rand::RngCore; + use rand::SeedableRng; + let mut bytes = vec![0u8; 256 * 1024]; + rand_xorshift::XorShiftRng::from_seed([0x42; 16]).fill_bytes(&mut bytes); + bytes +}); + +/// Generates an arbitrary instance of `T` from a deterministic seed. +/// Suitable for one-shot test instance creation. +#[cfg(feature = "arbitrary")] +pub fn test_arbitrary_instance<'a, T: arbitrary::Arbitrary<'a>>() -> T { + let mut u = arbitrary::Unstructured::new(&SEED); + T::arbitrary(&mut u).expect("sufficient bytes for arbitrary generation") +} + +/// Returns an `Unstructured` from a deterministic seed. +/// Use this when you need to pass an `Unstructured` to helpers like +/// `generate_rand_block_and_blobs`. +#[cfg(feature = "arbitrary")] +pub fn test_unstructured() -> arbitrary::Unstructured<'static> { + arbitrary::Unstructured::new(&SEED) +} use ssz::{Decode, Encode, ssz_encode}; use std::fmt::Debug; diff --git a/consensus/types/src/test_utils/test_random/address.rs b/consensus/types/src/test_utils/test_random/address.rs deleted file mode 100644 index 2f601cb91e..0000000000 --- a/consensus/types/src/test_utils/test_random/address.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::{core::Address, test_utils::TestRandom}; - -impl TestRandom for Address { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut key_bytes = vec![0; 20]; - rng.fill_bytes(&mut key_bytes); - Address::from_slice(&key_bytes[..]) - } -} diff --git a/consensus/types/src/test_utils/test_random/aggregate_signature.rs b/consensus/types/src/test_utils/test_random/aggregate_signature.rs deleted file mode 100644 index f9f3dd9567..0000000000 --- a/consensus/types/src/test_utils/test_random/aggregate_signature.rs +++ /dev/null @@ -1,12 +0,0 @@ -use bls::{AggregateSignature, Signature}; - -use crate::test_utils::TestRandom; - -impl TestRandom for AggregateSignature { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let signature = Signature::random_for_test(rng); - let mut aggregate_signature = AggregateSignature::infinity(); - aggregate_signature.add_assign(&signature); - aggregate_signature - } -} diff --git a/consensus/types/src/test_utils/test_random/bitfield.rs b/consensus/types/src/test_utils/test_random/bitfield.rs deleted file mode 100644 index 762f41eb34..0000000000 --- a/consensus/types/src/test_utils/test_random/bitfield.rs +++ /dev/null @@ -1,43 +0,0 @@ -use smallvec::smallvec; -use ssz_types::{BitList, BitVector}; -use typenum::Unsigned; - -use crate::test_utils::TestRandom; - -impl TestRandom for BitList { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let initial_len = std::cmp::max(1, N::to_usize().div_ceil(8)); - let mut raw_bytes = smallvec![0; initial_len]; - rng.fill_bytes(&mut raw_bytes); - - let non_zero_bytes = raw_bytes - .iter() - .enumerate() - .rev() - .find_map(|(i, byte)| (*byte > 0).then_some(i + 1)) - .unwrap_or(0); - - if non_zero_bytes < initial_len { - raw_bytes.truncate(non_zero_bytes); - } - - Self::from_bytes(raw_bytes).expect("we generate a valid BitList") - } -} - -impl TestRandom for BitVector { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut raw_bytes = smallvec![0; std::cmp::max(1, N::to_usize().div_ceil(8))]; - rng.fill_bytes(&mut raw_bytes); - // If N isn't divisible by 8 - // zero out bits greater than N - if let Some(last_byte) = raw_bytes.last_mut() { - let mut mask = 0; - for i in 0..N::to_usize() % 8 { - mask |= 1 << i; - } - *last_byte &= mask; - } - Self::from_bytes(raw_bytes).expect("we generate a valid BitVector") - } -} diff --git a/consensus/types/src/test_utils/test_random/hash256.rs b/consensus/types/src/test_utils/test_random/hash256.rs deleted file mode 100644 index 4d7570fb55..0000000000 --- a/consensus/types/src/test_utils/test_random/hash256.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::{core::Hash256, test_utils::TestRandom}; - -impl TestRandom for Hash256 { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut key_bytes = vec![0; 32]; - rng.fill_bytes(&mut key_bytes); - Hash256::from_slice(&key_bytes[..]) - } -} diff --git a/consensus/types/src/test_utils/test_random/kzg_commitment.rs b/consensus/types/src/test_utils/test_random/kzg_commitment.rs deleted file mode 100644 index 31e316a198..0000000000 --- a/consensus/types/src/test_utils/test_random/kzg_commitment.rs +++ /dev/null @@ -1,9 +0,0 @@ -use kzg::KzgCommitment; - -use crate::test_utils::TestRandom; - -impl TestRandom for KzgCommitment { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - KzgCommitment(<[u8; 48] as TestRandom>::random_for_test(rng)) - } -} diff --git a/consensus/types/src/test_utils/test_random/kzg_proof.rs b/consensus/types/src/test_utils/test_random/kzg_proof.rs deleted file mode 100644 index 4465d5ab39..0000000000 --- a/consensus/types/src/test_utils/test_random/kzg_proof.rs +++ /dev/null @@ -1,11 +0,0 @@ -use kzg::{BYTES_PER_COMMITMENT, KzgProof}; - -use crate::test_utils::TestRandom; - -impl TestRandom for KzgProof { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut bytes = [0; BYTES_PER_COMMITMENT]; - rng.fill_bytes(&mut bytes); - Self(bytes) - } -} diff --git a/consensus/types/src/test_utils/test_random/mod.rs b/consensus/types/src/test_utils/test_random/mod.rs deleted file mode 100644 index 41812593fa..0000000000 --- a/consensus/types/src/test_utils/test_random/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -mod address; -mod aggregate_signature; -mod bitfield; -mod hash256; -mod kzg_commitment; -mod kzg_proof; -mod public_key; -mod public_key_bytes; -mod secret_key; -mod signature; -mod signature_bytes; -mod test_random; -mod uint256; - -pub use test_random::{TestRandom, test_random_instance}; diff --git a/consensus/types/src/test_utils/test_random/public_key.rs b/consensus/types/src/test_utils/test_random/public_key.rs deleted file mode 100644 index 9d287c23d7..0000000000 --- a/consensus/types/src/test_utils/test_random/public_key.rs +++ /dev/null @@ -1,9 +0,0 @@ -use bls::{PublicKey, SecretKey}; - -use crate::test_utils::TestRandom; - -impl TestRandom for PublicKey { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - SecretKey::random_for_test(rng).public_key() - } -} diff --git a/consensus/types/src/test_utils/test_random/public_key_bytes.rs b/consensus/types/src/test_utils/test_random/public_key_bytes.rs deleted file mode 100644 index 587c3baf8f..0000000000 --- a/consensus/types/src/test_utils/test_random/public_key_bytes.rs +++ /dev/null @@ -1,17 +0,0 @@ -use bls::{PUBLIC_KEY_BYTES_LEN, PublicKey, PublicKeyBytes}; - -use crate::test_utils::TestRandom; - -impl TestRandom for PublicKeyBytes { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - //50-50 chance for signature to be "valid" or invalid - if bool::random_for_test(rng) { - //valid signature - PublicKeyBytes::from(PublicKey::random_for_test(rng)) - } else { - //invalid signature, just random bytes - PublicKeyBytes::deserialize(&<[u8; PUBLIC_KEY_BYTES_LEN]>::random_for_test(rng)) - .unwrap() - } - } -} diff --git a/consensus/types/src/test_utils/test_random/secret_key.rs b/consensus/types/src/test_utils/test_random/secret_key.rs deleted file mode 100644 index a8295d968a..0000000000 --- a/consensus/types/src/test_utils/test_random/secret_key.rs +++ /dev/null @@ -1,11 +0,0 @@ -use bls::SecretKey; - -use crate::test_utils::TestRandom; - -impl TestRandom for SecretKey { - fn random_for_test(_rng: &mut impl rand::RngCore) -> Self { - // TODO: Not deterministic generation. Using `SecretKey::deserialize` results in - // `BlstError(BLST_BAD_ENCODING)`, need to debug with blst source on what encoding expects. - SecretKey::random() - } -} diff --git a/consensus/types/src/test_utils/test_random/signature.rs b/consensus/types/src/test_utils/test_random/signature.rs deleted file mode 100644 index 006aba9650..0000000000 --- a/consensus/types/src/test_utils/test_random/signature.rs +++ /dev/null @@ -1,12 +0,0 @@ -use bls::Signature; - -use crate::test_utils::TestRandom; - -impl TestRandom for Signature { - fn random_for_test(_rng: &mut impl rand::RngCore) -> Self { - // TODO: `SecretKey::random_for_test` does not return a deterministic signature. Since this - // signature will not pass verification we could just return the generator point or the - // generator point multiplied by a random scalar if we want disctint signatures. - Signature::infinity().expect("infinity signature is valid") - } -} diff --git a/consensus/types/src/test_utils/test_random/signature_bytes.rs b/consensus/types/src/test_utils/test_random/signature_bytes.rs deleted file mode 100644 index 6992e57467..0000000000 --- a/consensus/types/src/test_utils/test_random/signature_bytes.rs +++ /dev/null @@ -1,16 +0,0 @@ -use bls::{SIGNATURE_BYTES_LEN, Signature, SignatureBytes}; - -use crate::test_utils::TestRandom; - -impl TestRandom for SignatureBytes { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - //50-50 chance for signature to be "valid" or invalid - if bool::random_for_test(rng) { - //valid signature - SignatureBytes::from(Signature::random_for_test(rng)) - } else { - //invalid signature, just random bytes - SignatureBytes::deserialize(&<[u8; SIGNATURE_BYTES_LEN]>::random_for_test(rng)).unwrap() - } - } -} diff --git a/consensus/types/src/test_utils/test_random/test_random.rs b/consensus/types/src/test_utils/test_random/test_random.rs deleted file mode 100644 index 101fbec51b..0000000000 --- a/consensus/types/src/test_utils/test_random/test_random.rs +++ /dev/null @@ -1,140 +0,0 @@ -use std::{marker::PhantomData, sync::Arc}; - -use rand::{RngCore, SeedableRng}; -use rand_xorshift::XorShiftRng; -use smallvec::{SmallVec, smallvec}; -use ssz_types::VariableList; -use typenum::Unsigned; - -pub fn test_random_instance() -> T { - let mut rng = XorShiftRng::from_seed([0x42; 16]); - T::random_for_test(&mut rng) -} - -pub trait TestRandom { - fn random_for_test(rng: &mut impl RngCore) -> Self; -} - -impl TestRandom for PhantomData { - fn random_for_test(_rng: &mut impl RngCore) -> Self { - PhantomData - } -} - -impl TestRandom for bool { - fn random_for_test(rng: &mut impl RngCore) -> Self { - (rng.next_u32() % 2) == 1 - } -} - -impl TestRandom for u64 { - fn random_for_test(rng: &mut impl RngCore) -> Self { - rng.next_u64() - } -} - -impl TestRandom for u32 { - fn random_for_test(rng: &mut impl RngCore) -> Self { - rng.next_u32() - } -} - -impl TestRandom for u8 { - fn random_for_test(rng: &mut impl RngCore) -> Self { - rng.next_u32().to_be_bytes()[0] - } -} - -impl TestRandom for usize { - fn random_for_test(rng: &mut impl RngCore) -> Self { - rng.next_u32() as usize - } -} - -impl TestRandom for Vec -where - U: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - let mut output = vec![]; - - for _ in 0..(usize::random_for_test(rng) % 4) { - output.push(::random_for_test(rng)); - } - - output - } -} - -impl TestRandom for Arc -where - U: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - Arc::new(U::random_for_test(rng)) - } -} - -impl TestRandom for ssz_types::FixedVector -where - T: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - Self::new( - (0..N::to_usize()) - .map(|_| T::random_for_test(rng)) - .collect(), - ) - .expect("N items provided") - } -} - -impl TestRandom for VariableList -where - T: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - let mut output = vec![]; - - if N::to_usize() != 0 { - for _ in 0..(usize::random_for_test(rng) % std::cmp::min(4, N::to_usize())) { - output.push(::random_for_test(rng)); - } - } - - output.try_into().unwrap() - } -} - -impl TestRandom for SmallVec<[U; N]> -where - U: TestRandom, -{ - fn random_for_test(rng: &mut impl RngCore) -> Self { - let mut output = smallvec![]; - - for _ in 0..(usize::random_for_test(rng) % 4) { - output.push(::random_for_test(rng)); - } - - output - } -} - -macro_rules! impl_test_random_for_u8_array { - ($len: expr) => { - impl TestRandom for [u8; $len] { - fn random_for_test(rng: &mut impl RngCore) -> Self { - let mut bytes = [0; $len]; - rng.fill_bytes(&mut bytes); - bytes - } - } - }; -} - -impl_test_random_for_u8_array!(3); -impl_test_random_for_u8_array!(4); -impl_test_random_for_u8_array!(32); -impl_test_random_for_u8_array!(48); -impl_test_random_for_u8_array!(96); diff --git a/consensus/types/src/test_utils/test_random/uint256.rs b/consensus/types/src/test_utils/test_random/uint256.rs deleted file mode 100644 index eccf476595..0000000000 --- a/consensus/types/src/test_utils/test_random/uint256.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::{core::Uint256, test_utils::TestRandom}; - -impl TestRandom for Uint256 { - fn random_for_test(rng: &mut impl rand::RngCore) -> Self { - let mut key_bytes = [0; 32]; - rng.fill_bytes(&mut key_bytes); - Self::from_le_slice(&key_bytes[..]) - } -} diff --git a/consensus/types/src/validator/validator.rs b/consensus/types/src/validator/validator.rs index 5c5bfc761f..a56093c0b5 100644 --- a/consensus/types/src/validator/validator.rs +++ b/consensus/types/src/validator/validator.rs @@ -3,7 +3,6 @@ use context_deserialize::context_deserialize; use fixed_bytes::FixedBytesExtended; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ @@ -11,16 +10,13 @@ use crate::{ core::{Address, ChainSpec, Epoch, EthSpec, Hash256}, fork::ForkName, state::BeaconState, - test_utils::TestRandom, }; /// Information about a `BeaconChain` validator. /// /// Spec v0.12.1 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash, -)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct Validator { pub pubkey: PublicKeyBytes, diff --git a/consensus/types/src/withdrawal/pending_partial_withdrawal.rs b/consensus/types/src/withdrawal/pending_partial_withdrawal.rs index cd866369a4..0b3842808d 100644 --- a/consensus/types/src/withdrawal/pending_partial_withdrawal.rs +++ b/consensus/types/src/withdrawal/pending_partial_withdrawal.rs @@ -1,15 +1,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Epoch, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Epoch, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct PendingPartialWithdrawal { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/withdrawal/withdrawal.rs b/consensus/types/src/withdrawal/withdrawal.rs index d75bd4f501..da69227626 100644 --- a/consensus/types/src/withdrawal/withdrawal.rs +++ b/consensus/types/src/withdrawal/withdrawal.rs @@ -2,19 +2,15 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ core::{Address, EthSpec}, fork::ForkName, - test_utils::TestRandom, }; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct Withdrawal { #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/withdrawal/withdrawal_request.rs b/consensus/types/src/withdrawal/withdrawal_request.rs index 98a40016f9..a89fe9b825 100644 --- a/consensus/types/src/withdrawal/withdrawal_request.rs +++ b/consensus/types/src/withdrawal/withdrawal_request.rs @@ -3,15 +3,12 @@ use context_deserialize::context_deserialize; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::{core::Address, fork::ForkName, test_utils::TestRandom}; +use crate::{core::Address, fork::ForkName}; #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[derive( - Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, -)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct WithdrawalRequest { #[serde(with = "serde_utils::address_hex")] diff --git a/consensus/types/tests/state.rs b/consensus/types/tests/state.rs index 5e223092cf..2168da9afc 100644 --- a/consensus/types/tests/state.rs +++ b/consensus/types/tests/state.rs @@ -2,15 +2,14 @@ use std::ops::Mul; use std::sync::LazyLock; +use arbitrary::Arbitrary; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use bls::Keypair; use fixed_bytes::FixedBytesExtended; use milhouse::Vector; -use rand::SeedableRng; -use rand_xorshift::XorShiftRng; use ssz::Encode; use swap_or_not_shuffle::compute_shuffled_index; -use types::test_utils::{TestRandom, generate_deterministic_keypairs}; +use types::test_utils::generate_deterministic_keypairs; use types::*; pub const MAX_VALIDATOR_COUNT: usize = 129; @@ -315,7 +314,7 @@ fn decode_base_and_altair() { type E = MainnetEthSpec; let spec = E::default_spec(); - let rng = &mut XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let fork_epoch = spec.altair_fork_epoch.unwrap(); @@ -328,7 +327,7 @@ fn decode_base_and_altair() { { let good_base_state: BeaconState = BeaconState::Base(BeaconStateBase { slot: base_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have a base state with a slot higher than the fork slot. let bad_base_state = { @@ -351,7 +350,7 @@ fn decode_base_and_altair() { let good_altair_state: BeaconState = BeaconState::Altair(BeaconStateAltair { slot: altair_slot, - ..<_>::random_for_test(rng) + ..<_>::arbitrary(&mut u).unwrap() }); // It's invalid to have an Altair state with a slot lower than the fork slot. let bad_altair_state = { diff --git a/crypto/bls/src/macros.rs b/crypto/bls/src/macros.rs index 58b1ec7d6c..4f2be22dc3 100644 --- a/crypto/bls/src/macros.rs +++ b/crypto/bls/src/macros.rs @@ -165,13 +165,26 @@ macro_rules! impl_debug { /// Contains the functions required for an `Arbitrary` implementation. /// /// Does not include the `Impl` section since it gets very complicated when it comes to generics. +/// +/// For `GenericPublicKeyBytes` and `GenericSignatureBytes`, this implementation works correctly +/// without falling back to zeros. +/// +/// For `GenericPublicKey`, `GenericSignature` and `GenericAggregateSignature`, this implementation +/// will almost always fail and fallback to zeros. This matches the behavior of the previous +/// `TestRandom` impls. +/// +/// TODO: For proper fuzzing, this implementation needs more consideration on how to +/// arbitrarily construct valid types. #[cfg(feature = "arbitrary")] macro_rules! impl_arbitrary { ($byte_size: expr) => { fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { let mut bytes = [0u8; $byte_size]; u.fill_buffer(&mut bytes)?; - Self::deserialize(&bytes).map_err(|_| arbitrary::Error::IncorrectFormat) + Ok(Self::deserialize(&bytes).unwrap_or_else(|_| { + // All-zeros is the "empty" encoding accepted by every BLS type. + Self::deserialize(&[0u8; $byte_size]).expect("all-zeros is a valid encoding") + })) } }; } diff --git a/testing/execution_engine_integration/src/nethermind.rs b/testing/execution_engine_integration/src/nethermind.rs index 6a336161bd..e606f5558c 100644 --- a/testing/execution_engine_integration/src/nethermind.rs +++ b/testing/execution_engine_integration/src/nethermind.rs @@ -20,6 +20,7 @@ fn build_result(repo_dir: &Path) -> Output { .arg("src/Nethermind/Nethermind.sln") .arg("-c") .arg("Release") + .arg("-p:TreatWarningsAsErrors=false") .current_dir(repo_dir) .output() .expect("failed to make nethermind") diff --git a/validator_client/doppelganger_service/Cargo.toml b/validator_client/doppelganger_service/Cargo.toml index 66b27eb39d..a0c579e11f 100644 --- a/validator_client/doppelganger_service/Cargo.toml +++ b/validator_client/doppelganger_service/Cargo.toml @@ -19,5 +19,7 @@ types = { workspace = true } validator_store = { workspace = true } [dev-dependencies] +arbitrary = { workspace = true } futures = { workspace = true } logging = { workspace = true } +types = { workspace = true, features = ["arbitrary"] } diff --git a/validator_client/doppelganger_service/src/lib.rs b/validator_client/doppelganger_service/src/lib.rs index 600ae82c54..0842638bfa 100644 --- a/validator_client/doppelganger_service/src/lib.rs +++ b/validator_client/doppelganger_service/src/lib.rs @@ -598,14 +598,12 @@ impl DoppelgangerService { #[cfg(test)] mod test { use super::*; + use arbitrary::Arbitrary; use futures::executor::block_on; use slot_clock::TestingSlotClock; use std::future; use std::time::Duration; - use types::{ - MainnetEthSpec, - test_utils::{SeedableRng, TestRandom, XorShiftRng}, - }; + use types::MainnetEthSpec; use validator_store::DoppelgangerStatus; const DEFAULT_VALIDATORS: usize = 8; @@ -641,12 +639,12 @@ mod test { impl TestBuilder { fn build(self) -> TestScenario { - let mut rng = XorShiftRng::from_seed([42; 16]); + let mut u = types::test_utils::test_unstructured(); let slot_clock = TestingSlotClock::new(Slot::new(0), GENESIS_TIME, SLOT_DURATION); TestScenario { validators: (0..self.validator_count) - .map(|_| PublicKeyBytes::random_for_test(&mut rng)) + .map(|_| PublicKeyBytes::arbitrary(&mut u).unwrap()) .collect(), doppelganger: DoppelgangerService::default(), slot_clock, diff --git a/validator_client/lighthouse_validator_store/src/lib.rs b/validator_client/lighthouse_validator_store/src/lib.rs index 989624db3b..44b6e884e9 100644 --- a/validator_client/lighthouse_validator_store/src/lib.rs +++ b/validator_client/lighthouse_validator_store/src/lib.rs @@ -22,11 +22,12 @@ use types::{ AbstractExecPayload, Address, AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, Domain, Epoch, EthSpec, ExecutionPayloadEnvelope, Fork, FullPayload, Graffiti, Hash256, InclusionList, PayloadAttestationData, - PayloadAttestationMessage, SelectionProof, SignedAggregateAndProof, SignedBeaconBlock, - SignedContributionAndProof, SignedExecutionPayloadEnvelope, SignedInclusionList, SignedRoot, - SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, - SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, - ValidatorRegistrationData, VoluntaryExit, graffiti::GraffitiString, + PayloadAttestationMessage, ProposerPreferences, SelectionProof, SignedAggregateAndProof, + SignedBeaconBlock, SignedContributionAndProof, SignedExecutionPayloadEnvelope, + SignedInclusionList, SignedProposerPreferences, SignedRoot, SignedValidatorRegistrationData, + SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, + SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, + VoluntaryExit, graffiti::GraffitiString, }; use validator_store::{ AggregateToSign, AttestationToSign, ContributionToSign, DoppelgangerStatus, @@ -1509,4 +1510,32 @@ impl ValidatorStore for LighthouseValidatorS signature, }) } + + async fn sign_proposer_preferences( + &self, + validator_pubkey: PublicKeyBytes, + preferences: ProposerPreferences, + ) -> Result { + let signing_context = self.signing_context( + Domain::ProposerPreferences, + preferences.proposal_slot.epoch(E::slots_per_epoch()), + ); + + let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?; + + let signature = signing_method + .get_signature::>( + SignableMessage::ProposerPreferences(&preferences), + signing_context, + &self.spec, + &self.task_executor, + ) + .await + .map_err(Error::SpecificError)?; + + Ok(SignedProposerPreferences { + message: preferences, + signature, + }) + } } diff --git a/validator_client/signing_method/src/lib.rs b/validator_client/signing_method/src/lib.rs index 5de14e9549..a84193bd29 100644 --- a/validator_client/signing_method/src/lib.rs +++ b/validator_client/signing_method/src/lib.rs @@ -52,6 +52,7 @@ pub enum SignableMessage<'a, E: EthSpec, Payload: AbstractExecPayload = FullP InclusionList(&'a InclusionList), ExecutionPayloadEnvelope(&'a ExecutionPayloadEnvelope), PayloadAttestationData(&'a PayloadAttestationData), + ProposerPreferences(&'a ProposerPreferences), } impl> SignableMessage<'_, E, Payload> { @@ -76,6 +77,7 @@ impl> SignableMessage<'_, E, Payload SignableMessage::InclusionList(il) => il.signing_root(domain), SignableMessage::ExecutionPayloadEnvelope(e) => e.signing_root(domain), SignableMessage::PayloadAttestationData(d) => d.signing_root(domain), + SignableMessage::ProposerPreferences(p) => p.signing_root(domain), } } } @@ -246,6 +248,9 @@ impl SigningMethod { SignableMessage::PayloadAttestationData(d) => { Web3SignerObject::PayloadAttestationData(d) } + SignableMessage::ProposerPreferences(p) => { + Web3SignerObject::ProposerPreferences(p) + } }; // Determine the Web3Signer message type. diff --git a/validator_client/signing_method/src/web3signer.rs b/validator_client/signing_method/src/web3signer.rs index 98d39e25ca..4d32890905 100644 --- a/validator_client/signing_method/src/web3signer.rs +++ b/validator_client/signing_method/src/web3signer.rs @@ -23,6 +23,7 @@ pub enum MessageType { // TODO(gloas) verify w/ web3signer specs ExecutionPayloadEnvelope, PayloadAttestation, + ProposerPreferences, } #[derive(Debug, PartialEq, Copy, Clone, Serialize)] @@ -83,6 +84,7 @@ pub enum Web3SignerObject<'a, E: EthSpec, Payload: AbstractExecPayload> { InclusionList(&'a InclusionList), ExecutionPayloadEnvelope(&'a ExecutionPayloadEnvelope), PayloadAttestationData(&'a PayloadAttestationData), + ProposerPreferences(&'a ProposerPreferences), } impl<'a, E: EthSpec, Payload: AbstractExecPayload> Web3SignerObject<'a, E, Payload> { @@ -156,6 +158,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> Web3SignerObject<'a, E, Pa Web3SignerObject::InclusionList(_) => MessageType::InclusionList, Web3SignerObject::ExecutionPayloadEnvelope(_) => MessageType::ExecutionPayloadEnvelope, Web3SignerObject::PayloadAttestationData(_) => MessageType::PayloadAttestation, + Web3SignerObject::ProposerPreferences(_) => MessageType::ProposerPreferences, } } } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 6f0163a7b6..a99dc99a17 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -49,6 +49,7 @@ use validator_services::{ latency_service, payload_attestation_service::PayloadAttestationService, preparation_service::{PreparationService, PreparationServiceBuilder}, + proposer_preferences_service::ProposerPreferencesService, sync_committee_service::SyncCommitteeService, }; use validator_store::ValidatorStore as ValidatorStoreTrait; @@ -88,6 +89,8 @@ pub struct ProductionValidatorClient { sync_committee_service: SyncCommitteeService, SystemTimeSlotClock>, inclusion_list_service: InclusionListService, SystemTimeSlotClock>, payload_attestation_service: PayloadAttestationService, SystemTimeSlotClock>, + proposer_preferences_service: + ProposerPreferencesService, SystemTimeSlotClock>, doppelganger_service: Option>, preparation_service: PreparationService, SystemTimeSlotClock>, validator_store: Arc>, @@ -577,6 +580,15 @@ impl ProductionValidatorClient { context.eth2_config.spec.clone(), ); + let proposer_preferences_service = ProposerPreferencesService::new( + duties_service.clone(), + validator_store.clone(), + slot_clock.clone(), + beacon_nodes.clone(), + context.executor.clone(), + context.eth2_config.spec.clone(), + ); + Ok(Self { context, duties_service, @@ -585,6 +597,7 @@ impl ProductionValidatorClient { sync_committee_service, inclusion_list_service, payload_attestation_service, + proposer_preferences_service, doppelganger_service, preparation_service, validator_store, @@ -666,6 +679,11 @@ impl ProductionValidatorClient { .clone() .start_update_service() .map_err(|e| format!("Unable to start payload attestation service: {}", e))?; + + self.proposer_preferences_service + .clone() + .start_update_service() + .map_err(|e| format!("Unable to start proposer preferences service: {}", e))?; } self.preparation_service diff --git a/validator_client/validator_services/src/lib.rs b/validator_client/validator_services/src/lib.rs index 83a2abe2b7..0b4f12b6ec 100644 --- a/validator_client/validator_services/src/lib.rs +++ b/validator_client/validator_services/src/lib.rs @@ -6,5 +6,6 @@ pub mod latency_service; pub mod notifier_service; pub mod payload_attestation_service; pub mod preparation_service; +pub mod proposer_preferences_service; pub mod sync; pub mod sync_committee_service; diff --git a/validator_client/validator_services/src/payload_attestation_service.rs b/validator_client/validator_services/src/payload_attestation_service.rs index 24949edc1f..f41893941f 100644 --- a/validator_client/validator_services/src/payload_attestation_service.rs +++ b/validator_client/validator_services/src/payload_attestation_service.rs @@ -139,14 +139,22 @@ impl PayloadAttestationServ beacon_node .get_validator_payload_attestation_data(slot) .await - .map_err(|e| format!("Failed to get payload attestation data: {e:?}")) - .map(|resp| resp.into_data()) + .map(|opt| opt.map(|resp| resp.into_data())) }) .await { - Ok(data) => data, + Ok(Some(data)) => data, + Ok(None) => { + // Per the consensus spec, validators should not submit a + // payload attestation when no block has been seen for the slot. + debug!( + %slot, + "No block received for slot, skipping payload attestation" + ); + return; + } Err(e) => { - crit!( + error!( error = %e, %slot, "Failed to produce payload attestation data" diff --git a/validator_client/validator_services/src/proposer_preferences_service.rs b/validator_client/validator_services/src/proposer_preferences_service.rs new file mode 100644 index 0000000000..fbefdf5d96 --- /dev/null +++ b/validator_client/validator_services/src/proposer_preferences_service.rs @@ -0,0 +1,221 @@ +use crate::duties_service::DutiesService; +use beacon_node_fallback::BeaconNodeFallback; +use slot_clock::SlotClock; +use std::ops::Deref; +use std::sync::Arc; +use task_executor::TaskExecutor; +use tokio::time::sleep; +use tracing::{debug, error, info, warn}; +use types::{ChainSpec, Epoch, EthSpec, ForkName, ProposerPreferences}; +use validator_store::ValidatorStore; + +pub struct Inner { + duties_service: Arc>, + validator_store: Arc, + slot_clock: T, + beacon_nodes: Arc>, + executor: TaskExecutor, + chain_spec: Arc, +} + +pub struct ProposerPreferencesService { + inner: Arc>, +} + +impl Clone for ProposerPreferencesService { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl Deref for ProposerPreferencesService { + type Target = Inner; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl ProposerPreferencesService { + pub fn new( + duties_service: Arc>, + validator_store: Arc, + slot_clock: T, + beacon_nodes: Arc>, + executor: TaskExecutor, + chain_spec: Arc, + ) -> Self { + Self { + inner: Arc::new(Inner { + duties_service, + validator_store, + slot_clock, + beacon_nodes, + executor, + chain_spec, + }), + } + } + + pub fn start_update_service(self) -> Result<(), String> { + let slot_duration = self.chain_spec.get_slot_duration(); + info!("Proposer preferences service started"); + + let executor = self.executor.clone(); + + let interval_fut = async move { + loop { + let Some(current_slot) = self.slot_clock.now() else { + error!("Failed to read slot clock"); + sleep(slot_duration).await; + continue; + }; + + if !self + .chain_spec + .fork_name_at_slot::(current_slot) + .gloas_enabled() + { + let duration_to_next_epoch = self + .slot_clock + .duration_to_next_epoch(S::E::slots_per_epoch()) + .unwrap_or_else(|| slot_duration * S::E::slots_per_epoch() as u32); + sleep(duration_to_next_epoch).await; + continue; + } + + let current_epoch = current_slot.epoch(S::E::slots_per_epoch()); + let fork_name = self.chain_spec.fork_name_at_slot::(current_slot); + self.publish_proposer_preferences(current_epoch, fork_name) + .await; + + let duration_to_next_epoch = self + .slot_clock + .duration_to_next_epoch(S::E::slots_per_epoch()) + .unwrap_or_else(|| slot_duration * S::E::slots_per_epoch() as u32); + sleep(duration_to_next_epoch).await; + } + }; + + executor.spawn(interval_fut, "proposer_preferences_service"); + Ok(()) + } + + async fn publish_proposer_preferences(&self, current_epoch: Epoch, fork_name: ForkName) { + let (dependent_root, duties) = { + let proposers = self.duties_service.proposers.read(); + match proposers.get(¤t_epoch) { + Some((root, duties)) => (*root, duties.clone()), + None => return, + } + }; + + let preferences_to_sign: Vec<_> = { + let mut result = vec![]; + for duty in &duties { + let Some(proposal_data) = self.validator_store.proposal_data(&duty.pubkey) else { + warn!( + validator = ?duty.pubkey, + "Missing proposal data for proposer preferences" + ); + continue; + }; + let Some(fee_recipient) = proposal_data.fee_recipient else { + warn!( + validator = ?duty.pubkey, + "Missing fee recipient for proposer preferences" + ); + continue; + }; + result.push(( + duty.pubkey, + ProposerPreferences { + dependent_root, + proposal_slot: duty.slot, + validator_index: duty.validator_index, + fee_recipient, + gas_limit: proposal_data.gas_limit, + }, + )); + } + result + }; + + if preferences_to_sign.is_empty() { + return; + } + + debug!( + %current_epoch, + count = preferences_to_sign.len(), + "Signing proposer preferences" + ); + + let mut signed = Vec::with_capacity(preferences_to_sign.len()); + for (pubkey, preferences) in preferences_to_sign { + match self + .validator_store + .sign_proposer_preferences(pubkey, preferences) + .await + { + Ok(signed_prefs) => signed.push(signed_prefs), + Err(e) => { + error!( + error = ?e, + validator = ?pubkey, + "Failed to sign proposer preferences" + ); + } + } + } + + if signed.is_empty() { + return; + } + + let count = signed.len(); + let signed = Arc::new(signed); + let result = self + .beacon_nodes + .first_success(|beacon_node| { + let signed = signed.clone(); + async move { + match beacon_node + .post_validator_proposer_preferences_ssz(&signed, fork_name) + .await + { + Ok(()) => Ok(()), + Err(ssz_err) => { + debug!(error = ?ssz_err, "SSZ publish failed, falling back to JSON"); + beacon_node + .post_validator_proposer_preferences(&signed, fork_name) + .await + .map_err(|e| { + format!("Failed to publish proposer preferences: {e:?}") + }) + } + } + } + }) + .await; + + match result { + Ok(()) => { + info!( + %current_epoch, + %count, + "Successfully published proposer preferences" + ); + } + Err(e) => { + error!( + error = %e, + %current_epoch, + "Failed to publish proposer preferences" + ); + } + } + } +} diff --git a/validator_client/validator_store/src/lib.rs b/validator_client/validator_store/src/lib.rs index 7dbc75bb66..8e11a98f2c 100644 --- a/validator_client/validator_store/src/lib.rs +++ b/validator_client/validator_store/src/lib.rs @@ -8,8 +8,9 @@ use std::sync::Arc; use types::{ Address, Attestation, AttestationError, BlindedBeaconBlock, Epoch, EthSpec, ExecutionPayloadEnvelope, Graffiti, Hash256, InclusionList, PayloadAttestationData, - PayloadAttestationMessage, SelectionProof, SignedAggregateAndProof, SignedBlindedBeaconBlock, - SignedContributionAndProof, SignedExecutionPayloadEnvelope, SignedInclusionList, + PayloadAttestationMessage, ProposerPreferences, SelectionProof, SignedAggregateAndProof, + SignedBlindedBeaconBlock, SignedContributionAndProof, SignedExecutionPayloadEnvelope, + SignedInclusionList, SignedProposerPreferences, SignedValidatorRegistrationData, Slot, SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, }; @@ -219,6 +220,13 @@ pub trait ValidatorStore: Send + Sync { data: PayloadAttestationData, ) -> impl Future>> + Send; + /// Sign a `ProposerPreferences` message. + fn sign_proposer_preferences( + &self, + validator_pubkey: PublicKeyBytes, + preferences: ProposerPreferences, + ) -> impl Future>> + Send; + /// Returns `ProposalData` for the provided `pubkey` if it exists in `InitializedValidators`. /// `ProposalData` fields include defaulting logic described in `get_fee_recipient_defaulting`, /// `get_gas_limit_defaulting`, and `get_builder_proposals_defaulting`.