diff --git a/Cargo.lock b/Cargo.lock index a6dd3332d7..3ba431d62e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,10 +42,10 @@ dependencies = [ "regex", "rpassword", "serde", - "serde_yaml", "tracing", "types", "validator_dir", + "yaml_serde", "zeroize", ] @@ -1885,8 +1885,8 @@ dependencies = [ "hex", "serde", "serde_json", - "serde_yaml", "types", + "yaml_serde", ] [[package]] @@ -1917,7 +1917,6 @@ dependencies = [ "sensitive_url", "serde", "serde_json", - "serde_yaml", "slasher", "slasher_service", "slot_clock", @@ -1930,6 +1929,7 @@ dependencies = [ "tracing", "tracing-subscriber", "types", + "yaml_serde", ] [[package]] @@ -2855,7 +2855,6 @@ dependencies = [ "serde", "serde_json", "serde_repr", - "serde_yaml", "snap", "ssz_types", "state_processing", @@ -2864,6 +2863,7 @@ dependencies = [ "tree_hash_derive", "typenum", "types", + "yaml_serde", ] [[package]] @@ -3164,7 +3164,7 @@ dependencies = [ "hex", "num-bigint", "serde", - "serde_yaml", + "yaml_serde", ] [[package]] @@ -3216,13 +3216,13 @@ dependencies = [ "pretty_reqwest_error", "reqwest", "sensitive_url", - "serde_yaml", "sha2", "tempfile", "tokio", "tracing", "types", "url", + "yaml_serde", "zip", ] @@ -4882,7 +4882,6 @@ dependencies = [ "rayon", "serde", "serde_json", - "serde_yaml", "snap", "state_processing", "store", @@ -4891,6 +4890,7 @@ dependencies = [ "tree_hash", "types", "validator_dir", + "yaml_serde", ] [[package]] @@ -5320,6 +5320,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libyaml-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" + [[package]] name = "libz-rs-sys" version = "0.5.4" @@ -5374,7 +5380,6 @@ dependencies = [ "sensitive_url", "serde", "serde_json", - "serde_yaml", "slasher", "slashing_protection", "store", @@ -5388,6 +5393,7 @@ dependencies = [ "validator_client", "validator_dir", "validator_manager", + "yaml_serde", "zeroize", ] @@ -7017,9 +7023,9 @@ dependencies = [ "fixed_bytes", "safe_arith", "serde", - "serde_yaml", "superstruct", "types", + "yaml_serde", ] [[package]] @@ -8054,19 +8060,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.12.1", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "serdect" version = "0.2.0" @@ -9361,7 +9354,6 @@ dependencies = [ "safe_arith", "serde", "serde_json", - "serde_yaml", "smallvec", "ssz_types", "state_processing", @@ -9374,6 +9366,7 @@ dependencies = [ "tree_hash", "tree_hash_derive", "typenum", + "yaml_serde", ] [[package]] @@ -9461,12 +9454,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "unsigned-varint" version = "0.8.0" @@ -9969,7 +9956,6 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_yaml", "slashing_protection", "slot_clock", "ssz_types", @@ -9979,6 +9965,7 @@ dependencies = [ "types", "url", "validator_store", + "yaml_serde", "zip", ] @@ -10493,6 +10480,19 @@ dependencies = [ "hashlink 0.11.0", ] +[[package]] +name = "yaml_serde" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c7c1b1a6a7c8a6b2741a6c21a4f8918e51899b111cfa08d1288202656e3975" +dependencies = [ + "indexmap 2.12.1", + "itoa", + "libyaml-rs", + "ryu", + "serde", +] + [[package]] name = "yamux" version = "0.12.1" diff --git a/Cargo.toml b/Cargo.toml index 340b650bca..96d57e0210 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -213,7 +213,6 @@ sensitive_url = { version = "0.1", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_repr = "0.1" -serde_yaml = "0.9" sha2 = "0.10" signing_method = { path = "validator_client/signing_method" } slasher = { path = "slasher", default-features = false } @@ -260,6 +259,7 @@ warp = { version = "0.3.7", default-features = false, features = ["tls"] } warp_utils = { path = "common/warp_utils" } workspace_members = { path = "common/workspace_members" } xdelta3 = { git = "https://github.com/sigp/xdelta3-rs", rev = "fe3906605c87b6c0515bd7c8fc671f47875e3ccc" } +yaml_serde = "0.10" zeroize = { version = "1", features = ["zeroize_derive", "serde"] } zip = { version = "6.0", default-features = false, features = ["deflate"] } zstd = "0.13" diff --git a/beacon_node/beacon_chain/src/events.rs b/beacon_node/beacon_chain/src/events.rs index 276edc3fe6..80667cd399 100644 --- a/beacon_node/beacon_chain/src/events.rs +++ b/beacon_node/beacon_chain/src/events.rs @@ -25,6 +25,11 @@ pub struct ServerSentEventHandler { attester_slashing_tx: Sender>, bls_to_execution_change_tx: Sender>, block_gossip_tx: Sender>, + execution_payload_tx: Sender>, + execution_payload_gossip_tx: Sender>, + execution_payload_available_tx: Sender>, + execution_payload_bid_tx: Sender>, + payload_attestation_message_tx: Sender>, } impl ServerSentEventHandler { @@ -51,6 +56,11 @@ impl ServerSentEventHandler { let (attester_slashing_tx, _) = broadcast::channel(capacity); let (bls_to_execution_change_tx, _) = broadcast::channel(capacity); let (block_gossip_tx, _) = broadcast::channel(capacity); + let (execution_payload_tx, _) = broadcast::channel(capacity); + let (execution_payload_gossip_tx, _) = broadcast::channel(capacity); + let (execution_payload_available_tx, _) = broadcast::channel(capacity); + let (execution_payload_bid_tx, _) = broadcast::channel(capacity); + let (payload_attestation_message_tx, _) = broadcast::channel(capacity); Self { attestation_tx, @@ -71,6 +81,11 @@ impl ServerSentEventHandler { attester_slashing_tx, bls_to_execution_change_tx, block_gossip_tx, + execution_payload_tx, + execution_payload_gossip_tx, + execution_payload_available_tx, + execution_payload_bid_tx, + payload_attestation_message_tx, } } @@ -155,6 +170,26 @@ impl ServerSentEventHandler { .block_gossip_tx .send(kind) .map(|count| log_count("block gossip", count)), + EventKind::ExecutionPayload(_) => self + .execution_payload_tx + .send(kind) + .map(|count| log_count("execution payload", count)), + EventKind::ExecutionPayloadGossip(_) => self + .execution_payload_gossip_tx + .send(kind) + .map(|count| log_count("execution payload gossip", count)), + EventKind::ExecutionPayloadAvailable(_) => self + .execution_payload_available_tx + .send(kind) + .map(|count| log_count("execution payload available", count)), + EventKind::ExecutionPayloadBid(_) => self + .execution_payload_bid_tx + .send(kind) + .map(|count| log_count("execution payload bid", count)), + EventKind::PayloadAttestationMessage(_) => self + .payload_attestation_message_tx + .send(kind) + .map(|count| log_count("payload attestation message", count)), }; if let Err(SendError(event)) = result { trace!(?event, "No receivers registered to listen for event"); @@ -233,6 +268,26 @@ impl ServerSentEventHandler { self.block_gossip_tx.subscribe() } + pub fn subscribe_execution_payload(&self) -> Receiver> { + self.execution_payload_tx.subscribe() + } + + pub fn subscribe_execution_payload_gossip(&self) -> Receiver> { + self.execution_payload_gossip_tx.subscribe() + } + + pub fn subscribe_execution_payload_available(&self) -> Receiver> { + self.execution_payload_available_tx.subscribe() + } + + pub fn subscribe_execution_payload_bid(&self) -> Receiver> { + self.execution_payload_bid_tx.subscribe() + } + + pub fn subscribe_payload_attestation_message(&self) -> Receiver> { + self.payload_attestation_message_tx.subscribe() + } + pub fn has_attestation_subscribers(&self) -> bool { self.attestation_tx.receiver_count() > 0 } @@ -296,4 +351,24 @@ impl ServerSentEventHandler { pub fn has_block_gossip_subscribers(&self) -> bool { self.block_gossip_tx.receiver_count() > 0 } + + pub fn has_execution_payload_subscribers(&self) -> bool { + self.execution_payload_tx.receiver_count() > 0 + } + + pub fn has_execution_payload_gossip_subscribers(&self) -> bool { + self.execution_payload_gossip_tx.receiver_count() > 0 + } + + pub fn has_execution_payload_available_subscribers(&self) -> bool { + self.execution_payload_available_tx.receiver_count() > 0 + } + + pub fn has_execution_payload_bid_subscribers(&self) -> bool { + self.execution_payload_bid_tx.receiver_count() > 0 + } + + pub fn has_payload_attestation_message_subscribers(&self) -> bool { + self.payload_attestation_message_tx.receiver_count() > 0 + } } diff --git a/beacon_node/beacon_chain/src/payload_envelope_verification/gossip_verified_envelope.rs b/beacon_node/beacon_chain/src/payload_envelope_verification/gossip_verified_envelope.rs index 03a3a91ac5..9a4ed2d044 100644 --- a/beacon_node/beacon_chain/src/payload_envelope_verification/gossip_verified_envelope.rs +++ b/beacon_node/beacon_chain/src/payload_envelope_verification/gossip_verified_envelope.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use educe::Educe; +use eth2::types::{EventKind, SseExecutionPayloadGossip}; use parking_lot::{Mutex, RwLock}; use store::DatabaseBlock; use tracing::{Span, debug}; @@ -10,7 +11,7 @@ use types::{ }; use crate::{ - BeaconChain, BeaconChainError, BeaconChainTypes, BeaconStore, + BeaconChain, BeaconChainError, BeaconChainTypes, BeaconStore, ServerSentEventHandler, beacon_proposer_cache::{self, BeaconProposerCache}, canonical_head::CanonicalHead, payload_envelope_verification::{ @@ -28,6 +29,7 @@ pub struct GossipVerificationContext<'a, T: BeaconChainTypes> { pub beacon_proposer_cache: &'a Mutex, pub validator_pubkey_cache: &'a RwLock>, pub genesis_validators_root: Hash256, + pub event_handler: &'a Option>, } /// Verify that an execution payload envelope is consistent with its beacon block @@ -213,6 +215,20 @@ impl GossipVerifiedEnvelope { return Err(EnvelopeError::BadSignature); } + if let Some(event_handler) = ctx.event_handler.as_ref() + && event_handler.has_execution_payload_gossip_subscribers() + { + event_handler.register(EventKind::ExecutionPayloadGossip( + SseExecutionPayloadGossip { + slot: block.slot(), + builder_index, + block_hash: signed_envelope.message.payload.block_hash, + block_root: beacon_block_root, + state_root: signed_envelope.message.state_root, + }, + )); + } + Ok(Self { signed_envelope, block, @@ -235,6 +251,7 @@ impl BeaconChain { beacon_proposer_cache: &self.beacon_proposer_cache, validator_pubkey_cache: &self.validator_pubkey_cache, genesis_validators_root: self.genesis_validators_root, + event_handler: &self.event_handler, } } diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index c53c29438e..13dcf22108 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -223,7 +223,7 @@ pub fn test_da_checker( let slot_clock = TestingSlotClock::new( Slot::new(0), Duration::from_secs(0), - Duration::from_secs(spec.seconds_per_slot), + spec.get_slot_duration(), ); let kzg = get_kzg(&spec); let ordered_custody_column_indices = generate_data_column_indices_rand_order::(); diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 2b4152b550..fb5262b893 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -2910,7 +2910,7 @@ async fn reproduction_unaligned_checkpoint_sync_pruned_payload() { let slot_clock = TestingSlotClock::new( Slot::new(0), Duration::from_secs(harness.chain.genesis_time), - Duration::from_secs(spec.seconds_per_slot), + spec.get_slot_duration(), ); slot_clock.set_slot(harness.get_current_slot().as_u64()); @@ -5334,8 +5334,8 @@ async fn test_safely_backfill_data_column_custody_info() { .await; let epoch_before_increase = Epoch::new(start_epochs); - let effective_delay_slots = - CUSTODY_CHANGE_DA_EFFECTIVE_DELAY_SECONDS / harness.chain.spec.seconds_per_slot; + let effective_delay_slots = CUSTODY_CHANGE_DA_EFFECTIVE_DELAY_SECONDS + / harness.chain.spec.get_slot_duration().as_secs(); let cgc_change_slot = epoch_before_increase.end_slot(E::slots_per_epoch()); @@ -6131,7 +6131,7 @@ async fn bellatrix_produce_and_store_payloads() { .genesis_time() .safe_add( slot.as_u64() - .safe_mul(harness.spec.seconds_per_slot) + .safe_mul(harness.spec.get_slot_duration().as_secs()) .unwrap(), ) .unwrap(); diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 229816ba77..61b46ca1e6 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -424,7 +424,11 @@ pub enum Work { IgnoredRpcBlock { process_fn: BlockingFn, }, - ChainSegment(AsyncFn), + ChainSegment { + process_fn: AsyncFn, + /// (chain_id, batch_epoch) for test observability + process_id: (u32, u64), + }, ChainSegmentBackfill(BlockingFn), Status(BlockingFn), BlocksByRangeRequest(AsyncFn), @@ -1482,7 +1486,7 @@ impl BeaconProcessor { } => task_spawner.spawn_blocking(move || { process_batch(aggregates); }), - Work::ChainSegment(process_fn) => task_spawner.spawn_async(async move { + Work::ChainSegment { process_fn, .. } => task_spawner.spawn_async(async move { process_fn.await; }), Work::UnknownBlockAttestation { process_fn } diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 3c4b2572c9..50d76e8f19 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -42,6 +42,6 @@ types = { workspace = true } [dev-dependencies] operation_pool = { workspace = true } -serde_yaml = { workspace = true } state_processing = { workspace = true } tokio = { workspace = true } +yaml_serde = { workspace = true } diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index aeaa196df8..851eb5da6c 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -236,7 +236,7 @@ mod tests { fn serde() { let config = Config::default(); let serialized = - serde_yaml::to_string(&config).expect("should serde encode default config"); - serde_yaml::from_str::(&serialized).expect("should serde decode default config"); + yaml_serde::to_string(&config).expect("should serde encode default config"); + yaml_serde::from_str::(&serialized).expect("should serde decode default config"); } } diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 32090bccfc..774eac5fe2 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -5,7 +5,7 @@ use crate::http::{ ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, ENGINE_GET_PAYLOAD_V4, ENGINE_GET_PAYLOAD_V5, ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, - ENGINE_NEW_PAYLOAD_V4, + ENGINE_NEW_PAYLOAD_V4, ENGINE_NEW_PAYLOAD_V5, }; use eth2::types::{ BlobsBundle, SsePayloadAttributes, SsePayloadAttributesV1, SsePayloadAttributesV2, @@ -551,6 +551,7 @@ pub struct EngineCapabilities { pub new_payload_v2: bool, pub new_payload_v3: bool, pub new_payload_v4: bool, + pub new_payload_v5: bool, pub forkchoice_updated_v1: bool, pub forkchoice_updated_v2: bool, pub forkchoice_updated_v3: bool, @@ -581,6 +582,9 @@ impl EngineCapabilities { if self.new_payload_v4 { response.push(ENGINE_NEW_PAYLOAD_V4); } + if self.new_payload_v5 { + response.push(ENGINE_NEW_PAYLOAD_V5); + } if self.forkchoice_updated_v1 { response.push(ENGINE_FORKCHOICE_UPDATED_V1); } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index c421491f80..bcd95d1ae4 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -35,6 +35,7 @@ pub const ENGINE_NEW_PAYLOAD_V1: &str = "engine_newPayloadV1"; pub const ENGINE_NEW_PAYLOAD_V2: &str = "engine_newPayloadV2"; pub const ENGINE_NEW_PAYLOAD_V3: &str = "engine_newPayloadV3"; pub const ENGINE_NEW_PAYLOAD_V4: &str = "engine_newPayloadV4"; +pub const ENGINE_NEW_PAYLOAD_V5: &str = "engine_newPayloadV5"; pub const ENGINE_NEW_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(8); pub const ENGINE_GET_PAYLOAD_V1: &str = "engine_getPayloadV1"; @@ -74,6 +75,7 @@ pub static LIGHTHOUSE_CAPABILITIES: &[&str] = &[ ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V4, + ENGINE_NEW_PAYLOAD_V5, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, @@ -883,7 +885,7 @@ impl HttpJsonRpc { Ok(response.into()) } - pub async fn new_payload_v4_gloas( + pub async fn new_payload_v5_gloas( &self, new_payload_request_gloas: NewPayloadRequestGloas<'_, E>, ) -> Result { @@ -903,7 +905,7 @@ impl HttpJsonRpc { let response: JsonPayloadStatusV1 = self .rpc_request( - ENGINE_NEW_PAYLOAD_V4, + ENGINE_NEW_PAYLOAD_V5, params, ENGINE_NEW_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, ) @@ -1198,6 +1200,7 @@ impl HttpJsonRpc { new_payload_v2: capabilities.contains(ENGINE_NEW_PAYLOAD_V2), new_payload_v3: capabilities.contains(ENGINE_NEW_PAYLOAD_V3), new_payload_v4: capabilities.contains(ENGINE_NEW_PAYLOAD_V4), + new_payload_v5: capabilities.contains(ENGINE_NEW_PAYLOAD_V5), forkchoice_updated_v1: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V1), forkchoice_updated_v2: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V2), forkchoice_updated_v3: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V3), @@ -1353,10 +1356,10 @@ impl HttpJsonRpc { } } NewPayloadRequest::Gloas(new_payload_request_gloas) => { - if engine_capabilities.new_payload_v4 { - self.new_payload_v4_gloas(new_payload_request_gloas).await + if engine_capabilities.new_payload_v5 { + self.new_payload_v5_gloas(new_payload_request_gloas).await } else { - Err(Error::RequiredMethodUnsupported("engine_newPayloadV4")) + Err(Error::RequiredMethodUnsupported("engine_newPayloadV5")) } } } diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 7a81017b3f..e263e5402a 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -102,7 +102,8 @@ pub async fn handle_rpc( ENGINE_NEW_PAYLOAD_V1 | ENGINE_NEW_PAYLOAD_V2 | ENGINE_NEW_PAYLOAD_V3 - | ENGINE_NEW_PAYLOAD_V4 => { + | ENGINE_NEW_PAYLOAD_V4 + | ENGINE_NEW_PAYLOAD_V5 => { let request = match method { ENGINE_NEW_PAYLOAD_V1 => JsonExecutionPayload::Bellatrix( get_param::>(params, 0) @@ -118,17 +119,16 @@ pub async fn handle_rpc( ENGINE_NEW_PAYLOAD_V3 => get_param::>(params, 0) .map(|jep| JsonExecutionPayload::Deneb(jep)) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, - ENGINE_NEW_PAYLOAD_V4 => get_param::>(params, 0) - .map(|jep| JsonExecutionPayload::Gloas(jep)) - .or_else(|_| { - get_param::>(params, 0) - .map(|jep| JsonExecutionPayload::Fulu(jep)) - }) + ENGINE_NEW_PAYLOAD_V4 => get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::Fulu(jep)) .or_else(|_| { get_param::>(params, 0) .map(|jep| JsonExecutionPayload::Electra(jep)) }) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, + ENGINE_NEW_PAYLOAD_V5 => get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::Gloas(jep)) + .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, _ => unreachable!(), }; @@ -192,7 +192,7 @@ pub async fn handle_rpc( )); } } - ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { + ForkName::Electra | ForkName::Fulu => { if method == ENGINE_NEW_PAYLOAD_V1 || method == ENGINE_NEW_PAYLOAD_V2 || method == ENGINE_NEW_PAYLOAD_V3 @@ -230,6 +230,14 @@ pub async fn handle_rpc( )); } } + ForkName::Gloas => { + if method != ENGINE_NEW_PAYLOAD_V5 { + return Err(( + format!("{} called after Gloas fork!", method), + GENERIC_ERROR_CODE, + )); + } + } _ => unreachable!(), }; diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index d8e1e70e49..47e3c9064c 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -43,6 +43,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { new_payload_v2: true, new_payload_v3: true, new_payload_v4: true, + new_payload_v5: true, forkchoice_updated_v1: true, forkchoice_updated_v2: true, forkchoice_updated_v3: true, diff --git a/beacon_node/http_api/src/beacon/execution_payload_envelope.rs b/beacon_node/http_api/src/beacon/execution_payload_envelope.rs index 584ef40009..ea8c0d4b8a 100644 --- a/beacon_node/http_api/src/beacon/execution_payload_envelope.rs +++ b/beacon_node/http_api/src/beacon/execution_payload_envelope.rs @@ -1,15 +1,21 @@ +use crate::block_id::BlockId; use crate::task_spawner::{Priority, TaskSpawner}; use crate::utils::{ChainFilter, EthV1Filter, NetworkTxFilter, ResponseFilter, TaskSpawnerFilter}; +use crate::version::{ + ResponseIncludesVersion, add_consensus_version_header, add_ssz_content_type_header, + execution_optimistic_finalized_beacon_response, +}; use beacon_chain::payload_envelope_verification::gossip_verified_envelope::GossipVerifiedEnvelope; use beacon_chain::{ BeaconChain, BeaconChainTypes, NotifyExecutionLayer, payload_envelope_verification::EnvelopeError, }; use bytes::Bytes; +use eth2::types as api_types; use eth2::{CONTENT_TYPE_HEADER, SSZ_CONTENT_TYPE_HEADER}; use lighthouse_network::PubsubMessage; use network::NetworkMessage; -use ssz::Decode; +use ssz::{Decode, Encode}; use std::sync::Arc; use tokio::sync::mpsc::UnboundedSender; use tracing::{info, warn}; @@ -152,3 +158,72 @@ pub async fn publish_execution_payload_envelope( Ok(warp::reply().into_response()) } + +// TODO(gloas): add tests for this endpoint once we support importing payloads into the db +// GET beacon/execution_payload_envelope/{block_id} +pub(crate) fn get_beacon_execution_payload_envelope( + eth_v1: EthV1Filter, + block_id_or_err: impl Filter + + Clone + + Send + + Sync + + 'static, + task_spawner_filter: TaskSpawnerFilter, + chain_filter: ChainFilter, +) -> ResponseFilter { + eth_v1 + .and(warp::path("beacon")) + .and(warp::path("execution_payload_envelope")) + .and(block_id_or_err) + .and(warp::path::end()) + .and(task_spawner_filter) + .and(chain_filter) + .and(warp::header::optional::("accept")) + .then( + |block_id: BlockId, + task_spawner: TaskSpawner, + chain: Arc>, + accept_header: Option| { + task_spawner.blocking_response_task(Priority::P1, move || { + let (root, execution_optimistic, finalized) = block_id.root(&chain)?; + + let envelope = chain + .get_payload_envelope(&root) + .map_err(warp_utils::reject::unhandled_error)? + .ok_or_else(|| { + warp_utils::reject::custom_not_found(format!( + "execution payload envelope for block root {root}" + )) + })?; + + let fork_name = chain + .spec + .fork_name_at_slot::(envelope.message.slot); + + match accept_header { + Some(api_types::Accept::Ssz) => Response::builder() + .status(200) + .body(envelope.as_ssz_bytes().into()) + .map(|res: Response| add_ssz_content_type_header(res)) + .map_err(|e| { + warp_utils::reject::custom_server_error(format!( + "failed to create response: {}", + e + )) + }), + _ => { + let res = execution_optimistic_finalized_beacon_response( + ResponseIncludesVersion::Yes(fork_name), + execution_optimistic, + finalized, + &envelope, + )?; + Ok(warp::reply::json(&res).into_response()) + } + } + .map(|resp| add_consensus_version_header(resp, fork_name)) + }) + }, + ) + .boxed() +} diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 29e2d39aee..17d41cfbcd 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -35,7 +35,8 @@ mod validators; mod version; use crate::beacon::execution_payload_envelope::{ - post_beacon_execution_payload_envelope, post_beacon_execution_payload_envelope_ssz, + get_beacon_execution_payload_envelope, post_beacon_execution_payload_envelope, + post_beacon_execution_payload_envelope_ssz, }; use crate::beacon::pool::*; use crate::light_client::{get_light_client_bootstrap, get_light_client_updates}; @@ -1509,6 +1510,14 @@ pub fn serve( network_tx_filter.clone(), ); + // GET beacon/execution_payload_envelope/{block_id} + let get_beacon_execution_payload_envelope = get_beacon_execution_payload_envelope( + eth_v1.clone(), + block_id_or_err, + task_spawner_filter.clone(), + chain_filter.clone(), + ); + let beacon_rewards_path = eth_v1 .clone() .and(warp::path("beacon")) @@ -3158,6 +3167,21 @@ pub fn serve( api_types::EventTopic::BlockGossip => { event_handler.subscribe_block_gossip() } + api_types::EventTopic::ExecutionPayload => { + event_handler.subscribe_execution_payload() + } + api_types::EventTopic::ExecutionPayloadGossip => { + event_handler.subscribe_execution_payload_gossip() + } + api_types::EventTopic::ExecutionPayloadAvailable => { + event_handler.subscribe_execution_payload_available() + } + api_types::EventTopic::ExecutionPayloadBid => { + event_handler.subscribe_execution_payload_bid() + } + api_types::EventTopic::PayloadAttestationMessage => { + event_handler.subscribe_payload_attestation_message() + } }; receivers.push( @@ -3283,6 +3307,7 @@ pub fn serve( .uor(get_beacon_block_root) .uor(get_blob_sidecars) .uor(get_blobs) + .uor(get_beacon_execution_payload_envelope) .uor(get_beacon_pool_attestations) .uor(get_beacon_pool_attester_slashings) .uor(get_beacon_pool_proposer_slashings) diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index e0e4029875..15f61537a0 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -975,9 +975,10 @@ async fn proposer_duties_with_gossip_tolerance() { assert_eq!(harness.chain.slot().unwrap(), num_initial); // Set the clock to just before the next epoch. - harness.chain.slot_clock.advance_time( - Duration::from_secs(spec.seconds_per_slot) - spec.maximum_gossip_clock_disparity(), - ); + harness + .chain + .slot_clock + .advance_time(spec.get_slot_duration() - spec.maximum_gossip_clock_disparity()); assert_eq!( harness .chain @@ -1081,9 +1082,10 @@ async fn proposer_duties_v2_with_gossip_tolerance() { assert_eq!(harness.chain.slot().unwrap(), num_initial); // Set the clock to just before the next epoch. - harness.chain.slot_clock.advance_time( - Duration::from_secs(spec.seconds_per_slot) - spec.maximum_gossip_clock_disparity(), - ); + harness + .chain + .slot_clock + .advance_time(spec.get_slot_duration() - spec.maximum_gossip_clock_disparity()); assert_eq!( harness .chain diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index ca5710076b..1a1d348bb0 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -636,11 +636,14 @@ impl NetworkBeaconProcessor { // Back-sync batches are dispatched with a different `Work` variant so // they can be rate-limited. let work = match process_id { - ChainSegmentProcessId::RangeBatchId(_, _) => { + ChainSegmentProcessId::RangeBatchId(chain_id, epoch) => { let process_fn = async move { processor.process_chain_segment(process_id, blocks).await; }; - Work::ChainSegment(Box::pin(process_fn)) + Work::ChainSegment { + process_fn: Box::pin(process_fn), + process_id: (chain_id, epoch.as_u64()), + } } ChainSegmentProcessId::BackSyncBatchId(_) => { let process_fn = diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index b4586994e4..f6d4940121 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -125,6 +125,7 @@ impl NetworkBeaconProcessor { verified_envelope, NotifyExecutionLayer::Yes, BlockImportSource::Lookup, + #[allow(clippy::result_large_err)] || Ok(()), ) .await; diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index cd872df887..a26996ec5e 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -1,16 +1,18 @@ use super::*; use crate::NetworkMessage; -use crate::network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor}; +use crate::network_beacon_processor::{ + ChainSegmentProcessId, InvalidBlockStorage, NetworkBeaconProcessor, +}; use crate::sync::block_lookups::{BlockLookupSummary, PARENT_DEPTH_TOLERANCE}; use crate::sync::{ SyncMessage, - manager::{BlockProcessType, BlockProcessingResult, SyncManager}, + manager::{BatchProcessResult, BlockProcessType, BlockProcessingResult, SyncManager}, }; use beacon_chain::blob_verification::KzgVerifiedBlob; use beacon_chain::block_verification_types::LookupBlock; use beacon_chain::custody_context::NodeCustodyType; use beacon_chain::{ - AvailabilityProcessingStatus, BlockError, NotifyExecutionLayer, + AvailabilityProcessingStatus, BlockError, EngineState, NotifyExecutionLayer, block_verification_types::{AsBlock, AvailableBlockData}, data_availability_checker::Availability, test_utils::{ @@ -23,7 +25,7 @@ use educe::Educe; use itertools::Itertools; use lighthouse_network::discovery::CombinedKey; use lighthouse_network::{ - NetworkConfig, NetworkGlobals, PeerId, + NetworkConfig, NetworkGlobals, PeerAction, PeerId, rpc::{RPCError, RequestType}, service::api_types::{AppRequestId, SyncRequestId}, types::SyncState, @@ -64,14 +66,33 @@ pub struct SimulateConfig { Option Option + Send + Sync>>, // Import a block directly before processing it (for simulating race conditions) import_block_before_process: HashSet, + /// Number of range batch processing attempts that return FaultyFailure + range_faulty_failures: usize, + /// Number of range batch processing attempts that return NonFaultyFailure + range_non_faulty_failures: usize, + /// Number of BlocksByRange requests that return empty (no blocks) + return_no_range_blocks_n_times: usize, + /// Number of DataColumnsByRange requests that return empty (no columns) + return_no_range_columns_n_times: usize, + /// Number of DataColumnsByRange requests that return columns with unrequested indices + return_wrong_range_column_indices_n_times: usize, + /// Number of DataColumnsByRange requests that return columns with unrequested slots + return_wrong_range_column_slots_n_times: usize, + /// Number of DataColumnsByRange requests that return fewer columns than requested + /// (drops half the columns). Triggers CouplingError::DataColumnPeerFailure → retry_partial_batch + return_partial_range_columns_n_times: usize, + /// Set EE offline at start, bring back online after this many BlocksByRange responses + ee_offline_for_n_range_responses: Option, + /// Disconnect all peers after this many successful BlocksByRange responses. + successful_range_responses_before_disconnect: Option, } impl SimulateConfig { - fn new() -> Self { + pub(super) fn new() -> Self { Self::default() } - fn happy_path() -> Self { + pub(super) fn happy_path() -> Self { Self::default() } @@ -111,7 +132,7 @@ impl SimulateConfig { self } - fn return_rpc_error(mut self, error: RPCError) -> Self { + pub(super) fn return_rpc_error(mut self, error: RPCError) -> Self { self.return_rpc_error = Some(error); self } @@ -133,6 +154,51 @@ impl SimulateConfig { self.import_block_before_process.insert(block_root); self } + + pub(super) fn with_range_faulty_failures(mut self, n: usize) -> Self { + self.range_faulty_failures = n; + self + } + + pub(super) fn with_range_non_faulty_failures(mut self, n: usize) -> Self { + self.range_non_faulty_failures = n; + self + } + + pub(super) fn with_no_range_blocks_n_times(mut self, n: usize) -> Self { + self.return_no_range_blocks_n_times = n; + self + } + + pub(super) fn with_no_range_columns_n_times(mut self, n: usize) -> Self { + self.return_no_range_columns_n_times = n; + self + } + + pub(super) fn with_wrong_range_column_indices_n_times(mut self, n: usize) -> Self { + self.return_wrong_range_column_indices_n_times = n; + self + } + + pub(super) fn with_wrong_range_column_slots_n_times(mut self, n: usize) -> Self { + self.return_wrong_range_column_slots_n_times = n; + self + } + + pub(super) fn with_partial_range_columns_n_times(mut self, n: usize) -> Self { + self.return_partial_range_columns_n_times = n; + self + } + + pub(super) fn with_ee_offline_for_n_range_responses(mut self, n: usize) -> Self { + self.ee_offline_for_n_range_responses = Some(n); + self + } + + pub(super) fn with_disconnect_after_range_requests(mut self, n: usize) -> Self { + self.successful_range_responses_before_disconnect = Some(n); + self + } } fn genesis_fork() -> ForkName { @@ -256,6 +322,7 @@ impl TestRig { }) } + #[allow(dead_code)] pub fn with_custody_type(node_custody_type: NodeCustodyType) -> Self { Self::new(TestRigConfig { fulu_test_type: FuluTestType::WeFullnodeThemSupernode, @@ -267,13 +334,23 @@ impl TestRig { /// /// Processes events from sync_rx (sink), beacon processor, and network queues in fixed /// priority order each tick. Handles completed work before pulling new requests. - async fn simulate(&mut self, complete_strategy: SimulateConfig) { + pub(super) async fn simulate(&mut self, complete_strategy: SimulateConfig) { self.complete_strategy = complete_strategy; self.log(&format!( "Running simulate with config {:?}", self.complete_strategy )); + // Set EE offline at the start if configured + if self + .complete_strategy + .ee_offline_for_n_range_responses + .is_some() + { + self.sync_manager + .update_execution_engine_state(EngineState::Offline); + } + let mut i = 0; loop { @@ -352,9 +429,34 @@ impl TestRig { process_fn.await } } - Work::RpcBlobs { process_fn } - | Work::RpcCustodyColumn(process_fn) - | Work::ChainSegment(process_fn) => process_fn.await, + Work::RpcBlobs { process_fn } | Work::RpcCustodyColumn(process_fn) => { + process_fn.await + } + Work::ChainSegment { + process_fn, + process_id: (chain_id, batch_epoch), + } => { + let sync_type = + ChainSegmentProcessId::RangeBatchId(chain_id, batch_epoch.into()); + if self.complete_strategy.range_faulty_failures > 0 { + self.complete_strategy.range_faulty_failures -= 1; + self.push_sync_message(SyncMessage::BatchProcessed { + sync_type, + result: BatchProcessResult::FaultyFailure { + imported_blocks: 0, + penalty: PeerAction::LowToleranceError, + }, + }); + } else if self.complete_strategy.range_non_faulty_failures > 0 { + self.complete_strategy.range_non_faulty_failures -= 1; + self.push_sync_message(SyncMessage::BatchProcessed { + sync_type, + result: BatchProcessResult::NonFaultyFailure, + }); + } else { + process_fn.await; + } + } Work::Reprocess(_) => {} // ignore other => panic!("Unsupported Work event {}", other.str_id()), } @@ -573,15 +675,50 @@ impl TestRig { if self.complete_strategy.skip_by_range_routes { return; } - let blocks = (*req.start_slot()..req.start_slot() + req.count()) - .filter_map(|slot| { - self.network_blocks_by_slot - .get(&Slot::new(slot)) - .map(|block| block.block_cloned()) - }) - .collect::>(); - self.send_rpc_blocks_response(req_id, peer_id, &blocks); + // Check if we should disconnect all peers instead of continuing + if let Some(ref mut remaining) = self + .complete_strategy + .successful_range_responses_before_disconnect + { + if *remaining == 0 { + // Disconnect all peers — remaining responses become "late" + for peer in self.get_connected_peers() { + self.peer_disconnected(peer); + } + return; + } else { + *remaining -= 1; + } + } + + // Return empty response N times to simulate peer returning no blocks + if self.complete_strategy.return_no_range_blocks_n_times > 0 { + self.complete_strategy.return_no_range_blocks_n_times -= 1; + self.send_rpc_blocks_response(req_id, peer_id, &[]); + } else { + let blocks = (*req.start_slot()..req.start_slot() + req.count()) + .filter_map(|slot| { + self.network_blocks_by_slot + .get(&Slot::new(slot)) + .map(|block| block.block_cloned()) + }) + .collect::>(); + self.send_rpc_blocks_response(req_id, peer_id, &blocks); + } + + // Bring EE back online after N range responses + if let Some(ref mut remaining) = + self.complete_strategy.ee_offline_for_n_range_responses + { + if *remaining == 0 { + self.sync_manager + .update_execution_engine_state(EngineState::Online); + self.complete_strategy.ee_offline_for_n_range_responses = None; + } else { + *remaining -= 1; + } + } } (RequestType::BlobsByRange(req), AppRequestId::Sync(req_id)) => { @@ -605,10 +742,80 @@ impl TestRig { if self.complete_strategy.skip_by_range_routes { return; } - // Note: This function is permissive, blocks may have zero columns and it won't - // error. Some caveats: - // - The genesis block never has columns - // - Some blocks may not have columns as the blob count is random + + // Return empty columns N times + if self.complete_strategy.return_no_range_columns_n_times > 0 { + self.complete_strategy.return_no_range_columns_n_times -= 1; + self.send_rpc_columns_response(req_id, peer_id, &[]); + return; + } + + // Return columns with unrequested indices N times. + // Note: for supernodes this returns no columns since they custody all indices. + if self + .complete_strategy + .return_wrong_range_column_indices_n_times + > 0 + { + self.complete_strategy + .return_wrong_range_column_indices_n_times -= 1; + let wrong_columns = (req.start_slot..req.start_slot + req.count) + .filter_map(|slot| self.network_blocks_by_slot.get(&Slot::new(slot))) + .filter_map(|block| block.block_data().data_columns()) + .flat_map(|columns| { + columns + .into_iter() + .filter(|c| !req.columns.contains(c.index())) + }) + .collect::>(); + self.send_rpc_columns_response(req_id, peer_id, &wrong_columns); + return; + } + + // Return columns from an out-of-range slot N times + if self + .complete_strategy + .return_wrong_range_column_slots_n_times + > 0 + { + self.complete_strategy + .return_wrong_range_column_slots_n_times -= 1; + // Get a column from a slot AFTER the requested range + let wrong_slot = req.start_slot + req.count; + let wrong_columns = self + .network_blocks_by_slot + .get(&Slot::new(wrong_slot)) + .and_then(|block| block.block_data().data_columns()) + .into_iter() + .flat_map(|columns| { + columns + .into_iter() + .filter(|c| req.columns.contains(c.index())) + }) + .collect::>(); + self.send_rpc_columns_response(req_id, peer_id, &wrong_columns); + return; + } + + // Return only half the requested columns N times — triggers CouplingError + if self.complete_strategy.return_partial_range_columns_n_times > 0 { + self.complete_strategy.return_partial_range_columns_n_times -= 1; + let columns = (req.start_slot..req.start_slot + req.count) + .filter_map(|slot| self.network_blocks_by_slot.get(&Slot::new(slot))) + .filter_map(|block| block.block_data().data_columns()) + .flat_map(|columns| { + columns + .into_iter() + .filter(|c| req.columns.contains(c.index())) + }) + .enumerate() + .filter(|(i, _)| i % 2 == 0) // keep every other column + .map(|(_, c)| c) + .collect::>(); + self.send_rpc_columns_response(req_id, peer_id, &columns); + return; + } + let columns = (req.start_slot..req.start_slot + req.count) .filter_map(|slot| self.network_blocks_by_slot.get(&Slot::new(slot))) .filter_map(|block| block.block_data().data_columns()) @@ -726,7 +933,7 @@ impl TestRig { // Preparation steps /// Returns the block root of the tip of the built chain - async fn build_chain(&mut self, block_count: usize) -> Hash256 { + pub(super) async fn build_chain(&mut self, block_count: usize) -> Hash256 { let mut blocks = vec![]; // Initialise a new beacon chain @@ -947,6 +1154,30 @@ impl TestRig { self.trigger_with_last_block(); } + /// Import blocks for slots 1..=up_to_slot into the local chain (advance local head) + pub(super) async fn import_blocks_up_to_slot(&mut self, up_to_slot: u64) { + for slot in 1..=up_to_slot { + let rpc_block = self + .network_blocks_by_slot + .get(&Slot::new(slot)) + .unwrap_or_else(|| panic!("No block at slot {slot}")) + .clone(); + let block_root = rpc_block.canonical_root(); + self.harness + .chain + .process_block( + block_root, + rpc_block, + NotifyExecutionLayer::Yes, + BlockImportSource::Gossip, + || Ok(()), + ) + .await + .unwrap(); + } + self.harness.chain.recompute_head_at_current_slot().await; + } + /// Import a block directly into the chain without going through lookup sync async fn import_block_by_root(&mut self, block_root: Hash256) { let range_sync_block = self @@ -1000,23 +1231,32 @@ impl TestRig { // Post-test assertions - fn head_slot(&self) -> Slot { + pub(super) fn head_slot(&self) -> Slot { self.harness.chain.head().head_slot() } - fn assert_head_slot(&self, slot: u64) { + pub(super) fn assert_head_slot(&self, slot: u64) { assert_eq!(self.head_slot(), Slot::new(slot), "Unexpected head slot"); } - fn max_known_slot(&self) -> Slot { + pub(super) fn max_known_slot(&self) -> Slot { self.network_blocks_by_slot .keys() .max() .copied() - .expect("no blocks") + .unwrap_or_default() } - fn assert_penalties(&self, expected_penalties: &[&'static str]) { + pub(super) fn finalized_epoch(&self) -> types::Epoch { + self.harness + .chain + .canonical_head + .cached_head() + .finalized_checkpoint() + .epoch + } + + pub(super) fn assert_penalties(&self, expected_penalties: &[&'static str]) { let penalties = self .penalties .iter() @@ -1034,7 +1274,7 @@ impl TestRig { } } - fn assert_penalties_of_type(&self, expected_penalty: &'static str) { + pub(super) fn assert_penalties_of_type(&self, expected_penalty: &'static str) { if self.penalties.is_empty() { panic!("No penalties but expected some of type {expected_penalty}"); } @@ -1051,7 +1291,7 @@ impl TestRig { } } - fn assert_no_penalties(&mut self) { + pub(super) fn assert_no_penalties(&mut self) { if !self.penalties.is_empty() { panic!("Some downscore events: {:?}", self.penalties); } @@ -1102,7 +1342,7 @@ impl TestRig { } /// Assert there is at least one range sync chain created and that all sync chains completed - fn assert_successful_range_sync(&self) { + pub(super) fn assert_successful_range_sync(&self) { assert!( self.range_sync_chains_added() > 0, "No created range sync chains" @@ -1425,6 +1665,7 @@ impl TestRig { } } + #[allow(dead_code)] pub fn pop_received_processor_event) -> Option>( &mut self, predicate_transform: F, diff --git a/beacon_node/network/src/sync/tests/range.rs b/beacon_node/network/src/sync/tests/range.rs index c19ee8eb6d..891d9d1e97 100644 --- a/beacon_node/network/src/sync/tests/range.rs +++ b/beacon_node/network/src/sync/tests/range.rs @@ -1,110 +1,47 @@ +//! Range sync tests for `BlocksByRange`, `BlobsByRange`, `DataColumnsByRange`. +//! +//! Tests follow the pattern from `lookups.rs`: +//! ```ignore +//! async fn test_name() { +//! let mut r = TestRig::default(); +//! r.setup_xyz().await; +//! r.simulate(SimulateConfig::happy_path()).await; +//! r.assert_range_sync_completed(); +//! } +//! ``` +//! +//! Rules: +//! - Tests must be succinct and readable (3-10 lines per test body) +//! - All complex logic lives in helpers (setup, SimulateConfig, assert) +//! - Test bodies must not manually grab requests, send SyncMessages, or do anything overly specific +//! - All tests use `simulate()` if they need peers to fulfill requests +//! - Extend `SimulateConfig` for new range-specific behaviors +//! - Extend `simulate()` to support by_range methods + +use super::lookups::SimulateConfig; use super::*; -use crate::network_beacon_processor::ChainSegmentProcessId; use crate::status::ToStatusMessage; use crate::sync::SyncMessage; use crate::sync::manager::SLOT_IMPORT_TOLERANCE; -use crate::sync::network_context::RangeRequestId; use crate::sync::range_sync::RangeSyncType; -use beacon_chain::BeaconChain; -use beacon_chain::block_verification_types::AvailableBlockData; -use beacon_chain::custody_context::NodeCustodyType; -use beacon_chain::data_column_verification::CustodyDataColumn; -use beacon_chain::test_utils::{AttestationStrategy, BlockStrategy}; -use beacon_chain::{EngineState, NotifyExecutionLayer, block_verification_types::RangeSyncBlock}; -use beacon_processor::WorkType; -use lighthouse_network::rpc::RequestType; -use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, DataColumnsByRangeRequest, OldBlocksByRangeRequest, - OldBlocksByRangeRequestV2, StatusMessageV2, -}; -use lighthouse_network::service::api_types::{ - AppRequestId, BlobsByRangeRequestId, BlocksByRangeRequestId, DataColumnsByRangeRequestId, - SyncRequestId, -}; +use lighthouse_network::rpc::RPCError; +use lighthouse_network::rpc::methods::StatusMessageV2; use lighthouse_network::{PeerId, SyncInfo}; -use std::time::Duration; -use types::{ - BlobSidecarList, BlockImportSource, Epoch, EthSpec, Hash256, MinimalEthSpec as E, - SignedBeaconBlock, SignedBeaconBlockHash, Slot, -}; +use types::{Epoch, EthSpec, Hash256, MinimalEthSpec as E, Slot}; -const D: Duration = Duration::new(0, 0); - -pub(crate) enum DataSidecars { - Blobs(BlobSidecarList), - DataColumns(Vec>), -} - -enum ByRangeDataRequestIds { - PreDeneb, - PrePeerDAS(BlobsByRangeRequestId, PeerId), - PostPeerDAS(Vec<(DataColumnsByRangeRequestId, PeerId)>), -} - -/// Sync tests are usually written in the form: -/// - Do some action -/// - Expect a request to be sent -/// - Complete the above request -/// -/// To make writting tests succint, the machinery in this testing rig automatically identifies -/// _which_ request to complete. Picking the right request is critical for tests to pass, so this -/// filter allows better expressivity on the criteria to identify the right request. -#[derive(Default, Debug, Clone)] -struct RequestFilter { - peer: Option, - epoch: Option, -} - -impl RequestFilter { - fn peer(mut self, peer: PeerId) -> Self { - self.peer = Some(peer); - self - } - - fn epoch(mut self, epoch: u64) -> Self { - self.epoch = Some(epoch); - self - } -} - -fn filter() -> RequestFilter { - RequestFilter::default() -} +/// MinimalEthSpec has 8 slots per epoch +const SLOTS_PER_EPOCH: usize = 8; impl TestRig { - /// Produce a head peer with an advanced head fn add_head_peer(&mut self) -> PeerId { - self.add_head_peer_with_root(Hash256::random()) - } - - /// Produce a head peer with an advanced head - fn add_head_peer_with_root(&mut self, head_root: Hash256) -> PeerId { let local_info = self.local_info(); self.add_supernode_peer(SyncInfo { - head_root, + head_root: Hash256::random(), head_slot: local_info.head_slot + 1 + Slot::new(SLOT_IMPORT_TOLERANCE as u64), ..local_info }) } - // Produce a finalized peer with an advanced finalized epoch - fn add_finalized_peer(&mut self) -> PeerId { - self.add_finalized_peer_with_root(Hash256::random()) - } - - // Produce a finalized peer with an advanced finalized epoch - fn add_finalized_peer_with_root(&mut self, finalized_root: Hash256) -> PeerId { - let local_info = self.local_info(); - let finalized_epoch = local_info.finalized_epoch + 2; - self.add_supernode_peer(SyncInfo { - finalized_epoch, - finalized_root, - head_slot: finalized_epoch.start_slot(E::slots_per_epoch()), - head_root: Hash256::random(), - earliest_available_slot: None, - }) - } - fn finalized_remote_info_advanced_by(&self, advanced_epochs: Epoch) -> SyncInfo { let local_info = self.local_info(); let finalized_epoch = local_info.finalized_epoch + advanced_epochs; @@ -142,11 +79,7 @@ impl TestRig { } fn add_supernode_peer(&mut self, remote_info: SyncInfo) -> PeerId { - // Create valid peer known to network globals - // TODO(fulu): Using supernode peers to ensure we have peer across all column - // subnets for syncing. Should add tests connecting to full node peers. let peer_id = self.new_connected_supernode_peer(); - // Send peer to sync self.send_sync_message(SyncMessage::AddPeer(peer_id, remote_info)); peer_id } @@ -184,450 +117,362 @@ impl TestRig { ) } - #[track_caller] - fn assert_chain_segments(&mut self, count: usize) { - for i in 0..count { - self.pop_received_processor_event(|ev| { - (ev.work_type() == beacon_processor::WorkType::ChainSegment).then_some(()) - }) - .unwrap_or_else(|e| panic!("Expect ChainSegment work event count {i}: {e:?}")); - } + // -- Setup helpers -- + + /// Head sync: peers whose finalized root/epoch match ours (known to fork choice), + /// but whose head is ahead. Only head chain is created. + async fn setup_head_sync(&mut self) { + self.build_chain(SLOTS_PER_EPOCH).await; + self.add_head_peer(); + self.assert_state(RangeSyncType::Head); } - fn update_execution_engine_state(&mut self, state: EngineState) { - self.log(&format!("execution engine state updated: {state:?}")); - self.sync_manager.update_execution_engine_state(state); + /// Finalized sync: peers whose finalized epoch is advanced and head == finalized start slot. + /// Returns the remote SyncInfo (needed for blacklist tests). + async fn setup_finalized_sync(&mut self) -> SyncInfo { + let advanced_epochs = 5; + self.build_chain(advanced_epochs * SLOTS_PER_EPOCH).await; + let remote_info = self.finalized_remote_info_advanced_by((advanced_epochs as u64).into()); + self.add_fullnode_peers(remote_info.clone(), 100); + self.add_supernode_peer(remote_info.clone()); + self.assert_state(RangeSyncType::Finalized); + remote_info } - fn find_blocks_by_range_request( - &mut self, - request_filter: RequestFilter, - ) -> ((BlocksByRangeRequestId, PeerId), ByRangeDataRequestIds) { - let filter_f = |peer: PeerId, start_slot: u64| { - if let Some(expected_epoch) = request_filter.epoch { - let epoch = Slot::new(start_slot).epoch(E::slots_per_epoch()).as_u64(); - if epoch != expected_epoch { - return false; - } - } - if let Some(expected_peer) = request_filter.peer - && peer != expected_peer - { - return false; - } - - true + /// Finalized-to-head: peers whose finalized is advanced AND head is beyond finalized. + /// After finalized sync completes, head chains are created from awaiting_head_peers. + async fn setup_finalized_and_head_sync(&mut self) { + let finalized_epochs = 5; + let head_epochs = 7; + self.build_chain(head_epochs * SLOTS_PER_EPOCH).await; + let local_info = self.local_info(); + let finalized_epoch = local_info.finalized_epoch + Epoch::new(finalized_epochs as u64); + let head_slot = Slot::new((head_epochs * SLOTS_PER_EPOCH) as u64); + let remote_info = SyncInfo { + finalized_epoch, + finalized_root: Hash256::random(), + head_slot, + head_root: Hash256::random(), + earliest_available_slot: None, }; - - let block_req = self - .pop_received_network_event(|ev| match ev { - NetworkMessage::SendRequest { - peer_id, - request: - RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( - OldBlocksByRangeRequestV2 { start_slot, .. }, - )), - app_request_id: AppRequestId::Sync(SyncRequestId::BlocksByRange(id)), - } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id)), - _ => None, - }) - .unwrap_or_else(|e| { - panic!("Should have a BlocksByRange request, filter {request_filter:?}: {e:?}") - }); - - let by_range_data_requests = if self.is_after_fulu() { - let mut data_columns_requests = vec![]; - while let Ok(data_columns_request) = self.pop_received_network_event(|ev| match ev { - NetworkMessage::SendRequest { - peer_id, - request: - RequestType::DataColumnsByRange(DataColumnsByRangeRequest { - start_slot, .. - }), - app_request_id: AppRequestId::Sync(SyncRequestId::DataColumnsByRange(id)), - } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id)), - _ => None, - }) { - data_columns_requests.push(data_columns_request); - } - if data_columns_requests.is_empty() { - panic!("Found zero DataColumnsByRange requests, filter {request_filter:?}"); - } - ByRangeDataRequestIds::PostPeerDAS(data_columns_requests) - } else if self.is_after_deneb() { - let (id, peer) = self - .pop_received_network_event(|ev| match ev { - NetworkMessage::SendRequest { - peer_id, - request: RequestType::BlobsByRange(BlobsByRangeRequest { start_slot, .. }), - app_request_id: AppRequestId::Sync(SyncRequestId::BlobsByRange(id)), - } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id)), - _ => None, - }) - .unwrap_or_else(|e| { - panic!("Should have a blobs by range request, filter {request_filter:?}: {e:?}") - }); - ByRangeDataRequestIds::PrePeerDAS(id, peer) - } else { - ByRangeDataRequestIds::PreDeneb - }; - - (block_req, by_range_data_requests) + self.add_fullnode_peers(remote_info.clone(), 100); + self.add_supernode_peer(remote_info); + self.assert_state(RangeSyncType::Finalized); } - fn find_and_complete_blocks_by_range_request( - &mut self, - request_filter: RequestFilter, - ) -> RangeRequestId { - let ((blocks_req_id, block_peer), by_range_data_request_ids) = - self.find_blocks_by_range_request(request_filter); - - // Complete the request with a single stream termination - self.log(&format!( - "Completing BlocksByRange request {blocks_req_id:?} with empty stream" - )); - self.send_sync_message(SyncMessage::RpcBlock { - sync_request_id: SyncRequestId::BlocksByRange(blocks_req_id), - peer_id: block_peer, - beacon_block: None, - seen_timestamp: D, - }); - - match by_range_data_request_ids { - ByRangeDataRequestIds::PreDeneb => {} - ByRangeDataRequestIds::PrePeerDAS(id, peer_id) => { - // Complete the request with a single stream termination - self.log(&format!( - "Completing BlobsByRange request {id:?} with empty stream" - )); - self.send_sync_message(SyncMessage::RpcBlob { - sync_request_id: SyncRequestId::BlobsByRange(id), - peer_id, - blob_sidecar: None, - seen_timestamp: D, - }); - } - ByRangeDataRequestIds::PostPeerDAS(data_column_req_ids) => { - // Complete the request with a single stream termination - for (id, peer_id) in data_column_req_ids { - self.log(&format!( - "Completing DataColumnsByRange request {id:?} with empty stream" - )); - self.send_sync_message(SyncMessage::RpcDataColumn { - sync_request_id: SyncRequestId::DataColumnsByRange(id), - peer_id, - data_column: None, - seen_timestamp: D, - }); - } - } - } - - blocks_req_id.parent_request_id.requester + /// Finalized sync with only 1 fullnode peer (insufficient custody coverage). + /// Returns remote_info to pass to `add_remaining_finalized_peers`. + async fn setup_finalized_sync_insufficient_peers(&mut self) -> SyncInfo { + let advanced_epochs = 5; + self.build_chain(advanced_epochs * SLOTS_PER_EPOCH).await; + let remote_info = self.finalized_remote_info_advanced_by((advanced_epochs as u64).into()); + self.add_fullnode_peer(remote_info.clone()); + self.assert_state(RangeSyncType::Finalized); + remote_info } - fn find_and_complete_processing_chain_segment(&mut self, id: ChainSegmentProcessId) { - self.pop_received_processor_event(|ev| { - (ev.work_type() == WorkType::ChainSegment).then_some(()) - }) - .unwrap_or_else(|e| panic!("Expected chain segment work event: {e}")); - - self.log(&format!( - "Completing ChainSegment processing work {id:?} with success" - )); - self.send_sync_message(SyncMessage::BatchProcessed { - sync_type: id, - result: crate::sync::BatchProcessResult::Success { - sent_blocks: 8, - imported_blocks: 8, - }, - }); - } - - fn complete_and_process_range_sync_until( - &mut self, - last_epoch: u64, - request_filter: RequestFilter, - ) { - for epoch in 0..last_epoch { - // Note: In this test we can't predict the block peer - let id = - self.find_and_complete_blocks_by_range_request(request_filter.clone().epoch(epoch)); - if let RangeRequestId::RangeSync { batch_id, .. } = id { - assert_eq!(batch_id.as_u64(), epoch, "Unexpected batch_id"); - } else { - panic!("unexpected RangeRequestId {id:?}"); - } - - let id = match id { - RangeRequestId::RangeSync { chain_id, batch_id } => { - ChainSegmentProcessId::RangeBatchId(chain_id, batch_id) - } - RangeRequestId::BackfillSync { batch_id } => { - ChainSegmentProcessId::BackSyncBatchId(batch_id) - } - }; - - self.find_and_complete_processing_chain_segment(id); - if epoch < last_epoch - 1 { - self.assert_state(RangeSyncType::Finalized); - } else { - self.assert_no_chains_exist(); - self.assert_no_failed_chains(); - } - } - } - - async fn create_canonical_block(&mut self) -> (SignedBeaconBlock, Option>) { - self.harness.advance_slot(); - - let block_root = self - .harness - .extend_chain( - 1, - BlockStrategy::OnCanonicalHead, - AttestationStrategy::AllValidators, - ) + /// Finalized sync where local node already has blocks up to `local_epochs`. + /// Triggers optimistic start: the chain tries to download a batch at the local head + /// epoch concurrently with sequential processing from the start. + async fn setup_finalized_sync_with_local_head(&mut self, local_epochs: usize) { + let target_epochs = local_epochs + 3; // target beyond local head + self.build_chain(target_epochs * SLOTS_PER_EPOCH).await; + self.import_blocks_up_to_slot((local_epochs * SLOTS_PER_EPOCH) as u64) .await; - - let store = &self.harness.chain.store; - let block = store.get_full_block(&block_root).unwrap().unwrap(); - let fork = block.fork_name_unchecked(); - - let data_sidecars = if fork.fulu_enabled() { - store - .get_data_columns(&block_root, fork) - .unwrap() - .map(|columns| { - columns - .into_iter() - .map(CustodyDataColumn::from_asserted_custody) - .collect() - }) - .map(DataSidecars::DataColumns) - } else if fork.deneb_enabled() { - store - .get_blobs(&block_root) - .unwrap() - .blobs() - .map(DataSidecars::Blobs) - } else { - None - }; - - (block, data_sidecars) + let remote_info = self.finalized_remote_info_advanced_by((target_epochs as u64).into()); + self.add_fullnode_peers(remote_info.clone(), 100); + self.add_supernode_peer(remote_info); + self.assert_state(RangeSyncType::Finalized); } - async fn remember_block( - &mut self, - (block, data_sidecars): (SignedBeaconBlock, Option>), - ) { - // This code is kind of duplicated from Harness::process_block, but takes sidecars directly. - let block_root = block.canonical_root(); - self.harness.set_current_slot(block.slot()); - let _: SignedBeaconBlockHash = self - .harness - .chain - .process_block( - block_root, - build_range_sync_block(block.into(), &data_sidecars, self.harness.chain.clone()), - NotifyExecutionLayer::Yes, - BlockImportSource::RangeSync, - || Ok(()), - ) - .await - .unwrap() - .try_into() - .unwrap(); - self.harness.chain.recompute_head_at_current_slot().await; + /// Add enough peers to cover all custody columns (same chain as insufficient setup) + fn add_remaining_finalized_peers(&mut self, remote_info: SyncInfo) { + self.add_fullnode_peers(remote_info.clone(), 100); + self.add_supernode_peer(remote_info); + } + + // -- Assert helpers -- + + /// Assert range sync completed: chains created and removed, all blocks ingested, + /// finalized epoch advanced, no penalties, no leftover events. + fn assert_range_sync_completed(&mut self) { + self.assert_successful_range_sync(); + self.assert_no_failed_chains(); + assert_eq!( + self.head_slot(), + self.max_known_slot(), + "Head slot should match the last built block (all blocks ingested)" + ); + assert!( + self.finalized_epoch() > types::Epoch::new(0), + "Finalized epoch should have advanced past genesis, got {}", + self.finalized_epoch() + ); + self.assert_no_penalties(); + self.assert_empty_network(); + self.assert_empty_processor(); + } + + /// Assert head sync completed (no finalization expected for short ranges) + fn assert_head_sync_completed(&mut self) { + self.assert_successful_range_sync(); + self.assert_no_failed_chains(); + assert_eq!( + self.head_slot(), + self.max_known_slot(), + "Head slot should match the last built block (all blocks ingested)" + ); + self.assert_no_penalties(); + } + + /// Assert chain was removed and peers received faulty_chain penalty + fn assert_range_sync_chain_failed(&mut self) { + self.assert_no_chains_exist(); + assert!( + self.penalties.iter().any(|p| p.msg == "faulty_chain"), + "Expected faulty_chain penalty, got {:?}", + self.penalties + ); + } + + /// Assert range sync removed chains (e.g., all peers disconnected) + fn assert_range_sync_chain_removed(&mut self) { + self.assert_no_chains_exist(); + } + + /// Assert a new peer with a blacklisted root gets disconnected + fn assert_peer_blacklisted(&mut self, remote_info: SyncInfo) { + let new_peer = self.add_supernode_peer(remote_info); + self.pop_received_network_event(|ev| match ev { + NetworkMessage::GoodbyePeer { peer_id, .. } if *peer_id == new_peer => Some(()), + _ => None, + }) + .expect("Peer with blacklisted root should receive Goodbye"); } } -fn build_range_sync_block( - block: Arc>, - data_sidecars: &Option>, - chain: Arc>, -) -> RangeSyncBlock { - match data_sidecars { - Some(DataSidecars::Blobs(blobs)) => { - let block_data = AvailableBlockData::new_with_blobs(blobs.clone()); - RangeSyncBlock::new( - block, - block_data, - &chain.data_availability_checker, - chain.spec.clone(), - ) - .unwrap() - } - Some(DataSidecars::DataColumns(columns)) => { - let block_data = AvailableBlockData::new_with_data_columns( - columns - .iter() - .map(|c| c.as_data_column().clone()) - .collect::>(), - ); - RangeSyncBlock::new( - block, - block_data, - &chain.data_availability_checker, - chain.spec.clone(), - ) - .unwrap() - } - // Block has no data, expects zero columns - None => RangeSyncBlock::new( - block, - AvailableBlockData::NoData, - &chain.data_availability_checker, - chain.spec.clone(), - ) - .unwrap(), - } -} - -#[test] -fn head_chain_removed_while_finalized_syncing() { - // NOTE: this is a regression test. - // Added in PR https://github.com/sigp/lighthouse/pull/2821 - let mut rig = TestRig::default(); - - // Get a peer with an advanced head - let head_peer = rig.add_head_peer(); - rig.assert_state(RangeSyncType::Head); - - // Sync should have requested a batch, grab the request. - let _ = rig.find_blocks_by_range_request(filter().peer(head_peer)); - - // Now get a peer with an advanced finalized epoch. - let finalized_peer = rig.add_finalized_peer(); - rig.assert_state(RangeSyncType::Finalized); - - // Sync should have requested a batch, grab the request - let _ = rig.find_blocks_by_range_request(filter().peer(finalized_peer)); - - // Fail the head chain by disconnecting the peer. - rig.peer_disconnected(head_peer); - rig.assert_state(RangeSyncType::Finalized); -} +// ============================================================================================ +// Tests +// ============================================================================================ +/// Head sync: single peer slightly ahead → download batches → all blocks ingested. #[tokio::test] -async fn state_update_while_purging() { - // NOTE: this is a regression test. - // Added in PR https://github.com/sigp/lighthouse/pull/2827 - let mut rig = TestRig::with_custody_type(NodeCustodyType::SemiSupernode); - - // Create blocks on a separate harness - // SemiSupernode ensures enough columns are stored for sampling + custody RPC block validation - let mut rig_2 = TestRig::with_custody_type(NodeCustodyType::SemiSupernode); - // Need to create blocks that can be inserted into the fork-choice and fit the "known - // conditions" below. - let head_peer_block = rig_2.create_canonical_block().await; - let head_peer_root = head_peer_block.0.canonical_root(); - let finalized_peer_block = rig_2.create_canonical_block().await; - let finalized_peer_root = finalized_peer_block.0.canonical_root(); - - // Get a peer with an advanced head - let head_peer = rig.add_head_peer_with_root(head_peer_root); - rig.assert_state(RangeSyncType::Head); - - // Sync should have requested a batch, grab the request. - let _ = rig.find_blocks_by_range_request(filter().peer(head_peer)); - - // Now get a peer with an advanced finalized epoch. - let finalized_peer = rig.add_finalized_peer_with_root(finalized_peer_root); - rig.assert_state(RangeSyncType::Finalized); - - // Sync should have requested a batch, grab the request - let _ = rig.find_blocks_by_range_request(filter().peer(finalized_peer)); - - // Now the chain knows both chains target roots. - rig.remember_block(head_peer_block).await; - rig.remember_block(finalized_peer_block).await; - - // Add an additional peer to the second chain to make range update it's status - rig.add_finalized_peer(); -} - -#[test] -fn pause_and_resume_on_ee_offline() { - let mut rig = TestRig::default(); - - // add some peers - let peer1 = rig.add_head_peer(); - // make the ee offline - rig.update_execution_engine_state(EngineState::Offline); - // send the response to the request - rig.find_and_complete_blocks_by_range_request(filter().peer(peer1).epoch(0)); - // the beacon processor shouldn't have received any work - rig.assert_empty_processor(); - - // while the ee is offline, more peers might arrive. Add a new finalized peer. - let _peer2 = rig.add_finalized_peer(); - - // send the response to the request - // Don't filter requests and the columns requests may be sent to peer1 or peer2 - // We need to filter by epoch, because the previous batch eagerly sent requests for the next - // epoch for the other batch. So we can either filter by epoch of by sync type. - rig.find_and_complete_blocks_by_range_request(filter().epoch(0)); - // the beacon processor shouldn't have received any work - rig.assert_empty_processor(); - // make the beacon processor available again. - // update_execution_engine_state implicitly calls resume - // now resume range, we should have two processing requests in the beacon processor. - rig.update_execution_engine_state(EngineState::Online); - - // The head chain and finalized chain (2) should be in the processing queue - rig.assert_chain_segments(2); -} - -/// To attempt to finalize the peer's status finalized checkpoint we synced to its finalized epoch + -/// 2 epochs + 1 slot. -const EXTRA_SYNCED_EPOCHS: u64 = 2 + 1; - -#[test] -fn finalized_sync_enough_global_custody_peers_few_chain_peers() { - // Run for all forks +async fn head_sync_completes() { let mut r = TestRig::default(); - - let advanced_epochs: u64 = 2; - let remote_info = r.finalized_remote_info_advanced_by(advanced_epochs.into()); - - // Generate enough peers and supernodes to cover all custody columns - let peer_count = 100; - r.add_fullnode_peers(remote_info.clone(), peer_count); - r.add_supernode_peer(remote_info); - r.assert_state(RangeSyncType::Finalized); - - let last_epoch = advanced_epochs + EXTRA_SYNCED_EPOCHS; - r.complete_and_process_range_sync_until(last_epoch, filter()); + r.setup_head_sync().await; + r.simulate(SimulateConfig::happy_path()).await; + r.assert_head_sync_completed(); + r.assert_head_slot(SLOTS_PER_EPOCH as u64); } -#[test] -fn finalized_sync_not_enough_custody_peers_on_start() { +/// Peers with advanced finalized AND head beyond finalized. Finalized sync completes first, +/// then head chains are created from awaiting_head_peers to sync the remaining gap. +#[tokio::test] +async fn finalized_to_head_transition() { + let mut r = TestRig::default(); + r.setup_finalized_and_head_sync().await; + r.simulate(SimulateConfig::happy_path()).await; + r.assert_range_sync_completed(); + r.assert_head_slot(7 * SLOTS_PER_EPOCH as u64); +} + +/// Finalized sync happy path: all batches download and process, head advances to target, +/// finalized epoch advances past genesis. +#[tokio::test] +async fn finalized_sync_completes() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path()).await; + r.assert_range_sync_completed(); + r.assert_head_slot(5 * SLOTS_PER_EPOCH as u64); +} + +/// First BlocksByRange request gets an RPC error. Batch retries from another peer, +/// sync completes with no penalties (RPC errors are not penalized). +#[tokio::test] +async fn batch_rpc_error_retries() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().return_rpc_error(RPCError::UnsupportedProtocol)) + .await; + r.assert_range_sync_completed(); +} + +/// Peer returns zero blocks for a BlocksByRange request. Batch retries, sync completes. +#[tokio::test] +async fn batch_peer_returns_empty_then_succeeds() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_no_range_blocks_n_times(1)) + .await; + r.assert_successful_range_sync(); +} + +/// Peer returns zero columns for a DataColumnsByRange request. Batch retries, sync completes. +/// Only exercises column logic on fulu+. +#[tokio::test] +async fn batch_peer_returns_no_columns_then_succeeds() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_no_range_columns_n_times(1)) + .await; + r.assert_successful_range_sync(); +} + +/// Peer returns columns with indices it wasn't asked for → UnrequestedIndex verify error. +/// Batch retries from another peer, sync completes. +#[tokio::test] +async fn batch_peer_returns_wrong_column_indices_then_succeeds() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_wrong_range_column_indices_n_times(1)) + .await; + r.assert_successful_range_sync(); +} + +/// Peer returns columns from a slot outside the requested range → UnrequestedSlot verify error. +/// Batch retries from another peer, sync completes. +#[tokio::test] +async fn batch_peer_returns_wrong_column_slots_then_succeeds() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_wrong_range_column_slots_n_times(1)) + .await; + r.assert_successful_range_sync(); +} + +/// PeerDAS: peer returns only half the requested columns. Block-sidecar coupling detects +/// missing columns → CouplingError::DataColumnPeerFailure → retry_partial_batch from other peers. +#[tokio::test] +async fn batch_peer_returns_partial_columns_then_succeeds() { let mut r = TestRig::default(); - // Only run post-PeerDAS if !r.fork_name.fulu_enabled() { return; } - - let advanced_epochs: u64 = 2; - let remote_info = r.finalized_remote_info_advanced_by(advanced_epochs.into()); - - // Unikely that the single peer we added has enough columns for us. Tests are deterministic and - // this error should never be hit - r.add_fullnode_peer(remote_info.clone()); - r.assert_state(RangeSyncType::Finalized); - - // Because we don't have enough peers on all columns we haven't sent any request. - // NOTE: There's a small chance that this single peer happens to custody exactly the set we - // expect, in that case the test will fail. Find a way to make the test deterministic. - r.assert_empty_network(); - - // Generate enough peers and supernodes to cover all custody columns - let peer_count = 100; - r.add_fullnode_peers(remote_info.clone(), peer_count); - r.add_supernode_peer(remote_info); - - let last_epoch = advanced_epochs + EXTRA_SYNCED_EPOCHS; - r.complete_and_process_range_sync_until(last_epoch, filter()); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_partial_range_columns_n_times(1)) + .await; + r.assert_successful_range_sync(); +} + +/// Batch processing returns NonFaultyFailure (e.g. transient error). Batch goes back to +/// AwaitingDownload, retries without penalty, sync completes. +#[tokio::test] +async fn batch_non_faulty_failure_retries() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_range_non_faulty_failures(1)) + .await; + r.assert_range_sync_completed(); +} + +/// Batch processing returns FaultyFailure once. Peer penalized with "faulty_batch", +/// batch redownloaded from a different peer, sync completes. +#[tokio::test] +async fn batch_faulty_failure_redownloads() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_range_faulty_failures(1)) + .await; + r.assert_successful_range_sync(); + r.assert_penalties_of_type("faulty_batch"); +} + +/// Batch processing fails MAX_BATCH_PROCESSING_ATTEMPTS (3) times with FaultyFailure. +/// Chain removed, all peers penalized with "faulty_chain". +#[tokio::test] +async fn batch_max_failures_removes_chain() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_range_faulty_failures(3)) + .await; + r.assert_range_sync_chain_failed(); +} + +/// Chain fails via max faulty retries → finalized root added to failed_chains LRU. +/// A new peer advertising the same finalized root gets disconnected with GoodbyeReason. +#[tokio::test] +async fn failed_chain_blacklisted() { + let mut r = TestRig::default(); + let remote_info = r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_range_faulty_failures(3)) + .await; + r.assert_range_sync_chain_failed(); + r.assert_peer_blacklisted(remote_info); +} + +/// All peers disconnect before any request is fulfilled → chain removed (EmptyPeerPool). +#[tokio::test] +async fn all_peers_disconnect_removes_chain() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_disconnect_after_range_requests(0)) + .await; + r.assert_range_sync_chain_removed(); +} + +/// Peers disconnect after 1 request is served. Remaining in-flight responses arrive +/// for a chain that no longer exists — verified as a no-op (no crash). +#[tokio::test] +async fn late_response_for_removed_chain() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_disconnect_after_range_requests(1)) + .await; + r.assert_range_sync_chain_removed(); +} + +/// Execution engine goes offline at sync start. Batch responses complete but processing +/// is paused. After 2 responses, EE comes back online, queued batches process, sync completes. +#[tokio::test] +async fn ee_offline_then_online_resumes_sync() { + let mut r = TestRig::default(); + r.setup_finalized_sync().await; + r.simulate(SimulateConfig::happy_path().with_ee_offline_for_n_range_responses(2)) + .await; + r.assert_range_sync_completed(); +} + +/// Local node already has blocks up to epoch 3. Finalized sync starts targeting epoch 6. +/// The chain uses optimistic start: downloads a batch at the local head epoch concurrently +/// with sequential processing from the start. All blocks ingested. +#[tokio::test] +async fn finalized_sync_with_local_head_partial() { + let mut r = TestRig::default(); + r.setup_finalized_sync_with_local_head(3).await; + r.simulate(SimulateConfig::happy_path()).await; + r.assert_range_sync_completed(); +} + +/// Local node has all blocks except the last one. Finalized sync only needs to fill the +/// final gap. Tests optimistic start where local head is near the target. +#[tokio::test] +async fn finalized_sync_with_local_head_near_target() { + let mut r = TestRig::default(); + let target_epochs = 5; + let local_slots = (target_epochs * SLOTS_PER_EPOCH) - 1; // all blocks except last + r.build_chain(target_epochs * SLOTS_PER_EPOCH).await; + r.import_blocks_up_to_slot(local_slots as u64).await; + let remote_info = r.finalized_remote_info_advanced_by((target_epochs as u64).into()); + r.add_fullnode_peers(remote_info.clone(), 100); + r.add_supernode_peer(remote_info); + r.assert_state(RangeSyncType::Finalized); + r.simulate(SimulateConfig::happy_path()).await; + r.assert_range_sync_completed(); + r.assert_head_slot((target_epochs * SLOTS_PER_EPOCH) as u64); +} + +/// PeerDAS only: single fullnode peer doesn't cover all custody columns → no requests sent. +/// Once enough fullnodes + a supernode arrive, sync proceeds and completes. +#[tokio::test] +async fn not_enough_custody_peers_then_peers_arrive() { + let mut r = TestRig::default(); + if !r.fork_name.fulu_enabled() { + return; + } + let remote_info = r.setup_finalized_sync_insufficient_peers().await; + r.assert_empty_network(); + r.add_remaining_finalized_peers(remote_info); + r.simulate(SimulateConfig::happy_path()).await; + r.assert_range_sync_completed(); } diff --git a/common/account_utils/Cargo.toml b/common/account_utils/Cargo.toml index d0a3e487c4..b5c84bbb64 100644 --- a/common/account_utils/Cargo.toml +++ b/common/account_utils/Cargo.toml @@ -14,8 +14,8 @@ rand = { workspace = true } regex = { workspace = true } rpassword = "5.0.0" serde = { workspace = true } -serde_yaml = { workspace = true } tracing = { workspace = true } types = { workspace = true } validator_dir = { workspace = true } +yaml_serde = { workspace = true } zeroize = { workspace = true } diff --git a/common/account_utils/src/validator_definitions.rs b/common/account_utils/src/validator_definitions.rs index bffdfcc38b..0fc5bf5665 100644 --- a/common/account_utils/src/validator_definitions.rs +++ b/common/account_utils/src/validator_definitions.rs @@ -31,11 +31,11 @@ pub enum Error { /// The config file could not be opened. UnableToOpenFile(io::Error), /// The config file could not be parsed as YAML. - UnableToParseFile(serde_yaml::Error), + UnableToParseFile(yaml_serde::Error), /// There was an error whilst performing the recursive keystore search function. UnableToSearchForKeystores(io::Error), /// The config file could not be serialized as YAML. - UnableToEncodeFile(serde_yaml::Error), + UnableToEncodeFile(yaml_serde::Error), /// The config file or temp file could not be written to the filesystem. UnableToWriteFile(filesystem::Error), /// The public key from the keystore is invalid. @@ -248,7 +248,7 @@ impl ValidatorDefinitions { .create_new(false) .open(config_path) .map_err(Error::UnableToOpenFile)?; - serde_yaml::from_reader(file).map_err(Error::UnableToParseFile) + yaml_serde::from_reader(file).map_err(Error::UnableToParseFile) } /// Perform a recursive, exhaustive search through `validators_dir` and add any keystores @@ -376,7 +376,7 @@ impl ValidatorDefinitions { let config_path = validators_dir.as_ref().join(CONFIG_FILENAME); let temp_path = validators_dir.as_ref().join(CONFIG_TEMP_FILENAME); let mut bytes = vec![]; - serde_yaml::to_writer(&mut bytes, self).map_err(Error::UnableToEncodeFile)?; + yaml_serde::to_writer(&mut bytes, self).map_err(Error::UnableToEncodeFile)?; write_file_via_temporary(&config_path, &temp_path, &bytes) .map_err(Error::UnableToWriteFile)?; @@ -531,7 +531,7 @@ mod tests { voting_keystore_path: "" voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: ValidatorDefinition = serde_yaml::from_str(no_graffiti).unwrap(); + let def: ValidatorDefinition = yaml_serde::from_str(no_graffiti).unwrap(); assert!(def.graffiti.is_none()); let invalid_graffiti = r#"--- @@ -543,7 +543,7 @@ mod tests { voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: Result = serde_yaml::from_str(invalid_graffiti); + let def: Result = yaml_serde::from_str(invalid_graffiti); assert!(def.is_err()); let valid_graffiti = r#"--- @@ -555,7 +555,7 @@ mod tests { voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: ValidatorDefinition = serde_yaml::from_str(valid_graffiti).unwrap(); + let def: ValidatorDefinition = yaml_serde::from_str(valid_graffiti).unwrap(); assert_eq!( def.graffiti, Some(GraffitiString::from_str("mrfwashere").unwrap()) @@ -571,7 +571,7 @@ mod tests { voting_keystore_path: "" voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: ValidatorDefinition = serde_yaml::from_str(no_suggested_fee_recipient).unwrap(); + let def: ValidatorDefinition = yaml_serde::from_str(no_suggested_fee_recipient).unwrap(); assert!(def.suggested_fee_recipient.is_none()); let invalid_suggested_fee_recipient = r#"--- @@ -584,7 +584,7 @@ mod tests { "#; let def: Result = - serde_yaml::from_str(invalid_suggested_fee_recipient); + yaml_serde::from_str(invalid_suggested_fee_recipient); assert!(def.is_err()); let valid_suggested_fee_recipient = r#"--- @@ -596,7 +596,7 @@ mod tests { voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: ValidatorDefinition = serde_yaml::from_str(valid_suggested_fee_recipient).unwrap(); + let def: ValidatorDefinition = yaml_serde::from_str(valid_suggested_fee_recipient).unwrap(); assert_eq!( def.suggested_fee_recipient, Some(Address::from_str("0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d").unwrap()) @@ -613,7 +613,7 @@ mod tests { voting_keystore_path: "" voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: ValidatorDefinition = serde_yaml::from_str(no_gas_limit).unwrap(); + let def: ValidatorDefinition = yaml_serde::from_str(no_gas_limit).unwrap(); assert!(def.gas_limit.is_none()); let invalid_gas_limit = r#"--- @@ -626,7 +626,7 @@ mod tests { voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: Result = serde_yaml::from_str(invalid_gas_limit); + let def: Result = yaml_serde::from_str(invalid_gas_limit); assert!(def.is_err()); let valid_gas_limit = r#"--- @@ -639,7 +639,7 @@ mod tests { voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: ValidatorDefinition = serde_yaml::from_str(valid_gas_limit).unwrap(); + let def: ValidatorDefinition = yaml_serde::from_str(valid_gas_limit).unwrap(); assert_eq!(def.gas_limit, Some(35000000)); } @@ -653,7 +653,7 @@ mod tests { voting_keystore_path: "" voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: ValidatorDefinition = serde_yaml::from_str(no_builder_proposals).unwrap(); + let def: ValidatorDefinition = yaml_serde::from_str(no_builder_proposals).unwrap(); assert!(def.builder_proposals.is_none()); let invalid_builder_proposals = r#"--- @@ -666,7 +666,7 @@ mod tests { voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: Result = serde_yaml::from_str(invalid_builder_proposals); + let def: Result = yaml_serde::from_str(invalid_builder_proposals); assert!(def.is_err()); let valid_builder_proposals = r#"--- @@ -679,7 +679,7 @@ mod tests { voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7" "#; - let def: ValidatorDefinition = serde_yaml::from_str(valid_builder_proposals).unwrap(); + let def: ValidatorDefinition = yaml_serde::from_str(valid_builder_proposals).unwrap(); assert_eq!(def.builder_proposals, Some(true)); } } diff --git a/common/clap_utils/Cargo.toml b/common/clap_utils/Cargo.toml index f3c166bda9..02c9ac97f1 100644 --- a/common/clap_utils/Cargo.toml +++ b/common/clap_utils/Cargo.toml @@ -14,5 +14,5 @@ ethereum_ssz = { workspace = true } hex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -serde_yaml = { workspace = true } types = { workspace = true } +yaml_serde = { workspace = true } diff --git a/common/clap_utils/src/lib.rs b/common/clap_utils/src/lib.rs index bc904c78e3..e969ab95d4 100644 --- a/common/clap_utils/src/lib.rs +++ b/common/clap_utils/src/lib.rs @@ -159,7 +159,7 @@ where let chain_config = Config::from_chain_spec::(spec); let mut file = std::fs::File::create(dump_path) .map_err(|e| format!("Failed to open file for writing chain config: {:?}", e))?; - serde_yaml::to_writer(&mut file, &chain_config) + yaml_serde::to_writer(&mut file, &chain_config) .map_err(|e| format!("Error serializing config: {:?}", e))?; } Ok(()) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 40c5ef58a6..d5140a3878 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -2732,6 +2732,55 @@ impl BeaconNodeHttpClient { Ok(()) } + /// Path for `v1/beacon/execution_payload_envelope/{block_id}` + pub fn get_beacon_execution_payload_envelope_path( + &self, + block_id: BlockId, + ) -> Result { + let mut path = self.eth_path(V1)?; + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("execution_payload_envelope") + .push(&block_id.to_string()); + Ok(path) + } + + /// `GET v1/beacon/execution_payload_envelope/{block_id}` + /// + /// Returns `Ok(None)` on a 404 error. + pub async fn get_beacon_execution_payload_envelope( + &self, + block_id: BlockId, + ) -> Result< + Option>>, + Error, + > { + let path = self.get_beacon_execution_payload_envelope_path(block_id)?; + self.get_opt(path) + .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) + } + + /// `GET v1/beacon/execution_payload_envelope/{block_id}` in SSZ format + /// + /// Returns `Ok(None)` on a 404 error. + pub async fn get_beacon_execution_payload_envelope_ssz( + &self, + block_id: BlockId, + ) -> Result>, Error> { + let path = self.get_beacon_execution_payload_envelope_path(block_id)?; + let opt_response = self + .get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.get_beacon_blocks_ssz) + .await?; + match opt_response { + Some(bytes) => SignedExecutionPayloadEnvelope::from_ssz_bytes(&bytes) + .map(Some) + .map_err(Error::InvalidSsz), + None => Ok(None), + } + } + /// `GET v2/validator/blocks/{slot}` in ssz format pub async fn get_validator_blocks_ssz( &self, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 94dff95bc6..54e9c98b5b 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1070,6 +1070,33 @@ pub struct BlockGossip { pub slot: Slot, pub block: Hash256, } +#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] +pub struct SseExecutionPayload { + pub slot: Slot, + #[serde(with = "serde_utils::quoted_u64")] + pub builder_index: u64, + pub block_hash: ExecutionBlockHash, + pub block_root: Hash256, + pub state_root: Hash256, + pub execution_optimistic: bool, +} + +#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] +pub struct SseExecutionPayloadGossip { + pub slot: Slot, + #[serde(with = "serde_utils::quoted_u64")] + pub builder_index: u64, + pub block_hash: ExecutionBlockHash, + pub block_root: Hash256, + pub state_root: Hash256, +} + +#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] +pub struct SseExecutionPayloadAvailable { + pub slot: Slot, + pub block_root: Hash256, +} + #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] pub struct SseChainReorg { pub slot: Slot, @@ -1134,6 +1161,8 @@ pub struct SseExtendedPayloadAttributesGeneric { pub type SseExtendedPayloadAttributes = SseExtendedPayloadAttributesGeneric; pub type VersionedSsePayloadAttributes = ForkVersionedResponse; +pub type VersionedSseExecutionPayloadBid = ForkVersionedResponse>; +pub type VersionedSsePayloadAttestationMessage = ForkVersionedResponse; impl<'de> ContextDeserialize<'de, ForkName> for SsePayloadAttributes { fn context_deserialize(deserializer: D, context: ForkName) -> Result @@ -1210,6 +1239,11 @@ pub enum EventKind { AttesterSlashing(Box>), BlsToExecutionChange(Box), BlockGossip(Box), + ExecutionPayload(SseExecutionPayload), + ExecutionPayloadGossip(SseExecutionPayloadGossip), + ExecutionPayloadAvailable(SseExecutionPayloadAvailable), + ExecutionPayloadBid(Box>), + PayloadAttestationMessage(Box), } impl EventKind { @@ -1233,6 +1267,11 @@ impl EventKind { EventKind::AttesterSlashing(_) => "attester_slashing", EventKind::BlsToExecutionChange(_) => "bls_to_execution_change", EventKind::BlockGossip(_) => "block_gossip", + EventKind::ExecutionPayload(_) => "execution_payload", + EventKind::ExecutionPayloadGossip(_) => "execution_payload_gossip", + EventKind::ExecutionPayloadAvailable(_) => "execution_payload_available", + EventKind::ExecutionPayloadBid(_) => "execution_payload_bid", + EventKind::PayloadAttestationMessage(_) => "payload_attestation_message", } } @@ -1322,6 +1361,40 @@ impl EventKind { "block_gossip" => Ok(EventKind::BlockGossip(serde_json::from_str(data).map_err( |e| ServerError::InvalidServerSentEvent(format!("Block Gossip: {:?}", e)), )?)), + "execution_payload" => Ok(EventKind::ExecutionPayload( + serde_json::from_str(data).map_err(|e| { + ServerError::InvalidServerSentEvent(format!("Execution Payload: {:?}", e)) + })?, + )), + "execution_payload_gossip" => Ok(EventKind::ExecutionPayloadGossip( + serde_json::from_str(data).map_err(|e| { + ServerError::InvalidServerSentEvent(format!( + "Execution Payload Gossip: {:?}", + e + )) + })?, + )), + "execution_payload_available" => Ok(EventKind::ExecutionPayloadAvailable( + serde_json::from_str(data).map_err(|e| { + ServerError::InvalidServerSentEvent(format!( + "Execution Payload Available: {:?}", + e + )) + })?, + )), + "execution_payload_bid" => Ok(EventKind::ExecutionPayloadBid(Box::new( + serde_json::from_str(data).map_err(|e| { + ServerError::InvalidServerSentEvent(format!("Execution Payload Bid: {:?}", e)) + })?, + ))), + "payload_attestation_message" => Ok(EventKind::PayloadAttestationMessage(Box::new( + serde_json::from_str(data).map_err(|e| { + ServerError::InvalidServerSentEvent(format!( + "Payload Attestation Message: {:?}", + e + )) + })?, + ))), _ => Err(ServerError::InvalidServerSentEvent( "Could not parse event tag".to_string(), )), @@ -1357,6 +1430,11 @@ pub enum EventTopic { ProposerSlashing, BlsToExecutionChange, BlockGossip, + ExecutionPayload, + ExecutionPayloadGossip, + ExecutionPayloadAvailable, + ExecutionPayloadBid, + PayloadAttestationMessage, } impl FromStr for EventTopic { @@ -1382,6 +1460,11 @@ impl FromStr for EventTopic { "proposer_slashing" => Ok(EventTopic::ProposerSlashing), "bls_to_execution_change" => Ok(EventTopic::BlsToExecutionChange), "block_gossip" => Ok(EventTopic::BlockGossip), + "execution_payload" => Ok(EventTopic::ExecutionPayload), + "execution_payload_gossip" => Ok(EventTopic::ExecutionPayloadGossip), + "execution_payload_available" => Ok(EventTopic::ExecutionPayloadAvailable), + "execution_payload_bid" => Ok(EventTopic::ExecutionPayloadBid), + "payload_attestation_message" => Ok(EventTopic::PayloadAttestationMessage), _ => Err("event topic cannot be parsed.".to_string()), } } @@ -1408,6 +1491,15 @@ impl fmt::Display for EventTopic { EventTopic::ProposerSlashing => write!(f, "proposer_slashing"), EventTopic::BlsToExecutionChange => write!(f, "bls_to_execution_change"), EventTopic::BlockGossip => write!(f, "block_gossip"), + EventTopic::ExecutionPayload => write!(f, "execution_payload"), + EventTopic::ExecutionPayloadGossip => write!(f, "execution_payload_gossip"), + EventTopic::ExecutionPayloadAvailable => { + write!(f, "execution_payload_available") + } + EventTopic::ExecutionPayloadBid => write!(f, "execution_payload_bid"), + EventTopic::PayloadAttestationMessage => { + write!(f, "payload_attestation_message") + } } } } diff --git a/common/eth2_interop_keypairs/Cargo.toml b/common/eth2_interop_keypairs/Cargo.toml index 309ff233e6..7eed6032c9 100644 --- a/common/eth2_interop_keypairs/Cargo.toml +++ b/common/eth2_interop_keypairs/Cargo.toml @@ -12,7 +12,7 @@ ethereum_hashing = { workspace = true } hex = { workspace = true } num-bigint = "0.4.2" serde = { workspace = true } -serde_yaml = { workspace = true } +yaml_serde = { workspace = true } [dev-dependencies] base64 = "0.13.0" diff --git a/common/eth2_interop_keypairs/src/lib.rs b/common/eth2_interop_keypairs/src/lib.rs index 0d24eb92f4..d00984a2d1 100644 --- a/common/eth2_interop_keypairs/src/lib.rs +++ b/common/eth2_interop_keypairs/src/lib.rs @@ -118,7 +118,7 @@ fn string_to_bytes(string: &str) -> Result, String> { pub fn keypairs_from_yaml_file(path: PathBuf) -> Result, String> { let file = File::open(path).map_err(|e| format!("Unable to open YAML key file: {}", e))?; - serde_yaml::from_reader::<_, Vec>(file) + yaml_serde::from_reader::<_, Vec>(file) .map_err(|e| format!("Could not parse YAML: {:?}", e))? .into_iter() .map(TryInto::try_into) diff --git a/common/eth2_network_config/Cargo.toml b/common/eth2_network_config/Cargo.toml index 416ffb1975..d2bdfea1fa 100644 --- a/common/eth2_network_config/Cargo.toml +++ b/common/eth2_network_config/Cargo.toml @@ -15,11 +15,11 @@ kzg = { workspace = true } pretty_reqwest_error = { workspace = true } reqwest = { workspace = true } sensitive_url = { workspace = true } -serde_yaml = { workspace = true } sha2 = { workspace = true } tracing = { workspace = true } types = { workspace = true } url = { workspace = true } +yaml_serde = { workspace = true } [build-dependencies] eth2_config = { workspace = true } diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml index 5df6370abe..02bf37cb55 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml @@ -56,21 +56,18 @@ ELECTRA_FORK_EPOCH: 364032 # May 7, 2025, 10:05:11am UTC FULU_FORK_VERSION: 0x06000000 FULU_FORK_EPOCH: 411392 # December 3, 2025, 09:49:11pm UTC # Gloas -GLOAS_FORK_VERSION: 0x07000000 # temporary stub +GLOAS_FORK_VERSION: 0x07000000 GLOAS_FORK_EPOCH: 18446744073709551615 -# EIP7441 -EIP7441_FORK_VERSION: 0x08000000 # temporary stub -EIP7441_FORK_EPOCH: 18446744073709551615 -# EIP7805 -EIP7805_FORK_VERSION: 0x0a000000 # temporary stub -EIP7805_FORK_EPOCH: 18446744073709551615 +# Heze +HEZE_FORK_VERSION: 0x08000000 +HEZE_FORK_EPOCH: 18446744073709551615 # EIP7928 -EIP7928_FORK_VERSION: 0x0b000000 # temporary stub +EIP7928_FORK_VERSION: 0xe7928000 # temporary stub EIP7928_FORK_EPOCH: 18446744073709551615 # Time parameters # --------------------------------------------------------------- -# 12 seconds (*deprecated*) +# 12 seconds SECONDS_PER_SLOT: 12 # 12000 milliseconds SLOT_DURATION_MS: 12000 @@ -96,8 +93,8 @@ SYNC_MESSAGE_DUE_BPS: 3333 CONTRIBUTION_DUE_BPS: 6667 # Gloas -# 2**12 (= 4,096) epochs -MIN_BUILDER_WITHDRAWABILITY_DELAY: 4096 +# 2**6 (= 64) epochs +MIN_BUILDER_WITHDRAWABILITY_DELAY: 64 # 2500 basis points, 25% of SLOT_DURATION_MS ATTESTATION_DUE_BPS_GLOAS: 2500 # 5000 basis points, 50% of SLOT_DURATION_MS @@ -109,7 +106,7 @@ CONTRIBUTION_DUE_BPS_GLOAS: 5000 # 7500 basis points, 75% of SLOT_DURATION_MS PAYLOAD_ATTESTATION_DUE_BPS: 7500 -# EIP7805 +# Heze # 7500 basis points, 75% of SLOT_DURATION_MS VIEW_FREEZE_CUTOFF_BPS: 7500 # 6667 basis points, ~67% of SLOT_DURATION_MS @@ -166,8 +163,6 @@ MAX_PAYLOAD_SIZE: 10485760 MAX_REQUEST_BLOCKS: 1024 # 2**8 (= 256) epochs EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 -# MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2 (= 33,024) epochs -MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 # 2**5 (= 32) slots ATTESTATION_PROPAGATION_SLOT_RANGE: 32 # 500ms @@ -180,8 +175,6 @@ SUBNETS_PER_NODE: 2 ATTESTATION_SUBNET_COUNT: 64 # 0 bits ATTESTATION_SUBNET_EXTRA_BITS: 0 -# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS (= 6 + 0) bits -ATTESTATION_SUBNET_PREFIX_BITS: 6 # Deneb # 2**7 (= 128) blocks @@ -192,24 +185,18 @@ MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 BLOB_SIDECAR_SUBNET_COUNT: 6 # 6 blobs MAX_BLOBS_PER_BLOCK: 6 -# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK (= 128 * 6) sidecars -MAX_REQUEST_BLOB_SIDECARS: 768 # Electra # 9 subnets BLOB_SIDECAR_SUBNET_COUNT_ELECTRA: 9 # 9 blobs MAX_BLOBS_PER_BLOCK_ELECTRA: 9 -# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_ELECTRA (= 128 * 9) sidecars -MAX_REQUEST_BLOB_SIDECARS_ELECTRA: 1152 # Fulu # 2**7 (= 128) groups NUMBER_OF_CUSTODY_GROUPS: 128 # 2**7 (= 128) subnets DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 -# MAX_REQUEST_BLOCKS_DENEB * NUMBER_OF_COLUMNS (= 128 * 128) sidecars -MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 # 2**3 (= 8) samples SAMPLES_PER_SLOT: 8 # 2**2 (= 4) sidecars @@ -225,18 +212,13 @@ MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096 # 2**7 (= 128) payloads MAX_REQUEST_PAYLOADS: 128 -# EIP7441 -# 2**8 (= 256) epochs -EPOCHS_PER_SHUFFLING_PHASE: 256 -# 2**1 (= 2) epochs -PROPOSER_SELECTION_GAP: 2 - -# EIP7805 +# Heze # 2**4 (= 16) inclusion lists MAX_REQUEST_INCLUSION_LIST: 16 # 2**13 (= 8,192) bytes MAX_BYTES_PER_INCLUSION_LIST: 8192 + # Blob Scheduling # --------------------------------------------------------------- diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 6fd8567bed..408ce6135d 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -101,14 +101,14 @@ impl Eth2NetworkConfig { /// Instantiates `Self` from a `HardcodedNet`. fn from_hardcoded_net(net: &HardcodedNet) -> Result { - let config: Config = serde_yaml::from_reader(net.config) + let config: Config = yaml_serde::from_reader(net.config) .map_err(|e| format!("Unable to parse yaml config: {:?}", e))?; let kzg_trusted_setup = get_trusted_setup(); Ok(Self { - deposit_contract_deploy_block: serde_yaml::from_reader(net.deploy_block) + deposit_contract_deploy_block: yaml_serde::from_reader(net.deploy_block) .map_err(|e| format!("Unable to parse deploy block: {:?}", e))?, boot_enr: Some( - serde_yaml::from_reader(net.boot_enr) + yaml_serde::from_reader(net.boot_enr) .map_err(|e| format!("Unable to parse boot enr: {:?}", e))?, ), genesis_state_source: net.genesis_state_source, @@ -286,7 +286,7 @@ impl Eth2NetworkConfig { File::create(base_dir.join($file)) .map_err(|e| format!("Unable to create {}: {:?}", $file, e)) .and_then(|mut file| { - let yaml = serde_yaml::to_string(&$variable) + let yaml = yaml_serde::to_string(&$variable) .map_err(|e| format!("Unable to YAML encode {}: {:?}", $file, e))?; // Remove the doc header from the YAML file. @@ -334,7 +334,7 @@ impl Eth2NetworkConfig { File::open(base_dir.join($file)) .map_err(|e| format!("Unable to open {}: {:?}", $file, e)) .and_then(|file| { - serde_yaml::from_reader(file) + yaml_serde::from_reader(file) .map_err(|e| format!("Unable to parse {}: {:?}", $file, e)) })? }; diff --git a/consensus/proto_array/Cargo.toml b/consensus/proto_array/Cargo.toml index 782610e0d3..7419ad813b 100644 --- a/consensus/proto_array/Cargo.toml +++ b/consensus/proto_array/Cargo.toml @@ -14,6 +14,6 @@ ethereum_ssz_derive = { workspace = true } fixed_bytes = { workspace = true } safe_arith = { workspace = true } serde = { workspace = true } -serde_yaml = { workspace = true } superstruct = { workspace = true } types = { workspace = true } +yaml_serde = { workspace = true } diff --git a/consensus/proto_array/src/bin.rs b/consensus/proto_array/src/bin.rs index e1d307affb..38ba3150e7 100644 --- a/consensus/proto_array/src/bin.rs +++ b/consensus/proto_array/src/bin.rs @@ -22,5 +22,5 @@ fn main() { fn write_test_def_to_yaml(filename: &str, def: ForkChoiceTestDefinition) { let file = File::create(filename).expect("Should be able to open file"); - serde_yaml::to_writer(file, &def).expect("Should be able to write YAML to file"); + yaml_serde::to_writer(file, &def).expect("Should be able to write YAML to file"); } diff --git a/consensus/state_processing/src/per_epoch_processing/altair.rs b/consensus/state_processing/src/per_epoch_processing/altair.rs index d9e6964730..683d92d836 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair.rs @@ -51,8 +51,8 @@ pub fn process_epoch( // without loss of correctness. let current_epoch_progressive_balances = state.progressive_balances_cache().clone(); let current_epoch_total_active_balance = state.get_total_active_balance()?; - let participation_summary = - process_epoch_single_pass(state, spec, SinglePassConfig::default())?; + let epoch_result = process_epoch_single_pass(state, spec, SinglePassConfig::default())?; + let participation_summary = epoch_result.summary; // Reset eth1 data votes. process_eth1_data_reset(state)?; @@ -79,6 +79,13 @@ pub fn process_epoch( // Rotate the epoch caches to suit the epoch transition. state.advance_caches()?; + + // Install the lookahead committee cache (built during PTC window processing) as the Next + // cache. After advance_caches, the lookahead epoch becomes the Next relative epoch. + if let Some(cache) = epoch_result.lookahead_committee_cache { + state.set_committee_cache(RelativeEpoch::Next, cache)?; + } + update_progressive_balances_on_epoch_transition(state, spec)?; Ok(EpochProcessingSummary::Altair { diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index 4eb1e36628..976607aa76 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -12,12 +12,13 @@ use milhouse::{Cow, List, Vector}; use safe_arith::{SafeArith, SafeArithIter}; use std::cmp::{max, min}; use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; use tracing::instrument; use typenum::Unsigned; use types::{ ActivationQueue, BeaconState, BeaconStateError, BuilderPendingPayment, ChainSpec, Checkpoint, - DepositData, Epoch, EthSpec, ExitCache, ForkName, ParticipationFlags, PendingDeposit, - ProgressiveBalancesCache, RelativeEpoch, Validator, + CommitteeCache, DepositData, Epoch, EthSpec, ExitCache, ForkName, ParticipationFlags, + PendingDeposit, ProgressiveBalancesCache, RelativeEpoch, Validator, consts::altair::{ NUM_FLAG_INDICES, PARTICIPATION_FLAG_WEIGHTS, TIMELY_HEAD_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX, WEIGHT_DENOMINATOR, @@ -34,6 +35,7 @@ pub struct SinglePassConfig { pub effective_balance_updates: bool, pub proposer_lookahead: bool, pub builder_pending_payments: bool, + pub ptc_window: bool, } impl Default for SinglePassConfig { @@ -54,6 +56,7 @@ impl SinglePassConfig { effective_balance_updates: true, proposer_lookahead: true, builder_pending_payments: true, + ptc_window: true, } } @@ -68,6 +71,7 @@ impl SinglePassConfig { effective_balance_updates: false, proposer_lookahead: false, builder_pending_payments: false, + ptc_window: false, } } } @@ -139,12 +143,20 @@ impl ValidatorInfo { } } +/// Result of single-pass epoch processing. +pub struct SinglePassEpochResult { + pub summary: ParticipationEpochSummary, + /// Committee cache for the lookahead epoch, built during PTC window processing. + /// Can be installed as the Next committee cache after `advance_caches`. + pub lookahead_committee_cache: Option>, +} + #[instrument(skip_all)] pub fn process_epoch_single_pass( state: &mut BeaconState, spec: &ChainSpec, conf: SinglePassConfig, -) -> Result, Error> { +) -> Result, Error> { initialize_epoch_cache(state, spec)?; initialize_progressive_balances_cache(state, spec)?; state.build_exit_cache(spec)?; @@ -479,7 +491,16 @@ pub fn process_epoch_single_pass( process_proposer_lookahead(state, spec)?; } - Ok(summary) + let lookahead_committee_cache = if conf.ptc_window && fork_name.gloas_enabled() { + Some(process_ptc_window(state, spec)?) + } else { + None + }; + + Ok(SinglePassEpochResult { + summary, + lookahead_committee_cache, + }) } // TOOO(EIP-7917): use balances cache @@ -512,6 +533,53 @@ pub fn process_proposer_lookahead( Ok(()) } +/// Process the PTC window, returning the committee cache built for the lookahead epoch. +/// +/// The returned cache can be injected into the state's Next committee cache slot after +/// `advance_caches` is called during the epoch transition, avoiding redundant recomputation. +pub fn process_ptc_window( + state: &mut BeaconState, + spec: &ChainSpec, +) -> Result, Error> { + let slots_per_epoch = E::slots_per_epoch() as usize; + + // Convert Vector -> List to use tree-efficient pop_front. + let ptc_window = state.ptc_window()?.clone(); + let mut window: List<_, E::PtcWindowLength> = List::from(ptc_window); + + // Drop the oldest epoch from the front (reuses shared tree nodes). + window + .pop_front(slots_per_epoch) + .map_err(|e| Error::BeaconStateError(BeaconStateError::MilhouseError(e)))?; + + // Compute PTC for the new lookahead epoch + let next_epoch = state + .current_epoch() + .safe_add(spec.min_seed_lookahead.as_u64())? + .safe_add(1)?; + let start_slot = next_epoch.start_slot(E::slots_per_epoch()); + + // Build a committee cache for the lookahead epoch (beyond the normal Next bound) + let committee_cache = state.initialize_committee_cache_for_lookahead(next_epoch, spec)?; + + for i in 0..slots_per_epoch { + let slot = start_slot.safe_add(i as u64)?; + let ptc = state.compute_ptc_with_cache(slot, &committee_cache, spec)?; + let ptc_u64: Vec = ptc.into_iter().map(|v| v as u64).collect(); + let entry = ssz_types::FixedVector::new(ptc_u64) + .map_err(|e| Error::BeaconStateError(BeaconStateError::SszTypesError(e)))?; + window + .push(entry) + .map_err(|e| Error::BeaconStateError(BeaconStateError::MilhouseError(e)))?; + } + + // Convert List back to Vector. + *state.ptc_window_mut()? = Vector::try_from(window) + .map_err(|e| Error::BeaconStateError(BeaconStateError::MilhouseError(e)))?; + + Ok(committee_cache) +} + /// Calculate the quorum threshold for builder payments based on total active balance. fn get_builder_payment_quorum_threshold( state_ctxt: &StateContext, diff --git a/consensus/state_processing/src/upgrade/gloas.rs b/consensus/state_processing/src/upgrade/gloas.rs index 7a88383ab0..b39ee6048f 100644 --- a/consensus/state_processing/src/upgrade/gloas.rs +++ b/consensus/state_processing/src/upgrade/gloas.rs @@ -2,7 +2,9 @@ use crate::per_block_processing::{ is_valid_deposit_signature, process_operations::apply_deposit_for_builder, }; use milhouse::{List, Vector}; +use safe_arith::SafeArith; use ssz_types::BitVector; +use ssz_types::FixedVector; use std::collections::HashSet; use std::mem; use typenum::Unsigned; @@ -102,13 +104,11 @@ pub fn upgrade_state_to_gloas( vec![0xFFu8; E::SlotsPerHistoricalRoot::to_usize() / 8].into(), ) .map_err(|_| Error::InvalidBitfield)?, - builder_pending_payments: Vector::new(vec![ - BuilderPendingPayment::default(); - E::builder_pending_payments_limit() - ])?, + builder_pending_payments: Vector::from_elem(BuilderPendingPayment::default())?, builder_pending_withdrawals: List::default(), // Empty list initially, latest_block_hash: pre.latest_execution_payload_header.block_hash, payload_expected_withdrawals: List::default(), + ptc_window: Vector::from_elem(FixedVector::from_elem(0))?, // placeholder, will be initialized below // Caches total_active_balance: pre.total_active_balance, progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache), @@ -120,10 +120,45 @@ pub fn upgrade_state_to_gloas( }); // [New in Gloas:EIP7732] onboard_builders_from_pending_deposits(&mut post, spec)?; + initialize_ptc_window(&mut post, spec)?; Ok(post) } +/// Initialize the `ptc_window` field in the beacon state at fork transition. +/// +/// The window contains: +/// - One epoch of empty entries (previous epoch) +/// - Computed PTC for the current epoch through `1 + MIN_SEED_LOOKAHEAD` epochs +fn initialize_ptc_window( + state: &mut BeaconState, + spec: &ChainSpec, +) -> Result<(), Error> { + let slots_per_epoch = E::slots_per_epoch() as usize; + + let empty_previous_epoch = vec![FixedVector::::from_elem(0); slots_per_epoch]; + let mut ptcs = empty_previous_epoch; + + // Compute PTC for current epoch + lookahead epochs + let current_epoch = state.current_epoch(); + for e in 0..=spec.min_seed_lookahead.as_u64() { + let epoch = current_epoch.safe_add(e)?; + let committee_cache = state.initialize_committee_cache_for_lookahead(epoch, spec)?; + let start_slot = epoch.start_slot(E::slots_per_epoch()); + for i in 0..slots_per_epoch { + let slot = start_slot.safe_add(i as u64)?; + let ptc = state.compute_ptc_with_cache(slot, &committee_cache, spec)?; + let ptc_u64: Vec = ptc.into_iter().map(|v| v as u64).collect(); + let entry = FixedVector::new(ptc_u64)?; + ptcs.push(entry); + } + } + + *state.ptc_window_mut()? = Vector::new(ptcs)?; + + Ok(()) +} + /// Applies any pending deposit for builders, effectively onboarding builders at the fork. fn onboard_builders_from_pending_deposits( state: &mut BeaconState, diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index c09e3d6931..4aae4b7f39 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -53,7 +53,6 @@ rusqlite = { workspace = true, optional = true } safe_arith = { workspace = true } serde = { workspace = true, features = ["rc"] } serde_json = { workspace = true } -serde_yaml = { workspace = true } smallvec = { workspace = true } ssz_types = { workspace = true } superstruct = { workspace = true } @@ -64,6 +63,7 @@ tracing = { workspace = true } tree_hash = { workspace = true } tree_hash_derive = { workspace = true } typenum = { workspace = true } +yaml_serde = { workspace = true } [dev-dependencies] beacon_chain = { workspace = true } diff --git a/consensus/types/configs/mainnet.yaml b/consensus/types/configs/mainnet.yaml new file mode 100644 index 0000000000..ab85bd9e71 --- /dev/null +++ b/consensus/types/configs/mainnet.yaml @@ -0,0 +1,227 @@ +# Mainnet config + +# Extends the mainnet preset +PRESET_BASE: 'mainnet' + +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'sepolia' - testnet +# * 'holesky' - testnet +# * 'hoodi' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'mainnet' + +# Transition +# --------------------------------------------------------------- +# Estimated on Sept 15, 2022 +TERMINAL_TOTAL_DIFFICULTY: 58750000000000000000000 +# By default, don't use these params +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Genesis +# --------------------------------------------------------------- +# 2**14 (= 16,384) validators +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384 +# Dec 1, 2020, 12pm UTC +MIN_GENESIS_TIME: 1606824000 +# Initial fork version for mainnet +GENESIS_FORK_VERSION: 0x00000000 +# 7 * 24 * 3,600 (= 604,800) seconds, 7 days +GENESIS_DELAY: 604800 + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x01000000 +ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC +# Bellatrix +BELLATRIX_FORK_VERSION: 0x02000000 +BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC +# Capella +CAPELLA_FORK_VERSION: 0x03000000 +CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC +# Deneb +DENEB_FORK_VERSION: 0x04000000 +DENEB_FORK_EPOCH: 269568 # March 13, 2024, 01:55:35pm UTC +# Electra +ELECTRA_FORK_VERSION: 0x05000000 +ELECTRA_FORK_EPOCH: 364032 # May 7, 2025, 10:05:11am UTC +# Fulu +FULU_FORK_VERSION: 0x06000000 +FULU_FORK_EPOCH: 411392 # December 3, 2025, 09:49:11pm UTC +# Gloas +GLOAS_FORK_VERSION: 0x07000000 +GLOAS_FORK_EPOCH: 18446744073709551615 +# Heze +HEZE_FORK_VERSION: 0x08000000 +HEZE_FORK_EPOCH: 18446744073709551615 +# EIP7928 +EIP7928_FORK_VERSION: 0xe7928000 # temporary stub +EIP7928_FORK_EPOCH: 18446744073709551615 + +# Time parameters +# --------------------------------------------------------------- +# 12000 milliseconds +SLOT_DURATION_MS: 12000 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks +ETH1_FOLLOW_DISTANCE: 2048 +# 1667 basis points, ~17% of SLOT_DURATION_MS +PROPOSER_REORG_CUTOFF_BPS: 1667 +# 3333 basis points, ~33% of SLOT_DURATION_MS +ATTESTATION_DUE_BPS: 3333 +# 6667 basis points, ~67% of SLOT_DURATION_MS +AGGREGATE_DUE_BPS: 6667 + +# Altair +# 3333 basis points, ~33% of SLOT_DURATION_MS +SYNC_MESSAGE_DUE_BPS: 3333 +# 6667 basis points, ~67% of SLOT_DURATION_MS +CONTRIBUTION_DUE_BPS: 6667 + +# Gloas +# 2**6 (= 64) epochs +MIN_BUILDER_WITHDRAWABILITY_DELAY: 64 +# 2500 basis points, 25% of SLOT_DURATION_MS +ATTESTATION_DUE_BPS_GLOAS: 2500 +# 5000 basis points, 50% of SLOT_DURATION_MS +AGGREGATE_DUE_BPS_GLOAS: 5000 +# 2500 basis points, 25% of SLOT_DURATION_MS +SYNC_MESSAGE_DUE_BPS_GLOAS: 2500 +# 5000 basis points, 50% of SLOT_DURATION_MS +CONTRIBUTION_DUE_BPS_GLOAS: 5000 +# 7500 basis points, 75% of SLOT_DURATION_MS +PAYLOAD_ATTESTATION_DUE_BPS: 7500 + +# Heze +# 7500 basis points, 75% of SLOT_DURATION_MS +VIEW_FREEZE_CUTOFF_BPS: 7500 +# 6667 basis points, ~67% of SLOT_DURATION_MS +INCLUSION_LIST_SUBMISSION_DUE_BPS: 6667 +# 9167 basis points, ~92% of SLOT_DURATION_MS +PROPOSER_INCLUSION_LIST_CUTOFF_BPS: 9167 + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**2 (= 4) validators +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + +# Deneb +# 2**3 (= 8) (*deprecated*) +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 + +# Electra +# 2**7 * 10**9 (= 128,000,000,000) Gwei +MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 128000000000 +# 2**8 * 10**9 (= 256,000,000,000) Gwei +MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: 256000000000 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 +# 20% +REORG_HEAD_WEIGHT_THRESHOLD: 20 +# 160% +REORG_PARENT_WEIGHT_THRESHOLD: 160 +# 2 epochs +REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 + +# Deposit contract +# --------------------------------------------------------------- +# Ethereum PoW Mainnet +DEPOSIT_CHAIN_ID: 1 +DEPOSIT_NETWORK_ID: 1 +DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa + +# Networking +# --------------------------------------------------------------- +# 10 * 2**20 (= 10,485,760) bytes, 10 MiB +MAX_PAYLOAD_SIZE: 10485760 +# 2**10 (= 1,024) blocks +MAX_REQUEST_BLOCKS: 1024 +# 2**8 (= 256) epochs +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# 2**5 (= 32) slots +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**6 (= 64) subnets +ATTESTATION_SUBNET_COUNT: 64 +# 0 bits +ATTESTATION_SUBNET_EXTRA_BITS: 0 + +# Deneb +# 2**7 (= 128) blocks +MAX_REQUEST_BLOCKS_DENEB: 128 +# 2**12 (= 4,096) epochs +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# 6 subnets +BLOB_SIDECAR_SUBNET_COUNT: 6 +# 6 blobs +MAX_BLOBS_PER_BLOCK: 6 + +# Electra +# 9 subnets +BLOB_SIDECAR_SUBNET_COUNT_ELECTRA: 9 +# 9 blobs +MAX_BLOBS_PER_BLOCK_ELECTRA: 9 + +# Fulu +# 2**7 (= 128) groups +NUMBER_OF_CUSTODY_GROUPS: 128 +# 2**7 (= 128) subnets +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 +# 2**3 (= 8) samples +SAMPLES_PER_SLOT: 8 +# 2**2 (= 4) sidecars +CUSTODY_REQUIREMENT: 4 +# 2**3 (= 8) sidecars +VALIDATOR_CUSTODY_REQUIREMENT: 8 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000 +# 2**12 (= 4,096) epochs +MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096 + +# Gloas +# 2**7 (= 128) payloads +MAX_REQUEST_PAYLOADS: 128 + +# Heze +# 2**4 (= 16) inclusion lists +MAX_REQUEST_INCLUSION_LIST: 16 +# 2**13 (= 8,192) bytes +MAX_BYTES_PER_INCLUSION_LIST: 8192 + + +# Blob Scheduling +# --------------------------------------------------------------- + +BLOB_SCHEDULE: + - EPOCH: 412672 # December 9, 2025, 02:21:11pm UTC + MAX_BLOBS_PER_BLOCK: 15 + - EPOCH: 419072 # January 7, 2026, 01:01:11am UTC + MAX_BLOBS_PER_BLOCK: 21 diff --git a/consensus/types/configs/minimal.yaml b/consensus/types/configs/minimal.yaml new file mode 100644 index 0000000000..8c0d7254fe --- /dev/null +++ b/consensus/types/configs/minimal.yaml @@ -0,0 +1,220 @@ +# Minimal config + +# Extends the minimal preset +PRESET_BASE: 'minimal' + +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'minimal' - spec-testing +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'minimal' + +# Transition +# --------------------------------------------------------------- +# 2**256-2**10 for testing minimal network +TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 +# By default, don't use these params +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Genesis +# --------------------------------------------------------------- +# [customized] 2**6 (= 64) validators +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64 +# [customized] Jan 3, 2020, 12am UTC +MIN_GENESIS_TIME: 1578009600 +# [customized] Initial fork version for minimal +GENESIS_FORK_VERSION: 0x00000001 +# [customized] 5 * 60 (= 300) seconds +GENESIS_DELAY: 300 + +# Forking +# --------------------------------------------------------------- +# Values provided for illustrative purposes. +# Individual tests/testnets may set different values. + +# [customized] Altair +ALTAIR_FORK_VERSION: 0x01000001 +ALTAIR_FORK_EPOCH: 18446744073709551615 +# [customized] Bellatrix +BELLATRIX_FORK_VERSION: 0x02000001 +BELLATRIX_FORK_EPOCH: 18446744073709551615 +# [customized] Capella +CAPELLA_FORK_VERSION: 0x03000001 +CAPELLA_FORK_EPOCH: 18446744073709551615 +# [customized] Deneb +DENEB_FORK_VERSION: 0x04000001 +DENEB_FORK_EPOCH: 18446744073709551615 +# [customized] Electra +ELECTRA_FORK_VERSION: 0x05000001 +ELECTRA_FORK_EPOCH: 18446744073709551615 +# [customized] Fulu +FULU_FORK_VERSION: 0x06000001 +FULU_FORK_EPOCH: 18446744073709551615 +# [customized] Gloas +GLOAS_FORK_VERSION: 0x07000001 +GLOAS_FORK_EPOCH: 18446744073709551615 +# [customized] Heze +HEZE_FORK_VERSION: 0x08000001 +HEZE_FORK_EPOCH: 18446744073709551615 +# [customized] EIP7928 +EIP7928_FORK_VERSION: 0xe7928001 +EIP7928_FORK_EPOCH: 18446744073709551615 + +# Time parameters +# --------------------------------------------------------------- +# [customized] 6000 milliseconds +SLOT_DURATION_MS: 6000 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# [customized] 2**6 (= 64) epochs +SHARD_COMMITTEE_PERIOD: 64 +# [customized] 2**4 (= 16) Eth1 blocks +ETH1_FOLLOW_DISTANCE: 16 +# 1667 basis points, ~17% of SLOT_DURATION_MS +PROPOSER_REORG_CUTOFF_BPS: 1667 +# 3333 basis points, ~33% of SLOT_DURATION_MS +ATTESTATION_DUE_BPS: 3333 +# 6667 basis points, ~67% of SLOT_DURATION_MS +AGGREGATE_DUE_BPS: 6667 + +# Altair +# 3333 basis points, ~33% of SLOT_DURATION_MS +SYNC_MESSAGE_DUE_BPS: 3333 +# 6667 basis points, ~67% of SLOT_DURATION_MS +CONTRIBUTION_DUE_BPS: 6667 + +# Gloas +# [customized] 2**1 (= 2) epochs +MIN_BUILDER_WITHDRAWABILITY_DELAY: 2 +# 2500 basis points, 25% of SLOT_DURATION_MS +ATTESTATION_DUE_BPS_GLOAS: 2500 +# 5000 basis points, 50% of SLOT_DURATION_MS +AGGREGATE_DUE_BPS_GLOAS: 5000 +# 2500 basis points, 25% of SLOT_DURATION_MS +SYNC_MESSAGE_DUE_BPS_GLOAS: 2500 +# 5000 basis points, 50% of SLOT_DURATION_MS +CONTRIBUTION_DUE_BPS_GLOAS: 5000 +# 7500 basis points, 75% of SLOT_DURATION_MS +PAYLOAD_ATTESTATION_DUE_BPS: 7500 + +# Heze +# 7500 basis points, 75% of SLOT_DURATION_MS +VIEW_FREEZE_CUTOFF_BPS: 7500 +# 6667 basis points, ~67% of SLOT_DURATION_MS +INCLUSION_LIST_SUBMISSION_DUE_BPS: 6667 +# 9167 basis points, ~92% of SLOT_DURATION_MS +PROPOSER_INCLUSION_LIST_CUTOFF_BPS: 9167 + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# [customized] 2**1 (= 2) validators +MIN_PER_EPOCH_CHURN_LIMIT: 2 +# [customized] 2**5 (= 32) +CHURN_LIMIT_QUOTIENT: 32 + +# Deneb +# [customized] 2**2 (= 4) (*deprecated*) +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 4 + +# Electra +# [customized] 2**6 * 10**9 (= 64,000,000,000) Gwei +MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000 +# [customized] 2**7 * 10**9 (= 128,000,000,000) Gwei +MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: 128000000000 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 +# 20% +REORG_HEAD_WEIGHT_THRESHOLD: 20 +# 160% +REORG_PARENT_WEIGHT_THRESHOLD: 160 +# 2 epochs +REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 + +# Deposit contract +# --------------------------------------------------------------- +# Ethereum Goerli testnet +DEPOSIT_CHAIN_ID: 5 +DEPOSIT_NETWORK_ID: 5 +# Configured on a per testnet basis +DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 + +# Networking +# --------------------------------------------------------------- +# 10 * 2**20 (= 10,485,760) bytes, 10 MiB +MAX_PAYLOAD_SIZE: 10485760 +# 2**10 (= 1,024) blocks +MAX_REQUEST_BLOCKS: 1024 +# 2**8 (= 256) epochs +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# 2**5 (= 32) slots +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**6 (= 64) subnets +ATTESTATION_SUBNET_COUNT: 64 +# 0 bits +ATTESTATION_SUBNET_EXTRA_BITS: 0 + +# Deneb +# 2**7 (= 128) blocks +MAX_REQUEST_BLOCKS_DENEB: 128 +# 2**12 (= 4,096) epochs +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# 6 subnets +BLOB_SIDECAR_SUBNET_COUNT: 6 +# 6 blobs +MAX_BLOBS_PER_BLOCK: 6 + +# Electra +# 9 subnets +BLOB_SIDECAR_SUBNET_COUNT_ELECTRA: 9 +# 9 blobs +MAX_BLOBS_PER_BLOCK_ELECTRA: 9 + +# Fulu +# 2**7 (= 128) groups +NUMBER_OF_CUSTODY_GROUPS: 128 +# 2**7 (= 128) subnets +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 +# 2**3 (= 8) samples +SAMPLES_PER_SLOT: 8 +# 2**2 (= 4) sidecars +CUSTODY_REQUIREMENT: 4 +# 2**3 (= 8) sidecars +VALIDATOR_CUSTODY_REQUIREMENT: 8 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000 +# 2**12 (= 4,096) epochs +MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096 + +# Gloas +# 2**7 (= 128) payloads +MAX_REQUEST_PAYLOADS: 128 + +# Heze +# 2**4 (= 16) inclusion lists +MAX_REQUEST_INCLUSION_LIST: 16 +# 2**13 (= 8,192) bytes +MAX_BYTES_PER_INCLUSION_LIST: 8192 + + +# Blob Scheduling +# --------------------------------------------------------------- + +BLOB_SCHEDULE: [] diff --git a/consensus/types/src/core/chain_spec.rs b/consensus/types/src/core/chain_spec.rs index 458622d7e6..cc79d3fc29 100644 --- a/consensus/types/src/core/chain_spec.rs +++ b/consensus/types/src/core/chain_spec.rs @@ -96,8 +96,7 @@ pub struct ChainSpec { * Time parameters */ pub genesis_delay: u64, - // TODO deprecate seconds_per_slot - pub seconds_per_slot: u64, + seconds_per_slot: u64, // Private so that this value can't get changed except via the `set_slot_duration_ms` function. slot_duration_ms: u64, pub min_attestation_inclusion_delay: u64, @@ -829,15 +828,17 @@ impl ChainSpec { /// Returns the min epoch for blob / data column sidecar requests based on the current epoch. /// Switch to use the column sidecar config once the `blob_retention_epoch` has passed Fulu fork epoch. + /// Never uses the `blob_retention_epoch` for networks that started with Fulu enabled. pub fn min_epoch_data_availability_boundary(&self, current_epoch: Epoch) -> Option { - let fork_epoch = self.deneb_fork_epoch?; + let deneb_fork_epoch = self.deneb_fork_epoch?; let blob_retention_epoch = current_epoch.saturating_sub(self.min_epochs_for_blob_sidecars_requests); - match self.fulu_fork_epoch { - Some(fulu_fork_epoch) if blob_retention_epoch > fulu_fork_epoch => Some( - current_epoch.saturating_sub(self.min_epochs_for_data_column_sidecars_requests), - ), - _ => Some(std::cmp::max(fork_epoch, blob_retention_epoch)), + if let Some(fulu_fork_epoch) = self.fulu_fork_epoch + && blob_retention_epoch >= fulu_fork_epoch + { + Some(current_epoch.saturating_sub(self.min_epochs_for_data_column_sidecars_requests)) + } else { + Some(std::cmp::max(deneb_fork_epoch, blob_retention_epoch)) } } @@ -914,6 +915,7 @@ impl ChainSpec { /// Set the duration of a slot (in ms). pub fn set_slot_duration_ms(mut self, slot_duration_ms: u64) -> Self { self.slot_duration_ms = slot_duration_ms; + self.seconds_per_slot = slot_duration_ms.saturating_div(1000); self.compute_derived_values::() } @@ -1235,7 +1237,7 @@ impl ChainSpec { gloas_fork_epoch: None, builder_payment_threshold_numerator: 6, builder_payment_threshold_denominator: 10, - min_builder_withdrawability_delay: Epoch::new(4096), + min_builder_withdrawability_delay: Epoch::new(64), max_request_payloads: 128, /* @@ -1381,6 +1383,7 @@ impl ChainSpec { // Gloas gloas_fork_version: [0x07, 0x00, 0x00, 0x01], gloas_fork_epoch: None, + min_builder_withdrawability_delay: Epoch::new(2), /* * Derived time values (set by `compute_derived_values()`) @@ -1391,6 +1394,9 @@ impl ChainSpec { sync_message_due: Duration::from_millis(1999), contribution_and_proof_due: Duration::from_millis(4000), + // Networking Fulu + blob_schedule: BlobSchedule::default(), + // Other network_id: 2, // lighthouse testnet network id deposit_chain_id: 5, @@ -1631,7 +1637,7 @@ impl ChainSpec { gloas_fork_epoch: None, builder_payment_threshold_numerator: 6, builder_payment_threshold_denominator: 10, - min_builder_withdrawability_delay: Epoch::new(4096), + min_builder_withdrawability_delay: Epoch::new(64), max_request_payloads: 128, /* @@ -1908,8 +1914,9 @@ pub struct Config { #[serde(deserialize_with = "deserialize_fork_epoch")] pub gloas_fork_epoch: Option>, - #[serde(with = "serde_utils::quoted_u64")] - seconds_per_slot: u64, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + seconds_per_slot: Option>, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] slot_duration_ms: Option>, @@ -2064,6 +2071,10 @@ pub struct Config { #[serde(default = "default_contribution_due_bps")] #[serde(with = "serde_utils::quoted_u64")] contribution_due_bps: u64, + + #[serde(default = "default_min_builder_withdrawability_delay")] + #[serde(with = "serde_utils::quoted_u64")] + min_builder_withdrawability_delay: u64, } fn default_bellatrix_fork_version() -> [u8; 4] { @@ -2289,6 +2300,10 @@ const fn default_contribution_due_bps() -> u64 { 6667 } +const fn default_min_builder_withdrawability_delay() -> u64 { + 64 +} + fn max_blocks_by_root_request_common(max_request_blocks: u64) -> usize { let max_request_blocks = max_request_blocks as usize; RuntimeVariableList::::new( @@ -2459,7 +2474,9 @@ impl Config { .gloas_fork_epoch .map(|epoch| MaybeQuoted { value: epoch }), - seconds_per_slot: spec.seconds_per_slot, + seconds_per_slot: Some(MaybeQuoted { + value: spec.seconds_per_slot, + }), slot_duration_ms: Some(MaybeQuoted { value: spec.slot_duration_ms, }), @@ -2525,13 +2542,15 @@ impl Config { aggregate_due_bps: spec.aggregate_due_bps, sync_message_due_bps: spec.sync_message_due_bps, contribution_due_bps: spec.contribution_due_bps, + + min_builder_withdrawability_delay: spec.min_builder_withdrawability_delay.as_u64(), } } pub fn from_file(filename: &Path) -> Result { let f = File::open(filename) .map_err(|e| format!("Error opening spec at {}: {:?}", filename.display(), e))?; - serde_yaml::from_reader(f) + yaml_serde::from_reader(f) .map_err(|e| format!("Error parsing spec at {}: {:?}", filename.display(), e)) } @@ -2616,12 +2635,21 @@ impl Config { aggregate_due_bps, sync_message_due_bps, contribution_due_bps, + min_builder_withdrawability_delay, } = self; if preset_base != E::spec_name().to_string().as_str() { return None; } + // Fail if seconds_per_slot and slot_duration_ms are both set but are inconsistent. + if let (Some(seconds_per_slot), Some(slot_duration_ms)) = + (seconds_per_slot, slot_duration_ms) + && seconds_per_slot.value.saturating_mul(1000) != slot_duration_ms.value + { + return None; + } + let spec = ChainSpec { config_name: config_name.clone(), min_genesis_active_validator_count, @@ -2642,10 +2670,12 @@ impl Config { fulu_fork_version, gloas_fork_version, gloas_fork_epoch: gloas_fork_epoch.map(|q| q.value), - seconds_per_slot, + seconds_per_slot: seconds_per_slot + .map(|q| q.value) + .or_else(|| slot_duration_ms.and_then(|q| q.value.checked_div(1000)))?, slot_duration_ms: slot_duration_ms .map(|q| q.value) - .unwrap_or_else(|| seconds_per_slot.saturating_mul(1000)), + .or_else(|| seconds_per_slot.map(|q| q.value.saturating_mul(1000)))?, seconds_per_eth1_block, min_validator_withdrawability_delay, shard_committee_period, @@ -2705,6 +2735,8 @@ impl Config { sync_message_due_bps, contribution_due_bps, + min_builder_withdrawability_delay: Epoch::new(min_builder_withdrawability_delay), + ..chain_spec.clone() }; Some(spec.compute_derived_values::()) @@ -2853,6 +2885,9 @@ mod yaml_tests { use super::*; use crate::core::MinimalEthSpec; use paste::paste; + use std::collections::BTreeSet; + use std::env; + use std::path::PathBuf; use std::sync::Arc; use tempfile::NamedTempFile; @@ -2869,7 +2904,7 @@ mod yaml_tests { let yamlconfig = Config::from_chain_spec::(&minimal_spec); // write fresh minimal config to file - serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize"); + yaml_serde::to_writer(writer, &yamlconfig).expect("failed to write or serialize"); let reader = File::options() .read(true) @@ -2877,7 +2912,7 @@ mod yaml_tests { .open(tmp_file.as_ref()) .expect("error while opening the file"); // deserialize minimal config from file - let from: Config = serde_yaml::from_reader(reader).expect("error while deserializing"); + let from: Config = yaml_serde::from_reader(reader).expect("error while deserializing"); assert_eq!(from, yamlconfig); } @@ -2891,17 +2926,78 @@ mod yaml_tests { .expect("error opening file"); let mainnet_spec = ChainSpec::mainnet(); let yamlconfig = Config::from_chain_spec::(&mainnet_spec); - serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize"); + yaml_serde::to_writer(writer, &yamlconfig).expect("failed to write or serialize"); let reader = File::options() .read(true) .write(false) .open(tmp_file.as_ref()) .expect("error while opening the file"); - let from: Config = serde_yaml::from_reader(reader).expect("error while deserializing"); + let from: Config = yaml_serde::from_reader(reader).expect("error while deserializing"); assert_eq!(from, yamlconfig); } + #[test] + fn slot_duration_fallback_both_fields() { + let mainnet = ChainSpec::mainnet(); + let mut config = Config::from_chain_spec::(&mainnet); + config.seconds_per_slot = Some(MaybeQuoted { value: 12 }); + config.slot_duration_ms = Some(MaybeQuoted { value: 12000 }); + let spec = config + .apply_to_chain_spec::(&mainnet) + .unwrap(); + assert_eq!(spec.seconds_per_slot, 12); + assert_eq!(spec.slot_duration_ms, 12000); + } + + #[test] + fn slot_duration_fallback_both_fields_inconsistent() { + let mainnet = ChainSpec::mainnet(); + let mut config = Config::from_chain_spec::(&mainnet); + config.seconds_per_slot = Some(MaybeQuoted { value: 10 }); + config.slot_duration_ms = Some(MaybeQuoted { value: 12000 }); + assert_eq!(config.apply_to_chain_spec::(&mainnet), None); + } + + #[test] + fn slot_duration_fallback_seconds_only() { + let mainnet = ChainSpec::mainnet(); + let mut config = Config::from_chain_spec::(&mainnet); + config.seconds_per_slot = Some(MaybeQuoted { value: 12 }); + config.slot_duration_ms = None; + let spec = config + .apply_to_chain_spec::(&mainnet) + .unwrap(); + assert_eq!(spec.seconds_per_slot, 12); + assert_eq!(spec.slot_duration_ms, 12000); + } + + #[test] + fn slot_duration_fallback_ms_only() { + let mainnet = ChainSpec::mainnet(); + let mut config = Config::from_chain_spec::(&mainnet); + config.seconds_per_slot = None; + config.slot_duration_ms = Some(MaybeQuoted { value: 12000 }); + let spec = config + .apply_to_chain_spec::(&mainnet) + .unwrap(); + assert_eq!(spec.seconds_per_slot, 12); + assert_eq!(spec.slot_duration_ms, 12000); + } + + #[test] + fn slot_duration_fallback_neither() { + let mainnet = ChainSpec::mainnet(); + let mut config = Config::from_chain_spec::(&mainnet); + config.seconds_per_slot = None; + config.slot_duration_ms = None; + assert!( + config + .apply_to_chain_spec::(&mainnet) + .is_none() + ); + } + #[test] fn blob_schedule_max_blobs_per_block() { let spec_contents = r#" @@ -2960,7 +3056,7 @@ mod yaml_tests { MAX_BLOBS_PER_BLOCK: 20 "#; let config: Config = - serde_yaml::from_str(spec_contents).expect("error while deserializing"); + yaml_serde::from_str(spec_contents).expect("error while deserializing"); let spec = ChainSpec::from_config::(&config).expect("error while creating spec"); @@ -3042,11 +3138,11 @@ mod yaml_tests { assert_eq!(spec.max_blobs_per_block_within_fork(ForkName::Fulu), 20); // Check that serialization is in ascending order - let yaml = serde_yaml::to_string(&spec.blob_schedule).expect("should serialize"); + let yaml = yaml_serde::to_string(&spec.blob_schedule).expect("should serialize"); // Deserialize back to Vec to check order let deserialized: Vec = - serde_yaml::from_str(&yaml).expect("should deserialize"); + yaml_serde::from_str(&yaml).expect("should deserialize"); // Should be in ascending order by epoch assert!( @@ -3113,7 +3209,7 @@ mod yaml_tests { MAX_BLOBS_PER_BLOCK: 300 "#; let config: Config = - serde_yaml::from_str(spec_contents).expect("error while deserializing"); + yaml_serde::from_str(spec_contents).expect("error while deserializing"); let spec = ChainSpec::from_config::(&config).expect("error while creating spec"); @@ -3203,7 +3299,7 @@ mod yaml_tests { SAMPLES_PER_SLOT: 8 "#; - let chain_spec: Config = serde_yaml::from_str(spec).unwrap(); + let chain_spec: Config = yaml_serde::from_str(spec).unwrap(); // Asserts that `chain_spec.$name` and `default_$name()` are equal. macro_rules! check_default { @@ -3304,17 +3400,19 @@ mod yaml_tests { spec.min_epoch_data_availability_boundary(fulu_fork_epoch) ); - // `min_epochs_for_data_sidecar_requests` at fulu fork epoch + min_epochs_for_blob_sidecars_request - let blob_retention_epoch_after_fulu = fulu_fork_epoch + blob_retention_epochs; - let expected_blob_retention_epoch = blob_retention_epoch_after_fulu - blob_retention_epochs; + // Now, the blob retention period starts still before the fulu fork epoch, so the boundary + // should respect the blob retention period. + let half_blob_retention_epoch_after_fulu = fulu_fork_epoch + (blob_retention_epochs / 2); + let expected_blob_retention_epoch = + half_blob_retention_epoch_after_fulu - blob_retention_epochs; assert_eq!( Some(expected_blob_retention_epoch), - spec.min_epoch_data_availability_boundary(blob_retention_epoch_after_fulu) + spec.min_epoch_data_availability_boundary(half_blob_retention_epoch_after_fulu) ); - // After the final blob retention epoch, `min_epochs_for_data_sidecar_requests` should be calculated - // using `min_epochs_for_data_column_sidecars_request` - let current_epoch = blob_retention_epoch_after_fulu + 1; + // If the retention period starts with the fulu fork epoch, there are no more blobs to + // retain, and the return value will be based on the data column retention period. + let current_epoch = fulu_fork_epoch + blob_retention_epochs; let expected_data_column_retention_epoch = current_epoch - data_column_retention_epochs; assert_eq!( Some(expected_data_column_retention_epoch), @@ -3322,6 +3420,39 @@ mod yaml_tests { ); } + #[test] + fn min_epochs_for_data_sidecar_requests_fulu_genesis() { + type E = MainnetEthSpec; + let spec = { + // fulu active at genesis + let mut spec = ForkName::Fulu.make_genesis_spec(E::default_spec()); + // set a different value for testing purpose, 4096 / 2 = 2048 + spec.min_epochs_for_data_column_sidecars_requests = + spec.min_epochs_for_blob_sidecars_requests / 2; + Arc::new(spec) + }; + let blob_retention_epochs = spec.min_epochs_for_blob_sidecars_requests; + let data_column_retention_epochs = spec.min_epochs_for_data_column_sidecars_requests; + + // If Fulu is activated at genesis, the column retention period should always be used. + let assert_correct_boundary = |epoch| { + let epoch = Epoch::new(epoch); + assert_eq!( + Some(epoch.saturating_sub(data_column_retention_epochs)), + spec.min_epoch_data_availability_boundary(epoch) + ) + }; + + assert_correct_boundary(0); + assert_correct_boundary(1); + assert_correct_boundary(blob_retention_epochs - 1); + assert_correct_boundary(blob_retention_epochs); + assert_correct_boundary(blob_retention_epochs + 1); + assert_correct_boundary(data_column_retention_epochs - 1); + assert_correct_boundary(data_column_retention_epochs); + assert_correct_boundary(data_column_retention_epochs + 1); + } + #[test] fn proposer_shuffling_decision_root_around_epoch_boundary() { type E = MainnetEthSpec; @@ -3375,7 +3506,6 @@ mod yaml_tests { // Test slot duration let slot_duration = spec.get_slot_duration(); assert_eq!(slot_duration, Duration::from_millis(12000)); - assert_eq!(slot_duration, Duration::from_secs(spec.seconds_per_slot)); // Test edge cases with custom spec let mut custom_spec = spec.clone(); @@ -3485,4 +3615,133 @@ mod yaml_tests { spec.attestation_due_bps = 15000; spec.compute_derived_values::(); } + + fn configs_base_path() -> PathBuf { + env::var("CARGO_MANIFEST_DIR") + .expect("should know manifest dir") + .parse::() + .expect("should parse manifest dir as path") + .join("configs") + } + + /// Upstream config keys that Lighthouse intentionally does not include in its + /// `Config` struct. These are forks/features not yet implemented. Update this + /// list as new forks are added. + const UPSTREAM_KEYS_NOT_IN_LIGHTHOUSE: &[&str] = &[ + // Forks not yet implemented + "HEZE_FORK_VERSION", + "HEZE_FORK_EPOCH", + "EIP7928_FORK_VERSION", + "EIP7928_FORK_EPOCH", + // Gloas params not yet in Config + "ATTESTATION_DUE_BPS_GLOAS", + "AGGREGATE_DUE_BPS_GLOAS", + "SYNC_MESSAGE_DUE_BPS_GLOAS", + "CONTRIBUTION_DUE_BPS_GLOAS", + "PAYLOAD_ATTESTATION_DUE_BPS", + "MAX_REQUEST_PAYLOADS", + // Gloas fork choice params not yet in Config + "REORG_HEAD_WEIGHT_THRESHOLD", + "REORG_PARENT_WEIGHT_THRESHOLD", + "REORG_MAX_EPOCHS_SINCE_FINALIZATION", + // Heze networking + "VIEW_FREEZE_CUTOFF_BPS", + "INCLUSION_LIST_SUBMISSION_DUE_BPS", + "PROPOSER_INCLUSION_LIST_CUTOFF_BPS", + "MAX_REQUEST_INCLUSION_LIST", + "MAX_BYTES_PER_INCLUSION_LIST", + ]; + + /// Compare a `ChainSpec` against an upstream consensus-specs config YAML file. + /// + /// 1. Extracts keys from the raw YAML text (to avoid yaml_serde's inability + /// to parse integers > u64 into `Value`/`Mapping` types) and checks that + /// every key is either known to `Config` or explicitly listed in + /// `UPSTREAM_KEYS_NOT_IN_LIGHTHOUSE`. + /// 2. Deserializes the upstream YAML as `Config` (which has custom + /// deserializers for large values like `TERMINAL_TOTAL_DIFFICULTY`) and + /// compares against `Config::from_chain_spec`. + fn config_test(spec: &ChainSpec, config_name: &str) { + let file_path = configs_base_path().join(format!("{config_name}.yaml")); + let upstream_yaml = std::fs::read_to_string(&file_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", file_path.display())); + + // Extract top-level keys from the raw YAML text. We can't parse as + // yaml_serde::Mapping because yaml_serde cannot represent integers + // exceeding u64 (e.g. TERMINAL_TOTAL_DIFFICULTY). Config YAML uses a + // simple `KEY: value` format with no indentation for top-level keys. + let upstream_keys: BTreeSet = upstream_yaml + .lines() + .filter_map(|line| { + // Skip comments, blank lines, and indented lines (nested YAML). + if line.is_empty() + || line.starts_with('#') + || line.starts_with(' ') + || line.starts_with('\t') + { + return None; + } + line.split(':').next().map(|k| k.to_string()) + }) + .collect(); + + // Get the set of keys that Config knows about by serializing and collecting + // keys. Also include keys for optional fields that may be skipped during + // serialization (e.g. CONFIG_NAME). + let our_config = Config::from_chain_spec::(spec); + let our_yaml = yaml_serde::to_string(&our_config).expect("failed to serialize Config"); + let our_mapping: yaml_serde::Mapping = + yaml_serde::from_str(&our_yaml).expect("failed to re-parse our Config"); + let mut known_keys: BTreeSet = our_mapping + .keys() + .filter_map(|k| k.as_str().map(String::from)) + .collect(); + // Fields that Config knows but may skip during serialization. + known_keys.insert("CONFIG_NAME".to_string()); + + // Check for upstream keys that our Config doesn't know about. + let mut missing_keys: Vec<&String> = upstream_keys + .iter() + .filter(|k| { + !known_keys.contains(k.as_str()) + && !UPSTREAM_KEYS_NOT_IN_LIGHTHOUSE.contains(&k.as_str()) + }) + .collect(); + missing_keys.sort(); + + assert!( + missing_keys.is_empty(), + "Upstream {config_name} config has keys not present in Lighthouse Config \ + (add to Config or to UPSTREAM_KEYS_NOT_IN_LIGHTHOUSE): {missing_keys:?}" + ); + + // Compare values for all fields Config knows about. + let mut upstream_config: Config = yaml_serde::from_str(&upstream_yaml) + .unwrap_or_else(|e| panic!("failed to parse {config_name} as Config: {e}")); + + // CONFIG_NAME is network metadata (not a spec parameter), so align it + // before comparing. + upstream_config.config_name = our_config.config_name.clone(); + // SECONDS_PER_SLOT is deprecated upstream but we still emit it, so + // fill it in if the upstream YAML omitted it. + if upstream_config.seconds_per_slot.is_none() { + upstream_config.seconds_per_slot = our_config.seconds_per_slot; + } + assert_eq!( + upstream_config, our_config, + "Config mismatch for {config_name}" + ); + } + + #[test] + fn mainnet_config_consistent() { + let spec = ChainSpec::mainnet(); + config_test::(&spec, "mainnet"); + } + + #[test] + fn minimal_config_consistent() { + let spec = ChainSpec::minimal(); + config_test::(&spec, "minimal"); + } } diff --git a/consensus/types/src/core/config_and_preset.rs b/consensus/types/src/core/config_and_preset.rs index 5b8b27b02e..06f080e82b 100644 --- a/consensus/types/src/core/config_and_preset.rs +++ b/consensus/types/src/core/config_and_preset.rs @@ -174,7 +174,7 @@ mod test { yamlconfig.extra_fields_mut().insert(k3.into(), v3.into()); yamlconfig.extra_fields_mut().insert(k4.into(), v4); - serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize"); + yaml_serde::to_writer(writer, &yamlconfig).expect("failed to write or serialize"); let reader = File::options() .read(true) @@ -182,7 +182,7 @@ mod test { .open(tmp_file.as_ref()) .expect("error while opening the file"); let from: ConfigAndPresetGloas = - serde_yaml::from_reader(reader).expect("error while deserializing"); + yaml_serde::from_reader(reader).expect("error while deserializing"); assert_eq!(ConfigAndPreset::Gloas(from), yamlconfig); } diff --git a/consensus/types/src/core/eth_spec.rs b/consensus/types/src/core/eth_spec.rs index a4b22da3f8..36d61fbbf9 100644 --- a/consensus/types/src/core/eth_spec.rs +++ b/consensus/types/src/core/eth_spec.rs @@ -6,9 +6,9 @@ use std::{ use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use typenum::{ - U0, U1, U2, U4, U8, U16, U17, U32, U64, U128, U256, U512, U625, U1024, U2048, U4096, U8192, - U16384, U65536, U131072, U262144, U1048576, U16777216, U33554432, U134217728, U1073741824, - U1099511627776, UInt, Unsigned, bit::B0, + U0, U1, U2, U4, U8, U16, U17, U24, U32, U48, U64, U96, U128, U256, U512, U625, U1024, U2048, + U4096, U8192, U16384, U65536, U131072, U262144, U1048576, U16777216, U33554432, U134217728, + U1073741824, U1099511627776, UInt, Unsigned, bit::B0, }; use crate::core::{ChainSpec, Epoch}; @@ -176,6 +176,7 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq + * New in Gloas */ type PTCSize: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type PtcWindowLength: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxPayloadAttestations: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BuilderPendingPaymentsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BuilderPendingWithdrawalsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; @@ -428,6 +429,11 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq + Self::PTCSize::to_usize() } + /// Returns the `PtcWindowLength` constant for this specification. + fn ptc_window_length() -> usize { + Self::PtcWindowLength::to_usize() + } + /// Returns the `MaxPayloadAttestations` constant for this specification. fn max_payload_attestations() -> usize { Self::MaxPayloadAttestations::to_usize() @@ -515,6 +521,7 @@ impl EthSpec for MainnetEthSpec { type MaxWithdrawalRequestsPerPayload = U16; type MaxPendingDepositsPerEpoch = U16; type PTCSize = U512; + type PtcWindowLength = U96; // (2 + MIN_SEED_LOOKAHEAD) * SLOTS_PER_EPOCH type MaxPayloadAttestations = U4; type MaxBuildersPerWithdrawalsSweep = U16384; @@ -561,6 +568,7 @@ impl EthSpec for MinimalEthSpec { type ProposerLookaheadSlots = U16; // Derived from (MIN_SEED_LOOKAHEAD + 1) * SLOTS_PER_EPOCH type BuilderPendingPaymentsLimit = U16; // 2 * SLOTS_PER_EPOCH = 2 * 8 = 16 type PTCSize = U2; + type PtcWindowLength = U24; // (2 + MIN_SEED_LOOKAHEAD) * SLOTS_PER_EPOCH type MaxBuildersPerWithdrawalsSweep = U16; params_from_eth_spec!(MainnetEthSpec { @@ -668,6 +676,7 @@ impl EthSpec for GnosisEthSpec { type ProposerLookaheadSlots = U32; // Derived from (MIN_SEED_LOOKAHEAD + 1) * SLOTS_PER_EPOCH type BuilderRegistryLimit = U1099511627776; type PTCSize = U512; + type PtcWindowLength = U48; // (2 + MIN_SEED_LOOKAHEAD) * SLOTS_PER_EPOCH type MaxPayloadAttestations = U2; type MaxBuildersPerWithdrawalsSweep = U16384; @@ -694,6 +703,11 @@ mod test { E::proposer_lookahead_slots(), (spec.min_seed_lookahead.as_usize() + 1) * E::slots_per_epoch() as usize ); + assert_eq!( + E::ptc_window_length(), + (spec.min_seed_lookahead.as_usize() + 2) * E::slots_per_epoch() as usize, + "PtcWindowLength must equal (2 + MIN_SEED_LOOKAHEAD) * SLOTS_PER_EPOCH" + ); } #[test] diff --git a/consensus/types/src/core/preset.rs b/consensus/types/src/core/preset.rs index 5b1978f8e9..4fa7a28204 100644 --- a/consensus/types/src/core/preset.rs +++ b/consensus/types/src/core/preset.rs @@ -359,7 +359,7 @@ mod test { fn preset_from_file(preset_name: &str, filename: &str) -> T { let f = File::open(presets_base_path().join(preset_name).join(filename)) .expect("preset file exists"); - serde_yaml::from_reader(f).unwrap() + yaml_serde::from_reader(f).unwrap() } fn preset_test() { diff --git a/consensus/types/src/state/beacon_state.rs b/consensus/types/src/state/beacon_state.rs index f431055c5f..a033272b9d 100644 --- a/consensus/types/src/state/beacon_state.rs +++ b/consensus/types/src/state/beacon_state.rs @@ -667,6 +667,11 @@ where #[superstruct(only(Gloas))] pub payload_expected_withdrawals: List, + #[compare_fields(as_iter)] + #[test_random(default)] + #[superstruct(only(Gloas))] + pub ptc_window: Vector, E::PtcWindowLength>, + // Caching (not in the spec) #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] @@ -2431,6 +2436,18 @@ impl BeaconState { CommitteeCache::initialized(self, epoch, spec) } + /// Like [`initialize_committee_cache`](Self::initialize_committee_cache), but allows epochs + /// beyond `current_epoch + 1`. Only checks that the required randao seed is available. + /// + /// Used by PTC window computation which needs shufflings for lookahead epochs. + pub fn initialize_committee_cache_for_lookahead( + &self, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result, BeaconStateError> { + CommitteeCache::initialized_for_lookahead(self, epoch, spec) + } + /// Advances the cache for this state into the next epoch. /// /// This should be used if the `slot` of this state is advanced beyond an epoch boundary. @@ -2501,6 +2518,17 @@ impl BeaconState { .ok_or(BeaconStateError::CommitteeCachesOutOfBounds(index)) } + /// Set the committee cache for the given `relative_epoch` to `cache`. + pub fn set_committee_cache( + &mut self, + relative_epoch: RelativeEpoch, + cache: Arc, + ) -> Result<(), BeaconStateError> { + let i = Self::committee_cache_index(relative_epoch); + *self.committee_cache_at_index_mut(i)? = cache; + Ok(()) + } + /// Returns the cache for some `RelativeEpoch`. Returns an error if the cache has not been /// initialized. pub fn committee_cache( @@ -3084,12 +3112,55 @@ impl BeaconState { } } - /// Get the payload timeliness committee for the given `slot`. - /// - /// Requires the committee cache to be initialized. - /// TODO(EIP-7732): definitely gonna have to cache this.. + /// Get the payload timeliness committee for the given `slot` from the `ptc_window`. pub fn get_ptc(&self, slot: Slot, spec: &ChainSpec) -> Result, BeaconStateError> { + let ptc_window = self.ptc_window()?; + let epoch = slot.epoch(E::slots_per_epoch()); + let state_epoch = self.current_epoch(); + let slots_per_epoch = E::slots_per_epoch() as usize; + let slot_in_epoch = slot.as_usize().safe_rem(slots_per_epoch)?; + + let index = if epoch < state_epoch { + if epoch.safe_add(1)? != state_epoch { + return Err(BeaconStateError::SlotOutOfBounds); + } + slot_in_epoch + } else { + if epoch > state_epoch.safe_add(spec.min_seed_lookahead)? { + return Err(BeaconStateError::SlotOutOfBounds); + } + let offset = epoch + .safe_sub(state_epoch)? + .safe_add(1)? + .as_usize() + .safe_mul(slots_per_epoch)?; + offset.safe_add(slot_in_epoch)? + }; + + let entry = ptc_window + .get(index) + .ok_or(BeaconStateError::SlotOutOfBounds)?; + + // Convert from FixedVector to PTC (FixedVector) + let indices: Vec = entry.iter().map(|&v| v as usize).collect(); + Ok(PTC(FixedVector::new(indices)?)) + } + + /// Compute the payload timeliness committee for the given `slot` from scratch. + /// + /// Requires the committee cache to be initialized for the slot's epoch. + pub fn compute_ptc(&self, slot: Slot, spec: &ChainSpec) -> Result, BeaconStateError> { let committee_cache = self.committee_cache_at_slot(slot)?; + self.compute_ptc_with_cache(slot, committee_cache, spec) + } + + /// Compute the PTC for a slot using a specific committee cache. + pub fn compute_ptc_with_cache( + &self, + slot: Slot, + committee_cache: &CommitteeCache, + spec: &ChainSpec, + ) -> Result, BeaconStateError> { let committees = committee_cache.get_beacon_committees_at_slot(slot)?; let seed = self.get_ptc_attester_seed(slot, spec)?; diff --git a/consensus/types/src/state/committee_cache.rs b/consensus/types/src/state/committee_cache.rs index 4a28f3c689..2e74ab760c 100644 --- a/consensus/types/src/state/committee_cache.rs +++ b/consensus/types/src/state/committee_cache.rs @@ -62,6 +62,9 @@ fn compare_shuffling_positions(xs: &Vec, ys: &Vec( state: &BeaconState, @@ -81,12 +84,44 @@ impl CommitteeCache { || epoch > state .current_epoch() - .safe_add(1) + .safe_add(1u64) .map_err(BeaconStateError::ArithError)? { return Err(BeaconStateError::EpochOutOfBounds); } + Self::initialized_unchecked(state, epoch, spec) + } + + /// Return a new, fully initialized cache for a lookahead epoch. + /// + /// Like [`initialized`](Self::initialized), but allows epochs beyond `current_epoch + 1`. + /// The only bound enforced is that the required randao seed is available in the state. + /// + /// This is used by PTC window computation, which needs committee shufflings for + /// `current_epoch + 1 + MIN_SEED_LOOKAHEAD`. + pub fn initialized_for_lookahead( + state: &BeaconState, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result, BeaconStateError> { + let reqd_randao_epoch = epoch + .saturating_sub(spec.min_seed_lookahead) + .saturating_sub(1u64); + + if reqd_randao_epoch < state.min_randao_epoch() { + return Err(BeaconStateError::EpochOutOfBounds); + } + + Self::initialized_unchecked(state, epoch, spec) + } + + /// Core committee cache construction. Callers are responsible for bounds-checking `epoch`. + fn initialized_unchecked( + state: &BeaconState, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result, BeaconStateError> { // May cause divide-by-zero errors. if E::slots_per_epoch() == 0 { return Err(BeaconStateError::ZeroSlotsPerEpoch); diff --git a/deny.toml b/deny.toml index cf0cd7d3cd..015f2ec88b 100644 --- a/deny.toml +++ b/deny.toml @@ -12,6 +12,7 @@ deny = [ { crate = "ark-ff", reason = "present in Cargo.lock but not needed by Lighthouse" }, { crate = "openssl", reason = "non-Rust dependency, use rustls instead" }, { crate = "c-kzg", reason = "non-Rust dependency, use rust_eth_kzg instead" }, + { crate = "serde_yaml", reason = "deprecated, use yaml_serde instead" }, { crate = "strum", deny-multiple-versions = true, reason = "takes a long time to compile" }, { crate = "reqwest", deny-multiple-versions = true, reason = "takes a long time to compile" }, { crate = "aes", deny-multiple-versions = true, reason = "takes a long time to compile" }, diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index 43e361b60d..84525c05b9 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -35,7 +35,6 @@ network_utils = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -serde_yaml = { workspace = true } snap = { workspace = true } state_processing = { workspace = true } store = { workspace = true } @@ -44,6 +43,7 @@ tracing-subscriber = { workspace = true } tree_hash = { workspace = true } types = { workspace = true } validator_dir = { workspace = true } +yaml_serde = { workspace = true } [target.'cfg(not(target_os = "windows"))'.dependencies] malloc_utils = { workspace = true, features = ["jemalloc"] } diff --git a/lcli/src/parse_ssz.rs b/lcli/src/parse_ssz.rs index f1e5c5759a..cd739b2a9e 100644 --- a/lcli/src/parse_ssz.rs +++ b/lcli/src/parse_ssz.rs @@ -141,7 +141,7 @@ fn decode_and_print( OutputFormat::Yaml => { println!( "{}", - serde_yaml::to_string(&item) + yaml_serde::to_string(&item) .map_err(|e| format!("Unable to write object to YAML: {e:?}"))? ); } diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 000c6fd0da..3595cf04e7 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -62,7 +62,6 @@ opentelemetry-otlp = { workspace = true } opentelemetry_sdk = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -serde_yaml = { workspace = true } slasher = { workspace = true } store = { workspace = true } task_executor = { workspace = true } @@ -73,6 +72,7 @@ tracing_samplers = { workspace = true } types = { workspace = true } validator_client = { workspace = true } validator_manager = { path = "../validator_manager" } +yaml_serde = { workspace = true } [target.'cfg(not(target_os = "windows"))'.dependencies] malloc_utils = { workspace = true, features = ["jemalloc"] } diff --git a/lighthouse/tests/exec.rs b/lighthouse/tests/exec.rs index 5379912c13..a25558bc2f 100644 --- a/lighthouse/tests/exec.rs +++ b/lighthouse/tests/exec.rs @@ -65,7 +65,7 @@ pub trait CommandLineTestExec { let spec_file = File::open(tmp_chain_config_path).expect("Unable to open dumped chain spec"); let chain_config: Config = - serde_yaml::from_reader(spec_file).expect("Unable to deserialize config"); + yaml_serde::from_reader(spec_file).expect("Unable to deserialize config"); CompletedTest::new(config, chain_config, tmp_dir) } @@ -102,7 +102,7 @@ pub trait CommandLineTestExec { let spec_file = File::open(tmp_chain_config_path).expect("Unable to open dumped chain spec"); let chain_config: Config = - serde_yaml::from_reader(spec_file).expect("Unable to deserialize config"); + yaml_serde::from_reader(spec_file).expect("Unable to deserialize config"); CompletedTest::new(config, chain_config, tmp_dir) } diff --git a/scripts/local_testnet/network_params.yaml b/scripts/local_testnet/network_params.yaml index 0c36e5c49c..083f719c60 100644 --- a/scripts/local_testnet/network_params.yaml +++ b/scripts/local_testnet/network_params.yaml @@ -21,10 +21,13 @@ network_params: slot_duration_ms: 3000 snooper_enabled: false global_log_level: debug +tempo_params: + image: grafana/tempo:2.10.3 additional_services: - dora - spamoor - - prometheus_grafana + - prometheus + - grafana - tempo spamoor_params: image: ethpandaops/spamoor:master diff --git a/scripts/range-sync-coverage.sh b/scripts/range-sync-coverage.sh new file mode 100755 index 0000000000..df438c0c7f --- /dev/null +++ b/scripts/range-sync-coverage.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# Aggregate range sync test coverage across all forks +# Usage: ./scripts/range-sync-coverage.sh [--html] +set -e + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +TARGET_DIR="${CARGO_TARGET_DIR:-/mnt/ssd/builds/lighthouse-range-sync-tests}" +FORKS=(base altair bellatrix capella deneb electra fulu) +LCOV_DIR="/tmp/range-cov-forks" +MERGED="/tmp/range-cov-merged.lcov" + +rm -rf "$LCOV_DIR" +mkdir -p "$LCOV_DIR" + +echo "=== Running coverage for each fork ===" +for fork in "${FORKS[@]}"; do + echo "--- $fork ---" + CARGO_TARGET_DIR="$TARGET_DIR" FORK_NAME="$fork" \ + cargo llvm-cov --features "network/fake_crypto,network/fork_from_env" \ + -p network --lib --lcov --output-path "$LCOV_DIR/$fork.lcov" \ + -- "sync::tests::range" 2>&1 | grep -E "test result|running" +done + +echo "" +echo "=== Merging lcov files ===" + +# Merge all lcov files: for each source file, take max hit count per line +python3 - "$LCOV_DIR" "$MERGED" << 'PYEOF' +import sys, os, glob +from collections import defaultdict + +lcov_dir = sys.argv[1] +output = sys.argv[2] + +# Parse all lcov files: file -> line -> max hits +coverage = defaultdict(lambda: defaultdict(int)) +fn_coverage = defaultdict(lambda: defaultdict(int)) +current_sf = None + +for lcov_file in sorted(glob.glob(os.path.join(lcov_dir, "*.lcov"))): + with open(lcov_file) as f: + for line in f: + line = line.strip() + if line.startswith("SF:"): + current_sf = line[3:] + elif line.startswith("DA:") and current_sf: + parts = line[3:].split(",") + lineno = int(parts[0]) + hits = int(parts[1]) + coverage[current_sf][lineno] = max(coverage[current_sf][lineno], hits) + elif line.startswith("FNDA:") and current_sf: + parts = line[5:].split(",", 1) + hits = int(parts[0]) + fn_name = parts[1] + fn_coverage[current_sf][fn_name] = max(fn_coverage[current_sf][fn_name], hits) + +# Write merged lcov +with open(output, "w") as f: + for sf in sorted(coverage.keys()): + f.write(f"SF:{sf}\n") + for fn_name, hits in sorted(fn_coverage.get(sf, {}).items()): + f.write(f"FNDA:{hits},{fn_name}\n") + for lineno in sorted(coverage[sf].keys()): + f.write(f"DA:{lineno},{coverage[sf][lineno]}\n") + total = len(coverage[sf]) + covered = sum(1 for h in coverage[sf].values() if h > 0) + f.write(f"LH:{covered}\n") + f.write(f"LF:{total}\n") + f.write("end_of_record\n") + +print(f"Merged {len(glob.glob(os.path.join(lcov_dir, '*.lcov')))} lcov files -> {output}") +PYEOF + +echo "" +echo "=== Range sync coverage (merged across all forks) ===" + +# Extract and display range sync files +python3 - "$MERGED" << 'PYEOF' +import sys +from collections import defaultdict + +current_sf = None +files = {} # short_name -> (total_lines, covered_lines) +lines = defaultdict(dict) + +with open(sys.argv[1]) as f: + for line in f: + line = line.strip() + if line.startswith("SF:"): + current_sf = line[3:] + elif line.startswith("DA:") and current_sf: + parts = line[3:].split(",") + lineno, hits = int(parts[0]), int(parts[1]) + lines[current_sf][lineno] = hits + +# Filter to range sync files +targets = [ + "range_sync/chain.rs", + "range_sync/chain_collection.rs", + "range_sync/range.rs", + "requests/blocks_by_range.rs", + "requests/blobs_by_range.rs", + "requests/data_columns_by_range.rs", +] + +print(f"{'File':<45} {'Lines':>6} {'Covered':>8} {'Missed':>7} {'Coverage':>9}") +print("-" * 80) + +total_all = 0 +covered_all = 0 + +for sf in sorted(lines.keys()): + short = sf.split("sync/")[-1] if "sync/" in sf else sf.split("/")[-1] + if not any(t in sf for t in targets): + continue + total = len(lines[sf]) + covered = sum(1 for h in lines[sf].values() if h > 0) + missed = total - covered + pct = covered / total * 100 if total > 0 else 0 + total_all += total + covered_all += covered + print(f"{short:<45} {total:>6} {covered:>8} {missed:>7} {pct:>8.1f}%") + +print("-" * 80) +pct_all = covered_all / total_all * 100 if total_all > 0 else 0 +print(f"{'TOTAL':<45} {total_all:>6} {covered_all:>8} {total_all - covered_all:>7} {pct_all:>8.1f}%") +PYEOF + +if [ "$1" = "--html" ]; then + echo "" + echo "=== Generating HTML report ===" + genhtml "$MERGED" -o /tmp/range-cov-html --ignore-errors source 2>/dev/null + echo "HTML report: /tmp/range-cov-html/index.html" +fi diff --git a/testing/ef_tests/Cargo.toml b/testing/ef_tests/Cargo.toml index cef201ee91..9d09c3dfe6 100644 --- a/testing/ef_tests/Cargo.toml +++ b/testing/ef_tests/Cargo.toml @@ -32,7 +32,6 @@ rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_repr = { workspace = true } -serde_yaml = { workspace = true } snap = { workspace = true } ssz_types = { workspace = true } state_processing = { workspace = true } @@ -41,3 +40,4 @@ tree_hash = { workspace = true } tree_hash_derive = { workspace = true } typenum = { workspace = true } types = { workspace = true } +yaml_serde = { workspace = true } diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index 48378a4c95..ab24ea35a0 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,6 +1,6 @@ # To download/extract nightly tests, run: # CONSENSUS_SPECS_TEST_VERSION=nightly make -CONSENSUS_SPECS_TEST_VERSION ?= v1.7.0-alpha.3 +CONSENSUS_SPECS_TEST_VERSION ?= v1.7.0-alpha.4 REPO_NAME := consensus-spec-tests OUTPUT_DIR := ./$(REPO_NAME) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index dd6be14306..2daafada31 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -53,6 +53,8 @@ excluded_paths = [ "tests/.*/gloas/fork_choice/.*", # Ignore MatrixEntry SSZ tests for now. "tests/.*/.*/ssz_static/MatrixEntry/.*", + # TODO: partial data column not implemented yet + "tests/.*/.*/ssz_static/PartialDataColumn.*/.*", # TODO(gloas): Ignore Gloas light client stuff for now "tests/.*/gloas/ssz_static/LightClient.*/.*", # Execution payload header is irrelevant after Gloas, this type will probably be deleted. @@ -73,7 +75,9 @@ excluded_paths = [ "tests/.*/compute_verify_cell_kzg_proof_batch_challenge/.*", "tests/.*/compute_challenge/.*", # We don't need these manifest files at the moment. - "tests/.*/manifest.yaml" + "tests/.*/manifest.yaml", + # TODO: gossip condition tests not implemented yet + "tests/.*/.*/networking/.*" ] diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index 7a90fc70d0..a032aa917f 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -12,7 +12,7 @@ use state_processing::per_epoch_processing::effective_balance_updates::{ process_effective_balance_updates, process_effective_balance_updates_slow, }; use state_processing::per_epoch_processing::single_pass::{ - SinglePassConfig, process_epoch_single_pass, process_proposer_lookahead, + SinglePassConfig, process_epoch_single_pass, process_proposer_lookahead, process_ptc_window, }; use state_processing::per_epoch_processing::{ altair, base, @@ -80,6 +80,8 @@ pub struct ParticipationFlagUpdates; #[derive(Debug)] pub struct ProposerLookahead; #[derive(Debug)] +pub struct PtcWindow; +#[derive(Debug)] pub struct BuilderPendingPayments; type_name!( @@ -102,6 +104,7 @@ type_name!(SyncCommitteeUpdates, "sync_committee_updates"); type_name!(InactivityUpdates, "inactivity_updates"); type_name!(ParticipationFlagUpdates, "participation_flag_updates"); type_name!(ProposerLookahead, "proposer_lookahead"); +type_name!(PtcWindow, "ptc_window"); type_name!(BuilderPendingPayments, "builder_pending_payments"); impl EpochTransition for JustificationAndFinalization { @@ -296,6 +299,16 @@ impl EpochTransition for ProposerLookahead { } } +impl EpochTransition for PtcWindow { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + if state.fork_name_unchecked().gloas_enabled() { + process_ptc_window(state, spec).map(|_| ()) + } else { + Ok(()) + } + } +} + impl EpochTransition for BuilderPendingPayments { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { process_epoch_single_pass( @@ -373,7 +386,9 @@ impl> Case for EpochProcessing { return false; } - if !fork_name.gloas_enabled() && T::name() == "builder_pending_payments" { + if !fork_name.gloas_enabled() + && (T::name() == "builder_pending_payments" || T::name() == "ptc_window") + { return false; } diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 798c66b666..1399815763 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -717,11 +717,7 @@ impl> LoadCase for Operations { // Check BLS setting here before SSZ deserialization, as most types require signatures // to be valid. let operation_path = path.join(O::filename()); - let (operation, bls_error) = if !operation_path.is_file() { - // Some test cases (e.g. builder_voluntary_exit__success) have no operation file. - // TODO(gloas): remove this once the test vectors are fixed - (None, None) - } else if metadata.bls_setting.unwrap_or_default().check().is_ok() { + let (operation, bls_error) = if metadata.bls_setting.unwrap_or_default().check().is_ok() { match O::decode(&operation_path, fork_name, spec) { Ok(op) => (Some(op), None), Err(Error::InvalidBLSInput(error)) => (None, Some(error)), diff --git a/testing/ef_tests/src/decode.rs b/testing/ef_tests/src/decode.rs index 2074ffce23..f4aa17fb08 100644 --- a/testing/ef_tests/src/decode.rs +++ b/testing/ef_tests/src/decode.rs @@ -33,14 +33,14 @@ pub fn log_file_access>(file_accessed: P) { } pub fn yaml_decode(string: &str) -> Result { - serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) + yaml_serde::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) } pub fn context_yaml_decode<'de, T, C>(string: &'de str, context: C) -> Result where T: ContextDeserialize<'de, C>, { - let deserializer = serde_yaml::Deserializer::from_str(string); + let deserializer = yaml_serde::Deserializer::from_str(string); T::context_deserialize(deserializer, context) .map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) } diff --git a/testing/ef_tests/src/lib.rs b/testing/ef_tests/src/lib.rs index 94b19b6644..5587bbed41 100644 --- a/testing/ef_tests/src/lib.rs +++ b/testing/ef_tests/src/lib.rs @@ -3,7 +3,7 @@ pub use cases::{ BuilderPendingPayments, Case, EffectiveBalanceUpdates, Eth1DataReset, ExecutionPayloadBidBlock, FeatureName, HistoricalRootsUpdate, HistoricalSummariesUpdate, InactivityUpdates, JustificationAndFinalization, ParticipationFlagUpdates, ParticipationRecordUpdates, - PendingBalanceDeposits, PendingConsolidations, ProposerLookahead, RandaoMixesReset, + PendingBalanceDeposits, PendingConsolidations, ProposerLookahead, PtcWindow, RandaoMixesReset, RegistryUpdates, RewardsAndPenalties, Slashings, SlashingsReset, SyncCommitteeUpdates, WithdrawalsPayload, }; diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 3893df2ef7..3254bb6e90 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -960,6 +960,12 @@ fn epoch_processing_proposer_lookahead() { EpochProcessingHandler::::default().run(); } +#[test] +fn epoch_processing_ptc_window() { + EpochProcessingHandler::::default().run(); + EpochProcessingHandler::::default().run(); +} + #[test] fn epoch_processing_builder_pending_payments() { EpochProcessingHandler::::default().run(); diff --git a/testing/web3signer_tests/Cargo.toml b/testing/web3signer_tests/Cargo.toml index 3ef2e0f7f7..1cac45fe52 100644 --- a/testing/web3signer_tests/Cargo.toml +++ b/testing/web3signer_tests/Cargo.toml @@ -23,7 +23,6 @@ parking_lot = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -serde_yaml = { workspace = true } slashing_protection = { workspace = true } slot_clock = { workspace = true } ssz_types = { workspace = true } @@ -33,4 +32,5 @@ tokio = { workspace = true } types = { workspace = true } url = { workspace = true } validator_store = { workspace = true } +yaml_serde = { workspace = true } zip = { workspace = true } diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs index 1f36f8d4ce..1e1e83d339 100644 --- a/testing/web3signer_tests/src/lib.rs +++ b/testing/web3signer_tests/src/lib.rs @@ -210,7 +210,7 @@ mod tests { }; let key_config_file = File::create(keystore_dir.path().join("key-config.yaml")).unwrap(); - serde_yaml::to_writer(key_config_file, &key_config).unwrap(); + yaml_serde::to_writer(key_config_file, &key_config).unwrap(); let tls_keystore_file = tls_dir().join("web3signer").join("key.p12"); let tls_keystore_password_file = tls_dir().join("web3signer").join("password.txt");