From d7d56e6312e3e4d2fe6e3988853dd08f21c6a888 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:57:03 +0200 Subject: [PATCH 01/15] Delete unnecessary SyncMessage variants (#9379) - Simplification from https://github.com/sigp/lighthouse/pull/9155 Lookup sync does not cache sidecars, so sending the full network object adds unnecessary complexity. Sync only needs to know: We have received a header that has an unknown parent. Replace `UnknownParentDataColumn` and `UnknownParentPartialDataColumn` for `UnknownParentSidecarHeader` Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> Co-Authored-By: Eitan Seri-Levi --- .../gossip_methods.rs | 12 +++-- .../network/src/sync/block_lookups/mod.rs | 13 ++--- .../sync/block_lookups/single_block_lookup.rs | 2 +- beacon_node/network/src/sync/manager.rs | 53 +++---------------- beacon_node/network/src/sync/tests/lookups.rs | 13 ++++- 5 files changed, 31 insertions(+), 62 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 65c95eff35..df94b473a8 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -719,17 +719,19 @@ impl NetworkBeaconProcessor { MessageAcceptance::Accept, ); } - GossipDataColumnError::ParentUnknown { parent_root, .. } => { + GossipDataColumnError::ParentUnknown { parent_root, slot } => { debug!( action = "requesting parent", %block_root, %parent_root, "Unknown parent hash for column" ); - self.send_sync_message(SyncMessage::UnknownParentDataColumn( + self.send_sync_message(SyncMessage::UnknownParentSidecarHeader { peer_id, - column_sidecar, - )); + block_root, + parent_root, + slot, + }); } GossipDataColumnError::BlockRootUnknown { block_root: unknown_block_root, @@ -1047,7 +1049,7 @@ impl NetworkBeaconProcessor { %parent_root, "Unknown parent hash for partial column" ); - self.send_sync_message(SyncMessage::UnknownParentPartialDataColumn { + self.send_sync_message(SyncMessage::UnknownParentSidecarHeader { peer_id, block_root, parent_root, diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index ecaee7c0ec..0c3cb988a9 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -74,26 +74,23 @@ const LOOKUP_MAX_DURATION_NO_PEERS_SECS: u64 = 10; /// take at most 2 GB. 200 lookups allow 3 parallel chains of depth 64 (current maximum). const MAX_LOOKUPS: usize = 200; -/// The values for `Blob`, `DataColumn` and `PartialDataColumn` is the parent root of the column. +/// The value for `Sidecar` is the parent root of the sidecar. pub enum BlockComponent { Block(DownloadResult>>), - DataColumn(DownloadResult), - PartialDataColumn(DownloadResult), + Sidecar { parent_root: Hash256 }, } impl BlockComponent { fn parent_root(&self) -> Hash256 { match self { BlockComponent::Block(block) => block.value.parent_root(), - BlockComponent::DataColumn(parent_root) - | BlockComponent::PartialDataColumn(parent_root) => parent_root.value, + BlockComponent::Sidecar { parent_root } => *parent_root, } } fn get_type(&self) -> &'static str { match self { BlockComponent::Block(_) => "block", - BlockComponent::DataColumn(_) => "data_column", - BlockComponent::PartialDataColumn(_) => "partial_data_column", + BlockComponent::Sidecar { .. } => "sidecar", } } } @@ -207,7 +204,7 @@ impl BlockLookups { block_root, Some(block_component), Some(parent_root), - // On a `UnknownParentBlock` or `UnknownParentDataColumn` event the peer is not + // On a `UnknownParentBlock` or `UnknownParentSidecarHeader` event the peer is not // required to have the rest of the block components. Create the lookup with zero // peers to house the block components. &[], diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index dda58023be..38d6b2216d 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -151,7 +151,7 @@ impl SingleBlockLookup { .block_request_state .state .insert_verified_response(block), - BlockComponent::DataColumn(_) | BlockComponent::PartialDataColumn(_) => { + BlockComponent::Sidecar { .. } => { // For now ignore single blobs and columns, as the blob request state assumes all blobs are // attributed to the same peer = the peer serving the remaining blobs. Ignoring this // block component has a minor effect, causing the node to re-request this blob diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index ecbe6227cc..37d13c2eae 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -144,11 +144,9 @@ pub enum SyncMessage { /// A block with an unknown parent has been received. UnknownParentBlock(PeerId, Arc>, Hash256), - /// A data column with an unknown parent has been received. - UnknownParentDataColumn(PeerId, Arc>), - - /// A partial data column with an unknown parent has been received. - UnknownParentPartialDataColumn { + /// A sidecar (full/partial data column) with an unknown parent has been received. Carries only the header + /// info needed to trigger a parent lookup, decoupled from the concrete sidecar type. + UnknownParentSidecarHeader { peer_id: PeerId, block_root: Hash256, parent_root: Hash256, @@ -875,58 +873,19 @@ impl SyncManager { }), ); } - SyncMessage::UnknownParentDataColumn(peer_id, data_column) => { - let data_column_slot = data_column.slot(); - let block_root = data_column.block_root(); - match data_column.as_ref() { - DataColumnSidecar::Fulu(column) => { - let parent_root = column.block_parent_root(); - debug!(%block_root, %parent_root, "Received unknown parent data column message"); - self.handle_unknown_parent( - peer_id, - block_root, - parent_root, - data_column_slot, - BlockComponent::DataColumn(DownloadResult { - value: parent_root, - block_root, - seen_timestamp: self - .chain - .slot_clock - .now_duration() - .unwrap_or_default(), - peer_group: PeerGroup::from_single(peer_id), - }), - ); - } - DataColumnSidecar::Gloas(_) => { - // TODO(gloas): proper lookup sync for Gloas. Routing into - // `handle_unknown_block_root` here mixes column processing with the - // single-block-lookup path; the Gloas column-arrives-before-block - // case wants its own queue/wakeup. - debug!(%block_root, "Received unknown block data column message"); - self.handle_unknown_block_root(peer_id, block_root); - } - } - } - SyncMessage::UnknownParentPartialDataColumn { + SyncMessage::UnknownParentSidecarHeader { peer_id, block_root, parent_root, slot, } => { - debug!(%block_root, %parent_root, "Received unknown parent partial column message"); + debug!(%block_root, %parent_root, "Received unknown parent sidecar header message"); self.handle_unknown_parent( peer_id, block_root, parent_root, slot, - BlockComponent::PartialDataColumn(DownloadResult { - value: parent_root, - block_root, - seen_timestamp: self.chain.slot_clock.now_duration().unwrap_or_default(), - peer_group: PeerGroup::from_single(peer_id), - }), + BlockComponent::Sidecar { parent_root }, ); } SyncMessage::UnknownBlockHashFromAttestation(peer_id, block_root) => { diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index 1a60e4f243..3ec4d11da2 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -1365,7 +1365,18 @@ impl TestRig { peer_id: PeerId, data_column: Arc>, ) { - self.send_sync_message(SyncMessage::UnknownParentDataColumn(peer_id, data_column)); + let block_root = data_column.block_root(); + let slot = data_column.slot(); + let parent_root = match data_column.as_ref() { + DataColumnSidecar::Fulu(column) => column.block_parent_root(), + DataColumnSidecar::Gloas(_) => panic!("Gloas data column not supported in this test"), + }; + self.send_sync_message(SyncMessage::UnknownParentSidecarHeader { + peer_id, + block_root, + parent_root, + slot, + }); } fn trigger_unknown_block_from_attestation(&mut self, block_root: Hash256, peer_id: PeerId) { From c2ac519c69311527a23ea161b7bde5117755506e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 3 Jun 2026 09:05:31 +0100 Subject: [PATCH 02/15] Disable Mplex by default (#9365) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: João Oliveira --- beacon_node/lighthouse_network/src/config.rs | 4 ++ .../lighthouse_network/src/service/mod.rs | 10 +++-- .../lighthouse_network/src/service/utils.rs | 40 ++++++++++++------- beacon_node/src/cli.rs | 8 ++++ beacon_node/src/config.rs | 4 ++ book/src/help_bn.md | 3 ++ 6 files changed, 52 insertions(+), 17 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index db42d0cfa8..4d4d91a456 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -125,6 +125,9 @@ pub struct Config { /// Whether light client protocols should be enabled. pub enable_light_client_server: bool, + /// Whether to enable the deprecated mplex multiplexer alongside yamux. + pub enable_mplex: bool, + /// Configuration for the outbound rate limiter (requests made by this node). pub outbound_rate_limiter_config: Option, @@ -362,6 +365,7 @@ impl Default for Config { proposer_only: false, metrics_enabled: false, enable_light_client_server: true, + enable_mplex: false, outbound_rate_limiter_config: None, invalid_block_storage: None, inbound_rate_limiter_config: None, diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 41d937e324..f5e2442f86 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -466,9 +466,13 @@ impl Network { } }; - // Set up the transport - tcp/quic with noise and mplex - let transport = build_transport(local_keypair.clone(), !config.disable_quic_support) - .map_err(|e| format!("Failed to build transport: {:?}", e))?; + // Set up the transport - tcp/quic with noise and yamux (mplex optional) + let transport = build_transport( + local_keypair.clone(), + !config.disable_quic_support, + config.enable_mplex, + ) + .map_err(|e| format!("Failed to build transport: {:?}", e))?; // use the executor for libp2p struct Executor(task_executor::TaskExecutor); diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index c7dabcb391..47629f4fd3 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -34,27 +34,39 @@ pub struct Context<'a> { type BoxedTransport = Boxed<(PeerId, StreamMuxerBox)>; /// The implementation supports TCP/IP, QUIC (experimental) over UDP, noise as the encryption layer, and -/// mplex/yamux as the multiplexing layer (when using TCP). +/// yamux as the multiplexing layer (when using TCP). Mplex can be optionally enabled. pub fn build_transport( local_private_key: Keypair, quic_support: bool, + enable_mplex: bool, ) -> std::io::Result { - // mplex config - let mut mplex_config = libp2p_mplex::Config::new(); - mplex_config.set_max_buffer_size(256); - mplex_config.set_max_buffer_behaviour(libp2p_mplex::MaxBufferBehaviour::Block); - // yamux config let yamux_config = yamux::Config::default(); + // Creates the TCP transport layer - let tcp = libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)) - .upgrade(core::upgrade::Version::V1) - .authenticate(generate_noise_config(&local_private_key)) - .multiplex(core::upgrade::SelectUpgrade::new( - yamux_config, - mplex_config, - )) - .timeout(Duration::from_secs(10)); + let tcp: BoxedTransport = if enable_mplex { + // Enable both yamux and mplex. + let mut mplex_config = libp2p_mplex::Config::new(); + mplex_config.set_max_num_streams(32); + mplex_config.set_max_buffer_behaviour(libp2p_mplex::MaxBufferBehaviour::ResetStream); + libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)) + .upgrade(core::upgrade::Version::V1) + .authenticate(generate_noise_config(&local_private_key)) + .multiplex(core::upgrade::SelectUpgrade::new( + yamux_config, + mplex_config, + )) + .timeout(Duration::from_secs(10)) + .boxed() + } else { + // Yamux only + libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)) + .upgrade(core::upgrade::Version::V1) + .authenticate(generate_noise_config(&local_private_key)) + .multiplex(yamux_config) + .timeout(Duration::from_secs(10)) + .boxed() + }; let transport = if quic_support { // Enables Quic // The default quic configuration suits us for now. diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 647b5858cb..988e2d1fc5 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -387,6 +387,14 @@ pub fn cli_app() -> Command { .help("Disables the quic transport. The node will rely solely on the TCP transport for libp2p connections.") .display_order(0) ) + .arg( + Arg::new("enable-mplex") + .long("enable-mplex") + .action(ArgAction::SetTrue) + .help_heading(FLAG_HEADER) + .help("Enables mplex multiplexer alongside yamux. Yamux is preferred when both are available.") + .display_order(0) + ) .arg( Arg::new("disable-peer-scoring") .long("disable-peer-scoring") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 045b432dc9..ddf8d07c4e 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1443,6 +1443,10 @@ pub fn set_network_config( config.disable_quic_support = true; } + if parse_flag(cli_args, "enable-mplex") { + config.enable_mplex = true; + } + if parse_flag(cli_args, "disable-upnp") { config.upnp_enabled = false; } diff --git a/book/src/help_bn.md b/book/src/help_bn.md index 30163f1f0c..1f57db1b59 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -494,6 +494,9 @@ Flags: Sets the local ENR IP address and port to match those set for lighthouse. Specifically, the IP address will be the value of --listen-address and the UDP port will be --discovery-port. + --enable-mplex + Enables mplex multiplexer alongside yamux. Yamux is preferred when + both are available. --enable-partial-columns Enable partial messages for data columns. This can reduce the amount of data sent over the network. Enabled by default on Hoodi and From eab5163d68f2c1e88b43627956dcd496a321ab41 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:29:04 +0200 Subject: [PATCH 03/15] Remove RequestState trait from lookup sync (#9391) Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> --- .../network/src/sync/block_lookups/common.rs | 164 ------ .../network/src/sync/block_lookups/mod.rs | 275 ++-------- .../sync/block_lookups/single_block_lookup.rs | 510 ++++++++++-------- beacon_node/network/src/sync/manager.rs | 29 +- .../network/src/sync/network_context.rs | 29 +- .../src/sync/network_context/custody.rs | 12 +- 6 files changed, 369 insertions(+), 650 deletions(-) delete mode 100644 beacon_node/network/src/sync/block_lookups/common.rs diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs deleted file mode 100644 index 4306458615..0000000000 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ /dev/null @@ -1,164 +0,0 @@ -use crate::sync::block_lookups::single_block_lookup::{ - LookupRequestError, SingleBlockLookup, SingleLookupRequestState, -}; -use crate::sync::block_lookups::{BlockRequestState, CustodyRequestState, PeerId}; -use crate::sync::manager::BlockProcessType; -use crate::sync::network_context::{LookupRequestResult, SyncNetworkContext}; -use beacon_chain::BeaconChainTypes; -use lighthouse_network::service::api_types::Id; -use parking_lot::RwLock; -use std::collections::HashSet; -use std::sync::Arc; -use types::{DataColumnSidecarList, SignedBeaconBlock}; - -use super::SingleLookupId; -use super::single_block_lookup::{ComponentRequests, DownloadResult}; - -#[derive(Debug, Copy, Clone)] -pub enum ResponseType { - Block, - CustodyColumn, -} - -/// This trait unifies common single block lookup functionality across blocks and data columns. -/// This includes making requests, verifying responses, and handling processing results. A -/// `SingleBlockLookup` includes both a `BlockRequestState` and a `CustodyRequestState`, this trait -/// is implemented for each. -/// -/// The use of the `ResponseType` associated type gives us a degree of type -/// safety when handling a block/column response ensuring we only mutate the correct corresponding -/// state. -pub trait RequestState { - /// The type created after validation. - type VerifiedResponseType: Clone; - - /// Request the network context to prepare a request of a component of `block_root`. If the - /// request is not necessary because the component is already known / processed, return false. - /// Return true if it sent a request and we can expect an event back from the network. - fn make_request( - &self, - id: Id, - lookup_peers: Arc>>, - expected_blobs: usize, - cx: &mut SyncNetworkContext, - ) -> Result; - - /* Response handling methods */ - - /// Send the response to the beacon processor. - fn send_for_processing( - id: Id, - result: DownloadResult, - cx: &SyncNetworkContext, - ) -> Result<(), LookupRequestError>; - - /* Utility methods */ - - /// Returns the `ResponseType` associated with this trait implementation. Useful in logging. - fn response_type() -> ResponseType; - - /// A getter for the `BlockRequestState` or `CustodyRequestState` associated with this trait. - fn request_state_mut(request: &mut SingleBlockLookup) -> Result<&mut Self, &'static str>; - - /// A getter for a reference to the `SingleLookupRequestState` associated with this trait. - fn get_state(&self) -> &SingleLookupRequestState; - - /// A getter for a mutable reference to the SingleLookupRequestState associated with this trait. - fn get_state_mut(&mut self) -> &mut SingleLookupRequestState; -} - -impl RequestState for BlockRequestState { - type VerifiedResponseType = Arc>; - - fn make_request( - &self, - id: SingleLookupId, - lookup_peers: Arc>>, - _: usize, - cx: &mut SyncNetworkContext, - ) -> Result { - cx.block_lookup_request(id, lookup_peers, self.requested_block_root) - .map_err(LookupRequestError::SendFailedNetwork) - } - - fn send_for_processing( - id: SingleLookupId, - download_result: DownloadResult, - cx: &SyncNetworkContext, - ) -> Result<(), LookupRequestError> { - let DownloadResult { - value, - block_root, - seen_timestamp, - .. - } = download_result; - cx.send_block_for_processing(id, block_root, value, seen_timestamp) - .map_err(LookupRequestError::SendFailedProcessor) - } - - fn response_type() -> ResponseType { - ResponseType::Block - } - fn request_state_mut(request: &mut SingleBlockLookup) -> Result<&mut Self, &'static str> { - Ok(&mut request.block_request_state) - } - fn get_state(&self) -> &SingleLookupRequestState { - &self.state - } - fn get_state_mut(&mut self) -> &mut SingleLookupRequestState { - &mut self.state - } -} - -impl RequestState for CustodyRequestState { - type VerifiedResponseType = DataColumnSidecarList; - - fn make_request( - &self, - id: Id, - lookup_peers: Arc>>, - _: usize, - cx: &mut SyncNetworkContext, - ) -> Result { - cx.custody_lookup_request(id, self.block_root, self.slot, lookup_peers) - .map_err(LookupRequestError::SendFailedNetwork) - } - - fn send_for_processing( - id: Id, - download_result: DownloadResult, - cx: &SyncNetworkContext, - ) -> Result<(), LookupRequestError> { - let DownloadResult { - value, - block_root, - seen_timestamp, - .. - } = download_result; - cx.send_custody_columns_for_processing( - id, - block_root, - value, - seen_timestamp, - BlockProcessType::SingleCustodyColumn(id), - ) - .map_err(LookupRequestError::SendFailedProcessor) - } - - fn response_type() -> ResponseType { - ResponseType::CustodyColumn - } - fn request_state_mut(request: &mut SingleBlockLookup) -> Result<&mut Self, &'static str> { - match &mut request.component_requests { - ComponentRequests::WaitingForBlock => Err("waiting for block"), - ComponentRequests::ActiveCustodyRequest(request) => Ok(request), - ComponentRequests::NotNeeded { .. } => Err("not needed"), - } - } - fn get_state(&self) -> &SingleLookupRequestState { - &self.state - } - fn get_state_mut(&mut self) -> &mut SingleLookupRequestState { - &mut self.state - } -} diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 0c3cb988a9..a265373e3f 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -24,27 +24,23 @@ use self::parent_chain::{NodeChain, compute_parent_chains}; pub use self::single_block_lookup::DownloadResult; use self::single_block_lookup::{LookupRequestError, LookupResult, SingleBlockLookup}; use super::manager::{BlockProcessType, SLOT_IMPORT_TOLERANCE}; -use super::network_context::{PeerGroup, RpcResponseError, SyncNetworkContext}; +use super::network_context::{RpcResponseError, SyncNetworkContext}; use crate::metrics; use crate::network_beacon_processor::BlockProcessingResult; use crate::sync::SyncMessage; use crate::sync::block_lookups::parent_chain::find_oldest_fork_ancestor; use beacon_chain::BeaconChainTypes; -use beacon_chain::block_verification_types::AsBlock; -pub use common::RequestState; use fnv::FnvHashMap; use lighthouse_network::PeerId; use lighthouse_network::service::api_types::SingleLookupReqId; use lru_cache::LRUTimeCache; -pub use single_block_lookup::{BlockRequestState, CustodyRequestState}; use std::collections::hash_map::Entry; use std::sync::Arc; use std::time::Duration; use store::Hash256; use tracing::{debug, error, warn}; -use types::{EthSpec, SignedBeaconBlock}; +use types::{DataColumnSidecarList, EthSpec, SignedBeaconBlock}; -pub mod common; pub mod parent_chain; mod single_block_lookup; @@ -74,35 +70,17 @@ const LOOKUP_MAX_DURATION_NO_PEERS_SECS: u64 = 10; /// take at most 2 GB. 200 lookups allow 3 parallel chains of depth 64 (current maximum). const MAX_LOOKUPS: usize = 200; -/// The value for `Sidecar` is the parent root of the sidecar. +type BlockDownloadResponse = Result>>, RpcResponseError>; +type CustodyDownloadResponse = + Result>, RpcResponseError>; + pub enum BlockComponent { Block(DownloadResult>>), - Sidecar { parent_root: Hash256 }, -} - -impl BlockComponent { - fn parent_root(&self) -> Hash256 { - match self { - BlockComponent::Block(block) => block.value.parent_root(), - BlockComponent::Sidecar { parent_root } => *parent_root, - } - } - fn get_type(&self) -> &'static str { - match self { - BlockComponent::Block(_) => "block", - BlockComponent::Sidecar { .. } => "sidecar", - } - } + Sidecar, } pub type SingleLookupId = u32; -enum Action { - Retry, - ParentUnknown { parent_root: Hash256 }, - Continue, -} - pub struct BlockLookups { /// A cache of block roots that must be ignored for some time to prevent useless searches. For /// example if a chain is too long, its lookup chain is dropped, and range sync is expected to @@ -190,11 +168,10 @@ impl BlockLookups { &mut self, block_root: Hash256, block_component: BlockComponent, + parent_root: Hash256, peer_id: PeerId, cx: &mut SyncNetworkContext, ) -> bool { - let parent_root = block_component.parent_root(); - let parent_lookup_exists = self.search_parent_of_child(parent_root, block_root, &[peer_id], cx); // Only create the child lookup if the parent exists @@ -215,7 +192,7 @@ impl BlockLookups { } } - /// Seach a block whose parent root is unknown. + /// Search a block whose parent root is unknown. /// /// Returns true if the lookup is created or already exists #[must_use = "only reference the new lookup if returns true"] @@ -358,13 +335,9 @@ impl BlockLookups { .find(|(_id, lookup)| lookup.is_for_block(block_root)) { if let Some(block_component) = block_component { - let component_type = block_component.get_type(); let imported = lookup.add_child_components(block_component); if !imported { - debug!( - ?block_root, - component_type, "Lookup child component ignored" - ); + debug!(?block_root, "Lookup child component ignored"); } } @@ -436,88 +409,33 @@ impl BlockLookups { /* Lookup responses */ - /// Process a block or blob response received from a single lookup request. - pub fn on_download_response>( + /// Process a block response received from a single lookup request. + pub fn on_block_download_response( &mut self, id: SingleLookupReqId, - response: Result<(R::VerifiedResponseType, PeerGroup, Duration), RpcResponseError>, + response: BlockDownloadResponse, cx: &mut SyncNetworkContext, ) { - let result = self.on_download_response_inner::(id, response, cx); - self.on_lookup_result(id.lookup_id, result, "download_response", cx); + let Some(lookup) = self.single_block_lookups.get_mut(&id.lookup_id) else { + debug!(?id, "Block returned for single block lookup not present"); + return; + }; + let result = lookup.on_block_download_response(id.req_id, response, cx); + self.on_lookup_result(id.lookup_id, result, "block_download_response", cx); } - /// Process a block or blob response received from a single lookup request. - pub fn on_download_response_inner>( + pub fn on_custody_download_response( &mut self, id: SingleLookupReqId, - response: Result<(R::VerifiedResponseType, PeerGroup, Duration), RpcResponseError>, + response: CustodyDownloadResponse, cx: &mut SyncNetworkContext, - ) -> Result { - // Note: no need to downscore peers here, already downscored on network context - - let response_type = R::response_type(); + ) { let Some(lookup) = self.single_block_lookups.get_mut(&id.lookup_id) else { - // We don't have the ability to cancel in-flight RPC requests. So this can happen - // if we started this RPC request, and later saw the block/blobs via gossip. - debug!(?id, "Block returned for single block lookup not present"); - return Err(LookupRequestError::UnknownLookup); + debug!(?id, "Custody returned for single block lookup not present"); + return; }; - - let block_root = lookup.block_root(); - let request_state = R::request_state_mut(lookup) - .map_err(|e| LookupRequestError::BadState(e.to_owned()))? - .get_state_mut(); - - match response { - Ok((response, peer_group, seen_timestamp)) => { - debug!( - ?block_root, - ?id, - ?peer_group, - ?response_type, - "Received lookup download success" - ); - - // Here we could check if response extends a parent chain beyond its max length. - // However we defer that check to the handling of a processing error ParentUnknown. - // - // Here we could check if there's already a lookup for parent_root of `response`. In - // that case we know that sending the response for processing will likely result in - // a `ParentUnknown` error. However, for simplicity we choose to not implement this - // optimization. - - // Register the download peer here. Once we have received some data over the wire we - // attribute it to this peer for scoring latter regardless of how the request was - // done. - request_state.on_download_success( - id.req_id, - DownloadResult { - value: response, - block_root, - seen_timestamp, - peer_group, - }, - )?; - // continue_request will send for processing as the request state is AwaitingProcessing - } - Err(e) => { - // No need to log peer source here. When sending a DataColumnsByRoot request we log - // the peer and the request ID which is linked to this `id` value here. - debug!( - ?block_root, - ?id, - ?response_type, - error = ?e, - "Received lookup download failure" - ); - - request_state.on_download_failure(id.req_id)?; - // continue_request will retry a download as the request state is AwaitingDownload - } - } - - lookup.continue_requests(cx) + let result = lookup.on_custody_download_response(id.req_id, response, cx); + self.on_lookup_result(id.lookup_id, result, "custody_download_response", cx); } /* Error responses */ @@ -539,128 +457,29 @@ impl BlockLookups { result: BlockProcessingResult, cx: &mut SyncNetworkContext, ) { - let lookup_result = match process_type { - BlockProcessType::SingleBlock { id } => { - self.on_processing_result_inner::>(id, result, cx) - } - BlockProcessType::SingleCustodyColumn(id) => { - self.on_processing_result_inner::>(id, result, cx) - } - // TODO(gloas): route into the payload envelope lookup state machine. - BlockProcessType::SinglePayloadEnvelope(_) => Ok(LookupResult::Pending), - }; - self.on_lookup_result(process_type.id(), lookup_result, "processing_result", cx); - } - - pub fn on_processing_result_inner>( - &mut self, - lookup_id: SingleLookupId, - result: BlockProcessingResult, - cx: &mut SyncNetworkContext, - ) -> Result { + let lookup_id = process_type.id(); let Some(lookup) = self.single_block_lookups.get_mut(&lookup_id) else { debug!(id = lookup_id, "Unknown single block lookup"); - return Err(LookupRequestError::UnknownLookup); + return; }; - let block_root = lookup.block_root(); - let request_state = R::request_state_mut(lookup) - .map_err(|e| LookupRequestError::BadState(e.to_owned()))? - .get_state_mut(); - debug!( - component = ?R::response_type(), - ?block_root, + block_root = ?lookup.block_root(), id = lookup_id, + ?process_type, ?result, "Received lookup processing result" ); - let action = match result { - BlockProcessingResult::Imported(fully_imported, _info) => { - // `on_processing_success` is called here to ensure the request state is updated - // prior to checking if all components have been processed (relevant for - // MissingComponents). - request_state.on_processing_success()?; - - if fully_imported { - Action::Continue - } else if lookup.all_components_processed() { - // We don't request for other block components until being sure that the block has - // data. If we request blobs / columns to a peer we are sure those must exist. - // Therefore if all components are processed and we still receive `MissingComponents` - // it indicates an internal bug. - return Err(LookupRequestError::Failed( - "missing components after all processed".to_owned(), - )); - } else { - Action::Retry - } - } - BlockProcessingResult::ParentUnknown { parent_root } => { - // `BlockError::ParentUnknown` is only returned when processing blocks. Reverts - // the status of this request to `AwaitingProcessing` holding the downloaded - // data. A future call to `continue_requests` will re-submit it once there are - // no pending parent requests. - request_state.revert_to_awaiting_processing()?; - Action::ParentUnknown { parent_root } - } - BlockProcessingResult::Error { penalty, reason } => { - // Retry on every processing error: `on_processing_failure` increments the - // per-component failure counter, so `SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS` bounds the - // retry loop and eventually drops the lookup if the failure persists. Whether the - // peer should be downscored is the producer's call (encoded in `penalty`). - debug!( - ?block_root, - component = ?R::response_type(), - reason, - ?penalty, - "Lookup component processing failed; retrying" - ); - let peer_group = request_state.on_processing_failure()?; - if let Some((action_kind, whom, msg)) = penalty { - whom.apply(action_kind, &peer_group, msg, cx); - } - Action::Retry + let lookup_result = match process_type { + BlockProcessType::SingleBlock { .. } => lookup.on_block_processing_result(result, cx), + BlockProcessType::SingleCustodyColumn(_) => { + lookup.on_data_processing_result(result, cx) } + // TODO(gloas): route into the payload envelope lookup state machine. + BlockProcessType::SinglePayloadEnvelope(_) => Ok(LookupResult::Pending), }; - - match action { - Action::Retry => { - // Trigger download for all components in case `MissingComponents` failed the blob - // request. Also if blobs are `AwaitingProcessing` and need to be progressed - lookup.continue_requests(cx) - } - Action::ParentUnknown { parent_root } => { - let peers = lookup.all_peers(); - // Mark lookup as awaiting **before** creating the parent lookup. At this point the - // lookup maybe inconsistent. - lookup.set_awaiting_parent(parent_root); - let parent_lookup_exists = - self.search_parent_of_child(parent_root, block_root, &peers, cx); - if parent_lookup_exists { - // The parent lookup exist or has been created. It's safe for `lookup` to - // reference the parent as awaiting. - debug!( - id = lookup_id, - ?block_root, - ?parent_root, - "Marking lookup as awaiting parent" - ); - Ok(LookupResult::Pending) - } else { - // The parent lookup is faulty and was not created, we must drop the `lookup` as - // it's in an inconsistent state. We must drop all of its children too. - Err(LookupRequestError::Failed(format!( - "Parent lookup is faulty {parent_root:?}" - ))) - } - } - Action::Continue => { - // Drop this completed lookup only - Ok(LookupResult::Completed) - } - } + self.on_lookup_result(lookup_id, lookup_result, "processing_result", cx); } pub fn on_external_processing_result( @@ -757,7 +576,20 @@ impl BlockLookups { cx: &mut SyncNetworkContext, ) -> bool { match result { - Ok(LookupResult::Pending) => true, // no action + Ok(LookupResult::Pending) => true, + Ok(LookupResult::ParentUnknown { + parent_root, + block_root, + peers, + }) => { + if self.search_parent_of_child(parent_root, block_root, &peers, cx) { + true + } else { + self.drop_lookup_and_children(id, "Failed"); + self.update_metrics(); + false + } + } Ok(LookupResult::Completed) => { if let Some(lookup) = self.single_block_lookups.remove(&id) { debug!( @@ -923,6 +755,7 @@ impl BlockLookups { } /// Adds peers to a lookup and its ancestors recursively. + /// /// Note: Takes a `lookup_id` as argument to allow recursion on mutable lookups, without having /// to duplicate the code to add peers to a lookup fn add_peers_to_lookup_and_ancestors( @@ -949,12 +782,12 @@ impl BlockLookups { } if let Some(parent_root) = lookup.awaiting_parent() { - if let Some((&child_id, _)) = self + if let Some((&parent_id, _)) = self .single_block_lookups .iter() .find(|(_, l)| l.block_root() == parent_root) { - self.add_peers_to_lookup_and_ancestors(child_id, peers, cx) + self.add_peers_to_lookup_and_ancestors(parent_id, peers, cx) } else { Err(format!("Lookup references unknown parent {parent_root:?}")) } diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 38d6b2216d..8eb58da4e6 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -1,15 +1,17 @@ use super::{BlockComponent, PeerId, SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS}; -use crate::sync::block_lookups::common::RequestState; +use crate::network_beacon_processor::BlockProcessingResult; +use crate::sync::block_lookups::{BlockDownloadResponse, CustodyDownloadResponse}; +use crate::sync::manager::BlockProcessType; use crate::sync::network_context::{ - LookupRequestResult, PeerGroup, ReqId, RpcRequestSendError, SendErrorProcessor, - SyncNetworkContext, + LookupRequestResult, PeerGroup, ReqId, RpcRequestSendError, RpcResponseError, + SendErrorProcessor, SyncNetworkContext, }; -use beacon_chain::{BeaconChainTypes, BlockProcessStatus}; +use beacon_chain::BeaconChainTypes; +use beacon_chain::block_verification_types::AsBlock; use educe::Educe; use lighthouse_network::service::api_types::Id; use parking_lot::RwLock; use std::collections::HashSet; -use std::fmt::Debug; use std::sync::Arc; use std::time::{Duration, Instant}; use store::Hash256; @@ -24,15 +26,18 @@ pub enum LookupResult { Completed, /// Lookup is expecting some future event from the network Pending, + /// Block's parent is not known to fork-choice, a parent lookup is needed + ParentUnknown { + parent_root: Hash256, + block_root: Hash256, + peers: Vec, + }, } #[derive(Debug, PartialEq, Eq, IntoStaticStr)] pub enum LookupRequestError { /// Too many failed attempts - TooManyAttempts { - /// The failed attempts were primarily due to processing failures. - cannot_process: bool, - }, + TooManyAttempts, /// Error sending event to network SendFailedNetwork(RpcRequestSendError), /// Error sending event to processor @@ -52,33 +57,63 @@ pub enum LookupRequestError { }, } +#[derive(Debug)] +struct BlockRequest { + state: SingleLookupRequestState>>, +} + +impl BlockRequest { + fn new() -> Self { + Self { + state: SingleLookupRequestState::new(), + } + } + + fn is_complete(&self) -> bool { + self.state.is_processed() + } +} + +#[derive(Debug)] +enum DataRequest { + WaitingForBlock, + Request { + slot: Slot, + state: SingleLookupRequestState>, + }, + NoData, +} + +impl DataRequest { + fn is_complete(&self) -> bool { + match &self { + DataRequest::WaitingForBlock => false, + DataRequest::Request { state, .. } => state.is_processed(), + DataRequest::NoData => true, + } + } +} + +type PeerSet = Arc>>; + #[derive(Educe)] #[educe(Debug(bound(T: BeaconChainTypes)))] pub struct SingleBlockLookup { pub id: Id, - pub block_request_state: BlockRequestState, - pub component_requests: ComponentRequests, + block_root: Hash256, + block_request: BlockRequest, + data_request: DataRequest, /// Peers that claim to have imported this set of block components. This state is shared with /// the custody request to have an updated view of the peers that claim to have imported the /// block associated with this lookup. The peer set of a lookup can change rapidly, and faster /// than the lifetime of a custody request. #[educe(Debug(method(fmt_peer_set_as_len)))] - peers: Arc>>, - block_root: Hash256, + peers: PeerSet, awaiting_parent: Option, created: Instant, pub(crate) span: Span, } -#[derive(Debug)] -pub(crate) enum ComponentRequests { - WaitingForBlock, - ActiveCustodyRequest(CustodyRequestState), - // When printing in debug this state display the reason why it's not needed - #[allow(dead_code)] - NotNeeded(&'static str), -} - impl SingleBlockLookup { pub fn new( requested_block_root: Hash256, @@ -94,25 +129,25 @@ impl SingleBlockLookup { Self { id, - block_request_state: BlockRequestState::new(requested_block_root), - component_requests: ComponentRequests::WaitingForBlock, - peers: Arc::new(RwLock::new(HashSet::from_iter(peers.iter().copied()))), block_root: requested_block_root, + block_request: BlockRequest::new(), + data_request: DataRequest::WaitingForBlock, + peers: Arc::new(RwLock::new(peers.iter().copied().collect())), awaiting_parent, created: Instant::now(), span: lookup_span, } } - /// Reset the status of all internal requests + /// Reset the status of all requests (used on block processing failure) pub fn reset_requests(&mut self) { - self.block_request_state = BlockRequestState::new(self.block_root); - self.component_requests = ComponentRequests::WaitingForBlock; + self.block_request = BlockRequest::new(); + self.data_request = DataRequest::WaitingForBlock; } - /// Return the slot of this lookup's block if it's currently cached as `AwaitingProcessing` + /// Return the slot of this lookup's block if it's currently cached pub fn peek_downloaded_block_slot(&self) -> Option { - self.block_request_state + self.block_request .state .peek_downloaded_data() .map(|block| block.slot()) @@ -147,15 +182,12 @@ impl SingleBlockLookup { /// Maybe insert a verified response into this lookup. Returns true if imported pub fn add_child_components(&mut self, block_component: BlockComponent) -> bool { match block_component { - BlockComponent::Block(block) => self - .block_request_state - .state - .insert_verified_response(block), - BlockComponent::Sidecar { .. } => { - // For now ignore single blobs and columns, as the blob request state assumes all blobs are - // attributed to the same peer = the peer serving the remaining blobs. Ignoring this - // block component has a minor effect, causing the node to re-request this blob - // once the parent chain is successfully resolved + BlockComponent::Block(block) => { + self.block_request.state.insert_verified_response(block) + } + BlockComponent::Sidecar => { + // There's nothing to do here, there's no component to insert. The lookup downloads + // its required data columns itself once it has the block. false } } @@ -166,29 +198,14 @@ impl SingleBlockLookup { self.block_root() == block_root } - /// Returns true if the block has already been downloaded. - pub fn all_components_processed(&self) -> bool { - self.block_request_state.state.is_processed() - && match &self.component_requests { - ComponentRequests::WaitingForBlock => false, - ComponentRequests::ActiveCustodyRequest(request) => request.state.is_processed(), - ComponentRequests::NotNeeded { .. } => true, - } - } - /// Returns true if this request is expecting some event to make progress pub fn is_awaiting_event(&self) -> bool { self.awaiting_parent.is_some() - || self.block_request_state.state.is_awaiting_event() - || match &self.component_requests { - // If components are waiting for the block request to complete, here we should - // check if the`block_request_state.state.is_awaiting_event(). However we already - // checked that above, so `WaitingForBlock => false` is equivalent. - ComponentRequests::WaitingForBlock => false, - ComponentRequests::ActiveCustodyRequest(request) => { - request.state.is_awaiting_event() - } - ComponentRequests::NotNeeded { .. } => false, + || self.block_request.state.is_awaiting_event() + || match &self.data_request { + DataRequest::WaitingForBlock => true, + DataRequest::Request { state, .. } => state.is_awaiting_event(), + DataRequest::NoData => false, } } @@ -199,139 +216,167 @@ impl SingleBlockLookup { cx: &mut SyncNetworkContext, ) -> Result { let _guard = self.span.clone().entered(); - // TODO: Check what's necessary to download, specially for blobs - self.continue_request::>(cx, 0)?; - if let ComponentRequests::WaitingForBlock = self.component_requests { - let downloaded_block = self - .block_request_state - .state - .peek_downloaded_data() - .cloned(); - - if let Some(block) = downloaded_block.or_else(|| { - // If the block is already being processed or fully validated, retrieve how many blobs - // it expects. Consider any stage of the block. If the block root has been validated, we - // can assert that this is the correct value of `blob_kzg_commitments_count`. - match cx.chain.get_block_process_status(&self.block_root) { - BlockProcessStatus::Unknown => None, - BlockProcessStatus::NotValidated(block, _) - | BlockProcessStatus::ExecutionValidated(block) => Some(block.clone()), - } - }) { - let expected_blobs = block.num_expected_blobs(); - let block_epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); - if expected_blobs == 0 { - self.component_requests = ComponentRequests::NotNeeded("no data"); - } else if cx.chain.should_fetch_custody_columns(block_epoch) { - self.component_requests = ComponentRequests::ActiveCustodyRequest( - CustodyRequestState::new(self.block_root, block.slot()), - ); - } else { - self.component_requests = ComponentRequests::NotNeeded("outside da window"); - } - } else { - // Wait to download the block before downloading blobs. Then we can be sure that the - // block has data, so there's no need to do "blind" requests for all possible blobs and - // latter handle the case where if the peer sent no blobs, penalize. - // - // Lookup sync event safety: Reaching this code means that a block is not in any pre-import - // cache nor in the request state of this lookup. Therefore, the block must either: (1) not - // be downloaded yet or (2) the block is already imported into the fork-choice. - // In case (1) the lookup must either successfully download the block or get dropped. - // In case (2) the block will be downloaded, processed, reach `DuplicateFullyImported` - // and get dropped as completed. - } + // === Block request === + self.block_request.state.maybe_start_downloading(|| { + cx.block_lookup_request(self.id, self.peers.clone(), self.block_root) + })?; + if self.awaiting_parent.is_none() + && let Some(data) = self.block_request.state.maybe_start_processing() + { + cx.send_block_for_processing(self.id, self.block_root, data.value, data.seen_timestamp) + .map_err(LookupRequestError::SendFailedProcessor)?; } - match &self.component_requests { - ComponentRequests::WaitingForBlock => {} // do nothing - ComponentRequests::ActiveCustodyRequest(_) => { - self.continue_request::>(cx, 0)? + // === Data request === + loop { + match &mut self.data_request { + DataRequest::WaitingForBlock => { + if let Some(block) = self.block_request.state.peek_downloaded_data() { + let block_epoch = block + .slot() + .epoch(::EthSpec::slots_per_epoch()); + self.data_request = if block.num_expected_blobs() == 0 { + DataRequest::NoData + } else if cx.chain.should_fetch_custody_columns(block_epoch) { + DataRequest::Request { + slot: block.slot(), + state: SingleLookupRequestState::new(), + } + } else { + DataRequest::NoData + }; + } else { + break; + } + } + DataRequest::Request { slot, state } => { + state.maybe_start_downloading(|| { + cx.custody_lookup_request( + self.id, + self.block_root, + *slot, + self.peers.clone(), + ) + })?; + // Wait for the parent to be imported, data column processing result handle does + // not support `ParentUnknown`. + if self.awaiting_parent.is_none() + && let Some(data) = state.maybe_start_processing() + { + cx.send_custody_columns_for_processing( + self.id, + self.block_root, + data.value, + data.seen_timestamp, + BlockProcessType::SingleCustodyColumn(self.id), + ) + .map_err(LookupRequestError::SendFailedProcessor)?; + } + break; + } + DataRequest::NoData => break, } - ComponentRequests::NotNeeded { .. } => {} // do nothing } // If all components of this lookup are already processed, there will be no future events // that can make progress so it must be dropped. Consider the lookup completed. // This case can happen if we receive the components from gossip during a retry. - if self.all_components_processed() { - self.span = Span::none(); - Ok(LookupResult::Completed) - } else { - Ok(LookupResult::Pending) + if self.block_request.is_complete() && self.data_request.is_complete() { + return Ok(LookupResult::Completed); } + + Ok(LookupResult::Pending) } - /// Potentially makes progress on this request if it's in a progress-able state - fn continue_request>( + /// Handle block processing result. Advances the lookup state machine. + pub fn on_block_processing_result( &mut self, + result: BlockProcessingResult, cx: &mut SyncNetworkContext, - expected_blobs: usize, - ) -> Result<(), LookupRequestError> { - let id = self.id; - let awaiting_parent = self.awaiting_parent.is_some(); - let request = - R::request_state_mut(self).map_err(|e| LookupRequestError::BadState(e.to_owned()))?; - - // Attempt to progress awaiting downloads - if request.get_state().is_awaiting_download() { - // Verify the current request has not exceeded the maximum number of attempts. - let request_state = request.get_state(); - if request_state.failed_attempts() >= SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS { - let cannot_process = request_state.more_failed_processing_attempts(); - return Err(LookupRequestError::TooManyAttempts { cannot_process }); + ) -> Result { + match result { + BlockProcessingResult::Imported(_fully_imported, _info) => { + self.block_request.state.on_processing_success()?; } - - let peers = self.peers.clone(); - let request = R::request_state_mut(self) - .map_err(|e| LookupRequestError::BadState(e.to_owned()))?; - - match request.make_request(id, peers, expected_blobs, cx)? { - LookupRequestResult::RequestSent(req_id) => { - // Lookup sync event safety: If make_request returns `RequestSent`, we are - // guaranteed that `BlockLookups::on_download_response` will be called exactly - // with this `req_id`. - request.get_state_mut().on_download_start(req_id)? - } - LookupRequestResult::NoRequestNeeded(reason) => { - // Lookup sync event safety: Advances this request to the terminal `Processed` - // state. If all requests reach this state, the request is marked as completed - // in `Self::continue_requests`. - request.get_state_mut().on_completed_request(reason)? - } - // Sync will receive a future event to make progress on the request, do nothing now - LookupRequestResult::Pending(reason) => { - // Lookup sync event safety: Refer to the code paths constructing - // `LookupRequestResult::Pending` - request - .get_state_mut() - .update_awaiting_download_status(reason); - return Ok(()); + BlockProcessingResult::ParentUnknown { parent_root } => { + // `BlockError::ParentUnknown` is only returned when processing blocks. Revert the + // block request to `Downloaded` and park this lookup until the parent resolves; a + // future call to `continue_requests` will re-submit the block for processing once + // the parent lookup completes. + self.block_request.state.revert_to_awaiting_processing()?; + self.set_awaiting_parent(parent_root); + return Ok(LookupResult::ParentUnknown { + parent_root, + block_root: self.block_root, + peers: self.all_peers(), + }); + } + BlockProcessingResult::Error { penalty, .. } => { + let peers = self.block_request.state.on_processing_failure()?; + if let Some((action, whom, msg)) = penalty { + whom.apply(action, &peers, msg, cx); } } - - // Otherwise, attempt to progress awaiting processing - // If this request is awaiting a parent lookup to be processed, do not send for processing. - // The request will be rejected with unknown parent error. - } else if !awaiting_parent { - // maybe_start_processing returns Some if state == AwaitingProcess. This pattern is - // useful to conditionally access the result data. - if let Some(result) = request.get_state_mut().maybe_start_processing() { - // Lookup sync event safety: If `send_for_processing` returns Ok() we are guaranteed - // that `BlockLookups::on_processing_result` will be called exactly once with this - // lookup_id - return R::send_for_processing(id, result, cx); - } - // Lookup sync event safety: If the request is not in `AwaitingDownload` or - // `AwaitingProcessing` state it is guaranteed to receive some event to make progress. } + self.continue_requests(cx) + } - // Lookup sync event safety: If a lookup is awaiting a parent we are guaranteed to either: - // (1) attempt to make progress with `BlockLookups::continue_child_lookups` if the parent - // lookup completes, or (2) get dropped if the parent fails and is dropped. + /// Handle data processing result + pub fn on_data_processing_result( + &mut self, + result: BlockProcessingResult, + cx: &mut SyncNetworkContext, + ) -> Result { + let DataRequest::Request { state, .. } = &mut self.data_request else { + return Err(LookupRequestError::BadState("no data_request".to_owned())); + }; - Ok(()) + match result { + BlockProcessingResult::Imported(_fully_imported, _info) => { + state.on_processing_success()?; + } + BlockProcessingResult::ParentUnknown { .. } => { + return Err(LookupRequestError::BadState( + "data processing returned ParentUnknown".to_owned(), + )); + } + BlockProcessingResult::Error { penalty, .. } => { + let peers = state.on_processing_failure()?; + if let Some((action, whom, msg)) = penalty { + whom.apply(action, &peers, msg, cx); + } + } + } + self.continue_requests(cx) + } + + /// Handle a block download response. Updates download state and advances the lookup. + pub fn on_block_download_response( + &mut self, + req_id: ReqId, + result: BlockDownloadResponse, + cx: &mut SyncNetworkContext, + ) -> Result { + self.block_request + .state + .on_download_response(req_id, result)?; + self.continue_requests(cx) + } + + /// Handle a custody columns download response. Updates download state and advances the lookup. + pub fn on_custody_download_response( + &mut self, + req_id: ReqId, + result: CustodyDownloadResponse, + cx: &mut SyncNetworkContext, + ) -> Result { + let DataRequest::Request { state, .. } = &mut self.data_request else { + return Err(LookupRequestError::BadState("no data_request".to_owned())); + }; + + state.on_download_response(req_id, result)?; + self.continue_requests(cx) } /// Get all unique peers that claim to have imported this set of block components @@ -340,7 +385,7 @@ impl SingleBlockLookup { } /// Add peer to all request states. The peer must be able to serve this request. - /// Returns true if the peer was newly inserted into some request state. + /// Returns true if the peer was newly inserted into any peer set. pub fn add_peer(&mut self, peer_id: PeerId) -> bool { self.peers.write().insert(peer_id) } @@ -356,52 +401,23 @@ impl SingleBlockLookup { } } -/// The state of the custody request component of a `SingleBlockLookup`. -#[derive(Educe)] -#[educe(Debug)] -pub struct CustodyRequestState { - #[educe(Debug(ignore))] - pub block_root: Hash256, - pub slot: Slot, - pub state: SingleLookupRequestState>, -} - -impl CustodyRequestState { - pub fn new(block_root: Hash256, slot: Slot) -> Self { - Self { - block_root, - slot, - state: SingleLookupRequestState::new(), - } - } -} - -/// The state of the block request component of a `SingleBlockLookup`. -#[derive(Educe)] -#[educe(Debug)] -pub struct BlockRequestState { - #[educe(Debug(ignore))] - pub requested_block_root: Hash256, - pub state: SingleLookupRequestState>>, -} - -impl BlockRequestState { - pub fn new(block_root: Hash256) -> Self { - Self { - requested_block_root: block_root, - state: SingleLookupRequestState::new(), - } - } -} - #[derive(Debug, Clone)] pub struct DownloadResult { pub value: T, - pub block_root: Hash256, pub seen_timestamp: Duration, pub peer_group: PeerGroup, } +impl DownloadResult { + pub fn new(value: T, peer_group: PeerGroup, seen_timestamp: Duration) -> Self { + Self { + value, + seen_timestamp, + peer_group, + } + } +} + #[derive(IntoStaticStr)] pub enum State { AwaitingDownload(/* reason */ &'static str), @@ -410,7 +426,7 @@ pub enum State { /// Request is processing, sent by lookup sync Processing(DownloadResult), /// Request is processed - Processed(/* reason */ &'static str), + Processed(/* reason */ &'static str, T), } /// Object representing the state of a single block or blob lookup request. @@ -477,10 +493,29 @@ impl SingleLookupRequestState { State::Downloading { .. } => None, State::AwaitingProcess(result) => Some(&result.value), State::Processing(result) => Some(&result.value), - State::Processed { .. } => None, + State::Processed(_, value) => Some(value), } } + /// Drive download: check max attempts, issue request, handle result. + fn maybe_start_downloading( + &mut self, + request_fn: impl FnOnce() -> Result, RpcRequestSendError>, + ) -> Result<(), LookupRequestError> { + if self.is_awaiting_download() { + match request_fn().map_err(LookupRequestError::SendFailedNetwork)? { + LookupRequestResult::RequestSent(req_id) => self.on_download_start(req_id)?, + LookupRequestResult::NoRequestNeeded(reason, value) => { + self.on_completed_request(reason, value)? + } + LookupRequestResult::Pending(reason) => { + self.update_awaiting_download_status(reason) + } + } + } + Ok(()) + } + /// Switch to `AwaitingProcessing` if the request is in `AwaitingDownload` state, otherwise /// ignore. pub fn insert_verified_response(&mut self, result: DownloadResult) -> bool { @@ -513,6 +548,17 @@ impl SingleLookupRequestState { } } + pub fn on_download_response( + &mut self, + req_id: ReqId, + result: Result, RpcResponseError>, + ) -> Result<(), LookupRequestError> { + match result { + Ok(result) => self.on_download_success(req_id, result), + Err(_) => self.on_download_failure(req_id), + } + } + /// Registers a failure in downloading a block. This might be a peer disconnection or a wrong /// block. pub fn on_download_failure(&mut self, req_id: ReqId) -> Result<(), LookupRequestError> { @@ -525,6 +571,10 @@ impl SingleLookupRequestState { }); } self.failed_downloading = self.failed_downloading.saturating_add(1); + if self.failed_downloading >= SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS { + return Err(LookupRequestError::TooManyAttempts); + } + self.state = State::AwaitingDownload("not started"); Ok(()) } @@ -589,6 +639,9 @@ impl SingleLookupRequestState { State::Processing(result) => { let peers_source = result.peer_group.clone(); self.failed_processing = self.failed_processing.saturating_add(1); + if self.failed_processing >= SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS { + return Err(LookupRequestError::TooManyAttempts); + } self.state = State::AwaitingDownload("not started"); Ok(peers_source) } @@ -600,8 +653,8 @@ impl SingleLookupRequestState { pub fn on_processing_success(&mut self) -> Result<(), LookupRequestError> { match &self.state { - State::Processing(_) => { - self.state = State::Processed("processing success"); + State::Processing(data) => { + self.state = State::Processed("processing success", data.value.clone()); Ok(()) } other => Err(LookupRequestError::BadState(format!( @@ -611,10 +664,14 @@ impl SingleLookupRequestState { } /// Mark a request as complete without any download or processing - pub fn on_completed_request(&mut self, reason: &'static str) -> Result<(), LookupRequestError> { + pub fn on_completed_request( + &mut self, + reason: &'static str, + value: T, + ) -> Result<(), LookupRequestError> { match &self.state { State::AwaitingDownload { .. } => { - self.state = State::Processed(reason); + self.state = State::Processed(reason, value); Ok(()) } other => Err(LookupRequestError::BadState(format!( @@ -622,15 +679,6 @@ impl SingleLookupRequestState { ))), } } - - /// The total number of failures, whether it be processing or downloading. - pub fn failed_attempts(&self) -> u8 { - self.failed_processing + self.failed_downloading - } - - pub fn more_failed_processing_attempts(&self) -> bool { - self.failed_processing >= self.failed_downloading - } } // Display is used in the BadState assertions above @@ -647,15 +695,15 @@ impl std::fmt::Debug for State { match self { Self::AwaitingDownload(reason) => write!(f, "AwaitingDownload({})", reason), Self::Downloading(req_id) => write!(f, "Downloading({:?})", req_id), - Self::AwaitingProcess(d) => write!(f, "AwaitingProcess({:?})", d.peer_group), - Self::Processing(d) => write!(f, "Processing({:?})", d.peer_group), - Self::Processed(reason) => write!(f, "Processed({})", reason), + Self::AwaitingProcess(_) => write!(f, "AwaitingProcess"), + Self::Processing(_) => write!(f, "Processing"), + Self::Processed(reason, _) => write!(f, "Processed({})", reason), } } } fn fmt_peer_set_as_len( - peer_set: &Arc>>, + peer_set: &PeerSet, f: &mut std::fmt::Formatter, ) -> Result<(), std::fmt::Error> { write!(f, "{}", peer_set.read().len()) diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 37d13c2eae..166c65b6e1 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -45,9 +45,7 @@ use crate::network_beacon_processor::{ }; use crate::service::NetworkMessage; use crate::status::ToStatusMessage; -use crate::sync::block_lookups::{ - BlockComponent, BlockRequestState, CustodyRequestState, DownloadResult, -}; +use crate::sync::block_lookups::{BlockComponent, DownloadResult}; use crate::sync::custody_backfill_sync::CustodyBackFillSync; use crate::sync::network_context::{PeerGroup, RpcResponseResult}; use beacon_chain::block_verification_types::AsBlock; @@ -867,7 +865,6 @@ impl SyncManager { block_slot, BlockComponent::Block(DownloadResult { value: block.block_cloned(), - block_root, seen_timestamp: self.chain.slot_clock.now_duration().unwrap_or_default(), peer_group: PeerGroup::from_single(peer_id), }), @@ -885,7 +882,7 @@ impl SyncManager { block_root, parent_root, slot, - BlockComponent::Sidecar { parent_root }, + BlockComponent::Sidecar, ); } SyncMessage::UnknownBlockHashFromAttestation(peer_id, block_root) => { @@ -975,6 +972,7 @@ impl SyncManager { if self.block_lookups.search_child_and_parent( block_root, block_component, + parent_root, peer_id, &mut self.network, ) { @@ -1125,14 +1123,13 @@ impl SyncManager { block: RpcEvent>>, ) { if let Some(resp) = self.network.on_single_block_response(id, peer_id, block) { - self.block_lookups - .on_download_response::>( - id, - resp.map(|(value, seen_timestamp)| { - (value, PeerGroup::from_single(peer_id), seen_timestamp) - }), - &mut self.network, - ) + self.block_lookups.on_block_download_response( + id, + resp.map(|(value, seen_timestamp)| { + DownloadResult::new(value, PeerGroup::from_single(peer_id), seen_timestamp) + }), + &mut self.network, + ) } } @@ -1308,11 +1305,7 @@ impl SyncManager { response: CustodyByRootResult, ) { self.block_lookups - .on_download_response::>( - requester.0, - response, - &mut self.network, - ); + .on_custody_download_response(requester.0, response, &mut self.network); } /// Handles receiving a response for a range sync request that should have both blocks and diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 95ae84755c..1e35c0a72f 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -16,7 +16,7 @@ use crate::network_beacon_processor::TestBeaconChainType; use crate::service::NetworkMessage; use crate::status::ToStatusMessage; use crate::sync::batch::ByRangeRequestType; -use crate::sync::block_lookups::SingleLookupId; +use crate::sync::block_lookups::{DownloadResult, SingleLookupId}; use crate::sync::block_sidecar_coupling::CouplingError; use crate::sync::range_data_column_batch_request::RangeDataColumnBatchRequest; use beacon_chain::block_verification_types::LookupBlock; @@ -95,7 +95,7 @@ pub type RpcResponseResult = Result<(T, Duration), RpcResponseError>; /// Duration = latest seen timestamp of all received data columns pub type CustodyByRootResult = - Result<(DataColumnSidecarList, PeerGroup, Duration), RpcResponseError>; + Result>, RpcResponseError>; #[derive(Debug)] pub enum RpcResponseError { @@ -176,13 +176,13 @@ impl PeerGroup { /// Sequential ID that uniquely identifies ReqResp outgoing requests pub type ReqId = u32; -pub enum LookupRequestResult { +pub enum LookupRequestResult { /// A request is sent. Sync MUST receive an event from the network in the future for either: /// completed response or failed request RequestSent(I), /// No request is sent, and no further action is necessary to consider this request completed. /// Includes a reason why this request is not needed. - NoRequestNeeded(&'static str), + NoRequestNeeded(&'static str, T), /// No request is sent, but the request is not completed. Sync MUST receive some future event /// that makes progress on the request. For example: request is processing from a different /// source (i.e. block received from gossip) and sync MUST receive an event with that processing @@ -820,7 +820,7 @@ impl SyncNetworkContext { lookup_id: SingleLookupId, lookup_peers: Arc>>, block_root: Hash256, - ) -> Result { + ) -> Result>>, RpcRequestSendError> { let active_request_count_by_peer = self.active_request_count_by_peer(); let Some(peer_id) = lookup_peers .read() @@ -871,9 +871,10 @@ impl SyncNetworkContext { }, // Block is fully validated. If it's not yet imported it's waiting for missing block // components. Consider this request completed and do nothing. - BlockProcessStatus::ExecutionValidated { .. } => { + BlockProcessStatus::ExecutionValidated(block) => { return Ok(LookupRequestResult::NoRequestNeeded( "block execution validated", + block, )); } } @@ -937,12 +938,13 @@ impl SyncNetworkContext { lookup_id: SingleLookupId, lookup_peers: Arc>>, block_root: Hash256, - ) -> Result { + ) -> Result, RpcRequestSendError> { // Skip the download if fork-choice already saw this envelope (e.g. imported via gossip // before the lookup got here). if self.chain.envelope_is_known_to_fork_choice(&block_root) { return Ok(LookupRequestResult::NoRequestNeeded( "envelope already known to fork-choice", + (), )); } @@ -1011,7 +1013,7 @@ impl SyncNetworkContext { peer_id: PeerId, request: DataColumnsByRootSingleBlockRequest, expect_max_responses: bool, - ) -> Result, &'static str> { + ) -> Result, &'static str> { let id = DataColumnsByRootRequestId { id: self.next_id(), requester, @@ -1060,7 +1062,7 @@ impl SyncNetworkContext { block_root: Hash256, block_slot: Slot, lookup_peers: Arc>>, - ) -> Result { + ) -> Result>, RpcRequestSendError> { let custody_indexes_imported = self .chain .cached_data_column_indexes(&block_root, block_slot) @@ -1078,7 +1080,10 @@ impl SyncNetworkContext { if custody_indexes_to_fetch.is_empty() { // No indexes required, do not issue any request - return Ok(LookupRequestResult::NoRequestNeeded("no indices to fetch")); + return Ok(LookupRequestResult::NoRequestNeeded( + "no indices to fetch", + vec![], + )); } let id = SingleLookupReqId { @@ -1528,8 +1533,8 @@ impl SyncNetworkContext { // Convert a result from internal format of `ActiveCustodyRequest` (error first to use ?) to // an Option first to use in an `if let Some() { act on result }` block. match result.as_ref() { - Some(Ok((columns, peer_group, _))) => { - debug!(?id, count = columns.len(), peers = ?peer_group, "Custody request success, removing") + Some(Ok(data)) => { + debug!(?id, count = data.value.len(), peers = ?data.peer_group, "Custody request success, removing") } Some(Err(e)) => { debug!(?id, error = ?e, "Custody request failure, removing" ) diff --git a/beacon_node/network/src/sync/network_context/custody.rs b/beacon_node/network/src/sync/network_context/custody.rs index 2b96800e37..e74b74ec08 100644 --- a/beacon_node/network/src/sync/network_context/custody.rs +++ b/beacon_node/network/src/sync/network_context/custody.rs @@ -1,3 +1,4 @@ +use crate::sync::block_lookups::DownloadResult; use crate::sync::network_context::{ DataColumnsByRootRequestId, DataColumnsByRootSingleBlockRequest, }; @@ -56,8 +57,7 @@ struct ActiveBatchColumnsRequest { span: Span, } -pub type CustodyRequestResult = - Result, PeerGroup, Duration)>, Error>; +pub type CustodyRequestResult = Result>>, Error>; impl ActiveCustodyRequest { pub(crate) fn new( @@ -227,7 +227,11 @@ impl ActiveCustodyRequest { .into_iter() .max() .unwrap_or_else(|| cx.chain.slot_clock.now_duration().unwrap_or_default()); - return Ok(Some((columns, peer_group, max_seen_timestamp))); + return Ok(Some(DownloadResult::new( + columns, + peer_group, + max_seen_timestamp, + ))); } let active_request_count_by_peer = cx.active_request_count_by_peer(); @@ -343,7 +347,7 @@ impl ActiveCustodyRequest { }, ); } - LookupRequestResult::NoRequestNeeded(_) => unreachable!(), + LookupRequestResult::NoRequestNeeded(..) => unreachable!(), LookupRequestResult::Pending(_) => unreachable!(), } } From d617c826fe6c4983cd883d50b7f4df5bce31304f Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 3 Jun 2026 09:07:41 -0700 Subject: [PATCH 04/15] Gloas data column reprocess queue (#9339) When debugging ePBS with columns, we noticed that columns arriving before their block dont pass gossip verification checks and are dropped. This PR ensures that columns arriving before the block are sent to the reprocess queue. Once their block arrives, they are reprocessed. This isn't an issue pre-gloas because we don't make block root checks for fulu data columns. This allows us to gossip verify the column and send it to the DA cache before the block arrives. I think we also need to handle this edge case for partial data columns. Theres an existing TODO for that already. Co-Authored-By: Eitan Seri-Levi --- beacon_node/beacon_processor/src/lib.rs | 23 +- .../src/scheduler/work_queue.rs | 7 + .../src/scheduler/work_reprocessing_queue.rs | 216 +++++++++++++++--- .../gossip_methods.rs | 46 +++- .../src/network_beacon_processor/mod.rs | 2 + .../src/network_beacon_processor/tests.rs | 1 + beacon_node/network/src/router.rs | 1 + 7 files changed, 248 insertions(+), 48 deletions(-) diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index af3ff09c8a..d6233ebaf9 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -41,8 +41,8 @@ pub use crate::scheduler::BeaconProcessorQueueLengths; use crate::scheduler::work_queue::WorkQueues; use crate::work_reprocessing_queue::{ - QueuedBackfillBatch, QueuedColumnReconstruction, QueuedGossipBlock, QueuedGossipEnvelope, - ReprocessQueueMessage, + QueuedBackfillBatch, QueuedColumnReconstruction, QueuedGossipBlock, QueuedGossipDataColumn, + QueuedGossipEnvelope, ReprocessQueueMessage, }; use futures::stream::{Stream, StreamExt}; use futures::task::Poll; @@ -304,6 +304,10 @@ impl From for WorkEvent { work: Work::ColumnReconstruction(process_fn), } } + ReadyWork::DataColumn(QueuedGossipDataColumn { process_fn, .. }) => Self { + drop_during_sync: true, + work: Work::UnknownBlockDataColumn { process_fn }, + }, } } } @@ -369,6 +373,9 @@ pub enum Work { UnknownBlockAttestation { process_fn: BlockingFn, }, + UnknownBlockDataColumn { + process_fn: BlockingFn, + }, GossipAttestationBatch { attestations: GossipAttestationBatch, process_batch: Box, @@ -464,6 +471,7 @@ pub enum WorkType { GossipAttestation, GossipAttestationToConvert, UnknownBlockAttestation, + UnknownBlockDataColumn, GossipAttestationBatch, GossipAggregate, UnknownBlockAggregate, @@ -569,6 +577,7 @@ impl Work { Work::LightClientFinalityUpdateRequest(_) => WorkType::LightClientFinalityUpdateRequest, Work::LightClientUpdatesByRangeRequest(_) => WorkType::LightClientUpdatesByRangeRequest, Work::UnknownBlockAttestation { .. } => WorkType::UnknownBlockAttestation, + Work::UnknownBlockDataColumn { .. } => WorkType::UnknownBlockDataColumn, Work::UnknownBlockAggregate { .. } => WorkType::UnknownBlockAggregate, Work::UnknownLightClientOptimisticUpdate { .. } => { WorkType::UnknownLightClientOptimisticUpdate @@ -842,6 +851,9 @@ impl BeaconProcessor { Some(item) } else if let Some(item) = work_queues.gossip_data_column_queue.pop() { Some(item) + } else if let Some(item) = work_queues.unknown_block_data_column_queue.pop() + { + Some(item) } else if let Some(item) = work_queues.gossip_partial_data_column_queue.pop() { @@ -1238,6 +1250,9 @@ impl BeaconProcessor { Work::UnknownBlockAttestation { .. } => { work_queues.unknown_block_attestation_queue.push(work) } + Work::UnknownBlockDataColumn { .. } => work_queues + .unknown_block_data_column_queue + .push(work, work_id), Work::UnknownBlockAggregate { .. } => { work_queues.unknown_block_aggregate_queue.push(work) } @@ -1288,6 +1303,9 @@ impl BeaconProcessor { WorkType::UnknownBlockAttestation => { work_queues.unknown_block_attestation_queue.len() } + WorkType::UnknownBlockDataColumn => { + work_queues.unknown_block_data_column_queue.len() + } WorkType::GossipAttestationBatch => 0, // No queue WorkType::GossipAggregate => work_queues.aggregate_queue.len(), WorkType::UnknownBlockAggregate => { @@ -1504,6 +1522,7 @@ impl BeaconProcessor { }), Work::UnknownBlockAttestation { process_fn } | Work::UnknownBlockAggregate { process_fn } + | Work::UnknownBlockDataColumn { process_fn } | Work::UnknownLightClientOptimisticUpdate { process_fn, .. } => { task_spawner.spawn_blocking(process_fn) } diff --git a/beacon_node/beacon_processor/src/scheduler/work_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_queue.rs index ebd66e743d..cc03feac51 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_queue.rs @@ -111,6 +111,7 @@ pub struct BeaconProcessorQueueLengths { attestation_queue: usize, unknown_block_aggregate_queue: usize, unknown_block_attestation_queue: usize, + unknown_block_data_column_queue: usize, sync_message_queue: usize, sync_contribution_queue: usize, gossip_voluntary_exit_queue: usize, @@ -174,6 +175,8 @@ impl BeaconProcessorQueueLengths { Ok(Self { aggregate_queue: 4096, unknown_block_aggregate_queue: 1024, + // Capacity for two slot's worth of data columns for a supernode. + unknown_block_data_column_queue: 256, // Capacity for a full slot's worth of attestations if subscribed to all subnets attestation_queue: std::cmp::max( active_validator_count / slots_per_epoch, @@ -245,6 +248,7 @@ pub struct WorkQueues { pub attestation_debounce: TimeLatch, pub unknown_block_aggregate_queue: LifoQueue>, pub unknown_block_attestation_queue: LifoQueue>, + pub unknown_block_data_column_queue: FifoQueue>, pub sync_message_queue: LifoQueue>, pub sync_contribution_queue: LifoQueue>, pub gossip_voluntary_exit_queue: FifoQueue>, @@ -302,6 +306,8 @@ impl WorkQueues { LifoQueue::new(queue_lengths.unknown_block_aggregate_queue); let unknown_block_attestation_queue = LifoQueue::new(queue_lengths.unknown_block_attestation_queue); + let unknown_block_data_column_queue = + FifoQueue::new(queue_lengths.unknown_block_data_column_queue); let sync_message_queue = LifoQueue::new(queue_lengths.sync_message_queue); let sync_contribution_queue = LifoQueue::new(queue_lengths.sync_contribution_queue); @@ -383,6 +389,7 @@ impl WorkQueues { attestation_debounce, unknown_block_aggregate_queue, unknown_block_attestation_queue, + unknown_block_data_column_queue, sync_message_queue, sync_contribution_queue, gossip_voluntary_exit_queue, diff --git a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs index b1fa56af01..62ed86fbad 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs @@ -52,6 +52,10 @@ pub const QUEUED_ATTESTATION_DELAY: Duration = Duration::from_secs(12); /// For how long to queue light client updates for re-processing. pub const QUEUED_LIGHT_CLIENT_UPDATE_DELAY: Duration = Duration::from_secs(12); +/// Data column timeout as a multiplier of slot duration. Columns waiting for their block will be +/// sent for processing after this many slots worth of time, even if the block hasn't arrived. +const QUEUED_DATA_COLUMN_DELAY_SLOTS: u32 = 1; + /// Envelope timeout as a multiplier of slot duration. Envelopes waiting for their block will be /// sent for processing after this many slots worth of time, even if the block hasn't arrived. const QUEUED_ENVELOPE_DELAY_SLOTS: u32 = 1; @@ -76,6 +80,9 @@ const MAXIMUM_QUEUED_ENVELOPES: usize = 16; /// How many attestations we keep before new ones get dropped. const MAXIMUM_QUEUED_ATTESTATIONS: usize = 16_384; +/// How many columns we keep before new ones get dropped. +const MAXIMUM_QUEUED_DATA_COLUMNS: usize = 256; + /// How many light client updates we keep before new ones get dropped. const MAXIMUM_QUEUED_LIGHT_CLIENT_UPDATES: usize = 128; @@ -123,6 +130,8 @@ pub enum ReprocessQueueMessage { UnknownLightClientOptimisticUpdate(QueuedLightClientUpdate), /// A new backfill batch that needs to be scheduled for processing. BackfillSync(QueuedBackfillBatch), + /// A gossip data column that references an unknown block. + UnknownBlockDataColumn(QueuedGossipDataColumn), /// A delayed column reconstruction that needs checking DelayColumnReconstruction(QueuedColumnReconstruction), } @@ -138,6 +147,7 @@ pub enum ReadyWork { LightClientUpdate(QueuedLightClientUpdate), BackfillSync(QueuedBackfillBatch), ColumnReconstruction(QueuedColumnReconstruction), + DataColumn(QueuedGossipDataColumn), } /// An Attestation for which the corresponding block was not seen while processing, queued for @@ -200,6 +210,12 @@ pub struct QueuedColumnReconstruction { pub process_fn: AsyncFn, } +/// A gossip data column that references an unknown block, queued for later reprocessing. +pub struct QueuedGossipDataColumn { + pub beacon_block_root: Hash256, + pub process_fn: BlockingFn, +} + impl TryFrom> for QueuedBackfillBatch { type Error = WorkEvent; @@ -240,6 +256,8 @@ enum InboundEvent { ReadyBackfillSync(QueuedBackfillBatch), /// A column reconstruction that was queued is ready for processing. ReadyColumnReconstruction(QueuedColumnReconstruction), + /// A gossip data column that is ready for re-processing. + ReadyDataColumn(Hash256), /// A message sent to the `ReprocessQueue` Msg(ReprocessQueueMessage), } @@ -264,6 +282,8 @@ struct ReprocessQueue { lc_updates_delay_queue: DelayQueue, /// Queue to manage scheduled column reconstructions. column_reconstructions_delay_queue: DelayQueue, + /// Queue to manage gossip data column timeouts. + data_columns_delay_queue: DelayQueue, /* Queued items */ /// Queued blocks. @@ -284,6 +304,10 @@ struct ReprocessQueue { queued_column_reconstructions: HashMap>, /// Queued backfill batches queued_backfill_batches: Vec, + /// Queued gossip data columns awaiting their block, keyed by block root. + awaiting_data_columns_per_root: HashMap, DelayKey)>, + /// Total number of queued gossip data columns across all roots. + queued_data_columns_count: usize, /* Aux */ /// Next attestation id, used for both aggregated and unaggregated attestations @@ -294,6 +318,7 @@ struct ReprocessQueue { rpc_block_debounce: TimeLatch, attestation_delay_debounce: TimeLatch, lc_update_delay_debounce: TimeLatch, + data_column_delay_debounce: TimeLatch, next_backfill_batch_event: Option>>, slot_clock: Arc, } @@ -387,6 +412,13 @@ impl Stream for ReprocessQueue { Poll::Ready(None) | Poll::Pending => (), } + match self.data_columns_delay_queue.poll_expired(cx) { + Poll::Ready(Some(block_root)) => { + return Poll::Ready(Some(InboundEvent::ReadyDataColumn(block_root.into_inner()))); + } + Poll::Ready(None) | Poll::Pending => (), + } + if let Some(next_backfill_batch_event) = self.next_backfill_batch_event.as_mut() { match next_backfill_batch_event.as_mut().poll(cx) { Poll::Ready(_) => { @@ -455,6 +487,7 @@ impl ReprocessQueue { attestations_delay_queue: DelayQueue::new(), lc_updates_delay_queue: DelayQueue::new(), column_reconstructions_delay_queue: DelayQueue::new(), + data_columns_delay_queue: DelayQueue::new(), queued_gossip_block_roots: HashSet::new(), awaiting_envelopes_per_root: HashMap::new(), queued_lc_updates: FnvHashMap::default(), @@ -464,6 +497,8 @@ impl ReprocessQueue { awaiting_lc_updates_per_parent_root: HashMap::new(), queued_backfill_batches: Vec::new(), queued_column_reconstructions: HashMap::new(), + awaiting_data_columns_per_root: HashMap::new(), + queued_data_columns_count: 0, next_attestation: 0, next_lc_update: 0, early_block_debounce: TimeLatch::default(), @@ -471,6 +506,7 @@ impl ReprocessQueue { rpc_block_debounce: TimeLatch::default(), attestation_delay_debounce: TimeLatch::default(), lc_update_delay_debounce: TimeLatch::default(), + data_column_delay_debounce: TimeLatch::default(), next_backfill_batch_event: None, slot_clock, } @@ -551,22 +587,16 @@ impl ReprocessQueue { return; } - // When the queue is full, evict the oldest entry to make room for newer envelopes. + // When the queue is full, drop the new envelope. if self.awaiting_envelopes_per_root.len() >= MAXIMUM_QUEUED_ENVELOPES { if self.envelope_delay_debounce.elapsed() { warn!( queue_size = MAXIMUM_QUEUED_ENVELOPES, msg = "system resources may be saturated", - "Envelope delay queue is full, evicting oldest entry" + "Envelope delay queue is full, dropping envelope" ); } - if let Some(oldest_root) = - self.awaiting_envelopes_per_root.keys().next().copied() - && let Some((_envelope, delay_key)) = - self.awaiting_envelopes_per_root.remove(&oldest_root) - { - self.envelope_delay_queue.remove(&delay_key); - } + return; } // Register the timeout. @@ -688,6 +718,37 @@ impl ReprocessQueue { self.next_attestation += 1; } + InboundEvent::Msg(UnknownBlockDataColumn(queued_data_column)) => { + let block_root = queued_data_column.beacon_block_root; + + if self.queued_data_columns_count >= MAXIMUM_QUEUED_DATA_COLUMNS { + if self.data_column_delay_debounce.elapsed() { + warn!( + queue_size = MAXIMUM_QUEUED_DATA_COLUMNS, + msg = "system resources may be saturated", + "Data column delay queue is full, dropping column" + ); + } + return; + } + + if let Some((columns, _delay_key)) = + self.awaiting_data_columns_per_root.get_mut(&block_root) + { + // Append to existing entry; the timer for this root is already running. + columns.push(queued_data_column); + } else { + let delay_key = self.data_columns_delay_queue.insert( + block_root, + self.slot_clock.slot_duration() * QUEUED_DATA_COLUMN_DELAY_SLOTS, + ); + + self.awaiting_data_columns_per_root + .insert(block_root, (vec![queued_data_column], delay_key)); + } + + self.queued_data_columns_count += 1; + } InboundEvent::Msg(UnknownLightClientOptimisticUpdate( queued_light_client_optimistic_update, )) => { @@ -800,6 +861,25 @@ impl ReprocessQueue { ); } } + + // Unqueue the data columns we have for this root, if any. + if let Some((data_columns, delay_key)) = + self.awaiting_data_columns_per_root.remove(&block_root) + { + self.data_columns_delay_queue.remove(&delay_key); + self.queued_data_columns_count = self + .queued_data_columns_count + .saturating_sub(data_columns.len()); + for data_column in data_columns { + if self + .ready_work_tx + .try_send(ReadyWork::DataColumn(data_column)) + .is_err() + { + error!(?block_root, "Failed to send data column for reprocessing"); + } + } + } } InboundEvent::Msg(NewLightClientOptimisticUpdate { parent_root }) => { // Unqueue the light client optimistic updates we have for this root, if any. @@ -1053,6 +1133,27 @@ impl ReprocessQueue { ); } } + InboundEvent::ReadyDataColumn(block_root) => { + if let Some((data_columns, _)) = + self.awaiting_data_columns_per_root.remove(&block_root) + { + self.queued_data_columns_count = self + .queued_data_columns_count + .saturating_sub(data_columns.len()); + for data_column in data_columns { + if self + .ready_work_tx + .try_send(ReadyWork::DataColumn(data_column)) + .is_err() + { + error!( + hint = "system may be overloaded", + "Ignored expired gossip data column" + ); + } + } + } + } } metrics::set_gauge_vec( @@ -1581,48 +1682,87 @@ mod tests { assert_eq!(queue.envelope_delay_queue.len(), 1); } + /// Tests that a queued gossip data column is released when its block is imported. #[tokio::test] - async fn envelope_capacity_evicts_oldest() { + async fn data_column_released_on_block_imported() { + create_test_tracing_subscriber(); + + let config = BeaconProcessorConfig::default(); + let (ready_work_tx, mut ready_work_rx) = + mpsc::channel::(config.max_scheduled_work_queue_len); + let (_, reprocess_work_rx) = + mpsc::channel::(config.max_scheduled_work_queue_len); + let slot_clock = Arc::new(testing_slot_clock(12)); + let mut queue = ReprocessQueue::new(ready_work_tx, reprocess_work_rx, slot_clock); + + tokio::time::pause(); + + let beacon_block_root = Hash256::repeat_byte(0xbb); + + let msg = ReprocessQueueMessage::UnknownBlockDataColumn(QueuedGossipDataColumn { + beacon_block_root, + process_fn: Box::new(|| {}), + }); + queue.handle_message(InboundEvent::Msg(msg)); + + assert_eq!(queue.awaiting_data_columns_per_root.len(), 1); + assert!( + queue + .awaiting_data_columns_per_root + .contains_key(&beacon_block_root) + ); + assert_eq!(queue.data_columns_delay_queue.len(), 1); + + // Simulate block import. + queue.handle_message(InboundEvent::Msg(ReprocessQueueMessage::BlockImported { + block_root: beacon_block_root, + parent_root: Hash256::repeat_byte(0x00), + })); + + // Internal state should be cleaned up. + assert!(queue.awaiting_data_columns_per_root.is_empty()); + assert_eq!(queue.data_columns_delay_queue.len(), 0); + + // The column should have been sent to the ready_work channel. + let ready = ready_work_rx.try_recv().expect("column should be ready"); + assert!(matches!(ready, ReadyWork::DataColumn(_))); + } + + /// Tests that an expired gossip data column is pruned cleanly from all internal state. + #[tokio::test] + async fn prune_awaiting_data_columns_per_root() { create_test_tracing_subscriber(); let mut queue = test_queue(); - // Pause time so it only advances manually tokio::time::pause(); - // Fill the queue to capacity. - for i in 0..MAXIMUM_QUEUED_ENVELOPES { - let block_root = Hash256::repeat_byte(i as u8); - let msg = ReprocessQueueMessage::UnknownBlockForEnvelope(QueuedGossipEnvelope { - beacon_block_slot: Slot::new(1), - beacon_block_root: block_root, - process_fn: Box::pin(async {}), - }); - queue.handle_message(InboundEvent::Msg(msg)); - } - assert_eq!( - queue.awaiting_envelopes_per_root.len(), - MAXIMUM_QUEUED_ENVELOPES - ); + let beacon_block_root = Hash256::repeat_byte(0xcd); - // One more should evict the oldest and insert the new one. - let overflow_root = Hash256::repeat_byte(0xff); - let msg = ReprocessQueueMessage::UnknownBlockForEnvelope(QueuedGossipEnvelope { - beacon_block_slot: Slot::new(1), - beacon_block_root: overflow_root, - process_fn: Box::pin(async {}), + let msg = ReprocessQueueMessage::UnknownBlockDataColumn(QueuedGossipDataColumn { + beacon_block_root, + process_fn: Box::new(|| {}), }); queue.handle_message(InboundEvent::Msg(msg)); - // Queue should still be at capacity, with the new root present. - assert_eq!( - queue.awaiting_envelopes_per_root.len(), - MAXIMUM_QUEUED_ENVELOPES - ); + assert_eq!(queue.awaiting_data_columns_per_root.len(), 1); assert!( queue - .awaiting_envelopes_per_root - .contains_key(&overflow_root) + .awaiting_data_columns_per_root + .contains_key(&beacon_block_root) ); + + // Advance time past the delay so the entry expires. + advance_time( + &queue.slot_clock, + 2 * queue.slot_clock.slot_duration() * QUEUED_DATA_COLUMN_DELAY_SLOTS, + ) + .await; + let ready_msg = queue.next().await.unwrap(); + assert!(matches!(ready_msg, InboundEvent::ReadyDataColumn(_))); + queue.handle_message(ready_msg); + + // All internal state should be cleaned up. + assert!(queue.awaiting_data_columns_per_root.is_empty()); } } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index df94b473a8..9becfd4d59 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -61,8 +61,8 @@ use beacon_processor::work_reprocessing_queue::QueuedColumnReconstruction; use beacon_processor::{ DuplicateCache, GossipAggregatePackage, GossipAttestationBatch, work_reprocessing_queue::{ - QueuedAggregate, QueuedGossipBlock, QueuedGossipEnvelope, QueuedLightClientUpdate, - QueuedUnaggregate, ReprocessQueueMessage, + QueuedAggregate, QueuedGossipBlock, QueuedGossipDataColumn, QueuedGossipEnvelope, + QueuedLightClientUpdate, QueuedUnaggregate, ReprocessQueueMessage, }, }; @@ -657,6 +657,7 @@ impl NetworkBeaconProcessor { subnet_id: DataColumnSubnetId, column_sidecar: Arc>, seen_duration: Duration, + allow_reprocess: bool, ) { let slot = column_sidecar.slot(); let block_root = column_sidecar.block_root(); @@ -738,19 +739,48 @@ impl NetworkBeaconProcessor { .. } => { debug!( - action = "ignoring", + action = "queuing for reprocessing", %unknown_block_root, "Unknown block root for column" ); - // TODO(gloas): wire this into proper lookup sync. Sending - // `UnknownBlockHashFromAttestation` here is a Fulu-shaped fallback that - // mixes column processing with the attestation lookup path and is not - // the right primitive for Gloas column lookups. self.propagate_validation_result( - message_id, + message_id.clone(), peer_id, MessageAcceptance::Ignore, ); + + if allow_reprocess { + // Queue the column for reprocessing when the block arrives. + let processor = self.clone(); + let reprocess_msg = ReprocessQueueMessage::UnknownBlockDataColumn( + QueuedGossipDataColumn { + beacon_block_root: unknown_block_root, + process_fn: Box::new(move || { + let _ = processor.send_gossip_data_column_sidecar( + message_id, + peer_id, + subnet_id, + column_sidecar, + seen_duration, + false, // Do not reprocess this message again. + ); + }), + }, + ); + if self + .beacon_processor_send + .try_send(WorkEvent { + drop_during_sync: false, + work: Work::Reprocess(reprocess_msg), + }) + .is_err() + { + debug!( + %unknown_block_root, + "Failed to queue data column for reprocessing" + ); + } + } } GossipDataColumnError::InvalidVariant | GossipDataColumnError::PubkeyCacheTimeout diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index c2c8577046..f3c773eb25 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -201,6 +201,7 @@ impl NetworkBeaconProcessor { subnet_id: DataColumnSubnetId, column_sidecar: Arc>, seen_timestamp: Duration, + allow_reprocess: bool, ) -> Result<(), Error> { let processor = self.clone(); let process_fn = async move { @@ -211,6 +212,7 @@ impl NetworkBeaconProcessor { subnet_id, column_sidecar, seen_timestamp, + allow_reprocess, ) .await }; diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index c0b093e254..ad98851532 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -412,6 +412,7 @@ impl TestRig { DataColumnSubnetId::from_column_index(*data_column.index(), &self.chain.spec), data_column.clone(), Duration::from_secs(0), + true, ) .unwrap(); } diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index a8e5c9ae4a..277ece0aa8 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -422,6 +422,7 @@ impl Router { subnet_id, column_sidecar, seen_timestamp, + true, ), ) } From 91456fb2186b2557937d6db88ca244dd5586998a Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 4 Jun 2026 17:24:27 +1000 Subject: [PATCH 05/15] Regression test for range sync CGC race condition (#8039) Co-Authored-By: Jimmy Chen --- .../src/peer_manager/peerdb.rs | 58 +++++++++----- .../network/src/sync/backfill_sync/mod.rs | 2 +- beacon_node/network/src/sync/tests/lookups.rs | 34 ++++++++- beacon_node/network/src/sync/tests/range.rs | 76 ++++++++++++++++++- 4 files changed, 147 insertions(+), 23 deletions(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs index 11ce785350..23f47c67a7 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs @@ -793,12 +793,39 @@ impl PeerDB { ); } - /// Updates the connection state. MUST ONLY BE USED IN TESTS. - pub fn __add_connected_peer_testing_only( + /// Adds a connected peer to the PeerDB and sets the custody subnets. + /// WARNING: This updates the connection state. MUST ONLY BE USED IN TESTS. + pub fn __add_connected_peer_with_custody_subnets( &mut self, supernode: bool, spec: &ChainSpec, enr_key: CombinedKey, + ) -> PeerId { + let peer_id = self.__add_connected_peer(supernode, enr_key, spec); + + let subnets = if supernode { + (0..spec.data_column_sidecar_subnet_count) + .map(|subnet_id| subnet_id.into()) + .collect() + } else { + let node_id = peer_id_to_node_id(&peer_id).expect("convert peer_id to node_id"); + compute_subnets_for_node::(node_id.raw(), spec.custody_requirement, spec) + .expect("should compute custody subnets") + }; + + let peer_info = self.peers.get_mut(&peer_id).expect("peer exists"); + peer_info.set_custody_subnets(subnets); + + peer_id + } + + /// Adds a connected peer to the PeerDB and updates the connection state. + /// MUST ONLY BE USED IN TESTS. + pub fn __add_connected_peer( + &mut self, + supernode: bool, + enr_key: CombinedKey, + spec: &ChainSpec, ) -> PeerId { let mut enr = Enr::builder().build(&enr_key).unwrap(); let peer_id = enr.peer_id(); @@ -835,24 +862,21 @@ impl PeerDB { }, ); - if supernode { - let peer_info = self.peers.get_mut(&peer_id).expect("peer exists"); - let all_subnets = (0..spec.data_column_sidecar_subnet_count) - .map(|subnet_id| subnet_id.into()) - .collect(); - peer_info.set_custody_subnets(all_subnets); - } else { - let peer_info = self.peers.get_mut(&peer_id).expect("peer exists"); - let node_id = peer_id_to_node_id(&peer_id).expect("convert peer_id to node_id"); - let subnets = - compute_subnets_for_node::(node_id.raw(), spec.custody_requirement, spec) - .expect("should compute custody subnets"); - peer_info.set_custody_subnets(subnets); - } - peer_id } + /// MUST ONLY BE USED IN TESTS. + pub fn __set_custody_subnets( + &mut self, + peer_id: &PeerId, + custody_subnets: HashSet, + ) -> Result<(), String> { + self.peers + .get_mut(peer_id) + .map(|info| info.set_custody_subnets(custody_subnets)) + .ok_or_else(|| "Cannot set custody subnets, peer not found".to_string()) + } + /// The connection state of the peer has been changed. Modify the peer in the db to ensure all /// variables are in sync with libp2p. /// Updating the state can lead to a `BanOperation` which needs to be processed via the peer diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 0f80138d24..f3dab7f395 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -1247,7 +1247,7 @@ mod tests { let peer_id = network_globals .peers .write() - .__add_connected_peer_testing_only( + .__add_connected_peer_with_custody_subnets( true, &beacon_chain.spec, k256::ecdsa::SigningKey::random(&mut rng).into(), diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index 3ec4d11da2..5642f7846a 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -31,13 +31,14 @@ use lighthouse_network::{ types::SyncState, }; use slot_clock::{SlotClock, TestingSlotClock}; +use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use tracing::info; use types::{ - BlobSidecar, BlockImportSource, ColumnIndex, DataColumnSidecar, ForkContext, ForkName, Hash256, - MinimalEthSpec as E, SignedBeaconBlock, Slot, + BlobSidecar, BlockImportSource, ColumnIndex, DataColumnSidecar, DataColumnSubnetId, + ForkContext, ForkName, Hash256, MinimalEthSpec as E, SignedBeaconBlock, Slot, }; const D: Duration = Duration::new(0, 0); @@ -1454,7 +1455,7 @@ impl TestRig { .network_globals .peers .write() - .__add_connected_peer_testing_only(false, &self.harness.spec, key); + .__add_connected_peer_with_custody_subnets(false, &self.harness.spec, key); // Assumes custody subnet count == column count let custody_subnets = self @@ -1485,13 +1486,38 @@ impl TestRig { .network_globals .peers .write() - .__add_connected_peer_testing_only(true, &self.harness.spec, key); + .__add_connected_peer_with_custody_subnets(true, &self.harness.spec, key); self.log(&format!( "Added new peer for testing {peer_id:?}, custody: supernode" )); peer_id } + /// Add a connected supernode peer, but without setting the peers' custody subnet. + /// This is to simulate the real behaviour where metadata is only received some time after + /// a connection is established. + pub fn new_connected_supernode_peer_no_metadata_custody_subnet(&mut self) -> PeerId { + let key = self.determinstic_key(); + self.network_globals + .peers + .write() + .__add_connected_peer(true, key, &self.harness.spec) + } + + /// Update the peer's custody subnet in PeerDB and send a `UpdatedPeerCgc` message to sync. + pub fn send_peer_cgc_update_to_sync( + &mut self, + peer_id: &PeerId, + subnets: HashSet, + ) { + self.network_globals + .peers + .write() + .__set_custody_subnets(peer_id, subnets) + .unwrap(); + self.send_sync_message(SyncMessage::UpdatedPeerCgc(*peer_id)) + } + fn determinstic_key(&mut self) -> CombinedKey { k256::ecdsa::SigningKey::random(&mut self.rng_08).into() } diff --git a/beacon_node/network/src/sync/tests/range.rs b/beacon_node/network/src/sync/tests/range.rs index 891d9d1e97..1499ae5016 100644 --- a/beacon_node/network/src/sync/tests/range.rs +++ b/beacon_node/network/src/sync/tests/range.rs @@ -27,6 +27,7 @@ use crate::sync::range_sync::RangeSyncType; use lighthouse_network::rpc::RPCError; use lighthouse_network::rpc::methods::StatusMessageV2; use lighthouse_network::{PeerId, SyncInfo}; +use std::collections::HashSet; use types::{Epoch, EthSpec, Hash256, MinimalEthSpec as E, Slot}; /// MinimalEthSpec has 8 slots per epoch @@ -50,7 +51,7 @@ impl TestRig { finalized_root: Hash256::random(), head_slot: finalized_epoch.start_slot(E::slots_per_epoch()), head_root: Hash256::random(), - earliest_available_slot: None, + earliest_available_slot: Some(Slot::new(0)), } } @@ -476,3 +477,76 @@ async fn not_enough_custody_peers_then_peers_arrive() { r.simulate(SimulateConfig::happy_path()).await; r.assert_range_sync_completed(); } + +/// This is a regression test for the following race condition scenario: +/// 1. A node is connected to 3 supernode peers: peer 1 is synced, & peer 2 and 3 are advanced. +/// 2. No metadata has been received yet (i.e. no custody info), so the node cannot start data +/// column range sync yet. +/// 3. Now peer 1 sends the CGC via metadata response, we now have one peer on all custody subnets, +/// BUT not on the finalized syncing chain. +/// 4. The node tries to `send_batch` but fails repeatedly with `NoPeers`, as there's no peer +/// that is able to serve columns for the advanced epochs. The chain is removed after 5 failed attempts. +/// 5. Now peer 2 & 3 send CGC updates, BUT because there's no syncing chain, nothing happens - +/// sync is stuck until finding new peers. +/// +/// The expected behaviour in this scenario should be: +/// 4. not finding suitable peers, chain is kept and batch remains in AwaitingDownload +/// 5. finalized sync should resume as soon as CGC updates are received from peer 2 or 3. +#[tokio::test] +async fn finalized_sync_not_enough_custody_peers_resume_after_peer_cgc_update() { + let mut r = TestRig::default(); + if !r.fork_name.fulu_enabled() { + return; + } + + // GIVEN: the node is connected to 3 supernode peers: + let advanced_epochs: usize = 2; + let sync_epochs = advanced_epochs + 3; + let sync_slots = sync_epochs * SLOTS_PER_EPOCH - 1; + r.build_chain(sync_slots).await; + r.harness.set_current_slot(Slot::new(sync_slots as u64 + 1)); + + // Peer 1 is synced (same finalized epoch), but its earliest available slot means it + // cannot serve the batches needed for this sync. + let peer_1 = r.new_connected_supernode_peer_no_metadata_custody_subnet(); + let mut remote_info = r.local_info().clone(); + remote_info.earliest_available_slot = Some(Slot::new(sync_slots as u64)); + r.send_sync_message(SyncMessage::AddPeer(peer_1, remote_info)); + + // Peer 2 is advanced (local finalized epoch + 2) + let peer_2 = r.new_connected_supernode_peer_no_metadata_custody_subnet(); + let remote_info = r.finalized_remote_info_advanced_by((advanced_epochs as u64).into()); + r.send_sync_message(SyncMessage::AddPeer(peer_2, remote_info.clone())); + // We expect a finalized chain to be created with peer 2, but no requests sent out yet due to missing custody info. + r.assert_state(RangeSyncType::Finalized); + r.assert_empty_network(); + + // Peer 3 is connected and advanced + let peer_3 = r.new_connected_supernode_peer_no_metadata_custody_subnet(); + r.send_sync_message(SyncMessage::AddPeer(peer_3, remote_info)); + // We are still in finalized sync state (now with peer 3 added) + r.assert_state(RangeSyncType::Finalized); + + for (i, p) in [peer_1, peer_2, peer_3].iter().enumerate() { + let peer_idx = i + 1; + r.log(&format!("Peer {peer_idx}: {p:?}")); + } + + // WHEN: peer 1 sends its CGC via metadata response + let all_custody_subnets = (0..r.harness.spec.data_column_sidecar_subnet_count) + .map(|i| i.into()) + .collect::>(); + r.send_peer_cgc_update_to_sync(&peer_1, all_custody_subnets.clone()); + + // We still don't have any peers on the syncing chain with custody columns (only peer 1) + // The node won't send the batch and will remain in the finalized sync state (this was failing before!) + r.assert_state(RangeSyncType::Finalized); + r.assert_empty_network(); + + // Now we receive peer 2 & 3's CGC updates, the node will resume syncing from these two peers + r.send_peer_cgc_update_to_sync(&peer_2, all_custody_subnets.clone()); + r.send_peer_cgc_update_to_sync(&peer_3, all_custody_subnets); + + r.simulate(SimulateConfig::happy_path()).await; + r.assert_range_sync_completed(); +} From d98de9f8dd757ddde73bf1e48f55d362306fc94f Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:53:05 +0200 Subject: [PATCH 06/15] Reject importing Gloas block until parent's payload is imported (#9382) Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> Co-Authored-By: Michael Sproul --- .../beacon_chain/src/block_verification.rs | 63 +++++++------------ beacon_node/beacon_chain/src/lib.rs | 2 +- .../beacon_chain/tests/block_verification.rs | 38 ++++++++++- beacon_node/beacon_chain/tests/store_tests.rs | 8 +++ consensus/fork_choice/src/fork_choice.rs | 43 +++++++++++++ consensus/fork_choice/src/lib.rs | 6 +- 6 files changed, 116 insertions(+), 44 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 33317cbda7..de592e8dae 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -70,7 +70,7 @@ use bls::{PublicKey, PublicKeyBytes}; use educe::Educe; use eth2::types::{BlockGossip, EventKind}; use execution_layer::PayloadStatus; -pub use fork_choice::{AttestationFromBlock, PayloadVerificationStatus}; +pub use fork_choice::{AttestationFromBlock, ParentImportStatus, PayloadVerificationStatus}; use metrics::TryExt; use parking_lot::RwLockReadGuard; use proto_array::Block as ProtoBlock; @@ -870,7 +870,7 @@ impl GossipVerifiedBlock { let block_epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); let (parent_block, block) = - verify_parent_block_is_known::(&fork_choice_read_lock, block)?; + verify_parent_block_and_envelope_are_known::(&fork_choice_read_lock, block)?; // [New in Gloas]: Verify bid.parent_block_root matches block.parent_root. if let Ok(bid) = block.message().body().signed_execution_payload_bid() @@ -882,13 +882,6 @@ impl GossipVerifiedBlock { }); } - // TODO(gloas) The following validation can only be completed once fork choice has been implemented: - // The block's parent execution payload (defined by bid.parent_block_hash) has been seen - // (via gossip or non-gossip sources) (a client MAY queue blocks for processing - // once the parent payload is retrieved). If execution_payload verification of block's execution - // payload parent by an execution node is complete, verify the block's execution payload - // parent (defined by bid.parent_block_hash) passes all validation. - drop(fork_choice_read_lock); // Track the number of skip slots between the block and its parent. @@ -1381,32 +1374,23 @@ impl ExecutionPendingBlock { .observe_proposal(block_root, block.message()) .map_err(|e| BlockError::BeaconChainError(Box::new(e.into())))?; - if let Some(parent) = chain + match chain .canonical_head .fork_choice_read_lock() - .get_block(&block.parent_root()) + .get_parent_import_status(block.as_block()) { - // Reject any block where the parent has an invalid payload. It's impossible for a valid - // block to descend from an invalid parent. - if parent.execution_status.is_invalid() { - return Err(BlockError::ParentExecutionPayloadInvalid { + ParentImportStatus::Imported(parent) => { + if parent.execution_status.is_invalid() { + return Err(BlockError::ParentExecutionPayloadInvalid { + parent_root: block.parent_root(), + }); + } + } + ParentImportStatus::UnknownBlock | ParentImportStatus::UnknownPayload => { + return Err(BlockError::ParentUnknown { parent_root: block.parent_root(), }); } - } else { - // Reject any block if its parent is not known to fork choice. - // - // A block that is not in fork choice is either: - // - // - Not yet imported: we should reject this block because we should only import a child - // after its parent has been fully imported. - // - Pre-finalized: if the parent block is _prior_ to finalization, we should ignore it - // because it will revert finalization. Note that the finalized block is stored in fork - // choice, so we will not reject any child of the finalized block (this is relevant during - // genesis). - return Err(BlockError::ParentUnknown { - parent_root: block.parent_root(), - }); } /* @@ -1862,19 +1846,20 @@ pub fn get_block_header_root(block_header: &SignedBeaconBlockHeader) -> Hash256 block_root } -/// Verify the parent of `block` is known, returning some information about the parent block from -/// fork choice. +/// Verify the parent block — and, for a post-Gloas FULL child, the parent payload — are known to +/// fork choice; both missing cases return `ParentUnknown`. #[allow(clippy::type_complexity)] -fn verify_parent_block_is_known( +fn verify_parent_block_and_envelope_are_known( fork_choice_read_lock: &RwLockReadGuard>, block: Arc>, ) -> Result<(ProtoBlock, Arc>), BlockError> { - if let Some(proto_block) = fork_choice_read_lock.get_block(&block.parent_root()) { - Ok((proto_block, block)) - } else { - Err(BlockError::ParentUnknown { - parent_root: block.parent_root(), - }) + match fork_choice_read_lock.get_parent_import_status(&block) { + ParentImportStatus::Imported(parent) => Ok((parent, block)), + ParentImportStatus::UnknownBlock | ParentImportStatus::UnknownPayload => { + Err(BlockError::ParentUnknown { + parent_root: block.parent_root(), + }) + } } } @@ -1901,7 +1886,7 @@ fn load_parent>( if !chain .canonical_head .fork_choice_read_lock() - .contains_block(&block.parent_root()) + .is_parent_imported(block.as_block()) { return Err(BlockError::ParentUnknown { parent_root: block.parent_root(), diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 804268a613..774920fa45 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -85,7 +85,7 @@ pub use beacon_fork_choice_store::{ }; pub use block_verification::{ BlockError, ExecutionPayloadError, ExecutionPendingBlock, GossipVerifiedBlock, - IntoExecutionPendingBlock, IntoGossipVerifiedBlock, InvalidSignature, + IntoExecutionPendingBlock, IntoGossipVerifiedBlock, InvalidSignature, ParentImportStatus, PayloadVerificationOutcome, PayloadVerificationStatus, build_blob_data_column_sidecars, get_block_root, signature_verify_chain_segment, }; diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index e0c39c350b..deadafac36 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -9,7 +9,7 @@ use beacon_chain::{ custody_context::NodeCustodyType, test_utils::{ AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType, - MakeAttestationOptions, test_spec, + MakeAttestationOptions, fork_name_from_env, test_spec, }, }; use beacon_chain::{ @@ -359,6 +359,10 @@ fn update_data_column_signed_header( #[tokio::test] async fn chain_segment_full_segment() { + // TODO(gloas): re-enable for Gloas once range sync imports payload envelopes. + if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let harness = get_harness(VALIDATOR_COUNT, NodeCustodyType::Fullnode); let (chain_segment, chain_segment_blobs) = get_chain_segment().await; store_envelopes_for_chain_segment(&chain_segment, &harness); @@ -399,6 +403,10 @@ async fn chain_segment_full_segment() { #[tokio::test] async fn chain_segment_varying_chunk_size() { + // TODO(gloas): re-enable for Gloas once range sync imports payload envelopes. + if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let (chain_segment, chain_segment_blobs) = get_chain_segment().await; let harness = get_harness(VALIDATOR_COUNT, NodeCustodyType::Fullnode); let blocks: Vec> = @@ -679,6 +687,10 @@ async fn get_invalid_sigs_harness( } #[tokio::test] async fn invalid_signature_gossip_block() { + // TODO(gloas): re-enable for Gloas once range sync imports payload envelopes. + if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { // Ensure the block will be rejected if imported on its own (without gossip checking). @@ -735,6 +747,10 @@ async fn invalid_signature_gossip_block() { #[tokio::test] async fn invalid_signature_block_proposal() { + // TODO(gloas): re-enable for Gloas once range sync imports payload envelopes. + if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; @@ -774,6 +790,10 @@ async fn invalid_signature_block_proposal() { #[tokio::test] async fn invalid_signature_randao_reveal() { + // TODO(gloas): re-enable for Gloas once range sync imports payload envelopes. + if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; @@ -802,6 +822,10 @@ async fn invalid_signature_randao_reveal() { #[tokio::test] async fn invalid_signature_proposer_slashing() { + // TODO(gloas): re-enable for Gloas once range sync imports payload envelopes. + if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; @@ -844,6 +868,10 @@ async fn invalid_signature_proposer_slashing() { #[tokio::test] async fn invalid_signature_attester_slashing() { + // TODO(gloas): re-enable for Gloas once range sync imports payload envelopes. + if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; @@ -965,6 +993,10 @@ async fn invalid_signature_attester_slashing() { #[tokio::test] async fn invalid_signature_attestation() { + // TODO(gloas): re-enable for Gloas once range sync imports payload envelopes. + if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; let mut checked_attestation = false; @@ -1090,6 +1122,10 @@ async fn invalid_signature_deposit() { #[tokio::test] async fn invalid_signature_exit() { + // TODO(gloas): re-enable for Gloas once range sync imports payload envelopes. + if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { + return; + } let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 0ac77dcfaa..b70961c499 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -3148,6 +3148,14 @@ async fn weak_subjectivity_sync_test( .store .put_payload_envelope(&wss_block_root, &envelope) .unwrap(); + + // `from_anchor` doesn't mark the anchor's payload received, so do it here; otherwise the + // first forward block (a FULL child of the anchor) would be rejected with `ParentUnknown`. + beacon_chain + .canonical_head + .fork_choice_write_lock() + .on_valid_payload_envelope_received(wss_block_root) + .unwrap(); } // Apply blocks forward to reach head. diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 2de8ce7d81..edced9b246 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -207,6 +207,18 @@ pub enum InvalidPayloadAttestation { }, } +/// The import status of a block's parent, as seen by fork choice. +#[allow(clippy::large_enum_variant)] +pub enum ParentImportStatus { + /// The parent block is imported and the child's bid commits to a parent payload known to fork + /// choice. + Imported(ProtoBlock), + /// The parent block is not known to fork choice. + UnknownBlock, + /// The parent block is known, but the child's bid commits to a payload not known to fork choice. + UnknownPayload, +} + impl From for Error { fn from(e: String) -> Self { Error::ProtoArrayStringError(e) @@ -1537,6 +1549,37 @@ where && self.is_finalized_checkpoint_or_descendant(*block_root) } + /// Returns `true` if the block's parent is imported (and, for a post-Gloas FULL child, its + /// parent's payload is imported too). See [`Self::get_parent_import_status`]. + pub fn is_parent_imported(&self, block: &SignedBeaconBlock) -> bool { + matches!( + self.get_parent_import_status(block), + ParentImportStatus::Imported(_) + ) + } + + /// Returns the import status of the parent of `block`. + /// + /// A post-Gloas FULL child also requires the parent's payload (committed to by the child's bid) + /// to have been received by fork choice. + pub fn get_parent_import_status(&self, block: &SignedBeaconBlock) -> ParentImportStatus { + if let Some(parent_block) = self.get_block(&block.parent_root()) { + let Some(parent_block_hash) = parent_block.execution_payload_block_hash else { + // Pre-Gloas parent: payload is embedded in the block, so treat as imported. + return ParentImportStatus::Imported(parent_block); + }; + if block.is_parent_block_full(parent_block_hash) + && !self.is_payload_received(&block.parent_root()) + { + ParentImportStatus::UnknownPayload + } else { + ParentImportStatus::Imported(parent_block) + } + } else { + ParentImportStatus::UnknownBlock + } + } + /// Called by the proposer to decide whether to build on the full or empty parent. pub fn should_build_on_full( &self, diff --git a/consensus/fork_choice/src/lib.rs b/consensus/fork_choice/src/lib.rs index 159eab0ec0..dcc499547b 100644 --- a/consensus/fork_choice/src/lib.rs +++ b/consensus/fork_choice/src/lib.rs @@ -4,9 +4,9 @@ mod metrics; pub use crate::fork_choice::{ AttestationFromBlock, Error, ForkChoice, ForkChoiceView, ForkchoiceUpdateParameters, - InvalidAttestation, InvalidBlock, InvalidPayloadAttestation, PayloadVerificationStatus, - PersistedForkChoice, PersistedForkChoiceV28, PersistedForkChoiceV29, QueuedAttestation, - ResetPayloadStatuses, + InvalidAttestation, InvalidBlock, InvalidPayloadAttestation, ParentImportStatus, + PayloadVerificationStatus, PersistedForkChoice, PersistedForkChoiceV28, PersistedForkChoiceV29, + QueuedAttestation, ResetPayloadStatuses, }; pub use fork_choice_store::ForkChoiceStore; pub use proto_array::{ From eeae8514b1ddb35980e8b39c1113677c0193cf59 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 4 Jun 2026 21:14:32 +0200 Subject: [PATCH 07/15] Remove unused spec field from AvailableBlock (#9411) N/A Remove unused spec field from AvailableBlock Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> --- beacon_node/beacon_chain/src/data_availability_checker.rs | 4 ---- .../src/data_availability_checker/overflow_lru_cache.rs | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 4dfb476686..9829db0f1d 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -861,8 +861,6 @@ pub struct AvailableBlock { #[educe(Hash(ignore))] /// Timestamp at which this block first became available (UNIX timestamp, time since 1970). blobs_available_timestamp: Option, - #[educe(Hash(ignore))] - pub spec: Arc, } impl AvailableBlock { @@ -952,7 +950,6 @@ impl AvailableBlock { block, blob_data: block_data, blobs_available_timestamp: None, - spec: spec.clone(), }) } @@ -1007,7 +1004,6 @@ impl AvailableBlock { } }, blobs_available_timestamp: self.blobs_available_timestamp, - spec: self.spec.clone(), }) } } diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 3e325cec02..2254728850 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -200,7 +200,6 @@ impl PendingComponents { /// must be persisted in the DB along with the block. pub fn make_available( &self, - spec: &Arc, num_expected_columns_opt: Option, ) -> Result>, AvailabilityCheckError> { let Some(CachedBlock::Executed(block)) = &self.block else { @@ -271,7 +270,6 @@ impl PendingComponents { block: block.clone(), blob_data, blobs_available_timestamp, - spec: spec.clone(), }; self.span.in_scope(|| { @@ -529,7 +527,7 @@ impl DataAvailabilityCheckerInner { num_expected_columns_opt: Option, ) -> Result, AvailabilityCheckError> { if let Some(available_block) = - pending_components.make_available(&self.spec, num_expected_columns_opt)? + pending_components.make_available(num_expected_columns_opt)? { // Explicitly drop read lock before acquiring write lock drop(pending_components); From da42d37456b556d97fb888ab5a8773457c185751 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Thu, 4 Jun 2026 17:01:20 -0700 Subject: [PATCH 08/15] Ensure PTC votes accurately reflect data availability (#9412) Co-Authored-By: Eitan Seri-Levi --- beacon_node/beacon_chain/src/beacon_chain.rs | 7 +- .../tests/attestation_production.rs | 67 ++++++++++++++++ beacon_node/http_api/tests/tests.rs | 77 +++++++++++++++++++ 3 files changed, 149 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d826895a25..73f1cd43d3 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2197,8 +2197,11 @@ impl BeaconChain { slot_start.is_some_and(|start| observed.saturating_sub(start) < payload_due) }); - // TODO(EIP-7732): Check blob data availability. For now, default to true. - let blob_data_available = true; + // A payload is only imported into fork choice if its data was available. + let blob_data_available = self + .canonical_head + .fork_choice_read_lock() + .is_payload_received(&beacon_block_root); Ok(PayloadAttestationData { beacon_block_root, diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index 1b87fc041a..862c2a9fe8 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -8,6 +8,7 @@ use beacon_chain::test_utils::{ use beacon_chain::validator_monitor::UNAGGREGATED_ATTESTATION_LAG_SLOTS; use beacon_chain::{StateSkipConfig, WhenSlotSkipped, metrics}; use bls::{AggregateSignature, Keypair}; +use slot_clock::SlotClock; use std::sync::{Arc, LazyLock}; use tree_hash::TreeHash; use types::{Attestation, EthSpec, MainnetEthSpec, RelativeEpoch, Slot}; @@ -448,3 +449,69 @@ async fn gloas_attestation_index_payload_absent() { "gloas attestation to prior slot without payload should have index=0 (payload_absent)" ); } + +/// Verify that `produce_payload_attestation_data` reports `payload_present = true` but +/// `blob_data_available = false` when the envelope was observed on but not imported +/// because its data was unavailable. +/// +/// Setup: build a chain through slot 2, then at slot 3 import only the beacon block (no +/// envelope) and mark the envelope as observed on time. +#[tokio::test] +async fn gloas_payload_attestation_seen_but_data_unavailable() { + if fork_name_from_env().is_some_and(|f| !f.gloas_enabled()) { + return; + } + + let harness = BeaconChainHarness::builder(MainnetEthSpec) + .default_spec() + .keypairs(KEYPAIRS[..].to_vec()) + .fresh_ephemeral_store() + .mock_execution_layer() + .build(); + + let chain = &harness.chain; + + harness.advance_slot(); + harness + .extend_chain( + 2, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + // Slot 3: import the beacon block but withhold its envelope. + harness.advance_slot(); + let state = harness.get_current_state(); + let (block_contents, _envelope, _new_state) = + harness.make_block_with_envelope(state, Slot::new(3)).await; + let block_root = block_contents.0.canonical_root(); + harness + .process_block(Slot::new(3), block_root, block_contents) + .await + .expect("block should import without envelope"); + + assert_eq!(chain.head_snapshot().beacon_block.slot(), Slot::new(3)); + + // Mark the envelope as observed at the start of the slot, before its deadline. + let slot_start = chain.slot_clock.start_of(Slot::new(3)).unwrap(); + chain.envelope_times_cache.write().set_time_observed( + block_root, + Slot::new(3), + slot_start, + None, + ); + + let pa_data = chain + .produce_payload_attestation_data(Slot::new(3)) + .expect("should produce payload attestation data"); + + assert!( + pa_data.payload_present, + "envelope observed before the deadline should vote payload_present=true" + ); + assert!( + !pa_data.blob_data_available, + "unimported envelope data should vote blob_data_available=false" + ); +} diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 40cb2e592f..319229d5f1 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -4970,6 +4970,10 @@ impl ApiTester { "payload attestation should report payload_present=true after publishing \ the envelope via the HTTP API (slot {slot})" ); + assert!( + pa_data.blob_data_available, + "blob_data_available should be true once the envelope is imported (slot {slot})" + ); self.chain.slot_clock.set_slot(slot.as_u64() + 1); } @@ -4977,6 +4981,71 @@ impl ApiTester { self } + /// When a payload hasn't been seen, the payload attestation data + /// must report `payload_present = false` and `blob_data_available = false`. + pub async fn test_payload_attestation_unavailable_without_envelope(self) -> Self { + if !self.chain.spec.is_gloas_scheduled() { + return self; + } + + let fork = self.chain.canonical_head.cached_head().head_fork(); + let genesis_validators_root = self.chain.genesis_validators_root; + + for _ in 0..E::slots_per_epoch() * 3 { + let slot = self.chain.slot().unwrap(); + let epoch = self.chain.epoch().unwrap(); + let fork_name = self.chain.spec.fork_name_at_slot::(slot); + + if !fork_name.gloas_enabled() { + self.chain.slot_clock.set_slot(slot.as_u64() + 1); + continue; + } + + let (sk, randao_reveal) = self + .proposer_setup(slot, epoch, &fork, genesis_validators_root) + .await; + + // Produce and publish a block, but withhold its envelope. + let (response, _metadata) = self + .client + .get_validator_blocks_v4::(slot, &randao_reveal, None, None, None, None) + .await + .unwrap(); + let block = response.data; + let block_root = block.tree_hash_root(); + + let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); + let signed_block_request = + PublishBlockRequest::try_from(Arc::new(signed_block)).unwrap(); + self.client + .post_beacon_blocks_v2(&signed_block_request, None) + .await + .unwrap(); + + let pa_data = self + .client + .get_validator_payload_attestation_data(slot) + .await + .unwrap() + .expect("expected payload attestation data for slot with block") + .into_data(); + + assert_eq!(pa_data.beacon_block_root, block_root); + assert!( + !pa_data.payload_present, + "payload_present should be false when the envelope is withheld (slot {slot})" + ); + assert!( + !pa_data.blob_data_available, + "blob_data_available should be false when the envelope is not imported (slot {slot})" + ); + + return self; + } + + self + } + pub async fn test_get_validator_payload_attestation_data_pre_gloas(self) -> Self { let slot = self.chain.slot().unwrap(); @@ -8703,6 +8772,14 @@ async fn payload_attestation_present_after_envelope_publish() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn payload_attestation_unavailable_without_envelope() { + ApiTester::new_with_hard_forks() + .await + .test_payload_attestation_unavailable_without_envelope() + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn post_beacon_pool_payload_attestations_valid() { if !fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { From 494b00a3491e2c5e281f6972aa00694b17f16722 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 5 Jun 2026 03:24:49 +0200 Subject: [PATCH 09/15] =?UTF-8?q?Fix=20O(n=C2=B2)=20find=5Fhead=20and=20st?= =?UTF-8?q?ack=20overflow=20in=20filter=5Fblock=5Ftree=20(#9090)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> Co-Authored-By: Michael Sproul --- Cargo.lock | 1 + consensus/proto_array/Cargo.toml | 8 + consensus/proto_array/benches/find_head.rs | 118 ++++++++++ consensus/proto_array/src/proto_array.rs | 212 ++++++++++++------ .../src/proto_array_fork_choice.rs | 1 + consensus/proto_array/src/ssz_container.rs | 4 +- 6 files changed, 272 insertions(+), 72 deletions(-) create mode 100644 consensus/proto_array/benches/find_head.rs diff --git a/Cargo.lock b/Cargo.lock index a9fdfe70bd..40db8876cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7020,6 +7020,7 @@ dependencies = [ name = "proto_array" version = "0.2.0" dependencies = [ + "criterion", "ethereum_ssz", "ethereum_ssz_derive", "fixed_bytes", diff --git a/consensus/proto_array/Cargo.toml b/consensus/proto_array/Cargo.toml index ee86277f9c..c424c01f6c 100644 --- a/consensus/proto_array/Cargo.toml +++ b/consensus/proto_array/Cargo.toml @@ -19,3 +19,11 @@ superstruct = { workspace = true } typenum = { workspace = true } types = { workspace = true } yaml_serde = { workspace = true } + +[dev-dependencies] +criterion = { workspace = true } +fixed_bytes = { workspace = true } + +[[bench]] +name = "find_head" +harness = false diff --git a/consensus/proto_array/benches/find_head.rs b/consensus/proto_array/benches/find_head.rs new file mode 100644 index 0000000000..98077a7f97 --- /dev/null +++ b/consensus/proto_array/benches/find_head.rs @@ -0,0 +1,118 @@ +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use fixed_bytes::FixedBytesExtended; +use proto_array::{Block, ExecutionStatus, JustifiedBalances, ProtoArrayForkChoice}; +use std::collections::BTreeSet; +use std::time::Duration; +use types::{ + AttestationShufflingId, Checkpoint, Epoch, EthSpec, ExecutionBlockHash, Hash256, + MainnetEthSpec, Slot, +}; + +fn get_root(i: u64) -> Hash256 { + Hash256::from_low_u64_be(i) +} + +fn get_hash(i: u64) -> ExecutionBlockHash { + ExecutionBlockHash::from_root(get_root(i)) +} + +/// Build a linear chain of `num_blocks` blocks. +fn build_chain(num_blocks: u64, gloas: bool) -> (ProtoArrayForkChoice, types::ChainSpec) { + let mut spec = MainnetEthSpec::default_spec(); + let gloas_fork_slot = 32; + if gloas { + spec.gloas_fork_epoch = Some(Epoch::new(1)); + } + + let finalized_checkpoint = Checkpoint { + epoch: Epoch::new(0), + root: get_root(0), + }; + let junk_shuffling_id = AttestationShufflingId::from_components(Epoch::new(0), Hash256::zero()); + + let mut fork_choice = ProtoArrayForkChoice::new::( + Slot::new(0), + Slot::new(0), + Hash256::zero(), + finalized_checkpoint, + finalized_checkpoint, + junk_shuffling_id.clone(), + junk_shuffling_id.clone(), + ExecutionStatus::Optimistic(ExecutionBlockHash::zero()), + None, + None, + 0, + &spec, + ) + .expect("should create fork choice"); + + for i in 1..=num_blocks { + let is_gloas = gloas && i >= gloas_fork_slot; + let block = Block { + slot: Slot::new(i), + root: get_root(i), + parent_root: Some(get_root(i - 1)), + state_root: Hash256::zero(), + target_root: get_root(0), + current_epoch_shuffling_id: junk_shuffling_id.clone(), + next_epoch_shuffling_id: junk_shuffling_id.clone(), + justified_checkpoint: finalized_checkpoint, + finalized_checkpoint, + execution_status: ExecutionStatus::Optimistic(ExecutionBlockHash::zero()), + unrealized_justified_checkpoint: Some(finalized_checkpoint), + unrealized_finalized_checkpoint: Some(finalized_checkpoint), + execution_payload_parent_hash: if is_gloas { + Some(get_hash(i - 1)) + } else { + None + }, + execution_payload_block_hash: if is_gloas { Some(get_hash(i)) } else { None }, + proposer_index: Some(0), + }; + + fork_choice + .process_block::(block, Slot::new(i), &spec, Duration::ZERO) + .expect("should process block"); + } + + (fork_choice, spec) +} + +fn bench_find_head(c: &mut Criterion) { + let mut group = c.benchmark_group("find_head"); + let equivocating_indices = BTreeSet::new(); + let finalized_checkpoint = Checkpoint { + epoch: Epoch::new(0), + root: get_root(0), + }; + let balances = JustifiedBalances::from_effective_balances(vec![1; 64]).unwrap(); + + // 216k = ~1 month non-finality mainnet, 518k = ~1 month non-finality Gnosis. + // Must survive extended non-finality (500k+ blocks). + for (label, gloas) in [("pre_gloas", false), ("gloas", true)] { + for &num_blocks in &[100, 1_000, 10_000, 50_000, 216_000, 518_000] { + let (mut fork_choice, spec) = build_chain(num_blocks, gloas); + + group.bench_function(BenchmarkId::new(label, num_blocks), |b| { + b.iter(|| { + fork_choice + .find_head::( + finalized_checkpoint, + finalized_checkpoint, + &balances, + Hash256::zero(), + &equivocating_indices, + Slot::new(num_blocks), + &spec, + ) + .expect("should find head") + }); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, bench_find_head); +criterion_main!(benches); diff --git a/consensus/proto_array/src/proto_array.rs b/consensus/proto_array/src/proto_array.rs index 1e3303afbb..bd15bb4599 100644 --- a/consensus/proto_array/src/proto_array.rs +++ b/consensus/proto_array/src/proto_array.rs @@ -391,6 +391,10 @@ pub struct ProtoArray { pub prune_threshold: usize, pub nodes: Vec, pub indices: HashMap, + /// Cached parent→children index. `children[i]` holds the node indices of all children of + /// node `i`. Maintained incrementally by `on_block` and `maybe_prune`. + #[serde(skip)] + pub children: Vec>, } impl ProtoArray { @@ -673,6 +677,16 @@ impl ProtoArray { self.indices.insert(node.root(), node_index); self.nodes.push(node.clone()); + // Maintain cached children index. `parent_index` is already bounds-checked above + // against `self.nodes`, and `self.children` is kept in lockstep with `self.nodes`. + self.children.push(Vec::new()); + if let Some(parent_index) = node.parent() { + self.children + .get_mut(parent_index) + .ok_or(Error::InvalidNodeIndex(parent_index))? + .push(node_index); + } + if let Some(parent_index) = node.parent() && matches!(block.execution_status, ExecutionStatus::Valid(_)) { @@ -1095,6 +1109,22 @@ impl ProtoArray { Ok((best_fc_node.root, best_fc_node.payload_status)) } + /// Rebuild the cached `self.children` index from `self.nodes`. Called once after + /// deserialization to populate the transient field. + pub fn rebuild_children_index(&mut self) -> Result<(), Error> { + let mut children = vec![Vec::new(); self.nodes.len()]; + for (i, node) in self.nodes.iter().enumerate() { + if let Some(parent_idx) = node.parent() { + children + .get_mut(parent_idx) + .ok_or(Error::InvalidNodeIndex(parent_idx))? + .push(i); + } + } + self.children = children; + Ok(()) + } + /// Spec: `get_filtered_block_tree`. /// /// Returns the set of node indices on viable branches — those with at least @@ -1105,7 +1135,7 @@ impl ProtoArray { current_slot: Slot, best_justified_checkpoint: Checkpoint, best_finalized_checkpoint: Checkpoint, - ) -> HashSet { + ) -> Result, Error> { let mut viable = HashSet::new(); self.filter_block_tree::( start_index, @@ -1113,71 +1143,88 @@ impl ProtoArray { best_justified_checkpoint, best_finalized_checkpoint, &mut viable, - ); - viable + )?; + Ok(viable) } /// Spec: `filter_block_tree`. + /// + /// Proto_array stores nodes in insertion order — children always have higher + /// indices than their parents. A single reverse pass therefore processes every + /// child before its parent, matching the spec's recursive post-order semantics + /// without recursion (required to survive 500k+ blocks of non-finality). + /// + /// The spec removes execution-invalid blocks (and their entire subtrees) from + /// `store.blocks` before running. We replicate that here with a forward pass + /// propagating `excluded` from parent to child — V29 children of an invalidated + /// V17 ancestor are excluded transitively, since V29 nodes carry no + /// `execution_status` of their own. fn filter_block_tree( &self, - node_index: usize, + start_index: usize, current_slot: Slot, best_justified_checkpoint: Checkpoint, best_finalized_checkpoint: Checkpoint, viable: &mut HashSet, - ) -> bool { - let Some(node) = self.nodes.get(node_index) else { - return false; - }; + ) -> Result<(), Error> { + // Forward pass: a node is "excluded" if it (or any ancestor down to + // `start_index`) has an invalid execution status. + let mut excluded = vec![false; self.nodes.len()]; + for i in (start_index + 1)..self.nodes.len() { + let node = self.nodes.get(i).ok_or(Error::InvalidNodeIndex(i))?; + let parent_excluded = match node.parent() { + Some(p) => *excluded.get(p).ok_or(Error::InvalidNodeIndex(p))?, + None => false, + }; + let self_invalid = node.execution_status().is_ok_and(|s| s.is_invalid()); + excluded[i] = parent_excluded || self_invalid; + } - // Skip invalid children — they aren't in store.blocks in the spec. - let children: Vec = self - .nodes - .iter() - .enumerate() - .filter(|(_, child)| { - child.parent() == Some(node_index) - && !child - .execution_status() - .is_ok_and(|status| status.is_invalid()) - }) - .map(|(i, _)| i) - .collect(); - - if !children.is_empty() { - // Evaluate ALL children (no short-circuit) to mark all viable branches. - let any_viable = children - .iter() - .map(|&child_index| { - self.filter_block_tree::( - child_index, - current_slot, - best_justified_checkpoint, - best_finalized_checkpoint, - viable, - ) - }) - .collect::>() - .into_iter() - .any(|v| v); - if any_viable { - viable.insert(node_index); - return true; + for node_index in (start_index..self.nodes.len()).rev() { + // Spec: invalid subtree removed from `store.blocks` — skip entirely. + if *excluded + .get(node_index) + .ok_or(Error::InvalidNodeIndex(node_index))? + { + continue; } - return false; - } + let node = self + .nodes + .get(node_index) + .ok_or(Error::InvalidNodeIndex(node_index))?; - // Leaf node: check viability. - if self.node_is_viable_for_head::( - node, - current_slot, - best_justified_checkpoint, - best_finalized_checkpoint, - ) { - viable.insert(node_index); - return true; + // Spec: children = [root for root in blocks if blocks[root].parent_root == block_root] + let valid_children: Vec = self + .children + .get(node_index) + .ok_or(Error::InvalidNodeIndex(node_index))? + .iter() + .copied() + .filter_map(|i| match excluded.get(i) { + Some(false) => Some(Ok(i)), + Some(true) => None, + None => Some(Err(Error::InvalidNodeIndex(i))), + }) + .collect::>()?; + + if !valid_children.is_empty() { + // Spec: if any(children): if any(filter_block_tree_result): blocks[block_root] = block + if valid_children.iter().any(|c| viable.contains(c)) { + viable.insert(node_index); + } + } else { + // Spec: leaf — check correct_justified and correct_finalized + if self.node_is_viable_for_head::( + node, + current_slot, + best_justified_checkpoint, + best_finalized_checkpoint, + ) { + viable.insert(node_index); + } + } } - false + Ok(()) } /// Spec: `get_head`. @@ -1204,7 +1251,7 @@ impl ProtoArray { current_slot, best_justified_checkpoint, best_finalized_checkpoint, - ); + )?; // Compute once rather than per-child per-level. let apply_proposer_boost = @@ -1468,25 +1515,35 @@ impl ProtoArray { } Ok(children) } else { - Ok(self - .nodes + // Spec: [root for root in blocks.keys() if blocks[root].parent_root == node.root ...] + // (cached `self.children[i]` is the same set as the spec's filtered scan). + let indices = self + .children + .get(node.proto_node_index) + .ok_or(Error::InvalidNodeIndex(node.proto_node_index))?; + indices .iter() - .enumerate() - .filter(|(_, child_node)| { - child_node.parent() == Some(node.proto_node_index) - && child_node.get_parent_payload_status() == node.payload_status + .copied() + .filter_map(|i| { + self.nodes + .get(i) + .ok_or(Error::InvalidNodeIndex(i)) + .map(|child| { + // Spec: node.payload_status == get_parent_payload_status(store, blocks[root]) + (child.get_parent_payload_status() == node.payload_status).then(|| { + ( + IndexedForkChoiceNode { + root: child.root(), + proto_node_index: i, + payload_status: PayloadStatus::Pending, + }, + child.clone(), + ) + }) + }) + .transpose() }) - .map(|(child_index, child_node)| { - ( - IndexedForkChoiceNode { - root: child_node.root(), - proto_node_index: child_index, - payload_status: PayloadStatus::Pending, - }, - child_node.clone(), - ) - }) - .collect()) + .collect() } } @@ -1617,6 +1674,19 @@ impl ProtoArray { // Drop all the nodes prior to finalization. self.nodes = self.nodes.split_off(finalized_index); + // Drop pruned entries from children index and shift all remaining indices down. + // Invariant: child_index > parent_index, and all parents we kept have + // index >= finalized_index, so every remaining child_index is also + // >= finalized_index. + self.children = self.children.split_off(finalized_index); + for children in self.children.iter_mut() { + for child_index in children.iter_mut() { + *child_index = child_index + .checked_sub(finalized_index) + .ok_or(Error::IndexOverflow("children"))?; + } + } + // Adjust the indices map. for (_root, index) in self.indices.iter_mut() { *index = index diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index 96d2302266..2c1195b491 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -514,6 +514,7 @@ impl ProtoArrayForkChoice { prune_threshold: DEFAULT_PRUNE_THRESHOLD, nodes: Vec::with_capacity(1), indices: HashMap::with_capacity(1), + children: Vec::with_capacity(1), }; let block = Block { diff --git a/consensus/proto_array/src/ssz_container.rs b/consensus/proto_array/src/ssz_container.rs index 69efb35027..ec70e88a73 100644 --- a/consensus/proto_array/src/ssz_container.rs +++ b/consensus/proto_array/src/ssz_container.rs @@ -59,11 +59,13 @@ impl TryFrom<(SszContainerV29, JustifiedBalances)> for ProtoArrayForkChoice { type Error = Error; fn try_from((from, balances): (SszContainerV29, JustifiedBalances)) -> Result { - let proto_array = ProtoArray { + let mut proto_array = ProtoArray { prune_threshold: from.prune_threshold, nodes: from.nodes, indices: from.indices.into_iter().collect::>(), + children: Vec::new(), }; + proto_array.rebuild_children_index()?; Ok(Self { proto_array, From 6698872f8a4748fd7bbe936d06ddab1cd63484fe Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Fri, 5 Jun 2026 19:27:28 +0530 Subject: [PATCH 10/15] Improve error types for envelopes (#9414) N/A Currently, we have `EnvelopeError` having a `ImportError` wrapping a `BlockError`. I feel this is extremely unintuitive because most of the envelope processing functions can simply return an `EnvelopeError` that makes sense in the function's context. It revealed further ugliness when implementing range sync in #9362 This PR does 2 main things: 1. Removes `ImportError(BlockError)` variant 2. Adds `EnvelopeError(EnvelopeError)` variant to a `BlockError`. I feel this is more natural as there can be envelope errors when we try importing a Block but envelope errors can be contained to just envelope related errors. The main blocker to doing this was `PayloadVerificationHandle` returning a `BlockError`. It uses a very small subset of `BlockError` which I extracted to its own error type which can be converted into both a BlockError and EnvelopeError. This allows us to keep most of the pure envelope processing functions to just return EnvelopeErrors while we convert it to a `BlockError` only in import paths where we need to return a consolidated `BlockError`. Co-Authored-By: Pawan Dhananjay --- beacon_node/beacon_chain/src/beacon_chain.rs | 1 + .../beacon_chain/src/block_verification.rs | 79 ++++++++++++++++--- .../beacon_chain/src/execution_payload.rs | 8 +- beacon_node/beacon_chain/src/lib.rs | 4 +- .../payload_envelope_verification/import.rs | 52 +++++------- .../src/payload_envelope_verification/mod.rs | 37 +++++++-- .../payload_notifier.rs | 13 +-- .../src/beacon/execution_payload_envelope.rs | 2 +- .../gossip_methods.rs | 17 +++- .../network_beacon_processor/sync_methods.rs | 8 +- 10 files changed, 151 insertions(+), 70 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 73f1cd43d3..dc786fb7fb 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4104,6 +4104,7 @@ impl BeaconChain { publish_fn()?; self.import_available_execution_payload_envelope(available_envelope) .await + .map_err(Into::into) } PayloadAvailability::MissingComponents(block_root) => Ok( AvailabilityProcessingStatus::MissingComponents(slot, block_root), diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index de592e8dae..49ab4a06d2 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -59,6 +59,7 @@ use crate::execution_payload::{ }; use crate::kzg_utils::blobs_to_data_column_sidecars; use crate::observed_block_producers::SeenBlock; +use crate::payload_envelope_verification::EnvelopeError; use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS; use crate::validator_pubkey_cache::ValidatorPubkeyCache; use crate::{ @@ -122,7 +123,9 @@ pub enum BlockError { /// /// It's unclear if this block is valid, but it cannot be processed without already knowing /// its parent. - ParentUnknown { parent_root: Hash256 }, + ParentUnknown { + parent_root: Hash256, + }, /// The block slot is greater than the present slot. /// /// ## Peer scoring @@ -137,7 +140,10 @@ pub enum BlockError { /// ## Peer scoring /// /// The peer has incompatible state transition logic and is faulty. - StateRootMismatch { block: Hash256, local: Hash256 }, + StateRootMismatch { + block: Hash256, + local: Hash256, + }, /// The block was a genesis block, these blocks cannot be re-imported. GenesisBlock, /// The slot is finalized, no need to import. @@ -156,7 +162,9 @@ pub enum BlockError { /// /// It's unclear if this block is valid, but it conflicts with finality and shouldn't be /// imported. - NotFinalizedDescendant { block_parent_root: Hash256 }, + NotFinalizedDescendant { + block_parent_root: Hash256, + }, /// Block is already known and valid, no need to re-import. /// /// ## Peer scoring @@ -183,7 +191,10 @@ pub enum BlockError { /// ## Peer scoring /// /// The block is invalid and the peer is faulty. - IncorrectBlockProposer { block: u64, local_shuffling: u64 }, + IncorrectBlockProposer { + block: u64, + local_shuffling: u64, + }, /// The `block.proposal_index` is not known. /// /// ## Peer scoring @@ -201,7 +212,10 @@ pub enum BlockError { /// ## Peer scoring /// /// The block is invalid and the peer is faulty. - BlockIsNotLaterThanParent { block_slot: Slot, parent_slot: Slot }, + BlockIsNotLaterThanParent { + block_slot: Slot, + parent_slot: Slot, + }, /// At least one block in the chain segment did not have it's parent root set to the root of /// the prior block. /// @@ -257,7 +271,9 @@ pub enum BlockError { /// If it's actually our fault (e.g. our execution node database is corrupt) we have bigger /// problems to worry about than losing peers, and we're doing the network a favour by /// disconnecting. - ParentExecutionPayloadInvalid { parent_root: Hash256 }, + ParentExecutionPayloadInvalid { + parent_root: Hash256, + }, /// This is a known invalid block that was listed in Lighthouses configuration. /// At the moment this error is only relevant as part of the Holesky network recovery efforts. KnownInvalidExecutionPayload(Hash256), @@ -285,10 +301,6 @@ pub enum BlockError { /// TODO: We may need to penalize the peer that gave us a potentially invalid rpc blob. /// https://github.com/sigp/lighthouse/issues/4546 AvailabilityCheck(AvailabilityCheckError), - /// The payload envelope's block root is unknown. - EnvelopeBlockRootUnknown(Hash256), - /// Optimistic sync is not supported for Gloas payload envelopes. - OptimisticSyncNotSupported { block_root: Hash256 }, /// An internal error has occurred when processing the block or sidecars. /// /// ## Peer scoring @@ -315,6 +327,7 @@ pub enum BlockError { bid_parent_root: Hash256, block_parent_root: Hash256, }, + EnvelopeError(Box), } /// Which specific signature(s) are invalid in a SignedBeaconBlock @@ -487,6 +500,50 @@ pub struct PayloadVerificationOutcome { pub payload_verification_status: PayloadVerificationStatus, } +/// The set of errors that can occur while notifying the execution layer of a new payload. +/// +/// This is deliberately narrow: notifying the EL can only fail in these two ways. The type is +/// shared by both the pre-Gloas block import path and the Gloas payload envelope path so that +/// neither pipeline has to borrow the other's error enum. It converts cleanly into both +/// [`BlockError`] and [`EnvelopeError`](crate::payload_envelope_verification::EnvelopeError) at the +/// point where the verification handle is consumed. +#[derive(Debug)] +pub enum PayloadVerificationError { + /// The execution payload was rejected by, or could not be sent to, the execution engine. + ExecutionPayloadError(ExecutionPayloadError), + /// An internal error occurred while notifying the execution layer. + BeaconChainError(Box), +} + +impl From for PayloadVerificationError { + fn from(e: ExecutionPayloadError) -> Self { + PayloadVerificationError::ExecutionPayloadError(e) + } +} + +impl From for PayloadVerificationError { + fn from(e: BeaconChainError) -> Self { + PayloadVerificationError::BeaconChainError(Box::new(e)) + } +} + +impl From for PayloadVerificationError { + fn from(e: BeaconStateError) -> Self { + PayloadVerificationError::BeaconChainError(Box::new(BeaconChainError::BeaconStateError(e))) + } +} + +impl From for BlockError { + fn from(e: PayloadVerificationError) -> Self { + match e { + PayloadVerificationError::ExecutionPayloadError(e) => { + BlockError::ExecutionPayloadError(e) + } + PayloadVerificationError::BeaconChainError(e) => BlockError::BeaconChainError(e), + } + } +} + /// Information about invalid blocks which might still be slashable despite being invalid. #[allow(clippy::enum_variant_names)] pub enum BlockSlashInfo { @@ -657,7 +714,7 @@ pub struct SignatureVerifiedBlock { /// Used to await the result of executing payload with an EE. pub type PayloadVerificationHandle = - JoinHandle>>; + JoinHandle>>; /// A wrapper around a `SignedBeaconBlock` that indicates that this block is fully verified and /// ready to import into the `BeaconChain`. The validation includes: diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index c8976fc6a8..d8cd3e0287 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -9,7 +9,7 @@ use crate::{ BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, BlockProductionError, - ExecutionPayloadError, + ExecutionPayloadError, PayloadVerificationError, }; use execution_layer::{ BlockProposalContentsType, BuilderParams, NewPayloadRequest, PayloadAttributes, @@ -104,7 +104,9 @@ impl PayloadNotifier { }) } - pub async fn notify_new_payload(self) -> Result { + pub async fn notify_new_payload( + self, + ) -> Result { if let Some(precomputed_status) = self.payload_verification_status { Ok(precomputed_status) } else { @@ -133,7 +135,7 @@ pub async fn notify_new_payload( slot: Slot, parent_beacon_block_root: Hash256, new_payload_request: NewPayloadRequest<'_, T::EthSpec>, -) -> Result { +) -> Result { let execution_layer = chain .execution_layer .as_ref() diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 774920fa45..9795d360ca 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -86,8 +86,8 @@ pub use beacon_fork_choice_store::{ pub use block_verification::{ BlockError, ExecutionPayloadError, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, IntoGossipVerifiedBlock, InvalidSignature, ParentImportStatus, - PayloadVerificationOutcome, PayloadVerificationStatus, build_blob_data_column_sidecars, - get_block_root, signature_verify_chain_segment, + PayloadVerificationError, PayloadVerificationOutcome, PayloadVerificationStatus, + build_blob_data_column_sidecars, get_block_root, signature_verify_chain_segment, }; pub use block_verification_types::AvailabilityPendingExecutedBlock; pub use block_verification_types::ExecutedBlock; diff --git a/beacon_node/beacon_chain/src/payload_envelope_verification/import.rs b/beacon_node/beacon_chain/src/payload_envelope_verification/import.rs index 73ddb43273..a10372b933 100644 --- a/beacon_node/beacon_chain/src/payload_envelope_verification/import.rs +++ b/beacon_node/beacon_chain/src/payload_envelope_verification/import.rs @@ -33,6 +33,13 @@ impl BeaconChain { /// /// Returns an `Err` if the given payload envelope was invalid, or an error was encountered during /// verification. + /// + /// Note: Returns a `BlockError` even though its an envelope processing function. + /// The reason is that this function actually imports the envelope in `check_envelope_availability_and_import` + /// which is coupled tightly with the block and data column import functions. + /// These functions return one error type for consistency across function signatures. + /// In the future, we could make the import error types more generic and then + /// this function could return an `EnvelopeError` as well. #[instrument(skip_all, fields(block_root = ?block_root, envelope_source = %envelope_source))] pub async fn process_execution_payload_envelope( self: &Arc, @@ -41,7 +48,7 @@ impl BeaconChain { notify_execution_layer: NotifyExecutionLayer, envelope_source: BlockImportSource, publish_fn: impl FnOnce() -> Result<(), EnvelopeError>, - ) -> Result { + ) -> Result { let block_slot = unverified_envelope.signed_envelope.slot(); // Set observed time if not already set. Usually this should be set by gossip or RPC, @@ -83,13 +90,7 @@ impl BeaconChain { // about what the function actually does. let executed_envelope = chain .into_executed_payload_envelope(execution_pending) - .await - .map_err(|error| match error { - BlockError::ExecutionPayloadError(error) => { - EnvelopeError::ExecutionPayloadError(error) - } - error => EnvelopeError::ImportError(error), - })?; + .await?; // Record the time it took to wait for execution layer verification. if let Some(timestamp) = slot_clock.now_duration() { @@ -100,7 +101,6 @@ impl BeaconChain { self.check_envelope_availability_and_import(executed_envelope) .await - .map_err(EnvelopeError::ImportError) }; // Verify and import the payload envelope. @@ -128,28 +128,12 @@ impl BeaconChain { Ok(status) } - Err(EnvelopeError::BeaconChainError(e)) => { - if matches!(e.as_ref(), BeaconChainError::TokioJoin(_)) { - debug!(error = ?e, "Envelope processing cancelled"); - } else { - warn!(error = ?e, "Execution payload envelope rejected"); - } - Err(EnvelopeError::BeaconChainError(e)) - } - Err(EnvelopeError::ImportError(BlockError::BeaconChainError(e))) => { - if matches!(e.as_ref(), BeaconChainError::TokioJoin(_)) { - debug!(error = ?e, "Envelope processing cancelled"); - } else { - warn!(error = ?e, "Execution payload envelope rejected"); - } - Err(EnvelopeError::ImportError(BlockError::BeaconChainError(e))) - } - Err(other) => { + Err(err) => { warn!( - reason = other.to_string(), + reason = err.to_string(), "Execution payload envelope rejected" ); - Err(other) + Err(err) } } } @@ -175,7 +159,7 @@ impl BeaconChain { async fn into_executed_payload_envelope( self: Arc, pending_envelope: ExecutionPendingEnvelope, - ) -> Result, BlockError> { + ) -> Result, EnvelopeError> { let ExecutionPendingEnvelope { signed_envelope, block_root, @@ -192,7 +176,7 @@ impl BeaconChain { .payload_verification_status .is_optimistic() { - return Err(BlockError::OptimisticSyncNotSupported { block_root }); + return Err(EnvelopeError::OptimisticSyncNotSupported { block_root }); } Ok(AvailabilityPendingExecutedEnvelope::new( @@ -206,7 +190,7 @@ impl BeaconChain { pub async fn import_available_execution_payload_envelope( self: &Arc, envelope: Box>, - ) -> Result { + ) -> Result { let AvailableExecutedEnvelope { envelope, block_root, @@ -243,13 +227,13 @@ impl BeaconChain { signed_envelope: AvailableEnvelope, block_root: Hash256, payload_verification_status: PayloadVerificationStatus, - ) -> Result { + ) -> Result { // Everything in this initial section is on the hot path for processing the envelope. // Take an upgradable read lock on fork choice so we can check if this block has already // been imported. We don't want to repeat work importing a block that is already imported. let fork_choice_reader = self.canonical_head.fork_choice_upgradable_read_lock(); if !fork_choice_reader.contains_block(&block_root) { - return Err(BlockError::EnvelopeBlockRootUnknown(block_root)); + return Err(EnvelopeError::BlockRootNotInForkChoice(block_root)); } // TODO(gloas) add defensive check to see if payload envelope is already in fork choice @@ -264,7 +248,7 @@ impl BeaconChain { // node which can be eligible for head. fork_choice .on_valid_payload_envelope_received(block_root) - .map_err(|e| BlockError::InternalError(format!("{e:?}")))?; + .map_err(|e| EnvelopeError::InternalError(format!("{e:?}")))?; // TODO(gloas) emit SSE event if the payload became the new head payload diff --git a/beacon_node/beacon_chain/src/payload_envelope_verification/mod.rs b/beacon_node/beacon_chain/src/payload_envelope_verification/mod.rs index a1e4e34eb6..a1cbac35b3 100644 --- a/beacon_node/beacon_chain/src/payload_envelope_verification/mod.rs +++ b/beacon_node/beacon_chain/src/payload_envelope_verification/mod.rs @@ -30,7 +30,7 @@ use types::{ use crate::{ BeaconChainError, BeaconChainTypes, BeaconStore, BlockError, ExecutionPayloadError, - PayloadVerificationOutcome, + PayloadVerificationError, PayloadVerificationOutcome, }; pub mod execution_pending_envelope; @@ -155,15 +155,23 @@ pub enum EnvelopeError { latest_finalized_slot: Slot, }, /// Some Beacon Chain Error - BeaconChainError(Arc), + BeaconChainError(Box), /// Some Beacon State error BeaconStateError(BeaconStateError), /// Some EnvelopeProcessingError EnvelopeProcessingError(EnvelopeProcessingError), /// Error verifying the execution payload ExecutionPayloadError(ExecutionPayloadError), - /// An error from importing the envelope. - ImportError(BlockError), + /// Optimistic sync is not supported for Gloas payload envelopes. + OptimisticSyncNotSupported { block_root: Hash256 }, + /// The envelope's beacon block was not present in fork choice at import time. + /// + /// Unlike [`EnvelopeError::BlockRootUnknown`] (raised during gossip verification, where the + /// block may simply not have arrived yet), this is raised during import where the block is + /// expected to already be present, so it indicates an internal inconsistency. + BlockRootNotInForkChoice(Hash256), + /// An internal error occurred while importing the envelope (e.g. updating fork choice). + InternalError(String), } impl std::fmt::Display for EnvelopeError { @@ -174,7 +182,7 @@ impl std::fmt::Display for EnvelopeError { impl From for EnvelopeError { fn from(e: BeaconChainError) -> Self { - EnvelopeError::BeaconChainError(Arc::new(e)) + EnvelopeError::BeaconChainError(Box::new(e)) } } @@ -192,7 +200,24 @@ impl From for EnvelopeError { impl From for EnvelopeError { fn from(e: DBError) -> Self { - EnvelopeError::BeaconChainError(Arc::new(BeaconChainError::DBError(e))) + EnvelopeError::BeaconChainError(Box::new(BeaconChainError::DBError(e))) + } +} + +impl From for BlockError { + fn from(e: EnvelopeError) -> Self { + BlockError::EnvelopeError(Box::new(e)) + } +} + +impl From for EnvelopeError { + fn from(e: PayloadVerificationError) -> Self { + match e { + PayloadVerificationError::ExecutionPayloadError(e) => { + EnvelopeError::ExecutionPayloadError(e) + } + PayloadVerificationError::BeaconChainError(e) => EnvelopeError::BeaconChainError(e), + } } } diff --git a/beacon_node/beacon_chain/src/payload_envelope_verification/payload_notifier.rs b/beacon_node/beacon_chain/src/payload_envelope_verification/payload_notifier.rs index 0bbe32525a..8a47e4689a 100644 --- a/beacon_node/beacon_chain/src/payload_envelope_verification/payload_notifier.rs +++ b/beacon_node/beacon_chain/src/payload_envelope_verification/payload_notifier.rs @@ -7,7 +7,7 @@ use tracing::warn; use types::{SignedBeaconBlock, SignedExecutionPayloadEnvelope}; use crate::{ - BeaconChain, BeaconChainTypes, BlockError, NotifyExecutionLayer, + BeaconChain, BeaconChainTypes, NotifyExecutionLayer, PayloadVerificationError, execution_payload::notify_new_payload, payload_envelope_verification::EnvelopeError, }; @@ -31,8 +31,7 @@ impl PayloadNotifier { match notify_execution_layer { NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => { - let new_payload_request = Self::build_new_payload_request(&envelope, &block) - .map_err(EnvelopeError::ImportError)?; + let new_payload_request = Self::build_new_payload_request(&envelope, &block)?; // TODO(gloas): check and test RLP block hash calculation post-Gloas if let Err(e) = new_payload_request.perform_optimistic_sync_verifications() { warn!( @@ -58,7 +57,9 @@ impl PayloadNotifier { }) } - pub async fn notify_new_payload(self) -> Result { + pub async fn notify_new_payload( + self, + ) -> Result { if let Some(precomputed_status) = self.payload_verification_status { Ok(precomputed_status) } else { @@ -71,12 +72,12 @@ impl PayloadNotifier { fn build_new_payload_request<'a>( envelope: &'a SignedExecutionPayloadEnvelope, block: &'a SignedBeaconBlock, - ) -> Result, BlockError> { + ) -> Result, PayloadVerificationError> { let bid = &block .message() .body() .signed_execution_payload_bid() - .map_err(|e| BlockError::BeaconChainError(Box::new(e.into())))? + .map_err(|e| PayloadVerificationError::BeaconChainError(Box::new(e.into())))? .message; let versioned_hashes = bid diff --git a/beacon_node/http_api/src/beacon/execution_payload_envelope.rs b/beacon_node/http_api/src/beacon/execution_payload_envelope.rs index d8813b0db5..d12e4371a4 100644 --- a/beacon_node/http_api/src/beacon/execution_payload_envelope.rs +++ b/beacon_node/http_api/src/beacon/execution_payload_envelope.rs @@ -144,7 +144,7 @@ pub async fn publish_execution_payload_envelope( PubsubMessage::ExecutionPayload(Box::new(envelope_for_gossip)), ) .map_err(|_| { - EnvelopeError::BeaconChainError(Arc::new( + EnvelopeError::BeaconChainError(Box::new( beacon_chain::BeaconChainError::UnableToPublish, )) }) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 9becfd4d59..17e569206b 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1654,9 +1654,14 @@ impl NetworkBeaconProcessor { crit!(error = %e, "Internal block gossip validation error. Availability check during gossip validation"); return None; } - Err(e @ BlockError::InternalError(_)) - | Err(e @ BlockError::EnvelopeBlockRootUnknown(_)) - | Err(e @ BlockError::OptimisticSyncNotSupported { .. }) => { + // This error variant cannot be reached when doing gossip block validation: a block has + // no envelope to verify, and `BlockError::EnvelopeError` is only ever produced by the + // envelope import pipeline. + Err(e @ BlockError::EnvelopeError(_)) => { + crit!(error = %e, "Internal block gossip validation error. Envelope error during gossip validation"); + return None; + } + Err(e @ BlockError::InternalError(_)) => { error!(error = %e, "Internal block gossip validation error"); return None; } @@ -3748,7 +3753,11 @@ impl NetworkBeaconProcessor { EnvelopeError::PriorToFinalization { .. } | EnvelopeError::BeaconChainError(_) | EnvelopeError::BeaconStateError(_) - | EnvelopeError::ImportError(_) => { + // The following variants are produced during envelope import, not gossip + // verification, so they cannot be reached here. Ignore them to be safe. + | EnvelopeError::OptimisticSyncNotSupported { .. } + | EnvelopeError::BlockRootNotInForkChoice(_) + | EnvelopeError::InternalError(_) => { self.propagate_validation_result( message_id, peer_id, diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 87d11946bd..8245b5dc0c 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -367,7 +367,7 @@ impl NetworkBeaconProcessor { ) .await } - Err(e) => Err(e), + Err(e) => Err(e.into()), }; // TODO(gloas): structured penalty classification arrives with the envelope lookup state @@ -1025,6 +1025,10 @@ impl From> for BlockProcessingR None } } + BlockError::EnvelopeError(_) => { + // TODO(gloas): penalize correctly in range sync PR + None + } // Remaining invalid blocks: penalize the block peer. Listed explicitly so a // new `BlockError` variant forces a compile error here. BlockError::FutureSlot { .. } @@ -1044,8 +1048,6 @@ impl From> for BlockProcessingR | BlockError::ParentExecutionPayloadInvalid { .. } | BlockError::KnownInvalidExecutionPayload(_) | BlockError::Slashable - | BlockError::EnvelopeBlockRootUnknown(_) - | BlockError::OptimisticSyncNotSupported { .. } | BlockError::InvalidBlobCount { .. } | BlockError::BidParentRootMismatch { .. } => block_peer_penalty(&e), }; From e78e1d38babab042771868f7a36451dcc2cd9e8d Mon Sep 17 00:00:00 2001 From: Daniel Knopik <107140945+dknopik@users.noreply.github.com> Date: Fri, 5 Jun 2026 16:50:19 +0200 Subject: [PATCH 11/15] Update libp2p (#9331) Update libp2p to benefit from recent improvements, including partial messages bugfixes. Co-Authored-By: Daniel Knopik --- Cargo.lock | 686 +++++++++++++----- Cargo.toml | 5 +- beacon_node/lighthouse_network/Cargo.toml | 2 - beacon_node/lighthouse_network/src/config.rs | 4 +- .../lighthouse_network/src/service/mod.rs | 41 +- .../lighthouse_network/src/types/pubsub.rs | 11 +- .../gossip_methods.rs | 23 +- 7 files changed, 549 insertions(+), 223 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40db8876cf..1bfc32a7a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,7 +73,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -1758,7 +1758,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", ] [[package]] @@ -1768,7 +1779,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -1973,6 +1984,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compare_fields" version = "0.1.1" @@ -2009,7 +2030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" dependencies = [ "futures-core", - "prost", + "prost 0.13.5", "prost-types", "tonic 0.12.3", "tracing-core", @@ -2028,7 +2049,7 @@ dependencies = [ "hdrhistogram", "humantime", "hyper-util", - "prost", + "prost 0.13.5", "prost-types", "serde", "serde_json", @@ -2048,7 +2069,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bb320cac8a0750d7f25280aa97b09c26edfe161164238ecbbb31092b079e735" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "proptest", "serde_core", ] @@ -2146,15 +2167,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "cpufeatures" version = "0.2.17" @@ -2164,6 +2176,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -2319,7 +2340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -2910,9 +2931,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" dependencies = [ "serde", ] @@ -3051,18 +3072,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "enum-ordinalize" version = "4.3.2" @@ -3118,7 +3127,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -3265,7 +3274,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aa93f58bb1eb3d1e556e4f408ef1dac130bad01ac37db4e7ade45de40d1c86a" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "ring", "sha2", ] @@ -3724,7 +3733,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", ] @@ -3823,11 +3832,25 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasip2", + "wasip3", +] + [[package]] name = "ghash" version = "0.5.1" @@ -4081,24 +4104,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" [[package]] -name = "hickory-proto" -version = "0.25.2" +name = "hickory-net" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" dependencies = [ "async-trait", "cfg-if", "data-encoding", - "enum-as-inner", "futures-channel", "futures-io", "futures-util", + "hickory-proto", "idna", "ipnet", - "once_cell", - "rand 0.9.2", - "ring", - "socket2 0.5.10", + "jni", + "rand 0.10.1", "thiserror 2.0.17", "tinyvec", "tokio", @@ -4107,21 +4128,46 @@ dependencies = [ ] [[package]] -name = "hickory-resolver" -version = "0.25.2" +name = "hickory-proto" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.1", + "ring", + "thiserror 2.0.17", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" dependencies = [ "cfg-if", "futures-util", + "hickory-net", "hickory-proto", "ipconfig", + "ipnet", + "jni", "moka", + "ndk-context", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.10.1", "resolv-conf", "smallvec", + "system-configuration", "thiserror 2.0.17", "tokio", "tracing", @@ -4349,7 +4395,7 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "hyper-util", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -4388,7 +4434,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -4499,6 +4545,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -4590,6 +4642,26 @@ dependencies = [ "xmltree", ] +[[package]] +name = "igd-next" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de7238d487a9aff61f81b5ab41c0a841532a115a398b5fa92a2fadd0885e2581" +dependencies = [ + "attohttpc", + "bytes", + "futures", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "log", + "rand 0.10.1", + "tokio", + "url", + "xmltree", +] + [[package]] name = "impl-codec" version = "0.6.0" @@ -4707,6 +4779,9 @@ name = "ipnet" version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +dependencies = [ + "serde", +] [[package]] name = "iri-string" @@ -4766,6 +4841,55 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.17", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version 0.4.1", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -4778,9 +4902,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ "once_cell", "wasm-bindgen", @@ -4822,7 +4946,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -4918,6 +5042,12 @@ dependencies = [ "yaml_serde", ] +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "leveldb" version = "0.8.6" @@ -4943,9 +5073,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -4981,7 +5111,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.57.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "bytes", "either", @@ -5012,7 +5142,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.7.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5022,7 +5152,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.7.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5032,7 +5162,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.44.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "either", "fnv", @@ -5044,7 +5174,7 @@ dependencies = [ "multistream-select", "parking_lot", "pin-project", - "quick-protobuf", + "prost 0.14.3", "rand 0.8.5", "rw-stream-sink", "thiserror 2.0.17", @@ -5056,7 +5186,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.45.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "futures", "hickory-resolver", @@ -5070,7 +5200,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5088,8 +5218,8 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "prometheus-client", - "quick-protobuf", - "quick-protobuf-codec", + "prost 0.14.3", + "prost-codec", "rand 0.8.5", "regex", "sha2", @@ -5100,7 +5230,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.48.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "asynchronous-codec", "either", @@ -5110,8 +5240,8 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "quick-protobuf", - "quick-protobuf-codec", + "prost 0.14.3", + "prost-codec", "smallvec", "thiserror 2.0.17", "tracing", @@ -5119,9 +5249,9 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c7892c221730ba55f7196e98b0b8ba5e04b4155651736036628e9f73ed6fc3" +checksum = "9525f3831544f7ae497bde79adf114ef127b0fbbb97edbbf692a80408636421c" dependencies = [ "asn1_der", "bs58 0.5.1", @@ -5129,7 +5259,7 @@ dependencies = [ "hkdf", "k256", "multihash", - "quick-protobuf", + "prost 0.14.3", "rand 0.8.5", "sha2", "thiserror 2.0.17", @@ -5140,7 +5270,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.49.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "futures", "hickory-proto", @@ -5150,7 +5280,7 @@ dependencies = [ "libp2p-swarm", "rand 0.8.5", "smallvec", - "socket2 0.6.3", + "socket2 0.6.4", "tokio", "tracing", ] @@ -5158,7 +5288,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.18.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "futures", "libp2p-core", @@ -5174,7 +5304,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.44.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "asynchronous-codec", "bytes", @@ -5192,7 +5322,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.47.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "asynchronous-codec", "bytes", @@ -5201,7 +5331,7 @@ dependencies = [ "libp2p-identity", "multiaddr", "multihash", - "quick-protobuf", + "prost 0.14.3", "rand 0.8.5", "snow", "static_assertions", @@ -5214,7 +5344,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.14.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "futures", "futures-timer", @@ -5225,8 +5355,8 @@ dependencies = [ "quinn", "rand 0.8.5", "ring", - "rustls 0.23.35", - "socket2 0.6.3", + "rustls 0.23.40", + "socket2 0.6.4", "thiserror 2.0.17", "tokio", "tracing", @@ -5235,7 +5365,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.48.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "either", "fnv", @@ -5258,7 +5388,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.36.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "heck", "quote", @@ -5268,14 +5398,14 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.45.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "futures", "futures-timer", "if-watch", "libc", "libp2p-core", - "socket2 0.6.3", + "socket2 0.6.4", "tokio", "tracing", ] @@ -5283,7 +5413,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.7.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "futures", "futures-rustls", @@ -5291,21 +5421,21 @@ dependencies = [ "libp2p-identity", "rcgen", "ring", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-webpki 0.103.13", "thiserror 2.0.17", - "x509-parser", - "yasna", + "x509-parser 0.18.1", + "yasna 0.6.0", ] [[package]] name = "libp2p-upnp" version = "0.7.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "futures", "futures-timer", - "igd-next", + "igd-next 0.17.1", "libp2p-core", "libp2p-swarm", "tokio", @@ -5315,7 +5445,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.48.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "either", "futures", @@ -5448,7 +5578,6 @@ dependencies = [ "if-addrs 0.14.0", "itertools 0.14.0", "libp2p", - "libp2p-gossipsub", "libp2p-mplex", "lighthouse_version", "logging", @@ -5831,9 +5960,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -5985,18 +6114,17 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.3" +version = "0.19.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" dependencies = [ - "core2", "unsigned-varint", ] [[package]] name = "multistream-select" version = "0.14.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "bytes", "futures", @@ -6006,6 +6134,12 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "netlink-packet-core" version = "0.8.1" @@ -6077,7 +6211,7 @@ dependencies = [ "futures", "genesis", "hex", - "igd-next", + "igd-next 0.16.2", "itertools 0.14.0", "k256", "kzg", @@ -6196,7 +6330,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6410,7 +6544,7 @@ dependencies = [ "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", - "prost", + "prost 0.13.5", "reqwest", "thiserror 2.0.17", "tokio", @@ -6426,7 +6560,7 @@ checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc" dependencies = [ "opentelemetry", "opentelemetry_sdk", - "prost", + "prost 0.13.5", "tonic 0.13.1", ] @@ -6492,7 +6626,7 @@ dependencies = [ "sha1", "sha2", "thiserror 2.0.17", - "x509-parser", + "x509-parser 0.17.0", ] [[package]] @@ -6624,18 +6758,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -6754,7 +6888,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -6766,7 +6900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -6827,6 +6961,17 @@ dependencies = [ "termtree", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "pretty_reqwest_error" version = "0.1.0" @@ -6991,7 +7136,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive 0.14.3", +] + +[[package]] +name = "prost-codec" +version = "0.4.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" +dependencies = [ + "asynchronous-codec", + "bytes", + "prost 0.14.3", + "thiserror 2.0.17", + "unsigned-varint", ] [[package]] @@ -7007,13 +7174,26 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "prost-types" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost", + "prost 0.13.5", ] [[package]] @@ -7058,26 +7238,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-protobuf" -version = "0.8.1" -source = "git+https://github.com/sigp/quick-protobuf.git?rev=87c4ccb9bb2af494de375f5f6c62850badd26304#87c4ccb9bb2af494de375f5f6c62850badd26304" -dependencies = [ - "byteorder", -] - -[[package]] -name = "quick-protobuf-codec" -version = "0.4.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" -dependencies = [ - "asynchronous-codec", - "bytes", - "quick-protobuf", - "thiserror 2.0.17", - "unsigned-varint", -] - [[package]] name = "quinn" version = "0.11.9" @@ -7091,8 +7251,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.35", - "socket2 0.6.3", + "rustls 0.23.40", + "socket2 0.5.10", "thiserror 2.0.17", "tokio", "tracing", @@ -7111,7 +7271,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash 2.1.1", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "slab", "thiserror 2.0.17", @@ -7129,9 +7289,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7149,6 +7309,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "r2d2" version = "0.8.10" @@ -7200,6 +7366,17 @@ dependencies = [ "serde", ] +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20 0.10.0", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -7239,6 +7416,12 @@ dependencies = [ "serde", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_xorshift" version = "0.3.0" @@ -7305,7 +7488,7 @@ dependencies = [ "ring", "rustls-pki-types", "time", - "yasna", + "yasna 0.5.2", ] [[package]] @@ -7408,7 +7591,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "serde", "serde_json", @@ -7653,7 +7836,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7672,9 +7855,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "log", "once_cell", @@ -7759,7 +7942,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.5.0" -source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" +source = "git+https://github.com/libp2p/rust-libp2p.git#3e72d4c071d5ec8815d2f6f7ee3602600ff51798" dependencies = [ "futures", "pin-project", @@ -8088,7 +8271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -8099,7 +8282,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -8181,6 +8364,22 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version 0.4.1", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "similar" version = "2.7.0" @@ -8355,9 +8554,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.60.2", @@ -8705,7 +8904,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -8918,9 +9117,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -8928,7 +9127,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2 0.6.4", "tokio-macros", "tracing", "windows-sys 0.61.2", @@ -8936,9 +9135,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -8962,7 +9161,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.35", + "rustls 0.23.40", "tokio", ] @@ -9043,7 +9242,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "socket2 0.5.10", "tokio", "tokio-stream", @@ -9070,7 +9269,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "rustls-native-certs", "tokio", "tokio-rustls 0.26.4", @@ -9832,14 +10031,23 @@ version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.46.0", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ "cfg-if", "once_cell", @@ -9850,11 +10058,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-util", "js-sys", "once_cell", "wasm-bindgen", @@ -9863,9 +10072,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9873,9 +10082,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ "bumpalo", "proc-macro2", @@ -9886,13 +10095,35 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.12.1", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -9906,6 +10137,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.15.5", + "indexmap 2.12.1", + "semver 1.0.27", +] + [[package]] name = "wasmtimer" version = "0.4.3" @@ -9922,9 +10165,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", @@ -10016,7 +10259,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -10403,6 +10646,94 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap 2.12.1", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.10.0", + "indexmap 2.12.1", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.12.1", + "log", + "semver 1.0.27", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "workspace_members" version = "0.1.0" @@ -10466,6 +10797,23 @@ dependencies = [ "time", ] +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.17", + "time", +] + [[package]] name = "xdelta3" version = "0.1.5" @@ -10559,6 +10907,12 @@ dependencies = [ "time", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" + [[package]] name = "yoke" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index 71398530fe..50b1733232 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,7 +161,7 @@ initialized_validators = { path = "validator_client/initialized_validators" } int_to_bytes = { path = "consensus/int_to_bytes" } itertools = "0.14" kzg = { path = "crypto/kzg" } -libp2p = { git = "https://github.com/libp2p/rust-libp2p.git", default-features = false, features = ["identify", "yamux", "noise", "dns", "tcp", "tokio", "secp256k1", "macros", "metrics", "quic", "upnp", "gossipsub"] } +libp2p = { git = "https://github.com/libp2p/rust-libp2p.git", default-features = false, features = ["identify", "yamux", "noise", "dns", "tcp", "tokio", "secp256k1", "macros", "metrics", "quic", "upnp", "gossipsub", "gossipsub-partial-messages"] } libsecp256k1 = "0.7" lighthouse_network = { path = "beacon_node/lighthouse_network" } lighthouse_validator_store = { path = "validator_client/lighthouse_validator_store" } @@ -273,6 +273,3 @@ incremental = false inherits = "release" debug = true -[patch.crates-io] -quick-protobuf = { git = "https://github.com/sigp/quick-protobuf.git", rev = "87c4ccb9bb2af494de375f5f6c62850badd26304" } - diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index 44af8d7006..659886f0f1 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -21,8 +21,6 @@ ethereum_ssz_derive = { workspace = true } fixed_bytes = { workspace = true } fnv = { workspace = true } futures = { workspace = true } -# Enable partial messages feature -gossipsub = { package = "libp2p-gossipsub", git = "https://github.com/libp2p/rust-libp2p.git", features = ["partial_messages"] } hex = { workspace = true } if-addrs = "0.14" itertools = { workspace = true } diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 4d4d91a456..8f7c1dd8de 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -508,7 +508,9 @@ pub fn gossipsub_config( .fanout_ttl(Duration::from_secs(60)) .history_length(12) .flood_publish(false) - .max_messages_per_rpc(Some(500)) // Responses to IWANT can be quite large + .max_publish_messages(500) // Responses to IWANT can be quite large + .max_control_messages_sent(500) + .max_control_message_size(128 << 10) // 128KB .history_gossip(load.history_gossip) .validate_messages() // require validation before propagation .validation_mode(gossipsub::ValidationMode::Anonymous) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index f5e2442f86..097736a010 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -311,11 +311,8 @@ impl Network { let fork = ctx.chain_spec.fork_name_at_epoch(epoch); all_topics_at_fork::(fork, &ctx.chain_spec) .into_iter() - .map(|topic| { - Topic::new(GossipTopic::new(topic, GossipEncoding::default(), digest)) - .into() - }) - .collect::>() + .map(|topic| GossipTopic::new(topic, GossipEncoding::default(), digest)) + .collect::>() }) .collect::>(); @@ -368,11 +365,20 @@ impl Network { gossipsub.add_explicit_peer(&PeerId::from(explicit_peer.clone())); } + // Register topics with enabled partial messages + for topic in all_topics_for_digests.iter().flatten() { + if topic.kind().use_partial_messages(&config) { + gossipsub.enable_partials_for_topic(Topic::new(topic.clone()).hash(), true); + } + } + // If we are using metrics, then register which topics we want to make sure to keep // track of if ctx.libp2p_registry.is_some() { for topics in all_topics_for_digests { - gossipsub.register_topics_for_metrics(topics); + gossipsub.register_topics_for_metrics( + topics.into_iter().map(|t| Topic::new(t).hash()).collect(), + ); } } @@ -823,18 +829,9 @@ impl Network { .write() .insert(topic.clone()); - let partial = topic - .kind() - .use_partial_messages(self.network_globals.config.as_ref()); let topic: Topic = topic.into(); - let subscribe_result = if partial { - self.gossipsub_mut().subscribe_partial(&topic, true) - } else { - self.gossipsub_mut().subscribe(&topic) - }; - - match subscribe_result { + match self.gossipsub_mut().subscribe(&topic) { Err(e) => { warn!(%topic, error = ?e, "Failed to subscribe to topic"); false @@ -1381,9 +1378,9 @@ impl Network { /* Sub-behaviour event handling functions */ /// Handle a gossipsub event. - fn inject_gs_event(&mut self, event: gossipsub::Event) -> Option> { + fn inject_gs_event(&mut self, event: Event) -> Option> { match event { - gossipsub::Event::Message { + Event::Message { propagation_source, message_id: id, message: gs_msg, @@ -1461,7 +1458,7 @@ impl Network { } } } - gossipsub::Event::Subscribed { peer_id, topic } => { + Event::Subscribed { peer_id, topic, .. } => { if let Ok(topic) = GossipTopic::decode(topic.as_str()) { if let Some(subnet_id) = topic.subnet_id() { self.network_globals @@ -1513,7 +1510,7 @@ impl Network { } } } - gossipsub::Event::Unsubscribed { peer_id, topic } => { + Event::Unsubscribed { peer_id, topic } => { if let Some(subnet_id) = subnet_from_topic_hash(&topic) { self.network_globals .peers @@ -1521,7 +1518,7 @@ impl Network { .remove_subscription(&peer_id, &subnet_id); } } - gossipsub::Event::GossipsubNotSupported { peer_id } => { + Event::GossipsubNotSupported { peer_id } => { debug!(%peer_id, "Peer does not support gossipsub"); self.peer_manager_mut().report_peer( &peer_id, @@ -1531,7 +1528,7 @@ impl Network { "does_not_support_gossipsub", ); } - gossipsub::Event::SlowPeer { + Event::SlowPeer { peer_id, failed_messages, } => { diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 043d1cfb88..d486ca5129 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -1,7 +1,7 @@ //! Handles the encoding and decoding of pubsub messages. use crate::types::{GossipEncoding, GossipKind, GossipTopic}; -use gossipsub::TopicHash; +use libp2p::gossipsub::{DataTransform, Message, RawMessage, TopicHash}; use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; @@ -73,12 +73,9 @@ impl SnappyTransform { } } -impl gossipsub::DataTransform for SnappyTransform { +impl DataTransform for SnappyTransform { // Provides the snappy decompression from RawGossipsubMessages - fn inbound_transform( - &self, - raw_message: gossipsub::RawMessage, - ) -> Result { + fn inbound_transform(&self, raw_message: RawMessage) -> Result { // first check the size of the compressed payload if raw_message.data.len() > self.max_compressed_len { return Err(Error::new( @@ -99,7 +96,7 @@ impl gossipsub::DataTransform for SnappyTransform { let decompressed_data = decoder.decompress_vec(&raw_message.data)?; // Build the GossipsubMessage struct - Ok(gossipsub::Message { + Ok(Message { source: raw_message.source, data: decompressed_data, sequence_number: raw_message.sequence_number, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 17e569206b..68b41cab5e 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -171,17 +171,6 @@ impl FailedAtt { } } -/// `MessageAcceptance` doesn't implement clone so we do a manual match here. -/// TODO: remove this once `Clone` is available on this type: -/// https://github.com/libp2p/rust-libp2p/pull/6445 -fn clone_message_acceptance(a: &MessageAcceptance) -> MessageAcceptance { - match a { - MessageAcceptance::Accept => MessageAcceptance::Accept, - MessageAcceptance::Reject => MessageAcceptance::Reject, - MessageAcceptance::Ignore => MessageAcceptance::Ignore, - } -} - impl NetworkBeaconProcessor { /* Auxiliary functions */ @@ -2018,11 +2007,7 @@ impl NetworkBeaconProcessor { } }; - self.propagate_validation_result( - message_id, - peer_id, - clone_message_acceptance(&validation_result), - ); + self.propagate_validation_result(message_id, peer_id, validation_result); if let Some(slashing) = verified_slashing_opt { metrics::inc_counter(&metrics::BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL); @@ -2084,11 +2069,7 @@ impl NetworkBeaconProcessor { } }; - self.propagate_validation_result( - message_id, - peer_id, - clone_message_acceptance(&validation_result), - ); + self.propagate_validation_result(message_id, peer_id, validation_result); if let Some(slashing) = verified_slashing_opt { metrics::inc_counter(&metrics::BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL); From 42e678189ce75bcd34b8b74d40e54a3f87ed001b Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Fri, 5 Jun 2026 13:16:06 -0700 Subject: [PATCH 12/15] Update gloas api routes to match updated spec (#9418) tldr the routes got pluralized https://github.com/ethereum/beacon-APIs/pull/613 Co-Authored-By: Eitan Seri-Levi --- ...yload_bid.rs => execution_payload_bids.rs} | 12 ++--- ...lope.rs => execution_payload_envelopes.rs} | 20 +++---- beacon_node/http_api/src/beacon/mod.rs | 4 +- beacon_node/http_api/src/lib.rs | 48 ++++++++--------- ...lope.rs => execution_payload_envelopes.rs} | 6 +-- beacon_node/http_api/src/validator/mod.rs | 2 +- beacon_node/http_api/tests/tests.rs | 26 ++++----- common/eth2/src/lib.rs | 54 +++++++++---------- .../validator_services/src/block_service.rs | 4 +- 9 files changed, 88 insertions(+), 88 deletions(-) rename beacon_node/http_api/src/beacon/{execution_payload_bid.rs => execution_payload_bids.rs} (91%) rename beacon_node/http_api/src/beacon/{execution_payload_envelope.rs => execution_payload_envelopes.rs} (95%) rename beacon_node/http_api/src/validator/{execution_payload_envelope.rs => execution_payload_envelopes.rs} (95%) diff --git a/beacon_node/http_api/src/beacon/execution_payload_bid.rs b/beacon_node/http_api/src/beacon/execution_payload_bids.rs similarity index 91% rename from beacon_node/http_api/src/beacon/execution_payload_bid.rs rename to beacon_node/http_api/src/beacon/execution_payload_bids.rs index f6041b55c8..856670aa94 100644 --- a/beacon_node/http_api/src/beacon/execution_payload_bid.rs +++ b/beacon_node/http_api/src/beacon/execution_payload_bids.rs @@ -14,8 +14,8 @@ use tracing::{debug, warn}; use types::SignedExecutionPayloadBid; use warp::{Filter, Rejection, Reply, hyper::Body, hyper::Response}; -// POST /eth/v1/beacon/execution_payload_bid (SSZ) -pub(crate) fn post_beacon_execution_payload_bid_ssz( +// POST /eth/v1/beacon/execution_payload_bids (SSZ) +pub(crate) fn post_beacon_execution_payload_bids_ssz( eth_v1: EthV1Filter, task_spawner_filter: TaskSpawnerFilter, chain_filter: ChainFilter, @@ -23,7 +23,7 @@ pub(crate) fn post_beacon_execution_payload_bid_ssz( ) -> ResponseFilter { eth_v1 .and(warp::path("beacon")) - .and(warp::path("execution_payload_bid")) + .and(warp::path("execution_payload_bids")) .and(warp::path::end()) .and(warp::body::bytes()) .and(task_spawner_filter) @@ -46,8 +46,8 @@ pub(crate) fn post_beacon_execution_payload_bid_ssz( .boxed() } -// POST /eth/v1/beacon/execution_payload_bid -pub(crate) fn post_beacon_execution_payload_bid( +// POST /eth/v1/beacon/execution_payload_bids +pub(crate) fn post_beacon_execution_payload_bids( eth_v1: EthV1Filter, task_spawner_filter: TaskSpawnerFilter, chain_filter: ChainFilter, @@ -55,7 +55,7 @@ pub(crate) fn post_beacon_execution_payload_bid( ) -> ResponseFilter { eth_v1 .and(warp::path("beacon")) - .and(warp::path("execution_payload_bid")) + .and(warp::path("execution_payload_bids")) .and(warp::path::end()) .and(warp::body::json()) .and(task_spawner_filter) diff --git a/beacon_node/http_api/src/beacon/execution_payload_envelope.rs b/beacon_node/http_api/src/beacon/execution_payload_envelopes.rs similarity index 95% rename from beacon_node/http_api/src/beacon/execution_payload_envelope.rs rename to beacon_node/http_api/src/beacon/execution_payload_envelopes.rs index d12e4371a4..f8ab8cddc8 100644 --- a/beacon_node/http_api/src/beacon/execution_payload_envelope.rs +++ b/beacon_node/http_api/src/beacon/execution_payload_envelopes.rs @@ -24,8 +24,8 @@ use warp::{ hyper::{Body, Response}, }; -// POST beacon/execution_payload_envelope (SSZ) -pub(crate) fn post_beacon_execution_payload_envelope_ssz( +// POST beacon/execution_payload_envelopes (SSZ) +pub(crate) fn post_beacon_execution_payload_envelopes_ssz( eth_v1: EthV1Filter, task_spawner_filter: TaskSpawnerFilter, chain_filter: ChainFilter, @@ -33,7 +33,7 @@ pub(crate) fn post_beacon_execution_payload_envelope_ssz( ) -> ResponseFilter { eth_v1 .and(warp::path("beacon")) - .and(warp::path("execution_payload_envelope")) + .and(warp::path("execution_payload_envelopes")) .and(warp::path::end()) .and(warp::body::bytes()) .and(task_spawner_filter) @@ -57,8 +57,8 @@ pub(crate) fn post_beacon_execution_payload_envelope_ssz( .boxed() } -// POST beacon/execution_payload_envelope -pub(crate) fn post_beacon_execution_payload_envelope( +// POST beacon/execution_payload_envelopes +pub(crate) fn post_beacon_execution_payload_envelopes( eth_v1: EthV1Filter, task_spawner_filter: TaskSpawnerFilter, chain_filter: ChainFilter, @@ -66,7 +66,7 @@ pub(crate) fn post_beacon_execution_payload_envelope( ) -> ResponseFilter { eth_v1 .and(warp::path("beacon")) - .and(warp::path("execution_payload_envelope")) + .and(warp::path("execution_payload_envelopes")) .and(warp::path::end()) .and(warp::body::json()) .and(task_spawner_filter.clone()) @@ -85,7 +85,7 @@ pub(crate) fn post_beacon_execution_payload_envelope( .boxed() } /// Publishes a signed execution payload envelope to the network. Implements -/// `POST /eth/v1/beacon/execution_payload_envelope` per the in-flight beacon-APIs PR +/// `POST /eth/v1/beacon/execution_payload_envelopes` per the in-flight beacon-APIs PR /// . pub async fn publish_execution_payload_envelope( envelope: SignedExecutionPayloadEnvelope, @@ -292,8 +292,8 @@ fn build_gloas_data_columns( } // TODO(gloas): add tests for this endpoint once we support importing payloads into the db -// GET beacon/execution_payload_envelope/{block_id} -pub(crate) fn get_beacon_execution_payload_envelope( +// GET beacon/execution_payload_envelopes/{block_id} +pub(crate) fn get_beacon_execution_payload_envelopes( eth_v1: EthV1Filter, block_id_or_err: impl Filter + Clone @@ -305,7 +305,7 @@ pub(crate) fn get_beacon_execution_payload_envelope( ) -> ResponseFilter { eth_v1 .and(warp::path("beacon")) - .and(warp::path("execution_payload_envelope")) + .and(warp::path("execution_payload_envelopes")) .and(block_id_or_err) .and(warp::path::end()) .and(task_spawner_filter) diff --git a/beacon_node/http_api/src/beacon/mod.rs b/beacon_node/http_api/src/beacon/mod.rs index db0062c14f..31c4077540 100644 --- a/beacon_node/http_api/src/beacon/mod.rs +++ b/beacon_node/http_api/src/beacon/mod.rs @@ -1,4 +1,4 @@ -pub mod execution_payload_bid; -pub mod execution_payload_envelope; +pub mod execution_payload_bids; +pub mod execution_payload_envelopes; pub mod pool; pub mod states; diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index ff88c12925..94f2e3f1df 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -36,12 +36,12 @@ mod validator_inclusion; mod validators; mod version; -use crate::beacon::execution_payload_bid::{ - post_beacon_execution_payload_bid, post_beacon_execution_payload_bid_ssz, +use crate::beacon::execution_payload_bids::{ + post_beacon_execution_payload_bids, post_beacon_execution_payload_bids_ssz, }; -use crate::beacon::execution_payload_envelope::{ - get_beacon_execution_payload_envelope, post_beacon_execution_payload_envelope, - post_beacon_execution_payload_envelope_ssz, +use crate::beacon::execution_payload_envelopes::{ + get_beacon_execution_payload_envelopes, post_beacon_execution_payload_envelopes, + post_beacon_execution_payload_envelopes_ssz, }; use crate::beacon::pool::*; use crate::caches::DEFAULT_HISTORICAL_COMMITTEE_CACHE_SIZE; @@ -101,7 +101,7 @@ use types::{ BeaconStateError, Checkpoint, ConfigAndPreset, Epoch, EthSpec, ForkName, Hash256, SignedBlindedBeaconBlock, }; -use validator::execution_payload_envelope::get_validator_execution_payload_envelope; +use validator::execution_payload_envelopes::get_validator_execution_payload_envelopes; use version::{ ResponseIncludesVersion, V1, V2, add_consensus_version_header, add_ssz_content_type_header, execution_optimistic_finalized_beacon_response, inconsistent_fork_rejection, @@ -1542,40 +1542,40 @@ pub fn serve( network_tx_filter.clone(), ); - // POST beacon/execution_payload_envelope - let post_beacon_execution_payload_envelope = post_beacon_execution_payload_envelope( + // POST beacon/execution_payload_envelopes + let post_beacon_execution_payload_envelopes = post_beacon_execution_payload_envelopes( eth_v1.clone(), task_spawner_filter.clone(), chain_filter.clone(), network_tx_filter.clone(), ); - // POST beacon/execution_payload_envelope (SSZ) - let post_beacon_execution_payload_envelope_ssz = post_beacon_execution_payload_envelope_ssz( + // POST beacon/execution_payload_envelopes (SSZ) + let post_beacon_execution_payload_envelopes_ssz = post_beacon_execution_payload_envelopes_ssz( eth_v1.clone(), task_spawner_filter.clone(), chain_filter.clone(), network_tx_filter.clone(), ); - // POST beacon/execution_payload_bid - let post_beacon_execution_payload_bid = post_beacon_execution_payload_bid( + // POST beacon/execution_payload_bids + let post_beacon_execution_payload_bids = post_beacon_execution_payload_bids( eth_v1.clone(), task_spawner_filter.clone(), chain_filter.clone(), network_tx_filter.clone(), ); - // POST beacon/execution_payload_bid (SSZ) - let post_beacon_execution_payload_bid_ssz = post_beacon_execution_payload_bid_ssz( + // POST beacon/execution_payload_bids (SSZ) + let post_beacon_execution_payload_bids_ssz = post_beacon_execution_payload_bids_ssz( eth_v1.clone(), task_spawner_filter.clone(), chain_filter.clone(), network_tx_filter.clone(), ); - // GET beacon/execution_payload_envelope/{block_id} - let get_beacon_execution_payload_envelope = get_beacon_execution_payload_envelope( + // GET beacon/execution_payload_envelopes/{block_id} + let get_beacon_execution_payload_envelopes = get_beacon_execution_payload_envelopes( eth_v1.clone(), block_id_or_err, task_spawner_filter.clone(), @@ -2584,8 +2584,8 @@ pub fn serve( task_spawner_filter.clone(), ); - // GET validator/execution_payload_envelope/{slot}/{builder_index} - let get_validator_execution_payload_envelope = get_validator_execution_payload_envelope( + // GET validator/execution_payload_envelopes/{slot}/{builder_index} + let get_validator_execution_payload_envelopes = get_validator_execution_payload_envelopes( eth_v1.clone(), chain_filter.clone(), not_while_syncing_filter.clone(), @@ -3401,7 +3401,7 @@ pub fn serve( .uor(get_beacon_block_root) .uor(get_blob_sidecars) .uor(get_blobs) - .uor(get_beacon_execution_payload_envelope) + .uor(get_beacon_execution_payload_envelopes) .uor(get_beacon_pool_attestations) .uor(get_beacon_pool_attester_slashings) .uor(get_beacon_pool_proposer_slashings) @@ -3425,7 +3425,7 @@ pub fn serve( .uor(get_validator_duties_proposer) .uor(get_validator_blocks) .uor(get_validator_blinded_blocks) - .uor(get_validator_execution_payload_envelope) + .uor(get_validator_execution_payload_envelopes) .uor(get_validator_attestation_data) .uor(get_validator_payload_attestation_data) .uor(get_validator_aggregate_attestation) @@ -3463,8 +3463,8 @@ pub fn serve( .uor(post_beacon_blocks_v2_ssz) .uor(post_beacon_blinded_blocks_ssz) .uor(post_beacon_blinded_blocks_v2_ssz) - .uor(post_beacon_execution_payload_envelope_ssz) - .uor(post_beacon_execution_payload_bid_ssz) + .uor(post_beacon_execution_payload_envelopes_ssz) + .uor(post_beacon_execution_payload_bids_ssz) .uor(post_beacon_pool_payload_attestations_ssz) .uor(post_validator_proposer_preferences_ssz), ) @@ -3480,8 +3480,8 @@ pub fn serve( .uor(post_beacon_pool_payload_attestations) .uor(post_beacon_pool_bls_to_execution_changes) .uor(post_validator_proposer_preferences) - .uor(post_beacon_execution_payload_envelope) - .uor(post_beacon_execution_payload_bid) + .uor(post_beacon_execution_payload_envelopes) + .uor(post_beacon_execution_payload_bids) .uor(post_beacon_state_validators) .uor(post_beacon_state_validator_balances) .uor(post_beacon_state_validator_identities) diff --git a/beacon_node/http_api/src/validator/execution_payload_envelope.rs b/beacon_node/http_api/src/validator/execution_payload_envelopes.rs similarity index 95% rename from beacon_node/http_api/src/validator/execution_payload_envelope.rs rename to beacon_node/http_api/src/validator/execution_payload_envelopes.rs index 7a7a430414..3a20b37c9b 100644 --- a/beacon_node/http_api/src/validator/execution_payload_envelope.rs +++ b/beacon_node/http_api/src/validator/execution_payload_envelopes.rs @@ -12,8 +12,8 @@ use types::Slot; use warp::http::Response; use warp::{Filter, Rejection}; -// GET validator/execution_payload_envelope/{slot} -pub fn get_validator_execution_payload_envelope( +// GET validator/execution_payload_envelopes/{slot} +pub fn get_validator_execution_payload_envelopes( eth_v1: EthV1Filter, chain_filter: ChainFilter, not_while_syncing_filter: NotWhileSyncingFilter, @@ -21,7 +21,7 @@ pub fn get_validator_execution_payload_envelope( ) -> ResponseFilter { eth_v1 .and(warp::path("validator")) - .and(warp::path("execution_payload_envelope")) + .and(warp::path("execution_payload_envelopes")) .and(warp::path::param::().or_else(|_| async { Err(warp_utils::reject::custom_bad_request( "Invalid slot".to_string(), diff --git a/beacon_node/http_api/src/validator/mod.rs b/beacon_node/http_api/src/validator/mod.rs index 77df94bc36..8639914774 100644 --- a/beacon_node/http_api/src/validator/mod.rs +++ b/beacon_node/http_api/src/validator/mod.rs @@ -36,7 +36,7 @@ use types::{ use warp::{Filter, Rejection, Reply}; use warp_utils::reject::convert_rejection; -pub mod execution_payload_envelope; +pub mod execution_payload_envelopes; /// Uses the `chain.validator_pubkey_cache` to resolve a pubkey to a validator /// index and then ensures that the validator exists in the given `state`. diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 319229d5f1..455a957337 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -3085,12 +3085,12 @@ impl ApiTester { } /// JSON bid with a valid structure reaches gossip verification and is rejected with 400. - pub async fn test_post_beacon_execution_payload_bid_json(self) -> Self { + pub async fn test_post_beacon_execution_payload_bids_json(self) -> Self { let (bid, fork_name) = self.make_signed_execution_payload_bid(); let result = self .client - .post_beacon_execution_payload_bid(&bid, fork_name) + .post_beacon_execution_payload_bids(&bid, fork_name) .await; assert!( @@ -3102,12 +3102,12 @@ impl ApiTester { } /// SSZ bid with a valid structure reaches gossip verification and is rejected with 400. - pub async fn test_post_beacon_execution_payload_bid_ssz(self) -> Self { + pub async fn test_post_beacon_execution_payload_bids_ssz(self) -> Self { let (bid, fork_name) = self.make_signed_execution_payload_bid(); let result = self .client - .post_beacon_execution_payload_bid_ssz(&bid, fork_name) + .post_beacon_execution_payload_bids_ssz(&bid, fork_name) .await; assert!( @@ -4433,7 +4433,7 @@ impl ApiTester { let envelope = self .client - .get_validator_execution_payload_envelope::(slot) + .get_validator_execution_payload_envelopes::(slot) .await .unwrap() .data; @@ -4452,7 +4452,7 @@ impl ApiTester { let signed_envelope = self.sign_envelope(envelope, &sk, epoch, &fork, genesis_validators_root); self.client - .post_beacon_execution_payload_envelope(&signed_envelope, fork_name) + .post_beacon_execution_payload_envelopes(&signed_envelope, fork_name) .await .unwrap(); @@ -4495,7 +4495,7 @@ impl ApiTester { let envelope = self .client - .get_validator_execution_payload_envelope_ssz::(slot) + .get_validator_execution_payload_envelopes_ssz::(slot) .await .unwrap(); @@ -4513,7 +4513,7 @@ impl ApiTester { let signed_envelope = self.sign_envelope(envelope, &sk, epoch, &fork, genesis_validators_root); self.client - .post_beacon_execution_payload_envelope_ssz(&signed_envelope, fork_name) + .post_beacon_execution_payload_envelopes_ssz(&signed_envelope, fork_name) .await .unwrap(); @@ -4942,7 +4942,7 @@ impl ApiTester { // Retrieve and publish the envelope. let envelope = self .client - .get_validator_execution_payload_envelope::(slot) + .get_validator_execution_payload_envelopes::(slot) .await .unwrap() .data; @@ -4950,7 +4950,7 @@ impl ApiTester { let signed_envelope = self.sign_envelope(envelope, &sk, epoch, &fork, genesis_validators_root); self.client - .post_beacon_execution_payload_envelope(&signed_envelope, fork_name) + .post_beacon_execution_payload_envelopes(&signed_envelope, fork_name) .await .unwrap(); @@ -9558,14 +9558,14 @@ async fn post_validator_proposer_preferences() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn post_beacon_execution_payload_bid() { +async fn post_beacon_execution_payload_bids() { if !fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { return; } ApiTester::new_with_hard_forks() .await - .test_post_beacon_execution_payload_bid_json() + .test_post_beacon_execution_payload_bids_json() .await - .test_post_beacon_execution_payload_bid_ssz() + .test_post_beacon_execution_payload_bids_ssz() .await; } diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index e5c66ab5ff..572f9522ee 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -2755,8 +2755,8 @@ impl BeaconNodeHttpClient { opt_response.ok_or(Error::StatusCode(StatusCode::NOT_FOUND)) } - /// `GET v1/validator/execution_payload_envelope/{slot}` - pub async fn get_validator_execution_payload_envelope( + /// `GET v1/validator/execution_payload_envelopes/{slot}` + pub async fn get_validator_execution_payload_envelopes( &self, slot: Slot, ) -> Result>, Error> { @@ -2765,14 +2765,14 @@ impl BeaconNodeHttpClient { path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("validator") - .push("execution_payload_envelope") + .push("execution_payload_envelopes") .push(&slot.to_string()); self.get(path).await } - /// `GET v1/validator/execution_payload_envelope/{slot}` in SSZ format - pub async fn get_validator_execution_payload_envelope_ssz( + /// `GET v1/validator/execution_payload_envelopes/{slot}` in SSZ format + pub async fn get_validator_execution_payload_envelopes_ssz( &self, slot: Slot, ) -> Result, Error> { @@ -2781,7 +2781,7 @@ impl BeaconNodeHttpClient { path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("validator") - .push("execution_payload_envelope") + .push("execution_payload_envelopes") .push(&slot.to_string()); let opt_response = self @@ -2793,8 +2793,8 @@ impl BeaconNodeHttpClient { ExecutionPayloadEnvelope::from_ssz_bytes(&response_bytes).map_err(Error::InvalidSsz) } - /// `POST v1/beacon/execution_payload_envelope` - pub async fn post_beacon_execution_payload_envelope( + /// `POST v1/beacon/execution_payload_envelopes` + pub async fn post_beacon_execution_payload_envelopes( &self, envelope: &SignedExecutionPayloadEnvelope, fork_name: ForkName, @@ -2804,7 +2804,7 @@ impl BeaconNodeHttpClient { path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("beacon") - .push("execution_payload_envelope"); + .push("execution_payload_envelopes"); self.post_generic_with_consensus_version( path, @@ -2817,8 +2817,8 @@ impl BeaconNodeHttpClient { Ok(()) } - /// `POST v1/beacon/execution_payload_envelope` in SSZ format - pub async fn post_beacon_execution_payload_envelope_ssz( + /// `POST v1/beacon/execution_payload_envelopes` in SSZ format + pub async fn post_beacon_execution_payload_envelopes_ssz( &self, envelope: &SignedExecutionPayloadEnvelope, fork_name: ForkName, @@ -2828,7 +2828,7 @@ impl BeaconNodeHttpClient { path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("beacon") - .push("execution_payload_envelope"); + .push("execution_payload_envelopes"); self.post_generic_with_consensus_version_and_ssz_body( path, @@ -2841,8 +2841,8 @@ impl BeaconNodeHttpClient { Ok(()) } - /// `POST v1/beacon/execution_payload_bid` - pub async fn post_beacon_execution_payload_bid( + /// `POST v1/beacon/execution_payload_bids` + pub async fn post_beacon_execution_payload_bids( &self, bid: &SignedExecutionPayloadBid, fork_name: ForkName, @@ -2852,7 +2852,7 @@ impl BeaconNodeHttpClient { path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("beacon") - .push("execution_payload_bid"); + .push("execution_payload_bids"); self.post_generic_with_consensus_version( path, @@ -2865,8 +2865,8 @@ impl BeaconNodeHttpClient { Ok(()) } - /// `POST v1/beacon/execution_payload_bid` in SSZ format - pub async fn post_beacon_execution_payload_bid_ssz( + /// `POST v1/beacon/execution_payload_bids` in SSZ format + pub async fn post_beacon_execution_payload_bids_ssz( &self, bid: &SignedExecutionPayloadBid, fork_name: ForkName, @@ -2876,7 +2876,7 @@ impl BeaconNodeHttpClient { path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("beacon") - .push("execution_payload_bid"); + .push("execution_payload_bids"); self.post_generic_with_consensus_version_and_ssz_body( path, @@ -2889,8 +2889,8 @@ impl BeaconNodeHttpClient { Ok(()) } - /// Path for `v1/beacon/execution_payload_envelope/{block_id}` - pub fn get_beacon_execution_payload_envelope_path( + /// Path for `v1/beacon/execution_payload_envelopes/{block_id}` + pub fn get_beacon_execution_payload_envelopes_path( &self, block_id: BlockId, ) -> Result { @@ -2898,35 +2898,35 @@ impl BeaconNodeHttpClient { path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("beacon") - .push("execution_payload_envelope") + .push("execution_payload_envelopes") .push(&block_id.to_string()); Ok(path) } - /// `GET v1/beacon/execution_payload_envelope/{block_id}` + /// `GET v1/beacon/execution_payload_envelopes/{block_id}` /// /// Returns `Ok(None)` on a 404 error. - pub async fn get_beacon_execution_payload_envelope( + pub async fn get_beacon_execution_payload_envelopes( &self, block_id: BlockId, ) -> Result< Option>>, Error, > { - let path = self.get_beacon_execution_payload_envelope_path(block_id)?; + let path = self.get_beacon_execution_payload_envelopes_path(block_id)?; self.get_opt(path) .await .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } - /// `GET v1/beacon/execution_payload_envelope/{block_id}` in SSZ format + /// `GET v1/beacon/execution_payload_envelopes/{block_id}` in SSZ format /// /// Returns `Ok(None)` on a 404 error. - pub async fn get_beacon_execution_payload_envelope_ssz( + pub async fn get_beacon_execution_payload_envelopes_ssz( &self, block_id: BlockId, ) -> Result>, Error> { - let path = self.get_beacon_execution_payload_envelope_path(block_id)?; + let path = self.get_beacon_execution_payload_envelopes_path(block_id)?; let opt_response = self .get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.get_beacon_blocks_ssz) .await?; diff --git a/validator_client/validator_services/src/block_service.rs b/validator_client/validator_services/src/block_service.rs index 1dd1878f4c..06fd14360a 100644 --- a/validator_client/validator_services/src/block_service.rs +++ b/validator_client/validator_services/src/block_service.rs @@ -659,7 +659,7 @@ impl BlockService { .beacon_nodes .first_success(|beacon_node| async move { beacon_node - .get_validator_execution_payload_envelope_ssz::(slot) + .get_validator_execution_payload_envelopes_ssz::(slot) .await .map_err(|e| { BlockError::Recoverable(format!( @@ -702,7 +702,7 @@ impl BlockService { let signed_envelope = signed_envelope.clone(); async move { beacon_node - .post_beacon_execution_payload_envelope_ssz(&signed_envelope, fork_name) + .post_beacon_execution_payload_envelopes_ssz(&signed_envelope, fork_name) .await .map_err(|e| { BlockError::Recoverable(format!( From abe7ca20a9becd341ef226fb040be53eb7876dec Mon Sep 17 00:00:00 2001 From: Alleysira <56925051+Alleysira@users.noreply.github.com> Date: Sat, 6 Jun 2026 04:16:28 +0800 Subject: [PATCH 13/15] fix(network): clear ENR `nfd` field when no next fork is scheduled during runtime transitions (#9131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No. But related to #9009 and #8996 - Change the `ForkContext::next_fork_digest()` to return `[u8; 4]` (returning `[0u8; 4]` for "no next fork"). - Update the initialization path and runtime fork transition path accordingly. Added tests: - [x] `test_next_fork_digest` — existing test passes with non-Option return type - [x] `test_next_fork_digest_returns_zero_when_no_next_fork` — init at last BPO fork returns `[0u8; 4]` - [x] `test_next_fork_digest_zero_after_runtime_transition_to_last_fork` — simulates `update_current_fork` to last fork, then verifies zero Co-Authored-By: alleysira <1367108378@qq.com> Co-Authored-By: Alleysira <56925051+Alleysira@users.noreply.github.com> Co-Authored-By: chonghe <44791194+chong-he@users.noreply.github.com> --- .../lighthouse_network/src/discovery/enr.rs | 5 ++- .../lighthouse_network/src/service/mod.rs | 4 +- beacon_node/network/src/service.rs | 5 +-- consensus/types/src/fork/fork_context.rs | 45 +++++++++++++++++-- 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index 01a01d55ab..0735cbb37a 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -320,11 +320,12 @@ fn compare_enr(local_enr: &Enr, disk_enr: &Enr) -> bool { && (local_enr.udp4().is_none() || local_enr.udp4() == disk_enr.udp4()) && (local_enr.udp6().is_none() || local_enr.udp6() == disk_enr.udp6()) // we need the ATTESTATION_BITFIELD_ENR_KEY and SYNC_COMMITTEE_BITFIELD_ENR_KEY and - // PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY key to match, otherwise we use a new ENR. This will - // likely only be true for non-validating nodes. + // PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY and NEXT_FORK_DIGEST_ENR_KEY keys to match, + // otherwise we use a new ENR. This will likely only be true for non-validating nodes. && local_enr.get_decodable::(ATTESTATION_BITFIELD_ENR_KEY) == disk_enr.get_decodable(ATTESTATION_BITFIELD_ENR_KEY) && local_enr.get_decodable::(SYNC_COMMITTEE_BITFIELD_ENR_KEY) == disk_enr.get_decodable(SYNC_COMMITTEE_BITFIELD_ENR_KEY) && local_enr.get_decodable::(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY) == disk_enr.get_decodable(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY) + && local_enr.get_decodable::(NEXT_FORK_DIGEST_ENR_KEY) == disk_enr.get_decodable(NEXT_FORK_DIGEST_ENR_KEY) } /// Loads enr from the given directory diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 097736a010..93c8410490 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -201,9 +201,7 @@ impl Network { // set up a collection of variables accessible outside of the network crate // Create an ENR or load from disk if appropriate - // Per [spec](https://github.com/ethereum/consensus-specs/blob/1baa05e71148b0975e28918ac6022d2256b56f4a/specs/fulu/p2p-interface.md?plain=1#L636-L637) - // `nfd` must be zero-valued when no next fork is scheduled. - let next_fork_digest = ctx.fork_context.next_fork_digest().unwrap_or_default(); + let next_fork_digest = ctx.fork_context.next_fork_digest(); let advertised_cgc = config .advertise_false_custody_group_count diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index ce54ffc38f..c2e79fe9e8 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -883,10 +883,7 @@ impl NetworkService { fork_context.update_current_fork(*new_fork_name, new_fork_digest, current_epoch); if self.beacon_chain.spec.is_peer_das_scheduled() { - let next_fork_digest = fork_context - .next_fork_digest() - .unwrap_or_else(|| fork_context.current_fork_digest()); - self.libp2p.update_nfd(next_fork_digest); + self.libp2p.update_nfd(fork_context.next_fork_digest()); } self.libp2p.update_fork_version(new_enr_fork_id); diff --git a/consensus/types/src/fork/fork_context.rs b/consensus/types/src/fork/fork_context.rs index 3407689e79..f563578237 100644 --- a/consensus/types/src/fork/fork_context.rs +++ b/consensus/types/src/fork/fork_context.rs @@ -93,14 +93,16 @@ impl ForkContext { pub fn current_fork_digest(&self) -> [u8; 4] { self.current_fork.read().fork_digest } - - /// Returns the next fork digest. If there's no future fork, returns the current fork digest. - pub fn next_fork_digest(&self) -> Option<[u8; 4]> { + /// Per [spec](https://github.com/ethereum/consensus-specs/blob/1baa05e71148b0975e28918ac6022d2256b56f4a/specs/fulu/p2p-interface.md?plain=1#L636-L637) + /// `nfd` must be zero-valued when no next fork is scheduled. + /// Returns the next fork digest. If there's no future fork, returns zero-valued bytes. + pub fn next_fork_digest(&self) -> [u8; 4] { let current_fork_epoch = self.current_fork_epoch(); self.epoch_to_forks .range(current_fork_epoch..) .nth(1) .map(|(_, fork)| fork.fork_digest) + .unwrap_or_default() } /// Updates the `digest_epoch` field to a new digest epoch. @@ -222,11 +224,46 @@ mod tests { let context = ForkContext::new::(electra_slot, genesis_root, &spec); - let next_digest = context.next_fork_digest().unwrap(); + let next_digest = context.next_fork_digest(); let expected_digest = spec.compute_fork_digest(genesis_root, spec.fulu_fork_epoch.unwrap()); assert_eq!(next_digest, expected_digest); } + #[test] + fn test_next_fork_digest_returns_zero_when_no_next_fork() { + let spec = make_chain_spec(); + let genesis_root = Hash256::ZERO; + // Epoch 100 is the last BPO fork in make_chain_spec + let last_bpo_slot = Epoch::new(100).end_slot(E::slots_per_epoch()); + + let context = ForkContext::new::(last_bpo_slot, genesis_root, &spec); + + // No next fork after the last BPO epoch — must return zero bytes per spec + assert_eq!(context.next_fork_digest(), [0u8; 4]); + } + + #[test] + fn test_next_fork_digest_zero_after_runtime_transition_to_last_fork() { + let spec = make_chain_spec(); + let genesis_root = Hash256::ZERO; + // Start at Gloas (epoch 7) + let gloas_epoch = spec.gloas_fork_epoch.unwrap(); + let gloas_slot = gloas_epoch.end_slot(E::slots_per_epoch()); + + let context = ForkContext::new::(gloas_slot, genesis_root, &spec); + + // Before: next fork exists (BPO at epoch 50) + let bpo_50_digest = spec.compute_fork_digest(genesis_root, Epoch::new(50)); + assert_eq!(context.next_fork_digest(), bpo_50_digest); + + // Simulate runtime transition to the last BPO fork (epoch 100) + let last_digest = spec.compute_fork_digest(genesis_root, Epoch::new(100)); + context.update_current_fork(ForkName::Gloas, last_digest, Epoch::new(100)); + + // After: no next fork — must return zero bytes per spec + assert_eq!(context.next_fork_digest(), [0u8; 4]); + } + #[test] fn test_get_fork_from_context_bytes() { let spec = make_chain_spec(); From 65f1a832e44c29afaa1fb0d1e3d87a196ffe3b90 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Sat, 6 Jun 2026 06:51:33 +1000 Subject: [PATCH 14/15] Optimise slow block verification tests (#9274) Reduce CI time for `block_verification` tests that exceed 120s on Fulu/Gloas forks. - Cache the chain segment across tests using `LazyLock` (was rebuilt 10+ times independently) - Reduce chain length from 320 to 192 slots (10 epochs to 6 epochs) - Reduce `BLOCK_INDICES` from 7 to 3 - Reduce `chain_segment_varying_chunk_size` from 5 to 3 AI assisted, self reviewed. Before vs After comparison (1 sample only): * **Before**: 19 tests exceeded the 120s slow threshold. * **After**: zero exceed it. Overall: 1,890s down to 797s (-58%). ``` | Test | Before | After | Change | |------------------------------------------------------|---------|---------|---------| | chain_segment_varying_chunk_size | 239s | 98s | -59% | | invalid_signature_attester_slashing | 175s | 64s | -64% | | invalid_signature_exit | 173s | 62s | -64% | | invalid_signature_deposit | 170s | 60s | -65% | | invalid_signature_attestation | 165s | 62s | -63% | | invalid_signature_proposer_slashing | 161s | 56s | -66% | | block_gossip_verification | 154s | 91s | -41% | | invalid_signature_block_proposal | 151s | 58s | -61% | | invalid_signature_randao_reveal | 149s | 54s | -64% | | invalid_signature_gossip_block | 135s | 46s | -66% | |------------------------------------------------------|---------|---------|---------| | TOTAL | 1890s | 797s | -58% | ``` Co-Authored-By: Jimmy Chen Co-Authored-By: Michael Sproul --- .../beacon_chain/tests/block_verification.rs | 85 +++++++++++-------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index deadafac36..74a521a79c 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -35,20 +35,30 @@ type E = MainnetEthSpec; // Gloas requires >= 1 validator per slot for PTC committee computation, so >= 32 for MainnetEthSpec. const VALIDATOR_COUNT: usize = 32; -const CHAIN_SEGMENT_LENGTH: usize = 64 * 5; -const BLOCK_INDICES: &[usize] = &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT_LENGTH - 1]; +const CHAIN_SEGMENT_LENGTH: usize = 32 * 6; +const BLOCK_INDICES: &[usize] = &[1, 32, 64]; /// A cached set of keys. static KEYPAIRS: LazyLock> = LazyLock::new(|| types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT)); // TODO(#8633): Delete this unnecessary enum and refactor this file to use `AvailableBlockData` instead. +#[derive(Clone)] enum DataSidecars { Blobs(BlobSidecarList), DataColumns(Vec>), } -async fn get_chain_segment() -> (Vec>, Vec>>) { +type ChainSegmentData = (Vec>, Vec>>); + +static CHAIN_SEGMENT: LazyLock> = + LazyLock::new(tokio::sync::OnceCell::new); + +async fn get_chain_segment() -> &'static ChainSegmentData { + CHAIN_SEGMENT.get_or_init(build_chain_segment).await +} + +async fn build_chain_segment() -> ChainSegmentData { // The assumption that you can re-import a block based on what you have in your DB // is no longer true, as fullnodes stores less than what they sample. // We use a supernode here to build a chain segment. @@ -365,9 +375,9 @@ async fn chain_segment_full_segment() { } let harness = get_harness(VALIDATOR_COUNT, NodeCustodyType::Fullnode); let (chain_segment, chain_segment_blobs) = get_chain_segment().await; - store_envelopes_for_chain_segment(&chain_segment, &harness); + store_envelopes_for_chain_segment(chain_segment, &harness); let blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs, harness.chain.clone()) + chain_segment_blocks(chain_segment, chain_segment_blobs, harness.chain.clone()) .into_iter() .collect(); @@ -391,7 +401,7 @@ async fn chain_segment_full_segment() { .into_block_error() .expect("should import chain segment"); - update_fork_choice_with_envelopes(&chain_segment, &harness); + update_fork_choice_with_envelopes(chain_segment, &harness); harness.chain.recompute_head_at_current_slot().await; assert_eq!( @@ -410,13 +420,13 @@ async fn chain_segment_varying_chunk_size() { let (chain_segment, chain_segment_blobs) = get_chain_segment().await; let harness = get_harness(VALIDATOR_COUNT, NodeCustodyType::Fullnode); let blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs, harness.chain.clone()) + chain_segment_blocks(chain_segment, chain_segment_blobs, harness.chain.clone()) .into_iter() .collect(); - for chunk_size in &[1, 2, 31, 32, 33] { + for chunk_size in &[1, 32, 33] { let harness = get_harness(VALIDATOR_COUNT, NodeCustodyType::Fullnode); - store_envelopes_for_chain_segment(&chain_segment, &harness); + store_envelopes_for_chain_segment(chain_segment, &harness); harness .chain @@ -432,7 +442,7 @@ async fn chain_segment_varying_chunk_size() { .unwrap_or_else(|_| panic!("should import chain segment of len {}", chunk_size)); } - update_fork_choice_with_envelopes(&chain_segment, &harness); + update_fork_choice_with_envelopes(chain_segment, &harness); harness.chain.recompute_head_at_current_slot().await; assert_eq!( @@ -457,7 +467,7 @@ async fn chain_segment_non_linear_parent_roots() { * Test with a block removed. */ let mut blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs, harness.chain.clone()) + chain_segment_blocks(chain_segment, chain_segment_blobs, harness.chain.clone()) .into_iter() .collect(); blocks.remove(2); @@ -478,7 +488,7 @@ async fn chain_segment_non_linear_parent_roots() { * Test with a modified parent root. */ let mut blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs, harness.chain.clone()) + chain_segment_blocks(chain_segment, chain_segment_blobs, harness.chain.clone()) .into_iter() .collect(); @@ -520,7 +530,7 @@ async fn chain_segment_non_linear_slots() { */ let mut blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs, harness.chain.clone()) + chain_segment_blocks(chain_segment, chain_segment_blobs, harness.chain.clone()) .into_iter() .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); @@ -550,7 +560,7 @@ async fn chain_segment_non_linear_slots() { */ let mut blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs, harness.chain.clone()) + chain_segment_blocks(chain_segment, chain_segment_blobs, harness.chain.clone()) .into_iter() .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); @@ -694,7 +704,7 @@ async fn invalid_signature_gossip_block() { let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { // Ensure the block will be rejected if imported on its own (without gossip checking). - let harness = get_invalid_sigs_harness(&chain_segment).await; + let harness = get_invalid_sigs_harness(chain_segment).await; let mut snapshots = chain_segment.clone(); let (block, _) = snapshots[block_index] .beacon_block @@ -753,7 +763,7 @@ async fn invalid_signature_block_proposal() { } let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { - let harness = get_invalid_sigs_harness(&chain_segment).await; + let harness = get_invalid_sigs_harness(chain_segment).await; let mut snapshots = chain_segment.clone(); let (block, _) = snapshots[block_index] .beacon_block @@ -794,9 +804,10 @@ async fn invalid_signature_randao_reveal() { if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { return; } - let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; + let (chain_segment, ref_blobs) = get_chain_segment().await; + let mut chain_segment_blobs = ref_blobs.clone(); for &block_index in BLOCK_INDICES { - let harness = get_invalid_sigs_harness(&chain_segment).await; + let harness = get_invalid_sigs_harness(chain_segment).await; let mut snapshots = chain_segment.clone(); let (mut block, signature) = snapshots[block_index] .beacon_block @@ -809,7 +820,7 @@ async fn invalid_signature_randao_reveal() { update_parent_roots(&mut snapshots, &mut chain_segment_blobs); update_proposal_signatures(&mut snapshots, &harness); assert_invalid_signature( - &chain_segment, + chain_segment, &chain_segment_blobs, &harness, block_index, @@ -826,9 +837,10 @@ async fn invalid_signature_proposer_slashing() { if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { return; } - let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; + let (chain_segment, ref_blobs) = get_chain_segment().await; + let mut chain_segment_blobs = ref_blobs.clone(); for &block_index in BLOCK_INDICES { - let harness = get_invalid_sigs_harness(&chain_segment).await; + let harness = get_invalid_sigs_harness(chain_segment).await; let mut snapshots = chain_segment.clone(); let (mut block, signature) = snapshots[block_index] .beacon_block @@ -855,7 +867,7 @@ async fn invalid_signature_proposer_slashing() { update_parent_roots(&mut snapshots, &mut chain_segment_blobs); update_proposal_signatures(&mut snapshots, &harness); assert_invalid_signature( - &chain_segment, + chain_segment, &chain_segment_blobs, &harness, block_index, @@ -872,9 +884,10 @@ async fn invalid_signature_attester_slashing() { if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { return; } - let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; + let (chain_segment, ref_blobs) = get_chain_segment().await; + let mut chain_segment_blobs = ref_blobs.clone(); for &block_index in BLOCK_INDICES { - let harness = get_invalid_sigs_harness(&chain_segment).await; + let harness = get_invalid_sigs_harness(chain_segment).await; let mut snapshots = chain_segment.clone(); let fork_name = harness.chain.spec.fork_name_at_slot::(Slot::new(0)); @@ -980,7 +993,7 @@ async fn invalid_signature_attester_slashing() { update_parent_roots(&mut snapshots, &mut chain_segment_blobs); update_proposal_signatures(&mut snapshots, &harness); assert_invalid_signature( - &chain_segment, + chain_segment, &chain_segment_blobs, &harness, block_index, @@ -997,11 +1010,12 @@ async fn invalid_signature_attestation() { if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { return; } - let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; + let (chain_segment, ref_blobs) = get_chain_segment().await; + let mut chain_segment_blobs = ref_blobs.clone(); let mut checked_attestation = false; for &block_index in BLOCK_INDICES { - let harness = get_invalid_sigs_harness(&chain_segment).await; + let harness = get_invalid_sigs_harness(chain_segment).await; let mut snapshots = chain_segment.clone(); let (mut block, signature) = snapshots[block_index] .beacon_block @@ -1049,7 +1063,7 @@ async fn invalid_signature_attestation() { update_parent_roots(&mut snapshots, &mut chain_segment_blobs); update_proposal_signatures(&mut snapshots, &harness); assert_invalid_signature( - &chain_segment, + chain_segment, &chain_segment_blobs, &harness, block_index, @@ -1069,10 +1083,11 @@ async fn invalid_signature_attestation() { #[tokio::test] async fn invalid_signature_deposit() { - let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; + let (chain_segment, ref_blobs) = get_chain_segment().await; + let mut chain_segment_blobs = ref_blobs.clone(); for &block_index in BLOCK_INDICES { // Note: an invalid deposit signature is permitted! - let harness = get_invalid_sigs_harness(&chain_segment).await; + let harness = get_invalid_sigs_harness(chain_segment).await; let mut snapshots = chain_segment.clone(); let deposit = Deposit { proof: vec![Hash256::zero(); DEPOSIT_TREE_DEPTH + 1] @@ -1126,9 +1141,10 @@ async fn invalid_signature_exit() { if fork_name_from_env().is_some_and(|f| f.gloas_enabled()) { return; } - let (chain_segment, mut chain_segment_blobs) = get_chain_segment().await; + let (chain_segment, ref_blobs) = get_chain_segment().await; + let mut chain_segment_blobs = ref_blobs.clone(); for &block_index in BLOCK_INDICES { - let harness = get_invalid_sigs_harness(&chain_segment).await; + let harness = get_invalid_sigs_harness(chain_segment).await; let mut snapshots = chain_segment.clone(); let epoch = snapshots[block_index].beacon_state.current_epoch(); let (mut block, signature) = snapshots[block_index] @@ -1152,7 +1168,7 @@ async fn invalid_signature_exit() { update_parent_roots(&mut snapshots, &mut chain_segment_blobs); update_proposal_signatures(&mut snapshots, &harness); assert_invalid_signature( - &chain_segment, + chain_segment, &chain_segment_blobs, &harness, block_index, @@ -1173,7 +1189,8 @@ fn unwrap_err(result: Result) -> U { #[tokio::test] async fn block_gossip_verification() { let harness = get_harness(VALIDATOR_COUNT, NodeCustodyType::Fullnode); - let (chain_segment, chain_segment_blobs) = get_chain_segment().await; + let (chain_segment, ref_blobs) = get_chain_segment().await; + let chain_segment_blobs = ref_blobs.clone(); let block_index = CHAIN_SEGMENT_LENGTH - 2; From 8e4df4ababf68d7c58973177013d78447b9a208d Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Sat, 6 Jun 2026 01:52:45 +0200 Subject: [PATCH 15/15] Simplify lookup sync da_checker oracle (#9428) Implementing gloas lookup sync is currently incompatible with the `GossipBlockProcessResult` mechanism. Today it's implemented such that if we receive a sucessful `GossipBlockProcessResult` we directly mark the lookup as Complete and delete it. In Gloas we can't delete a lookup after block import, as we may still have FULL child awaiting the payload. IMO this `GossipBlockProcessResult` brings a lot of headache and edge cases that we can just live without. Also the `reset_request` business is nasty and can easily leave the lookup in a bad state. If we get rid of `GossipBlockProcessResult` we only pay the following performance penalty: - Lookup is created exactly while the block's payload is being execution validated - (new degradation) we download the block again - send the block for processing but the duplicate cache prevents double execution So in the worst case we spend a few KBs of extra download bandwidth. Remember each block is downloaded 8x times through gossip in the happy case. Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> Co-Authored-By: Pawan Dhananjay --- .../gossip_methods.rs | 23 ---- .../src/network_beacon_processor/tests.rs | 104 +---------------- .../network/src/sync/block_lookups/mod.rs | 33 ------ .../sync/block_lookups/single_block_lookup.rs | 6 - beacon_node/network/src/sync/manager.rs | 13 --- .../network/src/sync/network_context.rs | 33 ++---- beacon_node/network/src/sync/tests/lookups.rs | 107 ------------------ 7 files changed, 13 insertions(+), 306 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 68b41cab5e..29e43b18c2 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -922,14 +922,6 @@ impl NetworkBeaconProcessor { &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, processing_start_time.elapsed().as_millis() as i64, ); - - // If a block is in the da_checker, sync maybe awaiting for an event when block is finally - // imported. A block can become imported both after processing a block or data column. If - // importing a block results in `Imported`, notify. Do not notify of data column errors. - self.send_sync_message(SyncMessage::GossipBlockProcessResult { - block_root, - imported: true, - }); } AvailabilityProcessingStatus::MissingComponents(slot, block_root) => { trace!( @@ -1354,16 +1346,6 @@ impl NetworkBeaconProcessor { // contributing to the partial. } } - - // If a block is in the da_checker, sync maybe awaiting for an event when block is finally - // imported. A block can become imported both after processing a block or data column. If a - // importing a block results in `Imported`, notify. Do not notify of data column errors. - if matches!(result, Ok(AvailabilityProcessingStatus::Imported(_))) { - self.send_sync_message(SyncMessage::GossipBlockProcessResult { - block_root, - imported: true, - }); - } } async fn check_reconstruction_trigger(self: &Arc, slot: Slot, block_root: &Hash256) { @@ -1898,11 +1880,6 @@ impl NetworkBeaconProcessor { if let Err(e) = &result { self.maybe_store_invalid_block(&invalid_block_storage, block_root, &block, e); } - - self.send_sync_message(SyncMessage::GossipBlockProcessResult { - block_root, - imported: matches!(result, Ok(AvailabilityProcessingStatus::Imported(_))), - }); } pub fn process_gossip_voluntary_exit( diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index ad98851532..8ccfe38fa3 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -6,7 +6,7 @@ use crate::{ ChainSegmentProcessId, DuplicateCache, InvalidBlockStorage, NetworkBeaconProcessor, }, service::NetworkMessage, - sync::{SyncMessage, manager::BlockProcessType}, + sync::manager::BlockProcessType, }; use beacon_chain::block_verification_types::LookupBlock; use beacon_chain::custody_context::NodeCustodyType; @@ -76,7 +76,6 @@ struct TestRig { beacon_processor_tx: BeaconProcessorSend, work_journal_rx: mpsc::Receiver<&'static str>, network_rx: mpsc::UnboundedReceiver>, - sync_rx: mpsc::UnboundedReceiver>, duplicate_cache: DuplicateCache, network_beacon_processor: Arc>, _harness: BeaconChainHarness, @@ -270,7 +269,7 @@ impl TestRig { beacon_processor_rx, } = BeaconProcessorChannels::new(&beacon_processor_config); - let (sync_tx, sync_rx) = mpsc::unbounded_channel(); + let (sync_tx, _sync_rx) = mpsc::unbounded_channel(); // Default metadata let meta_data = if spec.is_peer_das_scheduled() { @@ -375,7 +374,6 @@ impl TestRig { beacon_processor_tx, work_journal_rx, network_rx, - sync_rx, duplicate_cache, network_beacon_processor, _harness: harness, @@ -844,45 +842,6 @@ impl TestRig { Some(events) } } - - /// Listen for sync messages and collect them for a specified duration or until reaching a count. - /// - /// Returns None if no messages were received, or Some(Vec) containing the received messages. - pub async fn receive_sync_messages_with_timeout( - &mut self, - timeout: Duration, - count: Option, - ) -> Option>> { - let mut events = vec![]; - - let timeout_future = tokio::time::sleep(timeout); - tokio::pin!(timeout_future); - - loop { - // Break if we've received the requested count of messages - if let Some(target_count) = count - && events.len() >= target_count - { - break; - } - - tokio::select! { - _ = &mut timeout_future => break, - maybe_msg = self.sync_rx.recv() => { - match maybe_msg { - Some(msg) => events.push(msg), - None => break, // Channel closed - } - } - } - } - - if events.is_empty() { - None - } else { - Some(events) - } - } } fn junk_peer_id() -> PeerId { @@ -1862,65 +1821,6 @@ async fn test_blobs_by_root_post_fulu_should_return_empty() { assert_eq!(0, actual_count); } -/// Ensure that data column processing that results in block import sends a sync notification -#[tokio::test] -async fn test_data_column_import_notifies_sync() { - if test_spec::().fulu_fork_epoch.is_none() { - return; - } - - let mut rig = TestRig::new(SMALL_CHAIN).await; - let block_root = rig.next_block.canonical_root(); - - // Enqueue the block first to prepare for data column processing - rig.enqueue_gossip_block(); - rig.assert_event_journal_completes(&[WorkType::GossipBlock]) - .await; - rig.receive_sync_messages_with_timeout(Duration::from_millis(100), Some(1)) - .await - .expect("should receive sync message"); - - // Enqueue data columns which should trigger block import when complete - let num_data_columns = rig.next_data_columns.as_ref().map(|c| c.len()).unwrap_or(0); - if num_data_columns > 0 { - for i in 0..num_data_columns { - rig.enqueue_gossip_data_columns(i); - rig.assert_event_journal_completes(&[WorkType::GossipDataColumnSidecar]) - .await; - } - - // Verify block import succeeded - assert_eq!( - rig.head_root(), - block_root, - "block should be imported and become head" - ); - - // Check that sync was notified of the successful import - let sync_messages = rig - .receive_sync_messages_with_timeout(Duration::from_millis(100), Some(1)) - .await - .expect("should receive sync message"); - - // Verify we received the expected GossipBlockProcessResult message - assert_eq!( - sync_messages.len(), - 1, - "should receive exactly one sync message" - ); - match &sync_messages[0] { - SyncMessage::GossipBlockProcessResult { - block_root: msg_block_root, - imported, - } => { - assert_eq!(*msg_block_root, block_root, "block root should match"); - assert!(*imported, "block should be marked as imported"); - } - other => panic!("expected GossipBlockProcessResult, got {:?}", other), - } - } -} - #[tokio::test] async fn test_data_columns_by_range_request_only_returns_requested_columns() { if test_spec::().fulu_fork_epoch.is_none() { diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index a265373e3f..0cbeb5ee4e 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -482,39 +482,6 @@ impl BlockLookups { self.on_lookup_result(lookup_id, lookup_result, "processing_result", cx); } - pub fn on_external_processing_result( - &mut self, - block_root: Hash256, - imported: bool, - cx: &mut SyncNetworkContext, - ) { - let Some((id, lookup)) = self - .single_block_lookups - .iter_mut() - .find(|(_, lookup)| lookup.is_for_block(block_root)) - else { - // Ok to ignore gossip process events - return; - }; - - let lookup_result = if imported { - Ok(LookupResult::Completed) - } else { - // A lookup may be in the following state: - // - Block awaiting processing from a different source - // - Blobs downloaded processed, and inserted into the da_checker - // - // At this point the block fails processing (e.g. execution engine offline) and it is - // removed from the da_checker. Note that ALL components are removed from the da_checker - // so when we re-download and process the block we get the error - // MissingComponentsAfterAllProcessed and get stuck. - lookup.reset_requests(); - lookup.continue_requests(cx) - }; - let id = *id; - self.on_lookup_result(id, lookup_result, "external_processing_result", cx); - } - /// Makes progress on the immediate children of `block_root` pub fn continue_child_lookups(&mut self, block_root: Hash256, cx: &mut SyncNetworkContext) { let mut lookup_results = vec![]; // < need to buffer lookup results to not re-borrow &mut self diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 8eb58da4e6..157da5d806 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -139,12 +139,6 @@ impl SingleBlockLookup { } } - /// Reset the status of all requests (used on block processing failure) - pub fn reset_requests(&mut self) { - self.block_request = BlockRequest::new(); - self.data_request = DataRequest::WaitingForBlock; - } - /// Return the slot of this lookup's block if it's currently cached pub fn peek_downloaded_block_slot(&self) -> Option { self.block_request diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 166c65b6e1..66bb13ae98 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -182,11 +182,6 @@ pub enum SyncMessage { process_type: BlockProcessType, result: BlockProcessingResult, }, - - /// A gossip-received component has completed processing and the block may now be imported. - /// In Fulu this is sent after block or blob processing. In Gloas this is also sent after - /// data column or payload envelope processing triggers availability. - GossipBlockProcessResult { block_root: Hash256, imported: bool }, } /// The type of processing specified for a received block. @@ -907,14 +902,6 @@ impl SyncManager { } => self .block_lookups .on_processing_result(process_type, result, &mut self.network), - SyncMessage::GossipBlockProcessResult { - block_root, - imported, - } => self.block_lookups.on_external_processing_result( - block_root, - imported, - &mut self.network, - ), SyncMessage::BatchProcessed { sync_type, result } => match sync_type { ChainSegmentProcessId::RangeBatchId(chain_id, epoch) => { self.range_sync.handle_block_process_result( diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 1e35c0a72f..dfeb8d8f12 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -53,8 +53,8 @@ use task_executor::TaskExecutor; use tokio::sync::mpsc; use tracing::{Span, debug, debug_span, error, warn}; use types::{ - BlobSidecar, BlockImportSource, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, - ForkContext, Hash256, SignedBeaconBlock, SignedExecutionPayloadEnvelope, Slot, + BlobSidecar, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, ForkContext, + Hash256, SignedBeaconBlock, SignedExecutionPayloadEnvelope, Slot, }; pub mod custody; @@ -849,26 +849,15 @@ impl SyncNetworkContext { match self.chain.get_block_process_status(&block_root) { // Unknown block, continue request to download BlockProcessStatus::Unknown => {} - // Block is known and currently processing. Imports from gossip and HTTP API insert the - // block in the da_cache. However, HTTP API is unable to notify sync when it completes - // block import. Returning `Pending` here will result in stuck lookups if the block is - // importing from sync. - BlockProcessStatus::NotValidated(_, source) => match source { - BlockImportSource::Gossip => { - // Lookup sync event safety: If the block is currently in the processing cache, we - // are guaranteed to receive a `SyncMessage::GossipBlockProcessResult` that will - // make progress on this lookup - return Ok(LookupRequestResult::Pending("block in processing cache")); - } - BlockImportSource::Lookup - | BlockImportSource::RangeSync - | BlockImportSource::HttpApi => { - // Lookup, RangeSync or HttpApi block import don't emit the GossipBlockProcessResult - // event. If a lookup happens to be created during block import from one of - // those sources just import the block twice. Otherwise the lookup will get - // stuck. Double imports are fine, they just waste resources. - } - }, + // Block is known but processing. The block may turn out to be invalid, so we want sync to + // NOT mark the request as complete yet. The ideal flow would be: + // - Wait for processing to complete + // - Only if there is an error re-download and re-process + // But implementing this introduces complexity and the risk for the lookup to get stuck. + // Instead we always re-download the block eagerly and de-duplicate the processing. So in + // the happy case we just download the block again if the lookup is created while execution + // processing the block. + BlockProcessStatus::NotValidated(..) => {} // Block is fully validated. If it's not yet imported it's waiting for missing block // components. Consider this request completed and do nothing. BlockProcessStatus::ExecutionValidated(block) => { diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index 5642f7846a..91227d77f8 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -1235,12 +1235,6 @@ impl TestRig { self.assert_empty_network(); } - fn assert_pending_lookup_sync(&self) { - assert!(self.created_lookups() > 0, "no created lookups"); - assert_eq!(self.dropped_lookups(), 0, "some dropped lookups"); - assert_eq!(self.completed_lookups(), 0, "some completed lookups"); - } - /// Assert there is at least one range sync chain created and that all sync chains completed pub(super) fn assert_successful_range_sync(&self) { assert!( @@ -1330,15 +1324,6 @@ impl TestRig { genesis_fork().fulu_enabled().then(Self::default) } - fn new_after_deneb_before_fulu() -> Option { - let fork = genesis_fork(); - if fork.deneb_enabled() && !fork.fulu_enabled() { - Some(Self::default()) - } else { - None - } - } - pub fn new_fulu_peer_test(fulu_test_type: FuluTestType) -> Option { genesis_fork().fulu_enabled().then(|| { Self::new(TestRigConfig { @@ -1673,56 +1658,6 @@ impl TestRig { } } - fn insert_block_to_da_checker_as_pre_execution(&mut self, block: Arc>) { - self.log(&format!( - "Inserting block to availability_cache as pre_execution_block {:?}", - block.canonical_root() - )); - self.harness - .chain - .data_availability_checker - .put_pre_execution_block(block.canonical_root(), block, BlockImportSource::Gossip) - .unwrap(); - } - - fn simulate_block_gossip_processing_becomes_invalid(&mut self, block_root: Hash256) { - self.log(&format!( - "Marking block {block_root:?} in da_checker as execution error" - )); - self.harness - .chain - .data_availability_checker - .remove_block_on_execution_error(&block_root); - - self.send_sync_message(SyncMessage::GossipBlockProcessResult { - block_root, - imported: false, - }); - } - - async fn simulate_block_gossip_processing_becomes_valid( - &mut self, - block: Arc>, - ) { - let block_root = block.canonical_root(); - - match self.import_block_to_da_checker(block).await { - AvailabilityProcessingStatus::Imported(block_root) => { - self.log(&format!( - "insert block to da_checker and it imported {block_root:?}" - )); - } - AvailabilityProcessingStatus::MissingComponents(_, _) => { - panic!("block not imported after adding to da_checker"); - } - } - - self.send_sync_message(SyncMessage::GossipBlockProcessResult { - block_root, - imported: false, - }); - } - fn requests_count(&self) -> HashMap<&'static str, usize> { let mut requests_count = HashMap::new(); for (request, _) in &self.requests { @@ -2294,48 +2229,6 @@ async fn block_in_da_checker_skips_download() { ); } -#[tokio::test] -async fn block_in_processing_cache_becomes_invalid() { - let Some(mut r) = TestRig::new_after_deneb_before_fulu() else { - return; - }; - r.build_chain(1).await; - let block = r.block_at_slot(1); - r.insert_block_to_da_checker_as_pre_execution(block.clone()); - r.trigger_with_last_block(); - r.simulate(SimulateConfig::happy_path()).await; - r.assert_pending_lookup_sync(); - // Here the only active lookup is waiting for the block to finish processing - - // Simulate invalid block, removing it from processing cache - r.simulate_block_gossip_processing_becomes_invalid(block.canonical_root()); - // Should download block, then issue blobs request - r.simulate(SimulateConfig::happy_path()).await; - r.assert_successful_lookup_sync(); -} - -#[tokio::test] -async fn block_in_processing_cache_becomes_valid_imported() { - let Some(mut r) = TestRig::new_after_deneb_before_fulu() else { - return; - }; - r.build_chain(1).await; - let block = r.block_at_slot(1); - r.insert_block_to_da_checker_as_pre_execution(block.clone()); - r.trigger_with_last_block(); - r.simulate(SimulateConfig::happy_path()).await; - r.assert_pending_lookup_sync(); - // Here the only active lookup is waiting for the block to finish processing - - // Resolve the block from processing step - r.simulate_block_gossip_processing_becomes_valid(block) - .await; - // Should not trigger block or blob request - r.assert_empty_network(); - // Resolve blob and expect lookup completed - r.assert_no_active_lookups(); -} - macro_rules! fulu_peer_matrix_tests { ( [$($name:ident => $variant:expr),+ $(,)?]