From 8396dc87d092609b231b4db2afb5f9c49639cbd4 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Thu, 28 May 2026 19:59:23 -0700 Subject: [PATCH] Deprecate gossip blobs (#9126) #9124 Deprecate unneeded pre-Fulu blob features - blob gossip - blob lookup sync - engine getBlobsV1 Also deprecates some tests and cleans up production code paths I think this is blocked until gnosis forks to fulu? Co-Authored-By: Eitan Seri-Levi Co-Authored-By: Eitan Seri- Levi Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> Co-Authored-By: Pawan Dhananjay Co-Authored-By: Michael Sproul Co-Authored-By: Daniel Knopik Co-Authored-By: Michael Sproul --- .github/workflows/test-suite.yml | 2 +- Cargo.lock | 1 + Makefile | 4 +- beacon_node/beacon_chain/src/beacon_chain.rs | 197 ++----- .../beacon_chain/src/blob_verification.rs | 482 +---------------- .../beacon_chain/src/block_verification.rs | 37 -- beacon_node/beacon_chain/src/builder.rs | 1 - .../beacon_chain/src/canonical_head.rs | 7 - .../src/data_availability_checker.rs | 22 +- .../overflow_lru_cache.rs | 142 +++-- .../fetch_blobs/fetch_blobs_beacon_adapter.rs | 40 +- .../beacon_chain/src/fetch_blobs/mod.rs | 160 +----- .../beacon_chain/src/fetch_blobs/tests.rs | 291 +---------- beacon_node/beacon_chain/src/test_utils.rs | 71 +-- .../beacon_chain/tests/block_verification.rs | 25 +- beacon_node/beacon_chain/tests/events.rs | 41 -- beacon_node/beacon_processor/src/lib.rs | 10 - .../src/scheduler/work_queue.rs | 7 +- beacon_node/execution_layer/src/engine_api.rs | 15 +- .../execution_layer/src/engine_api/http.rs | 17 - beacon_node/execution_layer/src/lib.rs | 19 +- .../test_utils/execution_block_generator.rs | 14 +- .../src/test_utils/handle_rpc.rs | 14 - .../execution_layer/src/test_utils/mod.rs | 1 - beacon_node/http_api/src/block_id.rs | 18 +- beacon_node/http_api/src/publish_blocks.rs | 129 +---- .../tests/broadcast_validation_tests.rs | 56 +- beacon_node/http_api/tests/tests.rs | 8 +- .../src/service/gossip_cache.rs | 7 - .../lighthouse_network/src/service/utils.rs | 4 - .../lighthouse_network/src/types/pubsub.rs | 40 +- .../lighthouse_network/src/types/topics.rs | 30 -- beacon_node/network/src/metrics.rs | 30 -- .../gossip_methods.rs | 491 ++++-------------- .../src/network_beacon_processor/mod.rs | 138 +---- .../src/network_beacon_processor/tests.rs | 38 -- beacon_node/network/src/router.rs | 13 - .../network/src/sync/block_lookups/mod.rs | 11 +- .../sync/block_lookups/single_block_lookup.rs | 4 +- beacon_node/network/src/sync/manager.rs | 21 - .../requests/blobs_by_range.rs | 1 + .../network_context/requests/blobs_by_root.rs | 2 + beacon_node/network/src/sync/tests/lookups.rs | 59 +-- testing/ef_tests/src/cases/fork_choice.rs | 72 +-- testing/simulator/Cargo.toml | 1 + testing/simulator/src/basic_sim.rs | 12 +- testing/simulator/src/fallback_sim.rs | 15 +- testing/simulator/src/local_network.rs | 11 + 48 files changed, 485 insertions(+), 2346 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 1d66bd30e7..3db4804bd1 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -304,7 +304,7 @@ jobs: cache-target: release - name: Create log dir run: mkdir ${{ runner.temp }}/basic_simulator_logs - - name: Run a basic beacon chain sim that starts from Deneb + - name: Run a basic beacon chain sim run: cargo run --release --bin simulator basic-sim --disable-stdout-logging --log-dir ${{ runner.temp }}/basic_simulator_logs - name: Upload logs if: always() diff --git a/Cargo.lock b/Cargo.lock index f246f2b353..a9fdfe70bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8202,6 +8202,7 @@ dependencies = [ name = "simulator" version = "0.2.0" dependencies = [ + "beacon_chain", "clap", "environment", "execution_layer", diff --git a/Makefile b/Makefile index dd57bb038e..3c00883ce9 100644 --- a/Makefile +++ b/Makefile @@ -33,11 +33,11 @@ PROFILE ?= release # List of all hard forks up to gloas. This list is used to set env variables for several tests so that # they run for different forks. # TODO(EIP-7732) Remove this once we extend network tests to support gloas and use RECENT_FORKS instead -RECENT_FORKS_BEFORE_GLOAS=electra fulu +RECENT_FORKS_BEFORE_GLOAS=fulu # List of all recent hard forks. This list is used to set env variables for http_api tests # Include phase0 to test the code paths in sync that are pre blobs -RECENT_FORKS=electra fulu gloas +RECENT_FORKS=fulu gloas # For network tests include phase0 to cover genesis syncing (blocks without blobs or columns) TEST_NETWORK_FORKS=phase0 $(RECENT_FORKS_BEFORE_GLOAS) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b3d258a2fb..d826895a25 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -5,7 +5,6 @@ use crate::attestation_verification::{ }; use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckCaches}; use crate::beacon_proposer_cache::{BeaconProposerCache, EpochBlockProposers}; -use crate::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use crate::block_times_cache::BlockTimesCache; use crate::block_verification::{ BlockError, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, @@ -26,16 +25,15 @@ use crate::data_availability_checker::{ use crate::data_availability_checker::DataAvailabilityChecker; use crate::data_column_verification::{ GossipDataColumnError, GossipPartialDataColumnError, GossipVerifiedDataColumn, - GossipVerifiedPartialDataColumnHeader, KzgVerifiedCustodyPartialDataColumn, - KzgVerifiedPartialDataColumn, PartialColumnVerificationResult, - validate_partial_data_column_sidecar_for_gossip, + GossipVerifiedPartialDataColumnHeader, KzgVerifiedCustodyDataColumn, + KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, + PartialColumnVerificationResult, validate_partial_data_column_sidecar_for_gossip, }; use crate::early_attester_cache::EarlyAttesterCache; use crate::envelope_times_cache::EnvelopeTimesCache; use crate::errors::{BeaconChainError as Error, BlockProductionError}; use crate::events::ServerSentEventHandler; use crate::execution_payload::{NotifyExecutionLayer, PreparePayloadHandle, get_execution_payload}; -use crate::fetch_blobs::EngineGetBlobsOutput; use crate::fork_choice_signal::{ForkChoiceSignalRx, ForkChoiceSignalTx}; use crate::graffiti_calculator::{GraffitiCalculator, GraffitiSettings}; use crate::light_client_finality_update_verification::{ @@ -431,8 +429,6 @@ pub struct BeaconChain { pub(crate) observed_payload_attesters: RwLock>, /// Maintains a record of which validators have proposed blocks for each slot. pub observed_block_producers: RwLock>, - /// Maintains a record of blob sidecars seen over the gossip network. - pub observed_blob_sidecars: RwLock, T::EthSpec>>, /// Maintains a record of column sidecars seen over the gossip network. pub observed_column_sidecars: RwLock, T::EthSpec>>, @@ -2453,19 +2449,6 @@ impl BeaconChain { ret } - #[instrument(skip_all, level = "trace")] - pub fn verify_blob_sidecar_for_gossip( - self: &Arc, - blob_sidecar: Arc>, - subnet_id: u64, - ) -> Result, GossipBlobError> { - metrics::inc_counter(&metrics::BLOBS_SIDECAR_PROCESSING_REQUESTS); - let _timer = metrics::start_timer(&metrics::BLOBS_SIDECAR_GOSSIP_VERIFICATION_TIMES); - GossipVerifiedBlob::new(blob_sidecar, subnet_id, self).inspect(|_| { - metrics::inc_counter(&metrics::BLOBS_SIDECAR_PROCESSING_SUCCESSES); - }) - } - /// Accepts some 'LightClientOptimisticUpdate' from the network and attempts to verify it pub fn verify_optimistic_update_for_gossip( self: &Arc, @@ -3253,35 +3236,6 @@ impl BeaconChain { .map_err(BeaconChainError::TokioJoin)? } - /// Cache the blob in the processing cache, process it, then evict it from the cache if it was - /// imported or errors. - #[instrument(skip_all, level = "debug")] - pub async fn process_gossip_blob( - self: &Arc, - blob: GossipVerifiedBlob, - ) -> Result { - let block_root = blob.block_root(); - - // If this block has already been imported to forkchoice it must have been available, so - // we don't need to process its blobs again. - if self - .canonical_head - .fork_choice_read_lock() - .contains_block(&block_root) - { - return Err(BlockError::DuplicateFullyImported(blob.block_root())); - } - - // No need to process and import blobs beyond the PeerDAS epoch. - if self.spec.is_peer_das_enabled_for_epoch(blob.epoch()) { - return Err(BlockError::BlobNotRequired(blob.slot())); - } - - self.emit_sse_blob_sidecar_events(&block_root, std::iter::once(blob.as_blob())); - - self.check_gossip_blob_availability_and_import(blob).await - } - /// Cache the data columns in the processing cache, process it, then evict it from the cache if it was /// imported or errors. /// Only accepts full columns. Partials are handled via PartialDataColumnAssembler. @@ -3428,19 +3382,21 @@ impl BeaconChain { return Err(BlockError::DuplicateFullyImported(block_root)); } - // Reject RPC blobs referencing unknown parents. Otherwise we allow potentially invalid data - // into the da_checker, where invalid = descendant of invalid blocks. - // Note: blobs should have at least one item and all items have the same parent root. - if let Some(parent_root) = blobs - .iter() - .filter_map(|b| b.as_ref().map(|b| b.block_parent_root())) - .next() - && !self - .canonical_head - .fork_choice_read_lock() - .contains_block(&parent_root) - { - return Err(BlockError::ParentUnknown { parent_root }); + for blob in &blobs { + if let Some(blob) = blob.as_ref() { + // Reject RPC blobs referencing unknown parents. Otherwise we allow potentially invalid data + // into the da_checker, where invalid = descendant of invalid blocks. + // Note: blobs should have at least one item and all items have the same parent root. + if !self + .canonical_head + .fork_choice_read_lock() + .contains_block(&blob.block_parent_root()) + { + return Err(BlockError::ParentUnknown { + parent_root: blob.block_parent_root(), + }); + } + } } self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().flatten().map(Arc::as_ref)); @@ -3454,7 +3410,7 @@ impl BeaconChain { self: &Arc, slot: Slot, block_root: Hash256, - engine_get_blobs_output: EngineGetBlobsOutput, + engine_get_blobs_output: Vec>, ) -> Result { // If this block has already been imported to forkchoice it must have been available, so // we don't need to process its blobs again. @@ -3466,17 +3422,12 @@ impl BeaconChain { return Err(BlockError::DuplicateFullyImported(block_root)); } - match &engine_get_blobs_output { - EngineGetBlobsOutput::Blobs(blobs) => { - self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().map(|b| b.as_blob())); - } - EngineGetBlobsOutput::CustodyColumns(columns) => { - self.emit_sse_data_column_sidecar_events( - &block_root, - columns.iter().map(|column| column.as_data_column()), - ); - } - } + self.emit_sse_data_column_sidecar_events( + &block_root, + engine_get_blobs_output + .iter() + .map(|column| column.as_data_column()), + ); self.check_engine_blobs_availability_and_import(slot, block_root, engine_get_blobs_output) .await @@ -3915,24 +3866,6 @@ impl BeaconChain { .await } - /// Checks if the provided blob can make any cached blocks available, and imports immediately - /// if so, otherwise caches the blob in the data availability checker. - async fn check_gossip_blob_availability_and_import( - self: &Arc, - blob: GossipVerifiedBlob, - ) -> Result { - let slot = blob.slot(); - if let Some(slasher) = self.slasher.as_ref() { - slasher.accept_block_header(blob.signed_block_header()); - } - let availability = self - .data_availability_checker - .put_gossip_verified_blobs(blob.block_root(), std::iter::once(blob))?; - - self.process_availability(slot, availability, || Ok(())) - .await - } - /// Checks if the provided data column can make any cached blocks available, and imports immediately /// if so, otherwise caches the data column in the data availability checker. /// Check gossip data columns for availability and import. Only accepts full columns. @@ -4015,7 +3948,7 @@ impl BeaconChain { ) -> Result { self.check_blob_header_signature_and_slashability( block_root, - blobs.iter().flatten().map(Arc::as_ref), + blobs.iter().flatten().map(|b| b.as_ref()), )?; let availability = self .data_availability_checker @@ -4030,56 +3963,36 @@ impl BeaconChain { self: &Arc, slot: Slot, block_root: Hash256, - engine_get_blobs_output: EngineGetBlobsOutput, + engine_get_blobs_output: Vec>, ) -> Result { - match engine_get_blobs_output { - EngineGetBlobsOutput::Blobs(blobs) => { - self.check_blob_header_signature_and_slashability( - block_root, - blobs.iter().map(|b| b.as_blob()), - )?; - let availability = self - .data_availability_checker - .put_kzg_verified_blobs(block_root, blobs) - .map_err(BlockError::from)?; - - Ok(self - .process_availability(slot, availability, || Ok(())) - .await?) - } - EngineGetBlobsOutput::CustodyColumns(data_columns) => { - // TODO(gloas) verify that this check is no longer relevant for gloas - self.check_data_column_sidecar_header_signature_and_slashability( - block_root, - data_columns - .iter() - .filter_map(|c| match c.as_data_column() { - DataColumnSidecar::Fulu(column) => Some(column), - _ => None, - }), - )?; - if self - .spec - .fork_name_at_slot::(slot) - .gloas_enabled() - { - let availability = self - .pending_payload_cache - .put_kzg_verified_custody_data_columns(block_root, &data_columns) - .map_err(BlockError::from)?; - Ok(self - .process_payload_envelope_availability(slot, availability, || Ok(())) - .await?) - } else { - let availability = self - .data_availability_checker - .put_kzg_verified_custody_data_columns(block_root, data_columns) - .map_err(BlockError::from)?; - Ok(self - .process_availability(slot, availability, || Ok(())) - .await?) - } - } + // TODO(gloas) verify that this check is no longer relevant for gloas + self.check_data_column_sidecar_header_signature_and_slashability( + block_root, + engine_get_blobs_output + .iter() + .filter_map(|c| match c.as_data_column() { + DataColumnSidecar::Fulu(column) => Some(column), + _ => None, + }), + )?; + if self + .spec + .fork_name_at_slot::(slot) + .gloas_enabled() + { + let availability = self + .pending_payload_cache + .put_kzg_verified_custody_data_columns(block_root, &engine_get_blobs_output) + .map_err(BlockError::from)?; + self.process_payload_envelope_availability(slot, availability, || Ok(())) + .await + } else { + let availability = self + .data_availability_checker + .put_kzg_verified_custody_data_columns(block_root, engine_get_blobs_output) + .map_err(BlockError::from)?; + self.process_availability(slot, availability, || Ok(())) + .await } } diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index e557a24369..79b2969645 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -1,249 +1,11 @@ -use educe::Educe; -use slot_clock::SlotClock; -use std::marker::PhantomData; -use std::sync::Arc; - -use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; -use crate::block_verification::{ - BlockSlashInfo, get_validator_pubkey_cache, process_block_slash_info, -}; use crate::kzg_utils::{validate_blob, validate_blobs}; -use crate::observed_data_sidecars::{ - Error as ObservedDataSidecarsError, ObservationStrategy, Observe, -}; -use crate::{BeaconChainError, metrics}; +use educe::Educe; use kzg::{Error as KzgError, Kzg, KzgCommitment}; use ssz_derive::{Decode, Encode}; +use std::sync::Arc; use std::time::Duration; -use tracing::{debug, instrument}; -use tree_hash::TreeHash; -use types::data::BlobIdentifier; -use types::{ - BeaconStateError, BlobSidecar, Epoch, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, -}; - -/// An error occurred while validating a gossip blob. -#[derive(Debug)] -pub enum GossipBlobError { - /// The blob sidecar is from a slot that is later than the current slot (with respect to the - /// gossip clock disparity). - /// - /// ## Peer scoring - /// - /// Assuming the local clock is correct, the peer has sent an invalid message. - FutureSlot { - message_slot: Slot, - latest_permissible_slot: Slot, - }, - - /// There was an error whilst processing the blob. It is not known if it is - /// valid or invalid. - /// - /// ## Peer scoring - /// - /// We were unable to process this blob due to an internal error. It's - /// unclear if the blob is valid. - BeaconChainError(Box), - - /// The `BlobSidecar` was gossiped over an incorrect subnet. - /// - /// ## Peer scoring - /// - /// The blob is invalid or the peer is faulty. - InvalidSubnet { expected: u64, received: u64 }, - - /// The sidecar corresponds to a slot older than the finalized head slot. - /// - /// ## Peer scoring - /// - /// It's unclear if this blob is valid, but this blob is for a finalized slot and is - /// therefore useless to us. - PastFinalizedSlot { - blob_slot: Slot, - finalized_slot: Slot, - }, - - /// The proposer index specified in the sidecar does not match the locally computed - /// proposer index. - /// - /// ## Peer scoring - /// - /// The blob is invalid and the peer is faulty. - ProposerIndexMismatch { sidecar: usize, local: usize }, - - /// The proposal signature in invalid. - /// - /// ## Peer scoring - /// - /// The blob is invalid and the peer is faulty. - ProposalSignatureInvalid, - - /// The proposal_index corresponding to blob.beacon_block_root is not known. - /// - /// ## Peer scoring - /// - /// The blob is invalid and the peer is faulty. - UnknownValidator(u64), - - /// The provided blob is not from a later slot than its parent. - /// - /// ## Peer scoring - /// - /// The blob is invalid and the peer is faulty. - BlobIsNotLaterThanParent { blob_slot: Slot, parent_slot: Slot }, - - /// The provided blob's parent block is unknown. - /// - /// ## Peer scoring - /// - /// We cannot process the blob without validating its parent, the peer isn't necessarily faulty. - ParentUnknown { parent_root: Hash256 }, - - /// Invalid kzg commitment inclusion proof - /// ## Peer scoring - /// - /// The blob sidecar is invalid and the peer is faulty - InvalidInclusionProof, - - /// A blob has already been seen for the given `(sidecar.block_root, sidecar.index)` tuple - /// over gossip or no gossip sources. - /// - /// ## Peer scoring - /// - /// The peer isn't faulty, but we do not forward it over gossip. - RepeatBlob { - proposer: u64, - slot: Slot, - index: u64, - }, - - /// The kzg verification failed. - /// - /// ## Peer scoring - /// - /// The blob sidecar is invalid and the peer is faulty. - KzgError(kzg::Error), - - /// The pubkey cache timed out. - /// - /// ## Peer scoring - /// - /// The blob sidecar may be valid, this is an internal error. - PubkeyCacheTimeout, - - /// The block conflicts with finalization, no need to propagate. - /// - /// ## Peer scoring - /// - /// It's unclear if this block is valid, but it conflicts with finality and shouldn't be - /// imported. - NotFinalizedDescendant { block_parent_root: Hash256 }, -} - -impl std::fmt::Display for GossipBlobError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) - } -} - -impl From for GossipBlobError { - fn from(e: BeaconChainError) -> Self { - GossipBlobError::BeaconChainError(e.into()) - } -} - -impl From for GossipBlobError { - fn from(e: BeaconStateError) -> Self { - GossipBlobError::BeaconChainError(BeaconChainError::BeaconStateError(e).into()) - } -} - -/// A wrapper around a `BlobSidecar` that indicates it has been approved for re-gossiping on -/// the p2p network. -#[derive(Debug)] -pub struct GossipVerifiedBlob { - block_root: Hash256, - blob: KzgVerifiedBlob, - _phantom: PhantomData, -} - -impl Clone for GossipVerifiedBlob { - fn clone(&self) -> Self { - Self { - block_root: self.block_root, - blob: self.blob.clone(), - _phantom: PhantomData, - } - } -} - -impl GossipVerifiedBlob { - pub fn new( - blob: Arc>, - subnet_id: u64, - chain: &BeaconChain, - ) -> Result { - let header = blob.signed_block_header.clone(); - // We only process slashing info if the gossip verification failed - // since we do not process the blob any further in that case. - validate_blob_sidecar_for_gossip::(blob, subnet_id, chain).map_err(|e| { - process_block_slash_info::<_, GossipBlobError>( - chain, - BlockSlashInfo::from_early_error_blob(header, e), - ) - }) - } - /// Construct a `GossipVerifiedBlob` that is assumed to be valid. - /// - /// This should ONLY be used for testing. - pub fn __assumed_valid(blob: Arc>) -> Self { - Self { - block_root: blob.block_root(), - blob: KzgVerifiedBlob { - blob, - seen_timestamp: Duration::from_secs(0), - }, - _phantom: PhantomData, - } - } - pub fn id(&self) -> BlobIdentifier { - BlobIdentifier { - block_root: self.block_root, - index: self.blob.blob_index(), - } - } - pub fn block_root(&self) -> Hash256 { - self.block_root - } - pub fn slot(&self) -> Slot { - self.blob.blob.slot() - } - pub fn epoch(&self) -> Epoch { - self.blob.blob.epoch() - } - pub fn index(&self) -> u64 { - self.blob.blob.index - } - pub fn kzg_commitment(&self) -> KzgCommitment { - self.blob.blob.kzg_commitment - } - pub fn signed_block_header(&self) -> SignedBeaconBlockHeader { - self.blob.blob.signed_block_header.clone() - } - pub fn block_proposer_index(&self) -> u64 { - self.blob.blob.block_proposer_index() - } - pub fn into_inner(self) -> KzgVerifiedBlob { - self.blob - } - pub fn as_blob(&self) -> &BlobSidecar { - self.blob.as_blob() - } - /// This is cheap as we're calling clone on an Arc - pub fn clone_blob(&self) -> Arc> { - self.blob.clone_blob() - } -} +use tracing::instrument; +use types::{BlobSidecar, EthSpec}; /// Wrapper over a `BlobSidecar` for which we have completed kzg verification. /// i.e. `verify_blob_kzg_proof(blob, commitment, proof) == true`. @@ -388,239 +150,3 @@ where .unzip(); validate_blobs::(kzg, commitments.as_slice(), blobs, proofs.as_slice()) } - -pub fn validate_blob_sidecar_for_gossip( - blob_sidecar: Arc>, - subnet: u64, - chain: &BeaconChain, -) -> Result, GossipBlobError> { - let blob_slot = blob_sidecar.slot(); - let blob_index = blob_sidecar.index; - let block_parent_root = blob_sidecar.block_parent_root(); - let blob_proposer_index = blob_sidecar.block_proposer_index(); - let block_root = blob_sidecar.block_root(); - let blob_epoch = blob_slot.epoch(T::EthSpec::slots_per_epoch()); - let signed_block_header = &blob_sidecar.signed_block_header; - - let seen_timestamp = chain.slot_clock.now_duration().unwrap_or_default(); - - // This condition is not possible if we have received the blob from the network - // since we only subscribe to `MaxBlobsPerBlock` subnets over gossip network. - // We include this check only for completeness. - // Getting this error would imply something very wrong with our networking decoding logic. - if blob_index >= chain.spec.max_blobs_per_block(blob_epoch) { - return Err(GossipBlobError::InvalidSubnet { - expected: subnet, - received: blob_index, - }); - } - - // Verify that the blob_sidecar was received on the correct subnet. - if blob_index != subnet { - return Err(GossipBlobError::InvalidSubnet { - expected: subnet, - received: blob_index, - }); - } - - // Verify that the sidecar is not from a future slot. - let latest_permissible_slot = chain - .slot_clock - .now_with_future_tolerance(chain.spec.maximum_gossip_clock_disparity()) - .ok_or(BeaconChainError::UnableToReadSlot)?; - if blob_slot > latest_permissible_slot { - return Err(GossipBlobError::FutureSlot { - message_slot: blob_slot, - latest_permissible_slot, - }); - } - - // Verify that the sidecar slot is greater than the latest finalized slot - let latest_finalized_slot = chain - .head() - .finalized_checkpoint() - .epoch - .start_slot(T::EthSpec::slots_per_epoch()); - if blob_slot <= latest_finalized_slot { - return Err(GossipBlobError::PastFinalizedSlot { - blob_slot, - finalized_slot: latest_finalized_slot, - }); - } - - // Verify that this is the first blob sidecar received for the tuple: - // (block_header.slot, block_header.proposer_index, blob_sidecar.index) - if chain - .observed_blob_sidecars - .read() - .observation_key_is_known(&blob_sidecar) - .map_err(|e| GossipBlobError::BeaconChainError(Box::new(e.into())))? - .is_some() - { - return Err(GossipBlobError::RepeatBlob { - proposer: blob_proposer_index, - slot: blob_slot, - index: blob_index, - }); - } - - // Verify the inclusion proof in the sidecar - let _timer = metrics::start_timer(&metrics::BLOB_SIDECAR_INCLUSION_PROOF_VERIFICATION); - if !blob_sidecar.verify_blob_sidecar_inclusion_proof() { - return Err(GossipBlobError::InvalidInclusionProof); - } - drop(_timer); - - let fork_choice = chain.canonical_head.fork_choice_read_lock(); - - // We have already verified that the blob is past finalization, so we can - // just check fork choice for the block's parent. - let Some(parent_block) = fork_choice.get_block(&block_parent_root) else { - return Err(GossipBlobError::ParentUnknown { - parent_root: block_parent_root, - }); - }; - - // Do not process a blob that does not descend from the finalized root. - // We just loaded the parent_block, so we can be sure that it exists in fork choice. - if !fork_choice.is_finalized_checkpoint_or_descendant(block_parent_root) { - return Err(GossipBlobError::NotFinalizedDescendant { block_parent_root }); - } - drop(fork_choice); - - if parent_block.slot >= blob_slot { - return Err(GossipBlobError::BlobIsNotLaterThanParent { - blob_slot, - parent_slot: parent_block.slot, - }); - } - - let proposer_shuffling_root = - parent_block.proposer_shuffling_root_for_child_block(blob_epoch, &chain.spec); - - let proposer = chain.with_proposer_cache( - proposer_shuffling_root, - blob_epoch, - |proposers| proposers.get_slot::(blob_slot), - || { - debug!( - %block_root, - index = %blob_index, - "Proposer shuffling cache miss for blob verification" - ); - // Blob verification is only relevant pre-Fulu and pre-Gloas, so `Pending` payload - // status is sufficient. - chain - .store - .get_advanced_hot_state(block_parent_root, blob_slot, parent_block.state_root) - .map_err(|e| GossipBlobError::BeaconChainError(Box::new(e.into())))? - .ok_or_else(|| { - GossipBlobError::BeaconChainError(Box::new(BeaconChainError::DBInconsistent( - format!("Missing state for parent block {block_parent_root:?}",), - ))) - }) - }, - )?; - let proposer_index = proposer.index; - let fork = proposer.fork; - - // Signature verify the signed block header. - let signature_is_valid = { - let pubkey_cache = - get_validator_pubkey_cache(chain).map_err(|_| GossipBlobError::PubkeyCacheTimeout)?; - - let pubkey = pubkey_cache - .get(proposer_index) - .ok_or_else(|| GossipBlobError::UnknownValidator(proposer_index as u64))?; - signed_block_header.verify_signature::( - pubkey, - &fork, - chain.genesis_validators_root, - &chain.spec, - ) - }; - - if !signature_is_valid { - return Err(GossipBlobError::ProposalSignatureInvalid); - } - - if proposer_index != blob_proposer_index as usize { - return Err(GossipBlobError::ProposerIndexMismatch { - sidecar: blob_proposer_index as usize, - local: proposer_index, - }); - } - - // Kzg verification for gossip blob sidecar - let kzg = chain.kzg.as_ref(); - - let kzg_verified_blob = KzgVerifiedBlob::new(blob_sidecar.clone(), kzg, seen_timestamp) - .map_err(GossipBlobError::KzgError)?; - let blob_sidecar = &kzg_verified_blob.blob; - - chain - .observed_slashable - .write() - .observe_slashable( - blob_sidecar.slot(), - blob_sidecar.block_proposer_index(), - block_root, - ) - .map_err(|e| GossipBlobError::BeaconChainError(Box::new(e.into())))?; - - if O::observe() { - observe_gossip_blob(&kzg_verified_blob.blob, chain)?; - } - - Ok(GossipVerifiedBlob { - block_root, - blob: kzg_verified_blob, - _phantom: PhantomData, - }) -} - -pub fn observe_gossip_blob( - blob_sidecar: &BlobSidecar, - chain: &BeaconChain, -) -> Result<(), GossipBlobError> { - // Now the signature is valid, store the proposal so we don't accept another blob sidecar - // with the same `BlobIdentifier`. It's important to double-check that the proposer still - // hasn't been observed so we don't have a race-condition when verifying two blocks - // simultaneously. - // - // Note: If this BlobSidecar goes on to fail full verification, we do not evict it from the - // seen_cache as alternate blob_sidecars for the same identifier can still be retrieved over - // rpc. Evicting them from this cache would allow faster propagation over gossip. So we - // allow retrieval of potentially valid blocks over rpc, but try to punish the proposer for - // signing invalid messages. Issue for more background - // https://github.com/ethereum/consensus-specs/issues/3261 - if chain - .observed_blob_sidecars - .write() - .observe_sidecar(blob_sidecar) - .map_err(|e: ObservedDataSidecarsError| { - GossipBlobError::BeaconChainError(Box::new(e.into())) - })? - .is_some() - { - return Err(GossipBlobError::RepeatBlob { - proposer: blob_sidecar.block_proposer_index(), - slot: blob_sidecar.slot(), - index: blob_sidecar.index, - }); - } - Ok(()) -} - -/// Returns the canonical root of the given `blob`. -/// -/// Use this function to ensure that we report the blob hashing time Prometheus metric. -pub fn get_blob_root(blob: &BlobSidecar) -> Hash256 { - let blob_root_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_BLOB_ROOT); - - let blob_root = blob.tree_hash_root(); - - metrics::stop_timer(blob_root_timer); - - blob_root -} diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 24f971f736..22e50e4185 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -49,7 +49,6 @@ #![allow(clippy::result_large_err)] use crate::beacon_snapshot::PreProcessingSnapshot; -use crate::blob_verification::GossipBlobError; use crate::block_verification_types::{AsBlock, BlockImportData, LookupBlock, RangeSyncBlock}; use crate::data_availability_checker::{ AvailabilityCheckError, AvailableBlock, AvailableBlockData, MaybeAvailableBlock, @@ -290,14 +289,6 @@ pub enum BlockError { EnvelopeBlockRootUnknown(Hash256), /// Optimistic sync is not supported for Gloas payload envelopes. OptimisticSyncNotSupported { block_root: Hash256 }, - /// A Blob with a slot after PeerDAS is received and is not required to be imported. - /// This can happen because we stay subscribed to the blob subnet after 2 epochs, as we could - /// still receive valid blobs from a Deneb epoch after PeerDAS is activated. - /// - /// ## Peer scoring - /// - /// This indicates the peer is sending an unexpected gossip blob and should be penalised. - BlobNotRequired(Slot), /// An internal error has occurred when processing the block or sidecars. /// /// ## Peer scoring @@ -520,17 +511,6 @@ impl BlockSlashInfo { } } -impl BlockSlashInfo { - pub fn from_early_error_blob(header: SignedBeaconBlockHeader, e: GossipBlobError) -> Self { - match e { - GossipBlobError::ProposalSignatureInvalid => BlockSlashInfo::SignatureInvalid(e), - // `InvalidSignature` could indicate any signature in the block, so we want - // to recheck the proposer signature alone. - _ => BlockSlashInfo::SignatureNotChecked(header, e), - } - } -} - impl BlockSlashInfo { pub fn from_early_error_data_column( header: SignedBeaconBlockHeader, @@ -2038,23 +2018,6 @@ impl BlockBlobError for BlockError { } } -impl BlockBlobError for GossipBlobError { - fn not_later_than_parent_error(blob_slot: Slot, parent_slot: Slot) -> Self { - GossipBlobError::BlobIsNotLaterThanParent { - blob_slot, - parent_slot, - } - } - - fn unknown_validator_error(validator_index: u64) -> Self { - GossipBlobError::UnknownValidator(validator_index) - } - - fn proposer_signature_invalid() -> Self { - GossipBlobError::ProposalSignatureInvalid - } -} - impl BlockBlobError for GossipDataColumnError { fn not_later_than_parent_error(data_column_slot: Slot, parent_slot: Slot) -> Self { GossipDataColumnError::IsNotLaterThanParent { diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index b8da2bcded..6df0b9c1a9 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1007,7 +1007,6 @@ where // TODO: allow for persisting and loading the pool from disk. observed_block_producers: <_>::default(), observed_column_sidecars: RwLock::new(ObservedDataSidecars::new(self.spec.clone())), - observed_blob_sidecars: RwLock::new(ObservedDataSidecars::new(self.spec.clone())), observed_slashable: <_>::default(), pending_payload_envelopes: <_>::default(), observed_voluntary_exits: <_>::default(), diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index b3ab2e6975..1eab7ccf7a 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -967,13 +967,6 @@ impl BeaconChain { .start_slot(T::EthSpec::slots_per_epoch()), ); - self.observed_blob_sidecars.write().prune( - new_view - .finalized_checkpoint - .epoch - .start_slot(T::EthSpec::slots_per_epoch()), - ); - self.observed_column_sidecars.write().prune( new_view .finalized_checkpoint diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index cfd8ee7d34..3c2ba13fed 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -1,6 +1,4 @@ -use crate::blob_verification::{ - GossipVerifiedBlob, KzgVerifiedBlob, KzgVerifiedBlobList, verify_kzg_for_blob_list, -}; +use crate::blob_verification::{KzgVerifiedBlob, KzgVerifiedBlobList, verify_kzg_for_blob_list}; use crate::block_verification_types::{AvailabilityPendingExecutedBlock, AvailableExecutedBlock}; use crate::data_availability_checker::overflow_lru_cache::{ DataAvailabilityCheckerInner, ReconstructColumnsDecision, @@ -364,24 +362,6 @@ impl DataAvailabilityChecker { .put_kzg_verified_data_columns(block_root, verified_custody_columns) } - /// Check if we've cached other blobs for this block. If it completes a set and we also - /// have a block cached, return the `Availability` variant triggering block import. - /// Otherwise cache the blob sidecar. - /// - /// This should only accept gossip verified blobs, so we should not have to worry about dupes. - #[instrument(skip_all, level = "trace")] - pub fn put_gossip_verified_blobs< - I: IntoIterator>, - O: ObservationStrategy, - >( - &self, - block_root: Hash256, - blobs: I, - ) -> Result, AvailabilityCheckError> { - self.availability_cache - .put_kzg_verified_blobs(block_root, blobs.into_iter().map(|b| b.into_inner())) - } - #[instrument(skip_all, level = "trace")] pub fn put_kzg_verified_blobs>>( &self, diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 8a80f835ab..2ce0b4cd4a 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -780,9 +780,11 @@ impl DataAvailabilityCheckerInner { mod test { use super::*; - use crate::test_utils::generate_data_column_indices_rand_order; + use crate::data_column_verification::{GossipVerifiedDataColumn, KzgVerifiedCustodyDataColumn}; + use crate::test_utils::{ + generate_data_column_indices_rand_order, generate_data_column_sidecars_from_block, + }; use crate::{ - blob_verification::GossipVerifiedBlob, block_verification::PayloadVerificationOutcome, block_verification_types::{AsBlock, BlockImportData}, custody_context::NodeCustodyType, @@ -794,8 +796,8 @@ mod test { use store::{HotColdDB, ItemStore, StoreConfig, database::interface::BeaconNodeBackend}; use tempfile::{TempDir, tempdir}; use tracing::info; - use types::MinimalEthSpec; use types::new_non_zero_usize; + use types::{DataColumnSubnetId, MinimalEthSpec}; const LOW_VALIDATOR_COUNT: usize = 32; @@ -819,21 +821,25 @@ mod test { .expect("disk store should initialize") } - // get a beacon chain harness advanced to just before deneb fork - async fn get_deneb_chain( + // get a beacon chain harness advanced to just before fulu fork + async fn get_fulu_chain( db_path: &TempDir, ) -> BeaconChainHarness> { let altair_fork_epoch = Epoch::new(0); let bellatrix_fork_epoch = Epoch::new(0); let capella_fork_epoch = Epoch::new(3); let deneb_fork_epoch = Epoch::new(4); - let deneb_fork_slot = deneb_fork_epoch.start_slot(E::slots_per_epoch()); + let electra_fork_epoch = Epoch::new(5); + let fulu_fork_epoch = Epoch::new(6); + let fulu_fork_slot = fulu_fork_epoch.start_slot(E::slots_per_epoch()); let mut spec = E::default_spec(); spec.altair_fork_epoch = Some(altair_fork_epoch); spec.bellatrix_fork_epoch = Some(bellatrix_fork_epoch); spec.capella_fork_epoch = Some(capella_fork_epoch); spec.deneb_fork_epoch = Some(deneb_fork_epoch); + spec.electra_fork_epoch = Some(electra_fork_epoch); + spec.fulu_fork_epoch = Some(fulu_fork_epoch); let spec = Arc::new(spec); let chain_store = get_store_with_spec::(db_path, spec.clone()); @@ -846,8 +852,10 @@ mod test { .mock_execution_layer() .build(); - // go right before deneb slot - harness.extend_to_slot(deneb_fork_slot - 1).await; + harness.execution_block_generator().set_min_blob_count(1); + + // go right before fulu slot + harness.extend_to_slot(fulu_fork_slot - 1).await; harness } @@ -856,7 +864,7 @@ mod test { harness: &BeaconChainHarness>, ) -> ( AvailabilityPendingExecutedBlock, - Vec>>, + Vec>>, ) where E: EthSpec, @@ -874,7 +882,7 @@ mod test { .expect("should get block") .expect("should have block"); - let (signed_beacon_block_hash, (block, maybe_blobs), state) = harness + let (signed_beacon_block_hash, (block, _maybe_blobs), state) = harness .add_block_at_slot(target_slot, parent_state) .await .expect("should add block"); @@ -892,27 +900,25 @@ mod test { .message() .body() .blob_kzg_commitments() - .expect("should be deneb fork") + .expect("should be fulu fork") .clone(), ) { info!(commitment = ?comm, "kzg commitment"); } info!("done printing kzg commitments"); - let gossip_verified_blobs = if let Some((kzg_proofs, blobs)) = maybe_blobs { - let sidecars = - BlobSidecar::build_sidecars(blobs, &block, kzg_proofs, &chain.spec).unwrap(); - Vec::from(sidecars) - .into_iter() - .map(|sidecar| { - let subnet = sidecar.index; - GossipVerifiedBlob::new(sidecar, subnet, &harness.chain) - .expect("should validate blob") - }) - .collect() - } else { - vec![] - }; + // Generate data columns from the block + let data_columns = generate_data_column_sidecars_from_block(&block, &harness.spec); + + let gossip_verified_columns: Vec<_> = data_columns + .into_iter() + .map(|sidecar| { + let subnet_id = + DataColumnSubnetId::from_column_index(*sidecar.index(), &harness.spec); + GossipVerifiedDataColumn::new(sidecar, subnet_id, &harness.chain) + .expect("should validate data column") + }) + .collect(); let slot = block.slot(); let consensus_context = ConsensusContext::::new(slot); @@ -933,7 +939,7 @@ mod test { payload_verification_outcome, }; - (availability_pending_block, gossip_verified_blobs) + (availability_pending_block, gossip_verified_columns) } async fn setup_harness_and_cache( @@ -953,7 +959,7 @@ mod test { { create_test_tracing_subscriber(); let chain_db_path = tempdir().expect("should get temp dir"); - let harness = get_deneb_chain(&chain_db_path).await; + let harness = get_fulu_chain(&chain_db_path).await; let spec = harness.spec.clone(); let capacity_non_zero = new_non_zero_usize(capacity); let custody_context = Arc::new(CustodyContext::new( @@ -979,20 +985,27 @@ mod test { let capacity = 4; let (harness, cache, _path) = setup_harness_and_cache::(capacity).await; - let (pending_block, blobs) = availability_pending_block(&harness).await; + let (pending_block, columns) = availability_pending_block(&harness).await; let root = pending_block.import_data.block_root; + let epoch = pending_block.block.epoch(); - let blobs_expected = pending_block.num_blobs_expected(); + let num_blobs_expected = pending_block.num_blobs_expected(); + let columns_expected = cache + .custody_context + .num_of_data_columns_to_sample(epoch, &harness.spec); + + // All columns are returned from availability_pending_block (E::number_of_columns()) + // but we only need custody columns assert_eq!( - blobs.len(), - blobs_expected, - "should have expected number of blobs" + columns.len(), + E::number_of_columns(), + "should have all data columns from block" ); assert!(cache.critical.read().is_empty(), "cache should be empty"); let availability = cache .put_executed_block(pending_block) .expect("should put block"); - if blobs_expected == 0 { + if num_blobs_expected == 0 { assert!( matches!(availability, Availability::Available(_)), "block doesn't have blobs, should be available" @@ -1005,7 +1018,7 @@ mod test { } else { assert!( matches!(availability, Availability::MissingComponents(_)), - "should be pending blobs" + "should be pending columns" ); assert_eq!( cache.critical.read().len(), @@ -1018,13 +1031,26 @@ mod test { ); } - let mut kzg_verified_blobs = Vec::new(); - for (blob_index, gossip_blob) in blobs.into_iter().enumerate() { - kzg_verified_blobs.push(gossip_blob.into_inner()); + // Get sampling column indices for this epoch + let sampling_column_indices = cache + .custody_context + .sampling_columns_for_epoch(epoch, &harness.spec); + + // Filter to only sampling columns + let sampling_columns: Vec<_> = columns + .into_iter() + .filter(|col| sampling_column_indices.contains(&col.index())) + .collect(); + + let mut kzg_verified_columns = Vec::new(); + for (col_index, gossip_column) in sampling_columns.into_iter().enumerate() { + kzg_verified_columns.push(KzgVerifiedCustodyDataColumn::from_asserted_custody( + gossip_column.into_inner(), + )); let availability = cache - .put_kzg_verified_blobs(root, kzg_verified_blobs.clone()) - .expect("should put blob"); - if blob_index == blobs_expected - 1 { + .put_kzg_verified_data_columns(root, kzg_verified_columns.clone()) + .expect("should put column"); + if col_index == columns_expected - 1 { assert!(matches!(availability, Availability::Available(_))); } else { assert!(matches!(availability, Availability::MissingComponents(_))); @@ -1032,20 +1058,36 @@ mod test { } } - let (pending_block, blobs) = availability_pending_block(&harness).await; - let blobs_expected = pending_block.num_blobs_expected(); + let (pending_block, columns) = availability_pending_block(&harness).await; + let _num_blobs_expected = pending_block.num_blobs_expected(); + let epoch = pending_block.block.epoch(); + // All columns returned assert_eq!( - blobs.len(), - blobs_expected, - "should have expected number of blobs" + columns.len(), + E::number_of_columns(), + "should have all data columns" ); let root = pending_block.import_data.block_root; - let mut kzg_verified_blobs = vec![]; - for gossip_blob in blobs { - kzg_verified_blobs.push(gossip_blob.into_inner()); + + // Get sampling column indices for this epoch + let sampling_column_indices = cache + .custody_context + .sampling_columns_for_epoch(epoch, &harness.spec); + + // Filter to only sampling columns + let sampling_columns: Vec<_> = columns + .into_iter() + .filter(|col| sampling_column_indices.contains(&col.index())) + .collect(); + + let mut kzg_verified_columns = vec![]; + for gossip_column in sampling_columns { + kzg_verified_columns.push(KzgVerifiedCustodyDataColumn::from_asserted_custody( + gossip_column.into_inner(), + )); let availability = cache - .put_kzg_verified_blobs(root, kzg_verified_blobs.clone()) - .expect("should put blob"); + .put_kzg_verified_data_columns(root, kzg_verified_columns.clone()) + .expect("should put column"); assert!( matches!(availability, Availability::MissingComponents(_)), "should be pending block" diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index f5ba647fce..b75fcdac5c 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -1,8 +1,9 @@ -use crate::fetch_blobs::{EngineGetBlobsOutput, FetchEngineBlobError}; +use crate::data_column_verification::KzgVerifiedCustodyDataColumn; +use crate::fetch_blobs::FetchEngineBlobError; use crate::observed_data_sidecars::ObservationKey; use crate::partial_data_column_assembler::PartialDataColumnAssembler; use crate::{AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes}; -use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; +use execution_layer::json_structures::{BlobAndProofV2, BlobAndProofV3}; use kzg::Kzg; #[cfg(test)] use mockall::automock; @@ -43,22 +44,6 @@ impl FetchBlobsBeaconAdapter { .cloned() } - pub(crate) async fn get_blobs_v1( - &self, - versioned_hashes: Vec, - ) -> Result>>, FetchEngineBlobError> { - let execution_layer = self - .chain - .execution_layer - .as_ref() - .ok_or(FetchEngineBlobError::ExecutionLayerMissing)?; - - execution_layer - .get_blobs_v1(versioned_hashes) - .await - .map_err(FetchEngineBlobError::RequestFailed) - } - pub(crate) async fn get_blobs_v2( &self, versioned_hashes: Vec, @@ -91,17 +76,6 @@ impl FetchBlobsBeaconAdapter { .map_err(FetchEngineBlobError::RequestFailed) } - pub(crate) fn blobs_known_for_observation_key( - &self, - observation_key: ObservationKey, - ) -> Option> { - self.chain - .observed_blob_sidecars - .read() - .known_for_observation_key(&observation_key) - .cloned() - } - pub(crate) fn data_column_known_for_observation_key( &self, observation_key: ObservationKey, @@ -113,12 +87,6 @@ impl FetchBlobsBeaconAdapter { .cloned() } - pub(crate) fn cached_blob_indexes(&self, block_root: &Hash256) -> Option> { - self.chain - .data_availability_checker - .cached_blob_indexes(block_root) - } - pub(crate) fn cached_data_column_indexes( &self, block_root: &Hash256, @@ -131,7 +99,7 @@ impl FetchBlobsBeaconAdapter { &self, slot: Slot, block_root: Hash256, - blobs: EngineGetBlobsOutput, + blobs: Vec>, ) -> Result { self.chain .process_engine_blobs(slot, block_root, blobs) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 351e35666a..158cef0003 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -12,7 +12,6 @@ mod fetch_blobs_beacon_adapter; #[cfg(test)] mod tests; -use crate::blob_verification::{GossipBlobError, KzgVerifiedBlob}; use crate::data_column_verification::{ KzgVerifiedCustodyDataColumn, KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, }; @@ -25,26 +24,15 @@ use crate::{ metrics, }; use execution_layer::Error as ExecutionLayerError; -use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; +use execution_layer::json_structures::{BlobAndProofV2, BlobAndProofV3}; use metrics::{TryExt, inc_counter}; #[cfg(test)] use mockall_double::double; -use slot_clock::timestamp_now; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; use tracing::{debug, instrument, warn}; use types::data::{BlobSidecarError, ColumnIndex, DataColumnSidecarError, PartialDataColumnHeader}; -use types::{BeaconStateError, BlobSidecar, EthSpec, Hash256, VersionedHash}; - -/// Result from engine get blobs to be passed onto `DataAvailabilityChecker` and published to the -/// gossip network. The blobs / data columns have not been marked as observed yet, as they may not -/// be published immediately. -#[derive(Debug)] -pub enum EngineGetBlobsOutput { - Blobs(Vec>), - /// A filtered list of custody data columns to be imported into the `DataAvailabilityChecker`. - CustodyColumns(Vec>), -} +use types::{BeaconStateError, EthSpec, Hash256, VersionedHash}; #[derive(Debug)] pub enum FetchEngineBlobError { @@ -55,22 +43,21 @@ pub enum FetchEngineBlobError { DataColumnSidecarError(DataColumnSidecarError), ExecutionLayerMissing, InternalError(String), - GossipBlob(GossipBlobError), KzgError(kzg::Error), RequestFailed(ExecutionLayerError), RuntimeShutdown, TokioJoin(tokio::task::JoinError), } -/// Fetches blobs from the EL mempool and processes them. It also broadcasts unseen blobs or -/// data columns (PeerDAS onwards) to the network, using the supplied `publish_fn`. +/// Fetches blobs from the EL mempool and processes them as data columns. It also broadcasts +/// unseen data columns to the network, using the supplied `publish_fn`. #[instrument(skip_all)] pub async fn fetch_and_process_engine_blobs( chain: Arc>, block_root: Hash256, header: Arc>, custody_columns: &[ColumnIndex], - publish_fn: impl Fn(EngineGetBlobsOutput) + Send + 'static, + publish_fn: impl Fn(Vec>) + Send + 'static, ) -> Result, FetchEngineBlobError> { fetch_and_process_engine_blobs_inner( FetchBlobsBeaconAdapter::new(chain), @@ -89,7 +76,7 @@ async fn fetch_and_process_engine_blobs_inner( block_root: Hash256, header: Arc>, custody_columns: &[ColumnIndex], - publish_fn: impl Fn(EngineGetBlobsOutput) + Send + 'static, + publish_fn: impl Fn(Vec>) + Send + 'static, ) -> Result, FetchEngineBlobError> { let versioned_hashes = header .kzg_commitments @@ -120,104 +107,12 @@ async fn fetch_and_process_engine_blobs_inner( ) .await } else { - fetch_and_process_blobs_v1( - chain_adapter, - block_root, - &header, - versioned_hashes, - publish_fn, - ) - .await + Err(FetchEngineBlobError::InternalError( + "fetch blobs v1 no longer supported".to_owned(), + )) } } -#[instrument(skip_all, level = "debug")] -async fn fetch_and_process_blobs_v1( - chain_adapter: FetchBlobsBeaconAdapter, - block_root: Hash256, - header: &PartialDataColumnHeader, - versioned_hashes: Vec, - publish_fn: impl Fn(EngineGetBlobsOutput) + Send + Sized, -) -> Result, FetchEngineBlobError> { - let num_expected_blobs = versioned_hashes.len(); - metrics::observe(&metrics::BLOBS_FROM_EL_EXPECTED, num_expected_blobs as f64); - debug!(num_expected_blobs, "Fetching blobs from the EL"); - let response = chain_adapter - .get_blobs_v1(versioned_hashes) - .await - .inspect_err(|_| { - inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); - })?; - - let num_fetched_blobs = response.iter().filter(|opt| opt.is_some()).count(); - metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); - - if num_fetched_blobs == 0 { - debug!(num_expected_blobs, "No blobs fetched from the EL"); - inc_counter(&metrics::BLOBS_FROM_EL_MISS_TOTAL); - return Ok(None); - } else { - debug!( - num_expected_blobs, - num_fetched_blobs, "Received blobs from the EL" - ); - inc_counter(&metrics::BLOBS_FROM_EL_HIT_TOTAL); - } - - if chain_adapter.fork_choice_contains_block(&block_root) { - // Avoid computing sidecars if the block has already been imported. - debug!( - info = "block has already been imported", - "Ignoring EL blobs response" - ); - return Ok(None); - } - - let mut blob_sidecar_list = build_blob_sidecars(header, response)?; - - let observation_key = ObservationKey::new_proposer_key( - header.signed_block_header.message.proposer_index, - header.slot(), - ); - - if let Some(observed_blobs) = chain_adapter.blobs_known_for_observation_key(observation_key) { - blob_sidecar_list.retain(|blob| !observed_blobs.contains(&blob.blob_index())); - if blob_sidecar_list.is_empty() { - debug!( - info = "blobs have already been seen on gossip", - "Ignoring EL blobs response" - ); - return Ok(None); - } - } - - if let Some(known_blobs) = chain_adapter.cached_blob_indexes(&block_root) { - blob_sidecar_list.retain(|blob| !known_blobs.contains(&blob.blob_index())); - if blob_sidecar_list.is_empty() { - debug!( - info = "blobs have already been imported into data availability checker", - "Ignoring EL blobs response" - ); - return Ok(None); - } - } - - // Up until this point we have not observed the blobs in the gossip cache, which allows them to - // arrive independently while this function is running. In `publish_fn` we will observe them - // and then publish any blobs that had not already been observed. - publish_fn(EngineGetBlobsOutput::Blobs(blob_sidecar_list.clone())); - - let availability_processing_status = chain_adapter - .process_engine_blobs( - header.slot(), - block_root, - EngineGetBlobsOutput::Blobs(blob_sidecar_list), - ) - .await?; - - Ok(Some(availability_processing_status)) -} - #[instrument(skip_all, level = "debug")] async fn fetch_and_process_blobs_v2_or_v3( chain_adapter: FetchBlobsBeaconAdapter, @@ -225,7 +120,7 @@ async fn fetch_and_process_blobs_v2_or_v3( header: Arc>, versioned_hashes: Vec, custody_columns_indices: &[ColumnIndex], - publish_fn: impl Fn(EngineGetBlobsOutput) + Send + 'static, + publish_fn: impl Fn(Vec>) + Send + 'static, ) -> Result, FetchEngineBlobError> { let num_expected_blobs = versioned_hashes.len(); let slot = header.slot(); @@ -354,7 +249,7 @@ async fn fetch_and_process_blobs_v2_or_v3( // Publish complete columns if !full_columns.is_empty() { - publish_fn(EngineGetBlobsOutput::CustodyColumns(full_columns.clone())); + publish_fn(full_columns.clone()); } // We publish all partials at the calling site, regardless of result, as previous publishs // have been blocked, waiting for the results of this call @@ -362,11 +257,7 @@ async fn fetch_and_process_blobs_v2_or_v3( // Process complete columns through DA checker let availability_processing_status = if !full_columns.is_empty() { chain_adapter - .process_engine_blobs( - slot, - block_root, - EngineGetBlobsOutput::CustodyColumns(full_columns), - ) + .process_engine_blobs(slot, block_root, full_columns) .await? } else { // No complete columns yet, still missing components @@ -461,30 +352,3 @@ async fn compute_custody_columns_to_import( .await .map_err(FetchEngineBlobError::TokioJoin)? } - -fn build_blob_sidecars( - header: &PartialDataColumnHeader, - response: Vec>>, -) -> Result>, FetchEngineBlobError> { - let mut sidecars = vec![]; - for (index, blob_and_proof) in response - .into_iter() - .enumerate() - .filter_map(|(index, opt_blob)| Some((index, opt_blob?))) - { - let blob_sidecar = BlobSidecar::new_with_existing_proof( - index, - blob_and_proof.blob, - header.clone(), - blob_and_proof.proof, - ) - .map_err(FetchEngineBlobError::BlobSidecarError)?; - - sidecars.push(KzgVerifiedBlob::from_execution_verified( - Arc::new(blob_sidecar), - timestamp_now(), - )); - } - - Ok(sidecars) -} diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index 37d40f3a27..99cb4b5a78 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -1,8 +1,7 @@ use crate::AvailabilityProcessingStatus; +use crate::data_column_verification::KzgVerifiedCustodyDataColumn; use crate::fetch_blobs::fetch_blobs_beacon_adapter::MockFetchBlobsBeaconAdapter; -use crate::fetch_blobs::{ - EngineGetBlobsOutput, FetchEngineBlobError, fetch_and_process_engine_blobs_inner, -}; +use crate::fetch_blobs::{FetchEngineBlobError, fetch_and_process_engine_blobs_inner}; use crate::partial_data_column_assembler::PartialDataColumnAssembler; use crate::test_utils::{EphemeralHarnessType, get_kzg}; use bls::Signature; @@ -226,7 +225,7 @@ mod get_blobs_v2 { assert!( matches!( published_columns, - EngineGetBlobsOutput::CustodyColumns(columns) if columns.len() == custody_columns.len() + columns if columns.len() == custody_columns.len() ), "should publish custody columns" ); @@ -251,284 +250,10 @@ mod get_blobs_v2 { } } -mod get_blobs_v1 { - use super::*; - use crate::block_verification_types::AsBlock; - use std::collections::HashSet; - use types::{ColumnIndex, FullPayload, PartialDataColumnHeader}; - - const ELECTRA_FORK: ForkName = ForkName::Electra; - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_fetch_blobs_v1_no_blobs_in_block() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); - let spec = mock_adapter.spec(); - let (publish_fn, _s) = mock_publish_fn(); - let block_no_blobs = SignedBeaconBlock::>::from_block( - BeaconBlock::empty(spec), - Signature::empty(), - ); - let block_root = block_no_blobs.canonical_root(); - - // Expectations: engine fetch blobs should not be triggered - mock_adapter.expect_get_blobs_v1().times(0); - - // WHEN: Trigger fetch blobs on the block - let custody_columns: [ColumnIndex; 3] = [0, 1, 2]; - let processing_status = fetch_and_process_engine_blobs_inner( - mock_adapter, - block_root, - Arc::new(PartialDataColumnHeader::try_from(&block_no_blobs).unwrap()), - &custody_columns, - publish_fn, - ) - .await - .expect("fetch blobs should succeed"); - - // THEN: No blob is processed - assert_eq!(processing_status, None); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_fetch_blobs_v1_no_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); - let (publish_fn, _) = mock_publish_fn(); - let (block, _blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); - let block_root = block.canonical_root(); - - // GIVEN: No blobs in EL response - let expected_blob_count = block.message().body().blob_kzg_commitments().unwrap().len(); - mock_get_blobs_v1_response(&mut mock_adapter, vec![None; expected_blob_count]); - - // WHEN: Trigger fetch blobs on the block - let custody_columns: [ColumnIndex; 3] = [0, 1, 2]; - let processing_status = fetch_and_process_engine_blobs_inner( - mock_adapter, - block_root, - Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), - &custody_columns, - publish_fn, - ) - .await - .expect("fetch blobs should succeed"); - - // THEN: No blob is processed - assert_eq!(processing_status, None); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_fetch_blobs_v1_partial_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); - let (publish_fn, publish_fn_args) = mock_publish_fn(); - let blob_count = 2; - let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, blob_count); - let block_slot = block.slot(); - let block_root = block.canonical_root(); - - // GIVEN: Missing a blob in EL response (remove 1 blob from response) - let mut blob_and_proof_opts = blobs_and_proofs.into_iter().map(Some).collect::>(); - blob_and_proof_opts.first_mut().unwrap().take(); - mock_get_blobs_v1_response(&mut mock_adapter, blob_and_proof_opts); - // AND block is not imported into fork choice - mock_fork_choice_contains_block(&mut mock_adapter, vec![]); - // AND all blobs have not yet been seen - mock_adapter - .expect_cached_blob_indexes() - .returning(|_| None); - mock_adapter - .expect_blobs_known_for_observation_key() - .returning(|_| None); - // Returned blobs should be processed - mock_process_engine_blobs_result( - &mut mock_adapter, - Ok(AvailabilityProcessingStatus::MissingComponents( - block_slot, block_root, - )), - ); - - // WHEN: Trigger fetch blobs on the block - let custody_columns: [ColumnIndex; 3] = [0, 1, 2]; - let processing_status = fetch_and_process_engine_blobs_inner( - mock_adapter, - block_root, - Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), - &custody_columns, - publish_fn, - ) - .await - .expect("fetch blobs should succeed"); - - // THEN: Returned blobs are processed and published - assert_eq!( - processing_status, - Some(AvailabilityProcessingStatus::MissingComponents( - block_slot, block_root, - )) - ); - assert!( - matches!( - extract_published_blobs(publish_fn_args), - EngineGetBlobsOutput::Blobs(blobs) if blobs.len() == blob_count - 1 - ), - "partial blob results should still be published" - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_fetch_blobs_v1_block_imported_after_el_response() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); - let (publish_fn, publish_fn_args) = mock_publish_fn(); - let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); - let block_root = block.canonical_root(); - - // GIVEN: All blobs returned, but fork choice already imported the block - let blob_and_proof_opts = blobs_and_proofs.into_iter().map(Some).collect::>(); - mock_get_blobs_v1_response(&mut mock_adapter, blob_and_proof_opts); - mock_fork_choice_contains_block(&mut mock_adapter, vec![block.canonical_root()]); - - // WHEN: Trigger fetch blobs on the block - let custody_columns: [ColumnIndex; 3] = [0, 1, 2]; - let processing_status = fetch_and_process_engine_blobs_inner( - mock_adapter, - block_root, - Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), - &custody_columns, - publish_fn, - ) - .await - .expect("fetch blobs should succeed"); - - // THEN: Returned blobs should NOT be processed or published. - assert_eq!(processing_status, None); - assert_eq!( - publish_fn_args.lock().unwrap().len(), - 0, - "no blobs should be published" - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_fetch_blobs_v1_no_new_blobs_to_import() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); - let (publish_fn, publish_fn_args) = mock_publish_fn(); - let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); - let block_root = block.canonical_root(); - - // **GIVEN**: - // All blobs returned - let blob_and_proof_opts = blobs_and_proofs.into_iter().map(Some).collect::>(); - let all_blob_indices = blob_and_proof_opts - .iter() - .enumerate() - .map(|(i, _)| i as u64) - .collect::>(); - - mock_get_blobs_v1_response(&mut mock_adapter, blob_and_proof_opts); - // block not yet imported into fork choice - mock_fork_choice_contains_block(&mut mock_adapter, vec![]); - // All blobs already seen on gossip - mock_adapter - .expect_cached_blob_indexes() - .returning(|_| None); - mock_adapter - .expect_blobs_known_for_observation_key() - .returning(move |_| Some(all_blob_indices.clone())); - - // **WHEN**: Trigger `fetch_blobs` on the block - let custody_columns: [ColumnIndex; 3] = [0, 1, 2]; - let processing_status = fetch_and_process_engine_blobs_inner( - mock_adapter, - block_root, - Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), - &custody_columns, - publish_fn, - ) - .await - .expect("fetch blobs should succeed"); - - // **THEN**: Should NOT be processed and no blobs should be published. - assert_eq!(processing_status, None); - assert_eq!( - publish_fn_args.lock().unwrap().len(), - 0, - "no blobs should be published" - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_fetch_blobs_v1_success() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); - let (publish_fn, publish_fn_args) = mock_publish_fn(); - let blob_count = 2; - let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, blob_count); - let block_root = block.canonical_root(); - - // All blobs returned, fork choice doesn't contain block - let blob_and_proof_opts = blobs_and_proofs.into_iter().map(Some).collect::>(); - mock_get_blobs_v1_response(&mut mock_adapter, blob_and_proof_opts); - mock_fork_choice_contains_block(&mut mock_adapter, vec![]); - mock_adapter - .expect_cached_blob_indexes() - .returning(|_| None); - mock_adapter - .expect_blobs_known_for_observation_key() - .returning(|_| None); - mock_process_engine_blobs_result( - &mut mock_adapter, - Ok(AvailabilityProcessingStatus::Imported(block_root)), - ); - - // Trigger fetch blobs on the block - let custody_columns: [ColumnIndex; 3] = [0, 1, 2]; - let processing_status = fetch_and_process_engine_blobs_inner( - mock_adapter, - block_root, - Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), - &custody_columns, - publish_fn, - ) - .await - .expect("fetch blobs should succeed"); - - // THEN all fetched blobs are processed and published - assert_eq!( - processing_status, - Some(AvailabilityProcessingStatus::Imported(block_root)) - ); - - let published_blobs = extract_published_blobs(publish_fn_args); - assert!( - matches!( - published_blobs, - EngineGetBlobsOutput::Blobs(blobs) if blobs.len() == blob_count - ), - "should publish fetched blobs" - ); - } - - fn mock_get_blobs_v1_response( - mock_adapter: &mut MockFetchBlobsBeaconAdapter, - blobs_and_proofs_opt: Vec>>, - ) { - let blobs_and_proofs_v1 = blobs_and_proofs_opt - .into_iter() - .map(|blob_and_proof_opt| { - blob_and_proof_opt.map(|blob_and_proof| match blob_and_proof { - BlobAndProof::V1(inner) => inner, - _ => panic!("BlobAndProofV1 not expected"), - }) - }) - .collect(); - mock_adapter - .expect_get_blobs_v1() - .return_once(move |_| Ok(blobs_and_proofs_v1)); - } -} - -/// Extract the `EngineGetBlobsOutput` passed to the `publish_fn`. +/// Extract the `Vec>` passed to the `publish_fn`. fn extract_published_blobs( - publish_fn_args: Arc>>>, -) -> EngineGetBlobsOutput { + publish_fn_args: Arc>>>>, +) -> Vec> { let mut calls = publish_fn_args.lock().unwrap(); assert_eq!(calls.len(), 1); calls.pop().unwrap() @@ -597,8 +322,8 @@ fn create_test_block_and_blobs( #[allow(clippy::type_complexity)] fn mock_publish_fn() -> ( - impl Fn(EngineGetBlobsOutput) + Send + 'static, - Arc>>>, + impl Fn(Vec>) + Send + 'static, + Arc>>>>, ) { // Keep track of the arguments captured by `publish_fn`. let captured_args = Arc::new(Mutex::new(vec![])); diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index c2ccad7d8c..919bb43bfd 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1,4 +1,3 @@ -use crate::blob_verification::GossipVerifiedBlob; use crate::block_verification_types::{AsBlock, AvailableBlockData, LookupBlock, RangeSyncBlock}; use crate::custody_context::NodeCustodyType; use crate::data_availability_checker::DataAvailabilityChecker; @@ -3696,55 +3695,39 @@ where Ok(()) } - /// Simulate some of the blobs / data columns being seen on gossip. - /// Converts the blobs to data columns if the slot is Fulu or later. - pub async fn process_gossip_blobs_or_columns<'a>( + /// Simulate the block's custody data columns (or those in `custody_columns_opt`) being + /// seen on gossip. Panics unless PeerDAS is enabled for the block's epoch. + pub async fn process_gossip_columns( &self, block: &SignedBeaconBlock, - blobs: impl Iterator>, - proofs: impl Iterator, custody_columns_opt: Option>, ) { - let is_peerdas_enabled = self.chain.spec.is_peer_das_enabled_for_epoch(block.epoch()); - if is_peerdas_enabled { - let custody_columns = custody_columns_opt.unwrap_or_else(|| { - let epoch = block.slot().epoch(E::slots_per_epoch()); - self.chain - .sampling_columns_for_epoch(epoch) - .iter() - .copied() - .collect() - }); + assert!(self.chain.spec.is_peer_das_enabled_for_epoch(block.epoch())); + let custody_columns = custody_columns_opt.unwrap_or_else(|| { + let epoch = block.slot().epoch(E::slots_per_epoch()); + self.chain + .sampling_columns_for_epoch(epoch) + .iter() + .copied() + .collect() + }); - let verified_columns = generate_data_column_sidecars_from_block(block, &self.spec) - .into_iter() - .filter(|c| custody_columns.contains(c.index())) - .map(|sidecar| { - let subnet_id = - DataColumnSubnetId::from_column_index(*sidecar.index(), &self.spec); - self.chain - .verify_data_column_sidecar_for_gossip(sidecar, subnet_id) - }) - .collect::, _>>() + let verified_columns = generate_data_column_sidecars_from_block(block, &self.spec) + .into_iter() + .filter(|c| custody_columns.contains(c.index())) + .map(|sidecar| { + let subnet_id = DataColumnSubnetId::from_column_index(*sidecar.index(), &self.spec); + self.chain + .verify_data_column_sidecar_for_gossip(sidecar, subnet_id) + }) + .collect::, _>>() + .unwrap(); + + if !verified_columns.is_empty() { + self.chain + .process_gossip_data_columns(verified_columns, || Ok(())) + .await .unwrap(); - - if !verified_columns.is_empty() { - self.chain - .process_gossip_data_columns(verified_columns, || Ok(())) - .await - .unwrap(); - } - } else { - for (i, (kzg_proof, blob)) in proofs.into_iter().zip(blobs).enumerate() { - let sidecar = - Arc::new(BlobSidecar::new(i, blob.clone(), block, *kzg_proof).unwrap()); - let gossip_blob = GossipVerifiedBlob::new(sidecar, i as u64, &self.chain) - .expect("should obtain gossip verified blob"); - self.chain - .process_gossip_blob(gossip_blob) - .await - .expect("should import valid gossip verified blob"); - } } } } diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 533ef61219..67fe0eaae0 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -1456,20 +1456,8 @@ async fn verify_and_process_gossip_data_sidecars( data_sidecars: DataSidecars, ) { match data_sidecars { - DataSidecars::Blobs(blob_sidecars) => { - for blob_sidecar in blob_sidecars { - let blob_index = blob_sidecar.index; - let gossip_verified = harness - .chain - .verify_blob_sidecar_for_gossip(blob_sidecar.clone(), blob_index) - .expect("should obtain gossip verified blob"); - - harness - .chain - .process_gossip_blob(gossip_verified) - .await - .expect("should import valid gossip verified blob"); - } + DataSidecars::Blobs(_blob_sidecars) => { + // Blob gossip is deprecated, blobs are available via RPC. } DataSidecars::DataColumns(column_sidecars) => { let gossip_verified = column_sidecars @@ -1521,14 +1509,9 @@ async fn verify_block_for_gossip_slashing_detection() { let verified_block = harness.chain.verify_block_for_gossip(block1).await.unwrap(); - if let Some((kzg_proofs, blobs)) = blobs1 { + if blobs1.is_some() { harness - .process_gossip_blobs_or_columns( - verified_block.block(), - blobs.iter(), - kzg_proofs.iter(), - None, - ) + .process_gossip_columns(verified_block.block(), None) .await; } harness diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index 29d0e38b93..baa6975303 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -1,12 +1,9 @@ use arbitrary::Arbitrary; -use beacon_chain::blob_verification::GossipVerifiedBlob; use beacon_chain::data_column_verification::GossipVerifiedDataColumn; use beacon_chain::test_utils::{ BeaconChainHarness, fork_name_from_env, generate_data_column_sidecars_from_block, test_spec, }; use eth2::types::{EventKind, SseBlobSidecar, SseDataColumnSidecar}; -use rand::SeedableRng; -use rand::rngs::StdRng; use std::sync::Arc; use types::data::FixedBlobSidecarList; use types::{ @@ -17,44 +14,6 @@ use types::{ type E = MinimalEthSpec; -/// Verifies that a blob event is emitted when a gossip verified blob is received via gossip or the publish block API. -#[tokio::test] -async fn blob_sidecar_event_on_process_gossip_blob() { - if fork_name_from_env().is_some_and(|f| !f.deneb_enabled() || f.fulu_enabled()) { - return; - }; - - let spec = Arc::new(test_spec::()); - let harness = BeaconChainHarness::builder(E::default()) - .spec(spec) - .deterministic_keypairs(8) - .fresh_ephemeral_store() - .mock_execution_layer() - .build(); - - // subscribe to blob sidecar events - let event_handler = harness.chain.event_handler.as_ref().unwrap(); - let mut blob_event_receiver = event_handler.subscribe_blob_sidecar(); - - // build and process a gossip verified blob - let kzg = harness.chain.kzg.as_ref(); - let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); - let sidecar = BlobSidecar::random_valid(&mut rng, kzg) - .map(Arc::new) - .unwrap(); - let gossip_verified_blob = GossipVerifiedBlob::__assumed_valid(sidecar); - let expected_sse_blobs = SseBlobSidecar::from_blob_sidecar(gossip_verified_blob.as_blob()); - - let _ = harness - .chain - .process_gossip_blob(gossip_verified_blob) - .await - .unwrap(); - - let sidecar_event = blob_event_receiver.try_recv().unwrap(); - assert_eq!(sidecar_event, EventKind::BlobSidecar(expected_sse_blobs)); -} - /// Verifies that a data column event is emitted when a gossip verified data column is received via gossip or the publish block API. #[tokio::test] async fn data_column_sidecar_event_on_process_gossip_data_column() { diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index ce3851ea54..af3ff09c8a 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -390,7 +390,6 @@ pub enum Work { process_batch: Box>) + Send + Sync>, }, GossipBlock(AsyncFn), - GossipBlobSidecar(AsyncFn), GossipDataColumnSidecar(AsyncFn), GossipPartialDataColumnSidecar(AsyncFn), DelayedImportBlock { @@ -471,7 +470,6 @@ pub enum WorkType { UnknownLightClientOptimisticUpdate, GossipAggregateBatch, GossipBlock, - GossipBlobSidecar, GossipDataColumnSidecar, GossipPartialDataColumnSidecar, DelayedImportBlock, @@ -528,7 +526,6 @@ impl Work { Work::GossipAggregate { .. } => WorkType::GossipAggregate, Work::GossipAggregateBatch { .. } => WorkType::GossipAggregateBatch, Work::GossipBlock(_) => WorkType::GossipBlock, - Work::GossipBlobSidecar(_) => WorkType::GossipBlobSidecar, Work::GossipDataColumnSidecar(_) => WorkType::GossipDataColumnSidecar, Work::GossipPartialDataColumnSidecar(_) => WorkType::GossipPartialDataColumnSidecar, Work::DelayedImportBlock { .. } => WorkType::DelayedImportBlock, @@ -843,8 +840,6 @@ impl BeaconProcessor { } else if let Some(item) = work_queues.gossip_execution_payload_queue.pop() { Some(item) - } else if let Some(item) = work_queues.gossip_blob_queue.pop() { - Some(item) } else if let Some(item) = work_queues.gossip_data_column_queue.pop() { Some(item) } else if let Some(item) = @@ -1157,9 +1152,6 @@ impl BeaconProcessor { Work::GossipBlock { .. } => { work_queues.gossip_block_queue.push(work, work_id) } - Work::GossipBlobSidecar { .. } => { - work_queues.gossip_blob_queue.push(work, work_id) - } Work::GossipDataColumnSidecar { .. } => { work_queues.gossip_data_column_queue.push(work, work_id) } @@ -1306,7 +1298,6 @@ impl BeaconProcessor { } WorkType::GossipAggregateBatch => 0, // No queue WorkType::GossipBlock => work_queues.gossip_block_queue.len(), - WorkType::GossipBlobSidecar => work_queues.gossip_blob_queue.len(), WorkType::GossipDataColumnSidecar => { work_queues.gossip_data_column_queue.len() } @@ -1536,7 +1527,6 @@ impl BeaconProcessor { | Work::ColumnReconstruction(process_fn) => task_spawner.spawn_async(process_fn), Work::IgnoredRpcBlock { process_fn } => task_spawner.spawn_blocking(process_fn), Work::GossipBlock(work) - | Work::GossipBlobSidecar(work) | Work::GossipDataColumnSidecar(work) | Work::GossipPartialDataColumnSidecar(work) | Work::GossipExecutionPayload(work) => task_spawner.spawn_async(async move { diff --git a/beacon_node/beacon_processor/src/scheduler/work_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_queue.rs index 2fdc15182c..ebd66e743d 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_queue.rs @@ -125,7 +125,6 @@ pub struct BeaconProcessorQueueLengths { chain_segment_queue: usize, backfill_chain_segment: usize, gossip_block_queue: usize, - gossip_blob_queue: usize, gossip_data_column_queue: usize, gossip_partial_data_column_queue: usize, delayed_block_queue: usize, @@ -202,7 +201,6 @@ impl BeaconProcessorQueueLengths { chain_segment_queue: 64, backfill_chain_segment: 64, gossip_block_queue: 1024, - gossip_blob_queue: 1024, gossip_data_column_queue: 1024, gossip_partial_data_column_queue: 1024, delayed_block_queue: 1024, @@ -218,7 +216,7 @@ impl BeaconProcessorQueueLengths { payload_envelopes_brange_queue: 1024, payload_envelopes_broots_queue: 1024, gossip_bls_to_execution_change_queue: 16384, - // TODO(EIP-7732): verify 1024 is preferable. I used same value as `gossip_block_queue` and `gossip_blob_queue` + // TODO(EIP-7732): verify 1024 is preferable. gossip_execution_payload_queue: 1024, // TODO(EIP-7732) how big should this queue be? gossip_execution_payload_bid_queue: 1024, @@ -261,7 +259,6 @@ pub struct WorkQueues { pub chain_segment_queue: FifoQueue>, pub backfill_chain_segment: FifoQueue>, pub gossip_block_queue: FifoQueue>, - pub gossip_blob_queue: FifoQueue>, pub gossip_data_column_queue: FifoQueue>, pub gossip_partial_data_column_queue: FifoQueue>, pub delayed_block_queue: FifoQueue>, @@ -332,7 +329,6 @@ impl WorkQueues { let chain_segment_queue = FifoQueue::new(queue_lengths.chain_segment_queue); let backfill_chain_segment = FifoQueue::new(queue_lengths.backfill_chain_segment); let gossip_block_queue = FifoQueue::new(queue_lengths.gossip_block_queue); - let gossip_blob_queue = FifoQueue::new(queue_lengths.gossip_blob_queue); let gossip_data_column_queue = FifoQueue::new(queue_lengths.gossip_data_column_queue); let gossip_partial_data_column_queue = FifoQueue::new(queue_lengths.gossip_partial_data_column_queue); @@ -401,7 +397,6 @@ impl WorkQueues { column_reconstruction_queue, backfill_chain_segment, gossip_block_queue, - gossip_blob_queue, gossip_data_column_queue, gossip_partial_data_column_queue, delayed_block_queue, diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 7337a29c8f..d9dd9aaf4c 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -1,12 +1,11 @@ use crate::engines::ForkchoiceState; use crate::http::{ ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, - ENGINE_FORKCHOICE_UPDATED_V4, ENGINE_GET_BLOBS_V1, ENGINE_GET_BLOBS_V2, - ENGINE_GET_CLIENT_VERSION_V1, ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, - ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, - ENGINE_GET_PAYLOAD_V3, ENGINE_GET_PAYLOAD_V4, ENGINE_GET_PAYLOAD_V5, ENGINE_GET_PAYLOAD_V6, - ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V4, - ENGINE_NEW_PAYLOAD_V5, + ENGINE_FORKCHOICE_UPDATED_V4, ENGINE_GET_BLOBS_V2, ENGINE_GET_CLIENT_VERSION_V1, + ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, + ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, ENGINE_GET_PAYLOAD_V4, + ENGINE_GET_PAYLOAD_V5, ENGINE_GET_PAYLOAD_V6, ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, + ENGINE_NEW_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V4, ENGINE_NEW_PAYLOAD_V5, }; use eth2::types::{ BlobsBundle, SsePayloadAttributes, SsePayloadAttributesV1, SsePayloadAttributesV2, @@ -607,7 +606,6 @@ pub struct EngineCapabilities { pub get_payload_v5: bool, pub get_payload_v6: bool, pub get_client_version_v1: bool, - pub get_blobs_v1: bool, pub get_blobs_v2: bool, pub get_blobs_v3: bool, } @@ -669,9 +667,6 @@ impl EngineCapabilities { if self.get_client_version_v1 { response.push(ENGINE_GET_CLIENT_VERSION_V1); } - if self.get_blobs_v1 { - response.push(ENGINE_GET_BLOBS_V1); - } if self.get_blobs_v2 { response.push(ENGINE_GET_BLOBS_V2); } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 7c63f78a22..8df7d2a54b 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -62,7 +62,6 @@ pub const ENGINE_EXCHANGE_CAPABILITIES_TIMEOUT: Duration = Duration::from_secs(1 pub const ENGINE_GET_CLIENT_VERSION_V1: &str = "engine_getClientVersionV1"; pub const ENGINE_GET_CLIENT_VERSION_TIMEOUT: Duration = Duration::from_secs(1); -pub const ENGINE_GET_BLOBS_V1: &str = "engine_getBlobsV1"; pub const ENGINE_GET_BLOBS_V2: &str = "engine_getBlobsV2"; pub const ENGINE_GET_BLOBS_V3: &str = "engine_getBlobsV3"; pub const ENGINE_GET_BLOBS_TIMEOUT: Duration = Duration::from_secs(1); @@ -92,7 +91,6 @@ pub static LIGHTHOUSE_CAPABILITIES: &[&str] = &[ ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, ENGINE_GET_CLIENT_VERSION_V1, - ENGINE_GET_BLOBS_V1, ENGINE_GET_BLOBS_V2, ENGINE_GET_BLOBS_V3, ]; @@ -717,20 +715,6 @@ impl HttpJsonRpc { } } - pub async fn get_blobs_v1( - &self, - versioned_hashes: Vec, - ) -> Result>>, Error> { - let params = json!([versioned_hashes]); - - self.rpc_request( - ENGINE_GET_BLOBS_V1, - params, - ENGINE_GET_BLOBS_TIMEOUT * self.execution_timeout_multiplier, - ) - .await - } - pub async fn get_blobs_v2( &self, versioned_hashes: Vec, @@ -1272,7 +1256,6 @@ impl HttpJsonRpc { get_payload_v5: capabilities.contains(ENGINE_GET_PAYLOAD_V5), get_payload_v6: capabilities.contains(ENGINE_GET_PAYLOAD_V6), get_client_version_v1: capabilities.contains(ENGINE_GET_CLIENT_VERSION_V1), - get_blobs_v1: capabilities.contains(ENGINE_GET_BLOBS_V1), get_blobs_v2: capabilities.contains(ENGINE_GET_BLOBS_V2), get_blobs_v3: capabilities.contains(ENGINE_GET_BLOBS_V3), }) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index b1b8b0deaa..78076bee6c 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -4,7 +4,7 @@ //! This crate only provides useful functionality for "The Merge", it does not provide any of the //! deposit-contract functionality that the `beacon_node/eth1` crate already provides. -use crate::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; +use crate::json_structures::{BlobAndProofV2, BlobAndProofV3}; use crate::payload_cache::PayloadCache; use arc_swap::ArcSwapOption; use auth::{Auth, JwtKey, strip_prefix}; @@ -1722,23 +1722,6 @@ impl ExecutionLayer { } } - pub async fn get_blobs_v1( - &self, - query: Vec, - ) -> Result>>, Error> { - let capabilities = self.get_engine_capabilities(None).await?; - - if capabilities.get_blobs_v1 { - self.engine() - .request(|engine| async move { engine.api.get_blobs_v1(query).await }) - .await - .map_err(Box::new) - .map_err(Error::EngineError) - } else { - Err(Error::GetBlobsNotSupported) - } - } - pub async fn get_blobs_v2( &self, query: Vec, diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 4a46ce0f88..b05db6e8bd 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -565,20 +565,16 @@ impl ExecutionBlockGenerator { self.insert_block(Block::PoS(payload))?; } - // Post-Gloas, the justified and finalized block hashes must be non-zero, since the - // CL always has a known parent_block_hash to reference. - if let Some(head_block) = self.blocks.get(&head_block_hash) - && self - .get_fork_at_timestamp(head_block.timestamp()) - .gloas_enabled() - { + // If Gloas was enabled from genesis, the justified and finalized block hashes must be + // non-zero, since the CL always has a known parent_block_hash to reference. + if self.get_fork_at_timestamp(0).gloas_enabled() { assert!( forkchoice_state.safe_block_hash != ExecutionBlockHash::zero(), - "post-Gloas safe_block_hash must not be zero" + "for Gloas genesis safe_block_hash must not be zero" ); assert!( forkchoice_state.finalized_block_hash != ExecutionBlockHash::zero(), - "post-Gloas finalized_block_hash must not be zero" + "for Gloas genesis finalized_block_hash must not be zero" ); } diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 64eecccc58..9924fbe474 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -494,20 +494,6 @@ pub async fn handle_rpc( _ => unreachable!(), } } - ENGINE_GET_BLOBS_V1 => { - let versioned_hashes = - get_param::>(params, 0).map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?; - let generator = ctx.execution_block_generator.read(); - // V1: per-element nullable array, positionally matching the request. - let response: Vec>> = versioned_hashes - .iter() - .map(|hash| match generator.get_blob_and_proof(hash) { - Some(BlobAndProof::V1(v1)) => Some(v1), - _ => None, - }) - .collect(); - Ok(serde_json::to_value(response).unwrap()) - } ENGINE_GET_BLOBS_V2 => { let versioned_hashes = get_param::>(params, 0).map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?; diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 4eb03778f8..570008b62a 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -57,7 +57,6 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { get_payload_v5: true, get_payload_v6: true, get_client_version_v1: true, - get_blobs_v1: true, get_blobs_v2: true, get_blobs_v3: true, }; diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index ca980b96a4..8843541c11 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -455,20 +455,18 @@ impl BlockId { warp_utils::reject::custom_not_found(format!("no blobs stored for block {root}")) })?; - let blob_sidecar_list_filtered = match indices { - Some(vec) => { - let list: Vec<_> = vec - .into_iter() - .flat_map(|index| blob_sidecar_list.get(index as usize).cloned()) - .collect(); + let blob_sidecar_list: Vec<_> = blob_sidecar_list.into_iter().collect(); - BlobSidecarList::new(list, max_blobs_per_block) - .map_err(|e| warp_utils::reject::custom_server_error(format!("{:?}", e)))? - } + let blob_sidecar_list = match indices { + Some(indices) => indices + .into_iter() + .filter_map(|i| blob_sidecar_list.get(i as usize).cloned()) + .collect(), None => blob_sidecar_list, }; - Ok(blob_sidecar_list_filtered) + BlobSidecarList::new(blob_sidecar_list, max_blobs_per_block) + .map_err(|e| warp_utils::reject::custom_server_error(format!("{:?}", e))) } fn get_blobs_from_data_columns( diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index ca4ab85524..b46576ddad 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -1,7 +1,6 @@ use crate::metrics; use std::future::Future; -use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::{AsBlock, LookupBlock}; use beacon_chain::data_column_verification::GossipVerifiedDataColumn; use beacon_chain::validator_monitor::get_block_delay_ms; @@ -26,13 +25,12 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; -use tracing::{Span, debug, debug_span, error, field, info, instrument, warn}; +use tracing::{Span, debug, error, field, info, instrument, warn}; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, BeaconBlockRef, BlobSidecar, BlobsList, BlockImportSource, - DataColumnSidecar, DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, - FullPayload, FullPayloadBellatrix, Hash256, KzgProofs, SignedBeaconBlock, - SignedBlindedBeaconBlock, + AbstractExecPayload, BeaconBlockRef, BlobsList, BlockImportSource, DataColumnSidecar, + DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, + FullPayloadBellatrix, Hash256, KzgProofs, SignedBeaconBlock, SignedBlindedBeaconBlock, }; use warp::{Rejection, Reply, reply::Response}; @@ -195,23 +193,8 @@ pub async fn publish_block>( Ok(()) }; - // Wait for blobs/columns to get gossip verified before proceeding further as we need them for import. - let (gossip_verified_blobs, gossip_verified_columns) = build_sidecar_task_handle.await?; - - for blob in gossip_verified_blobs.into_iter().flatten() { - publish_blob_sidecars(network_tx, &blob).map_err(|_| { - warp_utils::reject::custom_server_error("unable to publish blob sidecars".into()) - })?; - if let Err(e) = Box::pin(chain.process_gossip_blob(blob)).await { - let msg = format!("Invalid blob: {e}"); - return if let BroadcastValidation::Gossip = validation_level { - Err(warp_utils::reject::broadcast_without_import(msg)) - } else { - error!(reason = &msg, "Invalid blob provided to HTTP API"); - Err(warp_utils::reject::custom_bad_request(msg)) - }; - } - } + // Wait for columns to get gossip verified before proceeding further as we need them for import. + let gossip_verified_columns = build_sidecar_task_handle.await?; if !gossip_verified_columns.is_empty() { if let Some(data_column_publishing_delay) = data_column_publishing_delay_for_testing { @@ -342,18 +325,9 @@ pub async fn publish_block>( } } -type BuildDataSidecarTaskResult = Result< - ( - Vec>>, - Vec>, - ), - Rejection, ->; +type BuildDataSidecarTaskResult = Result>, Rejection>; -/// Convert blobs to either: -/// -/// 1. Blob sidecars if prior to peer DAS, or -/// 2. Data column sidecars if post peer DAS. +/// Convert blobs to data column sidecars. fn spawn_build_data_sidecar_task( chain: Arc>, block: Arc>>, @@ -365,22 +339,9 @@ fn spawn_build_data_sidecar_task( .spawn_blocking_handle( move || { let Some((kzg_proofs, blobs)) = proofs_and_blobs else { - return Ok((vec![], vec![])); + return Ok(vec![]); }; - let _span = debug_span!("build_data_sidecars").entered(); - - let peer_das_enabled = chain.spec.is_peer_das_enabled_for_epoch(block.epoch()); - if !peer_das_enabled { - // Pre-PeerDAS: construct blob sidecars for the network. - let gossip_verified_blobs = - build_gossip_verified_blobs(&chain, &block, blobs, kzg_proofs)?; - Ok((gossip_verified_blobs, vec![])) - } else { - // Post PeerDAS: construct data columns. - let gossip_verified_data_columns = - build_data_columns(&chain, &block, blobs, kzg_proofs)?; - Ok((vec![], gossip_verified_data_columns)) - } + build_data_columns(&chain, &block, blobs, kzg_proofs) }, "build_data_sidecars", ) @@ -424,76 +385,6 @@ fn build_data_columns( Ok(gossip_verified_data_columns) } -fn build_gossip_verified_blobs( - chain: &BeaconChain, - block: &SignedBeaconBlock>, - blobs: BlobsList, - kzg_proofs: KzgProofs, -) -> Result>>, Rejection> { - let slot = block.slot(); - let gossip_verified_blobs = kzg_proofs - .into_iter() - .zip(blobs) - .enumerate() - .map(|(i, (proof, unverified_blob))| { - let timer = metrics::start_timer( - &beacon_chain::metrics::BLOB_SIDECAR_INCLUSION_PROOF_COMPUTATION, - ); - let blob_sidecar = BlobSidecar::new(i, unverified_blob, block, proof) - .map(Arc::new) - .map_err(|e| { - error!( - error = ?e, - blob_index = i, - %slot, - "Invalid blob - not publishing block" - ); - warp_utils::reject::custom_bad_request(format!("{e:?}")) - })?; - drop(timer); - - let gossip_verified_blob = - GossipVerifiedBlob::new(blob_sidecar.clone(), blob_sidecar.index, chain); - - match gossip_verified_blob { - Ok(blob) => Ok(Some(blob)), - Err(GossipBlobError::RepeatBlob { proposer, .. }) => { - // Log the error but do not abort publication, we may need to publish the block - // or some of the other blobs if the block & blobs are only partially published - // by the other publisher. - debug!( - blob_index = blob_sidecar.index, - %slot, - proposer, - "Blob for publication already known" - ); - Ok(None) - } - Err(e) => { - error!( - blob_index = blob_sidecar.index, - %slot, - error = ?e, - "Blob for publication is gossip-invalid" - ); - Err(warp_utils::reject::custom_bad_request(e.to_string())) - } - } - }) - .collect::, Rejection>>()?; - - Ok(gossip_verified_blobs) -} - -fn publish_blob_sidecars( - sender_clone: &UnboundedSender>, - blob: &GossipVerifiedBlob, -) -> Result<(), BlockError> { - let pubsub_message = PubsubMessage::BlobSidecar(Box::new((blob.index(), blob.clone_blob()))); - crate::utils::publish_pubsub_message(sender_clone, pubsub_message) - .map_err(|_| BlockError::BeaconChainError(Box::new(BeaconChainError::UnableToPublish))) -} - pub(crate) fn publish_column_sidecars( sender_clone: &UnboundedSender>, data_column_sidecars: &[GossipVerifiedDataColumn], diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index a189be1cfc..98629a1c5e 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -1587,7 +1587,7 @@ pub async fn block_seen_on_gossip_without_blobs_or_columns() { let state = tester.harness.get_current_state(); let fork_name = state.fork_name(&tester.harness.spec).unwrap(); // Gloas blocks don't carry blobs (execution data comes via envelopes). - if !fork_name.deneb_enabled() || fork_name.gloas_enabled() { + if !fork_name.fulu_enabled() || fork_name.gloas_enabled() { return; } @@ -1647,7 +1647,7 @@ pub async fn block_seen_on_gossip_without_blobs_or_columns() { /// This test checks that an HTTP POST request with the block & blobs/columns succeeds with a 200 response /// even if the block has already been seen on gossip without all blobs/columns. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -pub async fn block_seen_on_gossip_with_some_blobs_or_columns() { +pub async fn block_seen_on_gossip_with_columns() { let validation_level: Option = Some(BroadcastValidation::Gossip); // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing @@ -1658,7 +1658,7 @@ pub async fn block_seen_on_gossip_with_some_blobs_or_columns() { let state = tester.harness.get_current_state(); let fork_name = state.fork_name(&tester.harness.spec).unwrap(); // Gloas blocks don't carry blobs (execution data comes via envelopes). - if !fork_name.deneb_enabled() || fork_name.gloas_enabled() { + if !fork_name.fulu_enabled() || fork_name.gloas_enabled() { return; } @@ -1690,9 +1690,6 @@ pub async fn block_seen_on_gossip_with_some_blobs_or_columns() { blobs.0.len() ); - let partial_kzg_proofs = [*blobs.0.first().unwrap()]; - let partial_blobs = [blobs.1.first().unwrap().clone()]; - // Simulate the block being seen on gossip. block .clone() @@ -1702,12 +1699,7 @@ pub async fn block_seen_on_gossip_with_some_blobs_or_columns() { // Simulate some of the blobs being seen on gossip. tester .harness - .process_gossip_blobs_or_columns( - &block, - partial_blobs.iter(), - partial_kzg_proofs.iter(), - Some(get_custody_columns(&tester, block.slot())), - ) + .process_gossip_columns(&block, Some(get_custody_columns(&tester, block.slot()))) .await; // It should not yet be added to fork choice because all blobs have not been seen. @@ -1740,7 +1732,7 @@ pub async fn block_seen_on_gossip_with_some_blobs_or_columns() { /// This test checks that an HTTP POST request with the block & blobs/columns succeeds with a 200 response /// even if the blobs/columns have already been seen on gossip. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -pub async fn blobs_or_columns_seen_on_gossip_without_block() { +pub async fn columns_seen_on_gossip_without_block() { let spec = test_spec::(); let validation_level: Option = Some(BroadcastValidation::Gossip); @@ -1752,7 +1744,7 @@ pub async fn blobs_or_columns_seen_on_gossip_without_block() { let state = tester.harness.get_current_state(); let fork_name = state.fork_name(&tester.harness.spec).unwrap(); // Gloas blocks don't carry blobs (execution data comes via envelopes). - if !fork_name.deneb_enabled() || fork_name.gloas_enabled() { + if !fork_name.fulu_enabled() || fork_name.gloas_enabled() { return; } @@ -1778,12 +1770,7 @@ pub async fn blobs_or_columns_seen_on_gossip_without_block() { // Simulate the blobs being seen on gossip. tester .harness - .process_gossip_blobs_or_columns( - &block, - blobs.iter(), - kzg_proofs.iter(), - Some(get_custody_columns(&tester, block.slot())), - ) + .process_gossip_columns(&block, Some(get_custody_columns(&tester, block.slot()))) .await; // It should not yet be added to fork choice because the block has not been seen. @@ -1816,7 +1803,7 @@ pub async fn blobs_or_columns_seen_on_gossip_without_block() { /// This test checks that an HTTP POST request with the block succeeds with a 200 response /// if just the blobs have already been seen on gossip. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn blobs_or_columns_seen_on_gossip_without_block_and_no_http_blobs_or_columns() { +async fn columns_seen_on_gossip_without_block_and_no_http_columns() { let validation_level: Option = Some(BroadcastValidation::Gossip); // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing @@ -1827,7 +1814,7 @@ async fn blobs_or_columns_seen_on_gossip_without_block_and_no_http_blobs_or_colu let state = tester.harness.get_current_state(); let fork_name = state.fork_name(&tester.harness.spec).unwrap(); // Gloas blocks don't carry blobs (execution data comes via envelopes). - if !fork_name.deneb_enabled() || fork_name.gloas_enabled() { + if !fork_name.fulu_enabled() || fork_name.gloas_enabled() { return; } @@ -1848,18 +1835,13 @@ async fn blobs_or_columns_seen_on_gossip_without_block_and_no_http_blobs_or_colu let state_a = tester.harness.get_current_state(); let ((block, blobs), _) = tester.harness.make_block(state_a, slot_b).await; - let (kzg_proofs, blobs) = blobs.expect("should have some blobs"); + let (_, blobs) = blobs.expect("should have some blobs"); assert!(!blobs.is_empty()); // Simulate the blobs being seen on gossip. tester .harness - .process_gossip_blobs_or_columns( - &block, - blobs.iter(), - kzg_proofs.iter(), - Some(get_custody_columns(&tester, block.slot())), - ) + .process_gossip_columns(&block, Some(get_custody_columns(&tester, block.slot()))) .await; // It should not yet be added to fork choice because the block has not been seen. @@ -1893,7 +1875,7 @@ async fn blobs_or_columns_seen_on_gossip_without_block_and_no_http_blobs_or_colu } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn slashable_blobs_or_columns_seen_on_gossip_cause_failure() { +async fn slashable_columns_seen_on_gossip_cause_failure() { let validation_level: Option = Some(BroadcastValidation::ConsensusAndEquivocation); @@ -1905,7 +1887,7 @@ async fn slashable_blobs_or_columns_seen_on_gossip_cause_failure() { let state = tester.harness.get_current_state(); let fork_name = state.fork_name(&tester.harness.spec).unwrap(); // Gloas blocks don't carry blobs (execution data comes via envelopes). - if !fork_name.deneb_enabled() || fork_name.gloas_enabled() { + if !fork_name.fulu_enabled() || fork_name.gloas_enabled() { return; } @@ -1926,19 +1908,13 @@ async fn slashable_blobs_or_columns_seen_on_gossip_cause_failure() { let state_a = tester.harness.get_current_state(); let ((block_a, blobs_a), _) = tester.harness.make_block(state_a.clone(), slot_b).await; - let ((block_b, blobs_b), _) = tester.harness.make_block(state_a, slot_b).await; + let ((block_b, _), _) = tester.harness.make_block(state_a, slot_b).await; let (kzg_proofs_a, blobs_a) = blobs_a.expect("should have some blobs"); - let (kzg_proofs_b, blobs_b) = blobs_b.expect("should have some blobs"); // Simulate the blobs of block B being seen on gossip. tester .harness - .process_gossip_blobs_or_columns( - &block_b, - blobs_b.iter(), - kzg_proofs_b.iter(), - Some(get_custody_columns(&tester, block_b.slot())), - ) + .process_gossip_columns(&block_b, Some(get_custody_columns(&tester, block_b.slot()))) .await; // It should not yet be added to fork choice because block B has not been seen. @@ -1984,7 +1960,7 @@ pub async fn duplicate_block_status_code() { // Gloas blocks don't carry blobs (execution data comes via envelopes). let spec = test_spec::(); let genesis_fork = spec.fork_name_at_slot::(Slot::new(0)); - if !genesis_fork.deneb_enabled() || genesis_fork.gloas_enabled() { + if !genesis_fork.fulu_enabled() || genesis_fork.gloas_enabled() { return; } diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 3da0841a4e..06b3a6197b 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -9167,11 +9167,17 @@ async fn builder_works_post_deneb() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn get_blob_sidecars() { - let mut config = ApiTesterConfig::default(); + let mut config = ApiTesterConfig { + retain_historic_states: false, + spec: E::default_spec(), + node_custody_type: NodeCustodyType::Supernode, + }; config.spec.altair_fork_epoch = Some(Epoch::new(0)); config.spec.bellatrix_fork_epoch = Some(Epoch::new(0)); config.spec.capella_fork_epoch = Some(Epoch::new(0)); config.spec.deneb_fork_epoch = Some(Epoch::new(0)); + config.spec.electra_fork_epoch = Some(Epoch::new(0)); + config.spec.fulu_fork_epoch = Some(Epoch::new(0)); ApiTester::new_from_config(config) .await diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index e9862e3f74..4b96fe884e 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -20,8 +20,6 @@ pub struct GossipCache { topic_msgs: HashMap, Key>>, /// Timeout for blocks. beacon_block: Option, - /// Timeout for blobs. - blob_sidecar: Option, /// Timeout for data columns. data_column_sidecar: Option, /// Timeout for aggregate attestations. @@ -59,8 +57,6 @@ pub struct GossipCacheBuilder { default_timeout: Option, /// Timeout for blocks. beacon_block: Option, - /// Timeout for blob sidecars. - blob_sidecar: Option, /// Timeout for data column sidecars. data_column_sidecar: Option, /// Timeout for aggregate attestations. @@ -195,7 +191,6 @@ impl GossipCacheBuilder { let GossipCacheBuilder { default_timeout, beacon_block, - blob_sidecar, data_column_sidecar, aggregates, attestation, @@ -216,7 +211,6 @@ impl GossipCacheBuilder { expirations: DelayQueue::default(), topic_msgs: HashMap::default(), beacon_block: beacon_block.or(default_timeout), - blob_sidecar: blob_sidecar.or(default_timeout), data_column_sidecar: data_column_sidecar.or(default_timeout), aggregates: aggregates.or(default_timeout), attestation: attestation.or(default_timeout), @@ -247,7 +241,6 @@ impl GossipCache { pub fn insert(&mut self, topic: GossipTopic, data: Vec) { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, - GossipKind::BlobSidecar(_) => self.blob_sidecar, GossipKind::DataColumnSidecar(_) => self.data_column_sidecar, GossipKind::BeaconAggregateAndProof => self.aggregates, GossipKind::Attestation(_) => self.attestation, diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index d235e4b28f..c7dabcb391 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -284,10 +284,6 @@ pub(crate) fn create_whitelist_filter( for id in 0..sync_committee_subnet_count { add(SyncCommitteeMessage(SyncSubnetId::new(id))); } - let blob_subnet_count = spec.blob_sidecar_subnet_count_max(); - for id in 0..blob_subnet_count { - add(BlobSidecar(id)); - } for id in 0..spec.data_column_sidecar_subnet_count { add(DataColumnSidecar(DataColumnSubnetId::new(id))); } diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index e5a703ff1e..043d1cfb88 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -7,10 +7,10 @@ use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; use types::{ - AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, - DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, Hash256, - LightClientFinalityUpdate, LightClientOptimisticUpdate, PartialDataColumn, - PartialDataColumnSidecar, PayloadAttestationMessage, ProposerSlashing, SignedAggregateAndProof, + AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, DataColumnSidecar, + DataColumnSubnetId, EthSpec, ForkContext, ForkName, Hash256, LightClientFinalityUpdate, + LightClientOptimisticUpdate, PartialDataColumn, PartialDataColumnSidecar, + PayloadAttestationMessage, ProposerSlashing, SignedAggregateAndProof, SignedAggregateAndProofBase, SignedAggregateAndProofElectra, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, @@ -24,8 +24,6 @@ use types::{ pub enum PubsubMessage { /// Gossipsub message providing notification of a new block. BeaconBlock(Arc>), - /// Gossipsub message providing notification of a [`BlobSidecar`] along with the subnet id where it was received. - BlobSidecar(Box<(u64, Arc>)>), /// Gossipsub message providing notification of a [`DataColumnSidecar`] along with the subnet id where it was received. DataColumnSidecar(Box<(DataColumnSubnetId, Arc>)>), /// Gossipsub message providing notification of a Aggregate attestation and associated proof. @@ -139,9 +137,6 @@ impl PubsubMessage { pub fn kind(&self) -> GossipKind { match self { PubsubMessage::BeaconBlock(_) => GossipKind::BeaconBlock, - PubsubMessage::BlobSidecar(blob_sidecar_data) => { - GossipKind::BlobSidecar(blob_sidecar_data.0) - } PubsubMessage::DataColumnSidecar(column_sidecar_data) => { GossipKind::DataColumnSidecar(column_sidecar_data.0) } @@ -266,26 +261,6 @@ impl PubsubMessage { }; Ok(PubsubMessage::BeaconBlock(Arc::new(beacon_block))) } - GossipKind::BlobSidecar(blob_index) => { - if let Some(fork_name) = - fork_context.get_fork_from_context_bytes(gossip_topic.fork_digest) - && fork_name.deneb_enabled() - { - let blob_sidecar = Arc::new( - BlobSidecar::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, - ); - return Ok(PubsubMessage::BlobSidecar(Box::new(( - *blob_index, - blob_sidecar, - )))); - } - - Err(format!( - "beacon_blobs_and_sidecar topic invalid for given fork digest {:?}", - gossip_topic.fork_digest - )) - } GossipKind::DataColumnSidecar(subnet_id) => { match fork_context.get_fork_from_context_bytes(gossip_topic.fork_digest) { Some(fork) if fork.fulu_enabled() => { @@ -444,7 +419,6 @@ impl PubsubMessage { // messages for us. match &self { PubsubMessage::BeaconBlock(data) => data.as_ssz_bytes(), - PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), @@ -502,12 +476,6 @@ impl std::fmt::Display for PubsubMessage { block.slot(), block.message().proposer_index() ), - PubsubMessage::BlobSidecar(data) => write!( - f, - "BlobSidecar: slot: {}, blob index: {}", - data.1.slot(), - data.1.index, - ), PubsubMessage::DataColumnSidecar(data) => write!( f, "DataColumnSidecar: slot: {}, column index: {}", diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index b51c459a80..1a5acd79b4 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -21,7 +21,6 @@ pub const SSZ_SNAPPY_ENCODING_POSTFIX: &str = "ssz_snappy"; pub const BEACON_BLOCK_TOPIC: &str = "beacon_block"; pub const BEACON_AGGREGATE_AND_PROOF_TOPIC: &str = "beacon_aggregate_and_proof"; pub const BEACON_ATTESTATION_PREFIX: &str = "beacon_attestation_"; -pub const BLOB_SIDECAR_PREFIX: &str = "blob_sidecar_"; pub const DATA_COLUMN_SIDECAR_PREFIX: &str = "data_column_sidecar_"; pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit"; pub const PROPOSER_SLASHING_TOPIC: &str = "proposer_slashing"; @@ -82,13 +81,6 @@ pub fn core_topics_to_subscribe( topics.push(GossipKind::BlsToExecutionChange); } - if fork_name.deneb_enabled() && !fork_name.fulu_enabled() { - // All of deneb blob topics are core topics - for i in 0..spec.blob_sidecar_subnet_count(fork_name) { - topics.push(GossipKind::BlobSidecar(i)); - } - } - if fork_name.fulu_enabled() { for subnet in &opts.sampling_subnets { topics.push(GossipKind::DataColumnSidecar(*subnet)); @@ -118,7 +110,6 @@ pub fn is_fork_non_core_topic(topic: &GossipTopic, _fork_name: ForkName) -> bool // All these topics are core-only GossipKind::BeaconBlock | GossipKind::BeaconAggregateAndProof - | GossipKind::BlobSidecar(_) | GossipKind::DataColumnSidecar(_) | GossipKind::VoluntaryExit | GossipKind::ProposerSlashing @@ -166,8 +157,6 @@ pub enum GossipKind { BeaconBlock, /// Topic for publishing aggregate attestations and proofs. BeaconAggregateAndProof, - /// Topic for publishing BlobSidecars. - BlobSidecar(u64), /// Topic for publishing DataColumnSidecars. DataColumnSidecar(DataColumnSubnetId), /// Topic for publishing raw attestations on a particular subnet. @@ -216,9 +205,6 @@ impl std::fmt::Display for GossipKind { GossipKind::SyncCommitteeMessage(subnet_id) => { write!(f, "sync_committee_{}", **subnet_id) } - GossipKind::BlobSidecar(blob_index) => { - write!(f, "{}{}", BLOB_SIDECAR_PREFIX, blob_index) - } GossipKind::DataColumnSidecar(column_subnet_id) => { write!(f, "{}{}", DATA_COLUMN_SIDECAR_PREFIX, **column_subnet_id) } @@ -349,9 +335,6 @@ impl std::fmt::Display for GossipTopic { GossipKind::SyncCommitteeMessage(index) => { format!("{}{}", SYNC_COMMITTEE_PREFIX_TOPIC, *index) } - GossipKind::BlobSidecar(blob_index) => { - format!("{}{}", BLOB_SIDECAR_PREFIX, blob_index) - } GossipKind::DataColumnSidecar(column_subnet_id) => { format!("{}{}", DATA_COLUMN_SIDECAR_PREFIX, *column_subnet_id) } @@ -401,8 +384,6 @@ fn subnet_topic_index(topic: &str) -> Option { return Some(GossipKind::SyncCommitteeMessage(SyncSubnetId::new( index.parse::().ok()?, ))); - } else if let Some(index) = topic.strip_prefix(BLOB_SIDECAR_PREFIX) { - return Some(GossipKind::BlobSidecar(index.parse::().ok()?)); } else if let Some(index) = topic.strip_prefix(DATA_COLUMN_SIDECAR_PREFIX) { return Some(GossipKind::DataColumnSidecar(DataColumnSubnetId::new( index.parse::().ok()?, @@ -576,17 +557,6 @@ mod tests { } } - #[test] - fn blobs_are_not_subscribed_in_peerdas() { - let spec = get_spec(); - let s = get_sampling_subnets(); - let topic_config = get_topic_config(&s); - assert!( - !core_topics_to_subscribe::(ForkName::Fulu, &topic_config, &spec,) - .contains(&GossipKind::BlobSidecar(0)) - ); - } - #[test] fn columns_are_subscribed_in_peerdas() { let spec = get_spec(); diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 4b34d7bfc0..c043133cee 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -128,13 +128,6 @@ pub static BEACON_PROCESSOR_GOSSIP_BLOCK_EARLY_SECONDS: LazyLock> = - LazyLock::new(|| { - try_create_int_counter( - "beacon_processor_gossip_blob_verified_total", - "Total number of gossip blob verified for propagation.", - ) - }); pub static BEACON_PROCESSOR_GOSSIP_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: LazyLock< Result, > = LazyLock::new(|| { @@ -600,12 +593,6 @@ pub static BEACON_BLOCK_DELAY_GOSSIP_ARRIVED_LATE_TOTAL: LazyLock> = LazyLock::new(|| { - try_create_int_gauge( - "beacon_blob_delay_gossip_last_delay", - "The first time we see this blob as a delay from the start of the slot", - ) -}); pub static BEACON_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: LazyLock< Result, @@ -664,14 +651,6 @@ pub static BEACON_USEFUL_FULL_COLUMNS_RECEIVED_TOTAL: LazyLock> = LazyLock::new( - || { - try_create_int_gauge( - "beacon_blob_delay_gossip_verification", - "Keeps track of the time delay from the start of the slot to the point we propagate the blob", - ) - }, -); pub static BEACON_BLOB_DELAY_FULL_VERIFICATION: LazyLock> = LazyLock::new(|| { try_create_int_gauge( "beacon_blob_last_full_verification_delay", @@ -695,15 +674,6 @@ pub static BEACON_BLOB_RPC_SLOT_START_DELAY_TIME: LazyLock> = }, ); -pub static BEACON_BLOB_GOSSIP_ARRIVED_LATE_TOTAL: LazyLock> = LazyLock::new( - || { - try_create_int_counter( - "beacon_blob_gossip_arrived_late_total", - "Count of times when a gossip blob arrived from the network later than the attestation deadline.", - ) - }, -); - /* * Light client update reprocessing queue metrics. */ diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 71216b47a7..65c95eff35 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -11,6 +11,9 @@ use beacon_chain::data_column_verification::{ PartialColumnVerificationResult, }; use beacon_chain::payload_bid_verification::PayloadBidError; +use beacon_chain::payload_envelope_verification::{ + EnvelopeError, gossip_verified_envelope::GossipVerifiedEnvelope, +}; use beacon_chain::proposer_preferences_verification::ProposerPreferencesError; use beacon_chain::store::Error; use beacon_chain::{ @@ -27,12 +30,6 @@ use beacon_chain::{ sync_committee_verification::{self, Error as SyncCommitteeError}, validator_monitor::{get_block_delay_ms, get_slot_delay_ms}, }; -use beacon_chain::{ - blob_verification::{GossipBlobError, GossipVerifiedBlob}, - payload_envelope_verification::{ - EnvelopeError, gossip_verified_envelope::GossipVerifiedEnvelope, - }, -}; use beacon_processor::{Work, WorkEvent}; use lighthouse_network::{ Client, GossipTopic, MessageAcceptance, MessageId, PeerAction, PeerId, PubsubMessage, @@ -50,13 +47,13 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tracing::{Instrument, Span, debug, error, info, instrument, trace, warn}; use types::{ - Attestation, AttestationData, AttestationRef, AttesterSlashing, BlobSidecar, ColumnIndex, - DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, IndexedAttestation, - LightClientFinalityUpdate, LightClientOptimisticUpdate, PartialDataColumn, - PartialDataColumnHeader, PayloadAttestationMessage, ProposerSlashing, SignedAggregateAndProof, - SignedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, - SignedExecutionPayloadBid, SignedExecutionPayloadEnvelope, SignedProposerPreferences, - SignedVoluntaryExit, SingleAttestation, Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, + Attestation, AttestationData, AttestationRef, AttesterSlashing, ColumnIndex, DataColumnSidecar, + DataColumnSubnetId, EthSpec, Hash256, IndexedAttestation, LightClientFinalityUpdate, + LightClientOptimisticUpdate, PartialDataColumn, PartialDataColumnHeader, + PayloadAttestationMessage, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, + SignedBlsToExecutionChange, SignedContributionAndProof, SignedExecutionPayloadBid, + SignedExecutionPayloadEnvelope, SignedProposerPreferences, SignedVoluntaryExit, + SingleAttestation, Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, block::BlockImportSource, }; @@ -844,13 +841,109 @@ impl NetworkBeaconProcessor { } } - #[instrument( - name = "lh_process_gossip_partial_data_column", - parent = None, - level = "debug", - skip_all, - fields(block_root = ?column.block_root, index = column.index), - )] + async fn process_gossip_verified_data_column( + self: &Arc, + peer_id: PeerId, + verified_data_column: GossipVerifiedDataColumn, + // This value is not used presently, but it might come in handy for debugging. + _seen_duration: Duration, + ) { + let processing_start_time = Instant::now(); + let block_root = verified_data_column.block_root(); + let data_column_slot = verified_data_column.slot(); + let data_column_index = verified_data_column.index(); + + // TODO(gloas): implement partial messages + if let DataColumnSidecar::Fulu(col) = verified_data_column.as_data_column() + && self + .chain + .data_availability_checker + .partial_assembler() + .is_some_and(|a| !a.is_complete(block_root, verified_data_column.index())) + { + metrics::inc_counter_vec( + &metrics::BEACON_USEFUL_FULL_COLUMNS_RECEIVED_TOTAL, + &[&data_column_index.to_string()], + ); + + match col.to_partial() { + Ok(mut column) => { + let header = column.sidecar.header.take(); + if let Some(header) = header { + self.send_network_message(NetworkMessage::PublishPartialColumns { + columns: vec![Arc::new(column)], + header: Arc::new(header), + }); + } else { + crit!("Converting from full to partial yielded headerless partial") + }; + } + Err(err) => crit!(?err, "Could not convert from full to partial"), + } + } + + let result = self + .chain + .process_gossip_data_columns(vec![verified_data_column], || Ok(())) + .await; + register_process_result_metrics(&result, metrics::BlockSource::Gossip, "data_column"); + + match result { + Ok(availability) => match availability { + AvailabilityProcessingStatus::Imported(block_root) => { + debug!( + %block_root, + "Gossipsub data column processed, imported fully available block" + ); + self.chain.recompute_head_at_current_slot().await; + + metrics::set_gauge( + &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, + processing_start_time.elapsed().as_millis() as i64, + ); + + // If a block is in the da_checker, sync maybe awaiting for an event when block is finally + // imported. A block can become imported both after processing a block or data column. If + // importing a block results in `Imported`, notify. Do not notify of data column errors. + self.send_sync_message(SyncMessage::GossipBlockProcessResult { + block_root, + imported: true, + }); + } + AvailabilityProcessingStatus::MissingComponents(slot, block_root) => { + trace!( + %slot, + %data_column_index, + %block_root, + "Processed data column, waiting for other components" + ); + + self.check_reconstruction_trigger(slot, &block_root).await; + } + }, + Err(BlockError::DuplicateFullyImported(_)) => { + debug!( + ?block_root, + data_column_index, "Ignoring gossip column already imported" + ); + } + Err(err) => { + debug!( + outcome = ?err, + ?block_root, + block_slot = %data_column_slot, + data_column_index, + "Invalid gossip data column" + ); + self.gossip_penalize_peer( + peer_id, + PeerAction::MidToleranceError, + "bad_gossip_data_column_ssz", + ); + } + } + } + pub async fn process_gossip_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, @@ -1008,7 +1101,6 @@ impl NetworkBeaconProcessor { %index, "Could not verify partial column for gossip. Rejecting the column sidecar" ); - // Prevent recurring behaviour by penalizing the peer slightly. self.gossip_penalize_peer( peer_id, PeerAction::LowToleranceError, @@ -1017,9 +1109,6 @@ impl NetworkBeaconProcessor { self.propagate_partial_validation_failure(peer_id, topic); } GossipDataColumnError::PriorKnown { .. } => { - // Data column is available via either the EL or reconstruction. - // Do not penalise the peer. - // Gossip filter should filter any duplicates received after this. debug!( %block_root, %index, @@ -1034,7 +1123,6 @@ impl NetworkBeaconProcessor { %index, "Could not verify column sidecar for gossip. Ignoring the partial column sidecar" ); - // Prevent recurring behaviour by penalizing the peer slightly. self.gossip_penalize_peer( peer_id, PeerAction::HighToleranceError, @@ -1119,357 +1207,6 @@ impl NetworkBeaconProcessor { } } - #[allow(clippy::too_many_arguments)] - #[instrument( - name = "lh_process_gossip_blob", - parent = None, - level = "debug", - skip_all, - fields( - slot = ?blob_sidecar.slot(), - block_root = ?blob_sidecar.block_root(), - index = blob_sidecar.index), - )] - pub async fn process_gossip_blob( - self: &Arc, - message_id: MessageId, - peer_id: PeerId, - _peer_client: Client, - blob_index: u64, - blob_sidecar: Arc>, - seen_duration: Duration, - ) { - let slot = blob_sidecar.slot(); - let root = blob_sidecar.block_root(); - let index = blob_sidecar.index; - let commitment = blob_sidecar.kzg_commitment; - let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); - // Log metrics to track delay from other nodes on the network. - metrics::set_gauge(&metrics::BEACON_BLOB_DELAY_GOSSIP, delay.as_millis() as i64); - match self - .chain - .verify_blob_sidecar_for_gossip(blob_sidecar.clone(), blob_index) - { - Ok(gossip_verified_blob) => { - metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOB_VERIFIED_TOTAL); - - if delay >= self.chain.spec.get_unaggregated_attestation_due() { - metrics::inc_counter(&metrics::BEACON_BLOB_GOSSIP_ARRIVED_LATE_TOTAL); - debug!( - block_root = ?gossip_verified_blob.block_root(), - proposer_index = gossip_verified_blob.block_proposer_index(), - slot = %gossip_verified_blob.slot(), - delay = ?delay, - commitment = %gossip_verified_blob.kzg_commitment(), - "Gossip blob arrived late" - ); - } - - debug!( - %slot, - %root, - %index, - commitment = %gossip_verified_blob.kzg_commitment(), - "Successfully verified gossip blob" - ); - - self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept); - - // Log metrics to keep track of propagation delay times. - if let Some(duration) = SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .and_then(|now| now.checked_sub(seen_duration)) - { - metrics::set_gauge( - &metrics::BEACON_BLOB_DELAY_GOSSIP_VERIFICATION, - duration.as_millis() as i64, - ); - } - self.process_gossip_verified_blob(peer_id, gossip_verified_blob, seen_duration) - .await - } - Err(err) => { - match err { - GossipBlobError::ParentUnknown { parent_root } => { - debug!( - action = "requesting parent", - block_root = %root, - parent_root = %parent_root, - %commitment, - "Unknown parent hash for blob" - ); - self.send_sync_message(SyncMessage::UnknownParentBlob( - peer_id, - blob_sidecar, - )); - } - GossipBlobError::PubkeyCacheTimeout | GossipBlobError::BeaconChainError(_) => { - crit!( - error = ?err, - "Internal error when verifying blob sidecar" - ) - } - GossipBlobError::ProposalSignatureInvalid - | GossipBlobError::UnknownValidator(_) - | GossipBlobError::ProposerIndexMismatch { .. } - | GossipBlobError::BlobIsNotLaterThanParent { .. } - | GossipBlobError::InvalidSubnet { .. } - | GossipBlobError::InvalidInclusionProof - | GossipBlobError::KzgError(_) - | GossipBlobError::NotFinalizedDescendant { .. } => { - warn!( - error = ?err, - %slot, - %root, - %index, - %commitment, - "Could not verify blob sidecar for gossip. Rejecting the blob sidecar" - ); - // Prevent recurring behaviour by penalizing the peer. - self.gossip_penalize_peer( - peer_id, - PeerAction::LowToleranceError, - "gossip_blob_low", - ); - self.propagate_validation_result( - message_id, - peer_id, - MessageAcceptance::Reject, - ); - } - GossipBlobError::RepeatBlob { .. } => { - // We may have received the blob from the EL. Do not penalise the peer. - // Gossip filter should filter any duplicates received after this. - debug!( - %slot, - %root, - %index, - "Received already available blob sidecar. Ignoring the blob sidecar" - ) - } - GossipBlobError::FutureSlot { .. } => { - debug!( - error = ?err, - %slot, - %root, - %index, - %commitment, - "Could not verify blob sidecar for gossip. Ignoring the blob sidecar" - ); - // Prevent recurring behaviour by penalizing the peer slightly. - self.gossip_penalize_peer( - peer_id, - PeerAction::HighToleranceError, - "gossip_blob_high", - ); - self.propagate_validation_result( - message_id, - peer_id, - MessageAcceptance::Ignore, - ); - } - GossipBlobError::PastFinalizedSlot { .. } => { - debug!( - error = ?err, - %slot, - %root, - %index, - %commitment, - "Could not verify blob sidecar for gossip. Ignoring the blob sidecar" - ); - // Prevent recurring behaviour by penalizing the peer. A low-tolerance - // error is fine because there's no reason for peers to be propagating old - // blobs on gossip, even if their view of finality is lagging. - self.gossip_penalize_peer( - peer_id, - PeerAction::LowToleranceError, - "gossip_blob_low", - ); - self.propagate_validation_result( - message_id, - peer_id, - MessageAcceptance::Ignore, - ); - } - } - } - } - } - - async fn process_gossip_verified_blob( - self: &Arc, - peer_id: PeerId, - verified_blob: GossipVerifiedBlob, - _seen_duration: Duration, - ) { - let processing_start_time = Instant::now(); - let block_root = verified_blob.block_root(); - let blob_slot = verified_blob.slot(); - let blob_index = verified_blob.id().index; - - let result = self.chain.process_gossip_blob(verified_blob).await; - register_process_result_metrics(&result, metrics::BlockSource::Gossip, "blob"); - - match &result { - Ok(AvailabilityProcessingStatus::Imported(block_root)) => { - debug!( - %block_root, - "Gossipsub blob processed - imported fully available block" - ); - self.chain.recompute_head_at_current_slot().await; - - metrics::set_gauge( - &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, - processing_start_time.elapsed().as_millis() as i64, - ); - } - Ok(AvailabilityProcessingStatus::MissingComponents(slot, block_root)) => { - debug!( - %slot, - %blob_index, - %block_root, - "Processed gossip blob - waiting for other components" - ); - } - Err(BlockError::DuplicateFullyImported(_)) => { - debug!( - ?block_root, - blob_index, "Ignoring gossip blob already imported" - ); - } - Err(err) => { - debug!( - outcome = ?err, - ?block_root, - %blob_slot, - blob_index, - "Invalid gossip blob" - ); - self.gossip_penalize_peer( - peer_id, - PeerAction::MidToleranceError, - "bad_gossip_blob_ssz", - ); - } - } - - // If a block is in the da_checker, sync maybe awaiting for an event when block is finally - // imported. A block can become imported both after processing a block or blob. If a - // importing a block results in `Imported`, notify. Do not notify of blob errors. - if matches!(result, Ok(AvailabilityProcessingStatus::Imported(_))) { - self.send_sync_message(SyncMessage::GossipBlockProcessResult { - block_root, - imported: true, - }); - } - } - - /// Process a gossip-verified full data column (not partial). - /// Partials are handled by process_gossip_verified_partial_data_column. - async fn process_gossip_verified_data_column( - self: &Arc, - peer_id: PeerId, - verified_data_column: GossipVerifiedDataColumn, - // This value is not used presently, but it might come in handy for debugging. - _seen_duration: Duration, - ) { - let processing_start_time = Instant::now(); - let block_root = verified_data_column.block_root(); - let data_column_slot = verified_data_column.slot(); - let data_column_index = verified_data_column.index(); - - // TODO(gloas): implement partial messages - if let DataColumnSidecar::Fulu(col) = verified_data_column.as_data_column() - && self - .chain - .data_availability_checker - .partial_assembler() - .is_some_and(|a| !a.is_complete(block_root, verified_data_column.index())) - { - metrics::inc_counter_vec( - &metrics::BEACON_USEFUL_FULL_COLUMNS_RECEIVED_TOTAL, - &[&data_column_index.to_string()], - ); - - match col.to_partial() { - Ok(mut column) => { - let header = column.sidecar.header.take(); - if let Some(header) = header { - self.send_network_message(NetworkMessage::PublishPartialColumns { - columns: vec![Arc::new(column)], - header: Arc::new(header), - }); - } else { - crit!("Converting from full to partial yielded headerless partial") - }; - } - Err(err) => crit!(?err, "Could not convert from full to partial"), - } - } - - let result = self - .chain - .process_gossip_data_columns(vec![verified_data_column], || Ok(())) - .await; - register_process_result_metrics(&result, metrics::BlockSource::Gossip, "data_column"); - - match result { - Ok(availability) => match availability { - AvailabilityProcessingStatus::Imported(block_root) => { - debug!( - %block_root, - "Gossipsub data column processed, imported fully available block" - ); - self.chain.recompute_head_at_current_slot().await; - - metrics::set_gauge( - &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, - processing_start_time.elapsed().as_millis() as i64, - ); - - // If a block is in the da_checker, sync maybe awaiting for an event when block is finally - // imported. A block can become imported both after processing a block or data column. If - // importing a block results in `Imported`, notify. Do not notify of data column errors. - self.send_sync_message(SyncMessage::GossipBlockProcessResult { - block_root, - imported: true, - }); - } - AvailabilityProcessingStatus::MissingComponents(slot, block_root) => { - trace!( - %slot, - %data_column_index, - %block_root, - "Processed data column, waiting for other components" - ); - - self.check_reconstruction_trigger(slot, &block_root).await; - } - }, - Err(BlockError::DuplicateFullyImported(_)) => { - debug!( - ?block_root, - data_column_index, "Ignoring gossip column already imported" - ); - } - Err(err) => { - debug!( - outcome = ?err, - ?block_root, - block_slot = %data_column_slot, - data_column_index, - "Invalid gossip data column" - ); - self.gossip_penalize_peer( - peer_id, - PeerAction::MidToleranceError, - "bad_gossip_data_column_ssz", - ); - } - } - } - /// Process a gossip-verified partial data column by merging it in the assembler async fn process_gossip_verified_partial_data_column( self: &Arc, @@ -1885,9 +1622,7 @@ impl NetworkBeaconProcessor { crit!(error = %e, "Internal block gossip validation error. Availability check during gossip validation"); return None; } - // BlobNotRequired is unreachable. Only constructed in `process_gossip_blob` Err(e @ BlockError::InternalError(_)) - | Err(e @ BlockError::BlobNotRequired(_)) | Err(e @ BlockError::EnvelopeBlockRootUnknown(_)) | Err(e @ BlockError::OptimisticSyncNotSupported { .. }) => { error!(error = %e, "Internal block gossip validation error"); diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index bbaafec4ea..97673aa8b8 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -1,12 +1,11 @@ use crate::sync::manager::BlockProcessType; use crate::{service::NetworkMessage, sync::manager::SyncMessage}; -use beacon_chain::blob_verification::{GossipBlobError, observe_gossip_blob}; use beacon_chain::block_verification_types::LookupBlock; use beacon_chain::block_verification_types::RangeSyncBlock; -use beacon_chain::data_column_verification::{GossipDataColumnError, observe_gossip_data_column}; -use beacon_chain::fetch_blobs::{ - EngineGetBlobsOutput, FetchEngineBlobError, fetch_and_process_engine_blobs, +use beacon_chain::data_column_verification::{ + GossipDataColumnError, KzgVerifiedCustodyDataColumn, observe_gossip_data_column, }; +use beacon_chain::fetch_blobs::{FetchEngineBlobError, fetch_and_process_engine_blobs}; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use beacon_chain::{AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes, BlockError}; use beacon_processor::{ @@ -70,9 +69,6 @@ pub struct NetworkBeaconProcessor { pub executor: TaskExecutor, } -// Publish blobs in batches of exponentially increasing size. -const BLOB_PUBLICATION_EXP_FACTOR: usize = 2; - impl NetworkBeaconProcessor { fn try_send(&self, event: BeaconWorkEvent) -> Result<(), Error> { self.beacon_processor_send.try_send(event) @@ -198,36 +194,6 @@ impl NetworkBeaconProcessor { }) } - /// Create a new `Work` event for some blob sidecar. - pub fn send_gossip_blob_sidecar( - self: &Arc, - message_id: MessageId, - peer_id: PeerId, - peer_client: Client, - blob_index: u64, - blob_sidecar: Arc>, - seen_timestamp: Duration, - ) -> Result<(), Error> { - let processor = self.clone(); - let process_fn = async move { - processor - .process_gossip_blob( - message_id, - peer_id, - peer_client, - blob_index, - blob_sidecar, - seen_timestamp, - ) - .await - }; - - self.try_send(BeaconWorkEvent { - drop_during_sync: false, - work: Work::GossipBlobSidecar(Box::pin(process_fn)), - }) - } - /// Create a new `Work` event for some data column sidecar. pub fn send_gossip_data_column_sidecar( self: &Arc, @@ -970,22 +936,12 @@ impl NetworkBeaconProcessor { let epoch = header.slot().epoch(T::EthSpec::slots_per_epoch()); let custody_columns = self.chain.sampling_columns_for_epoch(epoch); let self_cloned = self.clone(); - let publish_fn = move |blobs_or_data_column| { + let publish_fn = move |columns: Vec>| { if publish_blobs { - match blobs_or_data_column { - EngineGetBlobsOutput::Blobs(blobs) => { - self_cloned.publish_blobs_gradually( - blobs.into_iter().map(|b| b.to_blob()).collect(), - block_root, - ); - } - EngineGetBlobsOutput::CustodyColumns(columns) => { - self_cloned.publish_data_columns_gradually( - columns.into_iter().map(|c| c.clone_arc()).collect(), - block_root, - ); - } - }; + self_cloned.publish_data_columns_gradually( + columns.into_iter().map(|c| c.clone_arc()).collect(), + block_root, + ); } }; @@ -1103,84 +1059,6 @@ impl NetworkBeaconProcessor { } } - /// This function gradually publishes blobs to the network in randomised batches. - /// - /// This is an optimisation to reduce outbound bandwidth and ensures each blob is published - /// by some nodes on the network as soon as possible. Our hope is that some blobs arrive from - /// other nodes in the meantime, obviating the need for us to publish them. If no other - /// publisher exists for a blob, it will eventually get published here. - fn publish_blobs_gradually( - self: &Arc, - mut blobs: Vec>>, - block_root: Hash256, - ) { - let self_clone = self.clone(); - - self.executor.spawn( - async move { - let chain = self_clone.chain.clone(); - let publish_fn = |blobs: Vec>>| { - self_clone.send_network_message(NetworkMessage::Publish { - messages: blobs - .into_iter() - .map(|blob| PubsubMessage::BlobSidecar(Box::new((blob.index, blob)))) - .collect(), - }); - }; - - // Permute the blobs and split them into batches. - // The hope is that we won't need to publish some blobs because we will receive them - // on gossip from other nodes. - blobs.shuffle(&mut rand::rng()); - - let blob_publication_batch_interval = chain.config.blob_publication_batch_interval; - let mut publish_count = 0usize; - let blob_count = blobs.len(); - let mut blobs_iter = blobs.into_iter().peekable(); - let mut batch_size = 1usize; - - while blobs_iter.peek().is_some() { - let batch = blobs_iter.by_ref().take(batch_size); - let publishable = batch - .filter_map(|blob| match observe_gossip_blob(&blob, &chain) { - Ok(()) => Some(blob), - Err(GossipBlobError::RepeatBlob { .. }) => None, - Err(e) => { - warn!( - error = ?e, - "Previously verified blob is invalid" - ); - None - } - }) - .collect::>(); - - if !publishable.is_empty() { - debug!( - publish_count = publishable.len(), - ?block_root, - "Publishing blob batch" - ); - publish_count += publishable.len(); - publish_fn(publishable); - } - - tokio::time::sleep(blob_publication_batch_interval).await; - batch_size *= BLOB_PUBLICATION_EXP_FACTOR; - } - - debug!( - batch_interval = blob_publication_batch_interval.as_millis(), - blob_count, - publish_count, - ?block_root, - "Batch blob publication complete" - ) - }, - "gradual_blob_publication", - ); - } - /// This function gradually publishes data columns to the network in randomised batches. /// /// This is an optimisation to reduce outbound bandwidth and ensures each column is published diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index 18d34b40b3..42d3b8f33d 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -409,22 +409,6 @@ impl TestRig { .unwrap(); } - pub fn enqueue_gossip_blob(&self, blob_index: usize) { - if let Some(blobs) = self.next_blobs.as_ref() { - let blob = blobs.get(blob_index).unwrap(); - self.network_beacon_processor - .send_gossip_blob_sidecar( - junk_message_id(), - junk_peer_id(), - Client::default(), - blob.index, - blob.clone(), - Duration::from_secs(0), - ) - .unwrap(); - } - } - pub fn enqueue_gossip_data_columns(&self, col_index: usize) { if let Some(data_columns) = self.next_data_columns.as_ref() { let data_column = data_columns.get(col_index).unwrap(); @@ -1101,13 +1085,6 @@ async fn import_gossip_block_acceptably_early() { rig.assert_event_journal_completes(&[WorkType::GossipBlock]) .await; - let num_blobs = rig.next_blobs.as_ref().map(|b| b.len()).unwrap_or(0); - for i in 0..num_blobs { - rig.enqueue_gossip_blob(i); - rig.assert_event_journal_completes(&[WorkType::GossipBlobSidecar]) - .await; - } - let num_data_columns = rig.next_data_columns.as_ref().map(|c| c.len()).unwrap_or(0); for i in 0..num_data_columns { rig.enqueue_gossip_data_columns(i); @@ -1242,13 +1219,6 @@ async fn import_gossip_block_at_current_slot() { rig.assert_event_journal_completes(&[WorkType::GossipBlock]) .await; - let num_blobs = rig.next_blobs.as_ref().map(|b| b.len()).unwrap_or(0); - for i in 0..num_blobs { - rig.enqueue_gossip_blob(i); - rig.assert_event_journal_completes(&[WorkType::GossipBlobSidecar]) - .await; - } - let num_data_columns = rig.next_data_columns.as_ref().map(|c| c.len()).unwrap_or(0); for i in 0..num_data_columns { rig.enqueue_gossip_data_columns(i); @@ -1315,10 +1285,6 @@ async fn attestation_to_unknown_block_processed(import_method: BlockImportMethod BlockImportMethod::Gossip => { rig.enqueue_gossip_block(); events.push(WorkType::GossipBlock); - for i in 0..num_blobs { - rig.enqueue_gossip_blob(i); - events.push(WorkType::GossipBlobSidecar); - } for i in 0..num_data_columns { rig.enqueue_gossip_data_columns(i); events.push(WorkType::GossipDataColumnSidecar); @@ -1401,10 +1367,6 @@ async fn aggregate_attestation_to_unknown_block(import_method: BlockImportMethod BlockImportMethod::Gossip => { rig.enqueue_gossip_block(); events.push(WorkType::GossipBlock); - for i in 0..num_blobs { - rig.enqueue_gossip_blob(i); - events.push(WorkType::GossipBlobSidecar); - } for i in 0..num_data_columns { rig.enqueue_gossip_data_columns(i); events.push(WorkType::GossipDataColumnSidecar) diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 35939c6f39..d2098d341e 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -412,19 +412,6 @@ impl Router { seen_timestamp, ), ), - PubsubMessage::BlobSidecar(data) => { - let (blob_index, blob_sidecar) = *data; - self.handle_beacon_processor_send_result( - self.network_beacon_processor.send_gossip_blob_sidecar( - message_id, - peer_id, - self.network_globals.client(&peer_id), - blob_index, - blob_sidecar, - seen_timestamp, - ), - ) - } PubsubMessage::DataColumnSidecar(data) => { let (subnet_id, column_sidecar) = *data; self.handle_beacon_processor_send_result( diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index f10610c751..ff3bf6f998 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -80,7 +80,6 @@ const MAX_LOOKUPS: usize = 200; /// The values for `Blob`, `DataColumn` and `PartialDataColumn` is the parent root of the column. pub enum BlockComponent { Block(DownloadResult>>), - Blob(DownloadResult), DataColumn(DownloadResult), PartialDataColumn(DownloadResult), } @@ -89,15 +88,13 @@ impl BlockComponent { fn parent_root(&self) -> Hash256 { match self { BlockComponent::Block(block) => block.value.parent_root(), - BlockComponent::Blob(parent_root) - | BlockComponent::DataColumn(parent_root) + BlockComponent::DataColumn(parent_root) | BlockComponent::PartialDataColumn(parent_root) => parent_root.value, } } fn get_type(&self) -> &'static str { match self { BlockComponent::Block(_) => "block", - BlockComponent::Blob(_) => "blob", BlockComponent::DataColumn(_) => "data_column", BlockComponent::PartialDataColumn(_) => "partial_data_column", } @@ -214,9 +211,9 @@ impl BlockLookups { block_root, Some(block_component), Some(parent_root), - // On a `UnknownParentBlock` or `UnknownParentBlob` event the peer is not required - // to have the rest of the block components (refer to decoupled blob gossip). Create - // the lookup with zero peers to house the block components. + // On a `UnknownParentBlock` or `UnknownParentDataColumn` event the peer is not + // required to have the rest of the block components. Create the lookup with zero + // peers to house the block components. &[], cx, ) diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 23bfd531f0..d54480e8e5 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -156,9 +156,7 @@ impl SingleBlockLookup { .block_request_state .state .insert_verified_response(block), - BlockComponent::Blob(_) - | BlockComponent::DataColumn(_) - | BlockComponent::PartialDataColumn(_) => { + BlockComponent::DataColumn(_) | BlockComponent::PartialDataColumn(_) => { // For now ignore single blobs and columns, as the blob request state assumes all blobs are // attributed to the same peer = the peer serving the remaining blobs. Ignoring this // block component has a minor effect, causing the node to re-request this blob diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 14a38f0e72..534e0bc7c8 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -144,9 +144,6 @@ pub enum SyncMessage { /// A block with an unknown parent has been received. UnknownParentBlock(PeerId, Arc>, Hash256), - /// A blob with an unknown parent has been received. - UnknownParentBlob(PeerId, Arc>), - /// A data column with an unknown parent has been received. UnknownParentDataColumn(PeerId, Arc>), @@ -890,24 +887,6 @@ impl SyncManager { }), ); } - SyncMessage::UnknownParentBlob(peer_id, blob) => { - let blob_slot = blob.slot(); - let block_root = blob.block_root(); - let parent_root = blob.block_parent_root(); - debug!(%block_root, %parent_root, "Received unknown parent blob message"); - self.handle_unknown_parent( - peer_id, - block_root, - parent_root, - blob_slot, - BlockComponent::Blob(DownloadResult { - value: parent_root, - block_root, - seen_timestamp: self.chain.slot_clock.now_duration().unwrap_or_default(), - peer_group: PeerGroup::from_single(peer_id), - }), - ); - } SyncMessage::UnknownParentDataColumn(peer_id, data_column) => { let data_column_slot = data_column.slot(); let block_root = data_column.block_root(); diff --git a/beacon_node/network/src/sync/network_context/requests/blobs_by_range.rs b/beacon_node/network/src/sync/network_context/requests/blobs_by_range.rs index 9c6f516199..1da0fb52f7 100644 --- a/beacon_node/network/src/sync/network_context/requests/blobs_by_range.rs +++ b/beacon_node/network/src/sync/network_context/requests/blobs_by_range.rs @@ -33,6 +33,7 @@ impl ActiveRequestItems for BlobsByRangeRequestItems { if blob.index >= self.max_blobs_per_block { return Err(LookupVerifyError::UnrequestedIndex(blob.index)); } + if !blob.verify_blob_sidecar_inclusion_proof() { return Err(LookupVerifyError::InvalidInclusionProof); } diff --git a/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs b/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs index 556985c2b4..f0ff99867b 100644 --- a/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs +++ b/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs @@ -50,9 +50,11 @@ impl ActiveRequestItems for BlobsByRootRequestItems { if self.request.block_root != block_root { return Err(LookupVerifyError::UnrequestedBlockRoot(block_root)); } + if !blob.verify_blob_sidecar_inclusion_proof() { return Err(LookupVerifyError::InvalidInclusionProof); } + if !self.request.indices.contains(&blob.index) { return Err(LookupVerifyError::UnrequestedIndex(blob.index)); } diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index c1b2793491..5c9e18362c 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -1205,17 +1205,6 @@ impl TestRig { self.trigger_unknown_parent_block(peer_id, last_block); } - fn trigger_with_last_unknown_blob_parent(&mut self) { - let peer_id = self.new_connected_supernode_peer(); - let blobs = self - .get_last_block() - .block_data() - .blobs() - .expect("no blobs"); - let blob = blobs.first().expect("empty blobs"); - self.trigger_unknown_parent_blob(peer_id, blob.clone()); - } - fn trigger_with_last_unknown_data_column_parent(&mut self) { let peer_id = self.new_connected_supernode_peer(); let columns = self @@ -1224,7 +1213,7 @@ impl TestRig { .data_columns() .expect("No data columns"); let column = columns.first().expect("empty columns"); - self.trigger_unknown_parent_column(peer_id, column.clone()); + self.trigger_unknown_parent_data_column(peer_id, column.clone()); } // Post-test assertions @@ -1428,6 +1417,10 @@ impl TestRig { genesis_fork().deneb_enabled().then(Self::default) } + fn new_after_fulu() -> Option { + genesis_fork().fulu_enabled().then(Self::default) + } + fn new_after_deneb_before_fulu() -> Option { let fork = genesis_fork(); if fork.deneb_enabled() && !fork.fulu_enabled() { @@ -1463,16 +1456,12 @@ impl TestRig { self.send_sync_message(SyncMessage::UnknownParentBlock(peer_id, block, block_root)) } - fn trigger_unknown_parent_blob(&mut self, peer_id: PeerId, blob: Arc>) { - self.send_sync_message(SyncMessage::UnknownParentBlob(peer_id, blob)); - } - - fn trigger_unknown_parent_column( + fn trigger_unknown_parent_data_column( &mut self, peer_id: PeerId, - column: Arc>, + data_column: Arc>, ) { - self.send_sync_message(SyncMessage::UnknownParentDataColumn(peer_id, column)); + self.send_sync_message(SyncMessage::UnknownParentDataColumn(peer_id, data_column)); } fn trigger_unknown_block_from_attestation(&mut self, block_root: Hash256, peer_id: PeerId) { @@ -1757,9 +1746,9 @@ impl TestRig { ) .unwrap() { - Availability::Available(_) => panic!("blob removed from da_checker, available"), + Availability::Available(_) => panic!("column removed from da_checker, available"), Availability::MissingComponents(block_root) => { - self.log(&format!("inserted blob to da_checker {block_root:?}")) + self.log(&format!("inserted column to da_checker {block_root:?}")) } }; } @@ -1944,35 +1933,29 @@ async fn happy_path_unknown_block_parent(depth: usize) { } } -/// Assert that sync completes from a GossipUnknownParentBlob / UnknownDataColumnParent +/// Assert that sync completes from an UnknownDataColumnParent async fn happy_path_unknown_data_parent(depth: usize) { - let Some(mut r) = TestRig::new_after_deneb() else { + let Some(mut r) = TestRig::new_after_fulu() else { return; }; r.build_chain(depth).await; - if r.is_after_fulu() { - r.trigger_with_last_unknown_data_column_parent(); - } else if r.is_after_deneb() { - r.trigger_with_last_unknown_blob_parent(); - } + r.trigger_with_last_unknown_data_column_parent(); r.simulate(SimulateConfig::happy_path()).await; r.assert_successful_lookup_sync_parent_trigger(); } /// Assert that multiple trigger types don't create extra lookups async fn happy_path_multiple_triggers(depth: usize) { - let mut r = TestRig::default(); + let Some(mut r) = TestRig::new_after_fulu() else { + return; + }; // + 1, because the unknown parent trigger needs two new blocks r.build_chain(depth + 1).await; r.trigger_with_last_block(); r.trigger_with_last_block(); r.trigger_with_last_unknown_block_parent(); r.trigger_with_last_unknown_block_parent(); - if r.is_after_fulu() { - r.trigger_with_last_unknown_data_column_parent(); - } else if r.is_after_deneb() { - r.trigger_with_last_unknown_blob_parent(); - } + r.trigger_with_last_unknown_data_column_parent(); r.simulate(SimulateConfig::happy_path()).await; assert_eq!(r.created_lookups(), depth + 1, "Don't create extra lookups"); r.assert_successful_lookup_sync(); @@ -2105,18 +2088,14 @@ async fn too_many_processing_failures(depth: usize) { #[tokio::test] /// Assert that multiple trigger types don't create extra lookups async fn unknown_parent_does_not_add_peers_to_itself() { - let Some(mut r) = TestRig::new_after_deneb() else { + let Some(mut r) = TestRig::new_after_fulu() else { return; }; // 2, because the unknown parent trigger needs two new blocks r.build_chain(2).await; r.trigger_with_last_unknown_block_parent(); r.trigger_with_last_unknown_block_parent(); - if r.is_after_fulu() { - r.trigger_with_last_unknown_data_column_parent(); - } else if r.is_after_deneb() { - r.trigger_with_last_unknown_blob_parent(); - } + r.trigger_with_last_unknown_data_column_parent(); r.simulate(SimulateConfig::happy_path()).await; r.assert_peers_at_lookup_of_slot(2, 0); r.assert_peers_at_lookup_of_slot(1, 3); diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 2954ee7eb4..1736cd951f 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -2,7 +2,6 @@ use super::*; use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file}; use ::fork_choice::{AttestationFromBlock, PayloadVerificationStatus, ProposerHeadError}; use beacon_chain::beacon_proposer_cache::compute_proposer_duties_from_head; -use beacon_chain::blob_verification::GossipBlobError; use beacon_chain::block_verification_types::LookupBlock; use beacon_chain::chain_config::DisallowedReOrgOffsets; use beacon_chain::data_column_verification::GossipVerifiedDataColumn; @@ -12,7 +11,7 @@ use beacon_chain::{ attestation_verification::{ VerifiedAttestation, obtain_indexed_attestation_and_committees_per_slot, }, - blob_verification::GossipVerifiedBlob, + blob_verification::KzgVerifiedBlob, custody_context::NodeCustodyType, test_utils::{BeaconChainHarness, EphemeralHarnessType}, }; @@ -696,7 +695,6 @@ impl Tester { let mut blob_success = true; - // Convert blobs and kzg_proofs into sidecars, then plumb them into the availability tracker if let Some(blobs) = blobs.clone() { let proofs = kzg_proofs.unwrap(); let commitments = block @@ -709,37 +707,51 @@ impl Tester { // Zipping will stop when any of the zipped lists runs out, which is what we want. Some // of the tests don't provide enough proofs/blobs, and should fail the availability // check. - for (i, ((blob, kzg_proof), kzg_commitment)) in - blobs.into_iter().zip(proofs).zip(commitments).enumerate() - { - let blob_sidecar = Arc::new(BlobSidecar { - index: i as u64, - blob, - kzg_commitment, - kzg_proof, - signed_block_header: block.signed_block_header(), - kzg_commitment_inclusion_proof: block - .message() - .body() - .kzg_commitment_merkle_proof(i) - .unwrap(), - }); + let verified_blobs: Vec> = blobs + .into_iter() + .zip(proofs) + .zip(commitments) + .enumerate() + .filter_map(|(i, ((blob, kzg_proof), kzg_commitment))| { + let blob_sidecar = Arc::new(BlobSidecar { + index: i as u64, + blob, + kzg_commitment, + kzg_proof, + signed_block_header: block.signed_block_header(), + kzg_commitment_inclusion_proof: block + .message() + .body() + .kzg_commitment_merkle_proof(i) + .unwrap(), + }); - let chain = self.harness.chain.clone(); - let blob = - match GossipVerifiedBlob::new(blob_sidecar.clone(), blob_sidecar.index, &chain) - { - Ok(gossip_verified_blob) => gossip_verified_blob, - Err(GossipBlobError::KzgError(_)) => { + match KzgVerifiedBlob::new( + blob_sidecar.clone(), + &self.harness.chain.kzg, + Duration::default(), + ) { + Ok(verified) => Some(verified), + Err(_) => { blob_success = false; - GossipVerifiedBlob::__assumed_valid(blob_sidecar) + None } - Err(_) => GossipVerifiedBlob::__assumed_valid(blob_sidecar), - }; - let result = - self.block_on_dangerous(self.harness.chain.process_gossip_blob(blob))?; + } + }) + .collect(); + + if !verified_blobs.is_empty() { + let result = self + .harness + .chain + .data_availability_checker + .put_kzg_verified_blobs(block_root, verified_blobs); if valid { - assert!(result.is_ok()); + assert!( + result.is_ok(), + "put_kzg_verified_blobs failed: {:?}", + result + ); } } }; diff --git a/testing/simulator/Cargo.toml b/testing/simulator/Cargo.toml index a1b1b6f95d..29972648f3 100644 --- a/testing/simulator/Cargo.toml +++ b/testing/simulator/Cargo.toml @@ -6,6 +6,7 @@ edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +beacon_chain = { workspace = true } clap = { workspace = true } environment = { workspace = true } execution_layer = { workspace = true } diff --git a/testing/simulator/src/basic_sim.rs b/testing/simulator/src/basic_sim.rs index 79581ee529..688cfb31ec 100644 --- a/testing/simulator/src/basic_sim.rs +++ b/testing/simulator/src/basic_sim.rs @@ -30,9 +30,10 @@ const ALTAIR_FORK_EPOCH: u64 = 0; const BELLATRIX_FORK_EPOCH: u64 = 0; const CAPELLA_FORK_EPOCH: u64 = 0; const DENEB_FORK_EPOCH: u64 = 0; -const ELECTRA_FORK_EPOCH: u64 = 2; -// const FULU_FORK_EPOCH: u64 = 3; -// const GLOAS_FORK_EPOCH: u64 = 4; +const ELECTRA_FORK_EPOCH: u64 = 0; +const FULU_FORK_EPOCH: u64 = 0; +// TODO(gloas): enable Gloas in simulator, current blocker is lack of data column gossip verification +// const GLOAS_FORK_EPOCH: u64 = 2; const SUGGESTED_FEE_RECIPIENT: [u8; 20] = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]; @@ -171,8 +172,8 @@ pub fn run_basic_sim(matches: &ArgMatches) -> Result<(), String> { let genesis_delay = GENESIS_DELAY; // Convenience variables. Update these values when adding a newer fork. - let latest_fork_version = spec.electra_fork_version; - let latest_fork_start_epoch = ELECTRA_FORK_EPOCH; + let latest_fork_version = spec.fulu_fork_version; + let latest_fork_start_epoch = FULU_FORK_EPOCH; let mut slot_duration_ms = spec.get_slot_duration().as_millis() as u64; slot_duration_ms /= speed_up_factor; @@ -187,6 +188,7 @@ pub fn run_basic_sim(matches: &ArgMatches) -> Result<(), String> { spec.capella_fork_epoch = Some(Epoch::new(CAPELLA_FORK_EPOCH)); spec.deneb_fork_epoch = Some(Epoch::new(DENEB_FORK_EPOCH)); spec.electra_fork_epoch = Some(Epoch::new(ELECTRA_FORK_EPOCH)); + spec.fulu_fork_epoch = Some(Epoch::new(FULU_FORK_EPOCH)); let spec = Arc::new(spec); env.eth2_config.spec = spec.clone(); diff --git a/testing/simulator/src/fallback_sim.rs b/testing/simulator/src/fallback_sim.rs index 06f4478c5e..aed113eca0 100644 --- a/testing/simulator/src/fallback_sim.rs +++ b/testing/simulator/src/fallback_sim.rs @@ -25,11 +25,12 @@ const END_EPOCH: u64 = 16; const GENESIS_DELAY: u64 = 38; const ALTAIR_FORK_EPOCH: u64 = 0; const BELLATRIX_FORK_EPOCH: u64 = 0; -const CAPELLA_FORK_EPOCH: u64 = 1; -const DENEB_FORK_EPOCH: u64 = 2; -// const ELECTRA_FORK_EPOCH: u64 = 3; -// const FULU_FORK_EPOCH: u64 = 4; -// const GLOAS_FORK_EPOCH: u64 = 5; +const CAPELLA_FORK_EPOCH: u64 = 0; +const DENEB_FORK_EPOCH: u64 = 0; +const ELECTRA_FORK_EPOCH: u64 = 0; +const FULU_FORK_EPOCH: u64 = 0; +// TODO(gloas): enable Gloas in simulator, current blocker is lack of data column gossip verification +// const GLOAS_FORK_EPOCH: u64 = 2; // Since simulator tests are non-deterministic and there is a non-zero chance of missed // attestations, define an acceptable network-wide attestation performance. @@ -191,8 +192,8 @@ pub fn run_fallback_sim(matches: &ArgMatches) -> Result<(), String> { spec.bellatrix_fork_epoch = Some(Epoch::new(BELLATRIX_FORK_EPOCH)); spec.capella_fork_epoch = Some(Epoch::new(CAPELLA_FORK_EPOCH)); spec.deneb_fork_epoch = Some(Epoch::new(DENEB_FORK_EPOCH)); - //spec.electra_fork_epoch = Some(Epoch::new(ELECTRA_FORK_EPOCH)); - //spec.fulu_fork_epoch = Some(Epoch::new(FULU_FORK_EPOCH)); + spec.electra_fork_epoch = Some(Epoch::new(ELECTRA_FORK_EPOCH)); + spec.fulu_fork_epoch = Some(Epoch::new(FULU_FORK_EPOCH)); let spec = Arc::new(spec); env.eth2_config.spec = spec.clone(); diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index 2beb9c0efc..780a09e543 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -1,4 +1,5 @@ use crate::checks::epoch_delay; +use beacon_chain::custody_context::NodeCustodyType; use kzg::trusted_setup::get_trusted_setup; use node_test_rig::{ ClientConfig, ClientGenesis, LocalBeaconNode, LocalExecutionNode, LocalValidatorClient, @@ -46,6 +47,7 @@ fn default_client_config(network_params: LocalNetworkParams, genesis_time: u64) beacon_config.network.discv5_config.enable_packet_filter = false; beacon_config.chain.enable_light_client_server = true; beacon_config.chain.optimistic_finalized_sync = false; + beacon_config.chain.node_custody_type = NodeCustodyType::Supernode; beacon_config.trusted_setup = get_trusted_setup(); let el_config = execution_layer::Config { @@ -103,6 +105,15 @@ fn default_mock_execution_config( ) } + if let Some(gloas_fork_epoch) = spec.gloas_fork_epoch { + mock_execution_config.amsterdam_time = Some( + genesis_time + + (spec.get_slot_duration().as_secs()) + * E::slots_per_epoch() + * gloas_fork_epoch.as_u64(), + ) + } + mock_execution_config }