gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-06-15 09:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge unstable

Browse Source
This commit is contained in:
Eitan Seri-Levi
2025-02-02 00:18:00 +03:00
parent f31c0b69e3 ddb845d503
commit db8d659f2f
340 changed files with 10457 additions and 4957 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
beacon_node/beacon_chain/src/attestation_rewards.rs
View File

@@ -51,13 +51,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.get_state(&state_root, Some(state_slot))?
.ok_or(BeaconChainError::MissingBeaconState(state_root))?;
match state {
BeaconState::Base(_) => self.compute_attestation_rewards_base(state, validators),
BeaconState::Altair(_)
| BeaconState::Bellatrix(_)
| BeaconState::Capella(_)
| BeaconState::Deneb(_)
| BeaconState::Electra(_) => self.compute_attestation_rewards_altair(state, validators),
if state.fork_name_unchecked().altair_enabled() {
self.compute_attestation_rewards_altair(state, validators)
} else {
self.compute_attestation_rewards_base(state, validators)
}
}

13
beacon_node/beacon_chain/src/attestation_verification.rs
View File

@@ -62,7 +62,7 @@ use tree_hash::TreeHash;
use types::{
Attestation, AttestationRef, BeaconCommittee, BeaconStateError::NoCommitteeFound, ChainSpec,
CommitteeIndex, Epoch, EthSpec, Hash256, IndexedAttestation, SelectionProof,
SignedAggregateAndProof, Slot, SubnetId,
SignedAggregateAndProof, SingleAttestation, Slot, SubnetId,
};
pub use batch::{batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations};
@@ -317,12 +317,22 @@ pub struct VerifiedUnaggregatedAttestation<'a, T: BeaconChainTypes> {
attestation: AttestationRef<'a, T::EthSpec>,
indexed_attestation: IndexedAttestation<T::EthSpec>,
subnet_id: SubnetId,
validator_index: usize,
}
impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<'_, T> {
pub fn into_indexed_attestation(self) -> IndexedAttestation<T::EthSpec> {
self.indexed_attestation
}
pub fn single_attestation(&self) -> Option<SingleAttestation> {
Some(SingleAttestation {
committee_index: self.attestation.committee_index()?,
attester_index: self.validator_index as u64,
data: self.attestation.data().clone(),
signature: self.attestation.signature().clone(),
})
}
}
/// Custom `Clone` implementation is to avoid the restrictive trait bounds applied by the usual derive
@@ -1035,6 +1045,7 @@ impl<'a, T: BeaconChainTypes> VerifiedUnaggregatedAttestation<'a, T> {
attestation,
indexed_attestation,
subnet_id,
validator_index: validator_index as usize,
})
}

9
beacon_node/beacon_chain/src/beacon_block_streamer.rs
View File

@@ -15,7 +15,7 @@ use types::{
};
use types::{
ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadElectra,
ExecutionPayloadHeader,
ExecutionPayloadFulu, ExecutionPayloadHeader,
};
#[derive(PartialEq)]
@@ -99,6 +99,7 @@ fn reconstruct_default_header_block<E: EthSpec>(
ForkName::Capella => ExecutionPayloadCapella::default().into(),
ForkName::Deneb => ExecutionPayloadDeneb::default().into(),
ForkName::Electra => ExecutionPayloadElectra::default().into(),
ForkName::Fulu => ExecutionPayloadFulu::default().into(),
ForkName::Base | ForkName::Altair => {
return Err(Error::PayloadReconstruction(format!(
"Block with fork variant {} has execution payload",
@@ -742,13 +743,14 @@ mod tests {
}
#[tokio::test]
async fn check_all_blocks_from_altair_to_electra() {
async fn check_all_blocks_from_altair_to_fulu() {
let slots_per_epoch = MinimalEthSpec::slots_per_epoch() as usize;
let num_epochs = 10;
let num_epochs = 12;
let bellatrix_fork_epoch = 2usize;
let capella_fork_epoch = 4usize;
let deneb_fork_epoch = 6usize;
let electra_fork_epoch = 8usize;
let fulu_fork_epoch = 10usize;
let num_blocks_produced = num_epochs * slots_per_epoch;
let mut spec = test_spec::<MinimalEthSpec>();
@@ -757,6 +759,7 @@ mod tests {
spec.capella_fork_epoch = Some(Epoch::new(capella_fork_epoch as u64));
spec.deneb_fork_epoch = Some(Epoch::new(deneb_fork_epoch as u64));
spec.electra_fork_epoch = Some(Epoch::new(electra_fork_epoch as u64));
spec.fulu_fork_epoch = Some(Epoch::new(fulu_fork_epoch as u64));
let spec = Arc::new(spec);
let harness = get_harness(VALIDATOR_COUNT, spec.clone());

354
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@@ -34,6 +34,7 @@ use crate::execution_payload::{get_execution_payload, NotifyExecutionLayer, Prep
use crate::fork_choice_signal::{ForkChoiceSignalRx, ForkChoiceSignalTx, ForkChoiceWaitResult};
use crate::graffiti_calculator::GraffitiCalculator;
use crate::head_tracker::{HeadTracker, HeadTrackerReader, SszHeadTracker};
use crate::kzg_utils::reconstruct_blobs;
use crate::light_client_finality_update_verification::{
Error as LightClientFinalityUpdateError, VerifiedLightClientFinalityUpdate,
};
@@ -117,10 +118,11 @@ use std::sync::Arc;
use std::time::Duration;
use store::iter::{BlockRootsIterator, ParentRootBlockIterator, StateRootsIterator};
use store::{
DatabaseBlock, Error as DBError, HotColdDB, KeyValueStore, KeyValueStoreOp, StoreItem, StoreOp,
BlobSidecarListFromRoot, DatabaseBlock, Error as DBError, HotColdDB, KeyValueStore,
KeyValueStoreOp, StoreItem, StoreOp,
};
use task_executor::{ShutdownReason, TaskExecutor};
use tokio::sync::mpsc::Receiver;
use tokio::sync::oneshot;
use tokio_stream::Stream;
use tree_hash::TreeHash;
use types::blob_sidecar::FixedBlobSidecarList;
@@ -575,7 +577,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.start_slot(T::EthSpec::slots_per_epoch());
let is_canonical = self
.block_root_at_slot(block_slot, WhenSlotSkipped::None)?
.map_or(false, |canonical_root| block_root == &canonical_root);
.is_some_and(|canonical_root| block_root == &canonical_root);
Ok(block_slot <= finalized_slot && is_canonical)
}
@@ -606,7 +608,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let slot_is_finalized = state_slot <= finalized_slot;
let canonical = self
.state_root_at_slot(state_slot)?
.map_or(false, |canonical_root| state_root == &canonical_root);
.is_some_and(|canonical_root| state_root == &canonical_root);
Ok(FinalizationAndCanonicity {
slot_is_finalized,
canonical,
@@ -1149,9 +1151,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
pub fn get_blobs_checking_early_attester_cache(
&self,
block_root: &Hash256,
) -> Result<BlobSidecarList<T::EthSpec>, Error> {
) -> Result<BlobSidecarListFromRoot<T::EthSpec>, Error> {
self.early_attester_cache
.get_blobs(*block_root)
.map(Into::into)
.map_or_else(|| self.get_blobs(block_root), Ok)
}
@@ -1242,10 +1245,59 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
///
/// ## Errors
/// May return a database error.
pub fn get_blobs(&self, block_root: &Hash256) -> Result<BlobSidecarList<T::EthSpec>, Error> {
match self.store.get_blobs(block_root)? {
Some(blobs) => Ok(blobs),
None => Ok(BlobSidecarList::default()),
pub fn get_blobs(
&self,
block_root: &Hash256,
) -> Result<BlobSidecarListFromRoot<T::EthSpec>, Error> {
self.store.get_blobs(block_root).map_err(Error::from)
}
/// Returns the data columns at the given root, if any.
///
/// ## Errors
/// May return a database error.
pub fn get_data_columns(
&self,
block_root: &Hash256,
) -> Result<Option<DataColumnSidecarList<T::EthSpec>>, Error> {
self.store.get_data_columns(block_root).map_err(Error::from)
}
/// Returns the blobs at the given root, if any.
///
/// Uses the `block.epoch()` to determine whether to retrieve blobs or columns from the store.
///
/// If at least 50% of columns are retrieved, blobs will be reconstructed and returned,
/// otherwise an error `InsufficientColumnsToReconstructBlobs` is returned.
///
/// ## Errors
/// May return a database error.
pub fn get_or_reconstruct_blobs(
&self,
block_root: &Hash256,
) -> Result<Option<BlobSidecarList<T::EthSpec>>, Error> {
let Some(block) = self.store.get_blinded_block(block_root)? else {
return Ok(None);
};
if self.spec.is_peer_das_enabled_for_epoch(block.epoch()) {
if let Some(columns) = self.store.get_data_columns(block_root)? {
let num_required_columns = self.spec.number_of_columns / 2;
let reconstruction_possible = columns.len() >= num_required_columns as usize;
if reconstruction_possible {
reconstruct_blobs(&self.kzg, &columns, None, &block, &self.spec)
.map(Some)
.map_err(Error::FailedToReconstructBlobs)
} else {
Err(Error::InsufficientColumnsToReconstructBlobs {
columns_found: columns.len(),
})
}
} else {
Ok(None)
}
} else {
self.get_blobs(block_root).map(|b| b.blobs())
}
}
@@ -2161,10 +2213,30 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
|v| {
// This method is called for API and gossip attestations, so this covers all unaggregated attestation events
if let Some(event_handler) = self.event_handler.as_ref() {
if event_handler.has_single_attestation_subscribers() {
let current_fork = self
.spec
.fork_name_at_slot::<T::EthSpec>(v.attestation().data().slot);
if current_fork.electra_enabled() {
// I don't see a situation where this could return None. The upstream unaggregated attestation checks
// should have already verified that this is an attestation with a single committee bit set.
if let Some(single_attestation) = v.single_attestation() {
event_handler.register(EventKind::SingleAttestation(Box::new(
single_attestation,
)));
}
}
}
if event_handler.has_attestation_subscribers() {
event_handler.register(EventKind::Attestation(Box::new(
v.attestation().clone_as_attestation(),
)));
let current_fork = self
.spec
.fork_name_at_slot::<T::EthSpec>(v.attestation().data().slot);
if !current_fork.electra_enabled() {
event_handler.register(EventKind::Attestation(Box::new(
v.attestation().clone_as_attestation(),
)));
}
}
}
metrics::inc_counter(&metrics::UNAGGREGATED_ATTESTATION_PROCESSING_SUCCESSES);
@@ -3214,7 +3286,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
slot: Slot,
block_root: Hash256,
blobs: FixedBlobSidecarList<T::EthSpec>,
data_column_recv: Option<Receiver<DataColumnSidecarList<T::EthSpec>>>,
data_column_recv: Option<oneshot::Receiver<DataColumnSidecarList<T::EthSpec>>>,
) -> Result<AvailabilityProcessingStatus, BlockError> {
// If this block has already been imported to forkchoice it must have been available, so
// we don't need to process its blobs again.
@@ -3342,7 +3414,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
};
let r = self
.process_availability(slot, availability, None, || Ok(()))
.process_availability(slot, availability, || Ok(()))
.await;
self.remove_notified(&block_root, r)
.map(|availability_processing_status| {
@@ -3470,7 +3542,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
match executed_block {
ExecutedBlock::Available(block) => {
self.import_available_block(Box::new(block), None).await
self.import_available_block(Box::new(block)).await
}
ExecutedBlock::AvailabilityPending(block) => {
self.check_block_availability_and_import(block).await
@@ -3602,7 +3674,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let availability = self
.data_availability_checker
.put_pending_executed_block(block)?;
self.process_availability(slot, availability, None, || Ok(()))
self.process_availability(slot, availability, || Ok(()))
.await
}
@@ -3618,7 +3690,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
}
let availability = self.data_availability_checker.put_gossip_blob(blob)?;
self.process_availability(slot, availability, None, || Ok(()))
self.process_availability(slot, availability, || Ok(()))
.await
}
@@ -3641,7 +3713,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.data_availability_checker
.put_gossip_data_columns(block_root, data_columns)?;
self.process_availability(slot, availability, None, publish_fn)
self.process_availability(slot, availability, publish_fn)
.await
}
@@ -3685,7 +3757,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.data_availability_checker
.put_rpc_blobs(block_root, blobs)?;
self.process_availability(slot, availability, None, || Ok(()))
self.process_availability(slot, availability, || Ok(()))
.await
}
@@ -3694,14 +3766,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
slot: Slot,
block_root: Hash256,
blobs: FixedBlobSidecarList<T::EthSpec>,
data_column_recv: Option<Receiver<DataColumnSidecarList<T::EthSpec>>>,
data_column_recv: Option<oneshot::Receiver<DataColumnSidecarList<T::EthSpec>>>,
) -> Result<AvailabilityProcessingStatus, BlockError> {
self.check_blobs_for_slashability(block_root, &blobs)?;
let availability = self
.data_availability_checker
.put_engine_blobs(block_root, blobs)?;
let availability =
self.data_availability_checker
.put_engine_blobs(block_root, blobs, data_column_recv)?;
self.process_availability(slot, availability, data_column_recv, || Ok(()))
self.process_availability(slot, availability, || Ok(()))
.await
}
@@ -3741,7 +3813,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.data_availability_checker
.put_rpc_custody_columns(block_root, custody_columns)?;
self.process_availability(slot, availability, None, || Ok(()))
self.process_availability(slot, availability, || Ok(()))
.await
}
@@ -3753,14 +3825,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
self: &Arc<Self>,
slot: Slot,
availability: Availability<T::EthSpec>,
recv: Option<Receiver<DataColumnSidecarList<T::EthSpec>>>,
publish_fn: impl FnOnce() -> Result<(), BlockError>,
) -> Result<AvailabilityProcessingStatus, BlockError> {
match availability {
Availability::Available(block) => {
publish_fn()?;
// Block is fully available, import into fork choice
self.import_available_block(block, recv).await
self.import_available_block(block).await
}
Availability::MissingComponents(block_root) => Ok(
AvailabilityProcessingStatus::MissingComponents(slot, block_root),
@@ -3771,7 +3842,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
pub async fn import_available_block(
self: &Arc<Self>,
block: Box<AvailableExecutedBlock<T::EthSpec>>,
data_column_recv: Option<Receiver<DataColumnSidecarList<T::EthSpec>>>,
) -> Result<AvailabilityProcessingStatus, BlockError> {
let AvailableExecutedBlock {
block,
@@ -3786,6 +3856,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
parent_eth1_finalization_data,
confirmed_state_roots,
consensus_context,
data_column_recv,
} = import_data;
// Record the time at which this block's blobs became available.
@@ -3852,7 +3923,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
parent_block: SignedBlindedBeaconBlock<T::EthSpec>,
parent_eth1_finalization_data: Eth1FinalizationData,
mut consensus_context: ConsensusContext<T::EthSpec>,
data_column_recv: Option<Receiver<DataColumnSidecarList<T::EthSpec>>>,
data_column_recv: Option<oneshot::Receiver<DataColumnSidecarList<T::EthSpec>>>,
) -> Result<Hash256, BlockError> {
// ----------------------------- BLOCK NOT YET ATTESTABLE ----------------------------------
// Everything in this initial section is on the hot path between processing the block and
@@ -4020,44 +4091,32 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// end up with blocks in fork choice that are missing from disk.
// See https://github.com/sigp/lighthouse/issues/2028
let (_, signed_block, blobs, data_columns) = signed_block.deconstruct();
// TODO(das) we currently store all subnet sampled columns. Tracking issue to exclude non
// custody columns: https://github.com/sigp/lighthouse/issues/6465
let custody_columns_count = self.data_availability_checker.get_sampling_column_count();
// if block is made available via blobs, dropped the data columns.
let data_columns = data_columns.filter(|columns| columns.len() == custody_columns_count);
let data_columns = match (data_columns, data_column_recv) {
// If the block was made available via custody columns received from gossip / rpc, use them
// since we already have them.
(Some(columns), _) => Some(columns),
// Otherwise, it means blobs were likely available via fetching from EL, in this case we
// wait for the data columns to be computed (blocking).
(None, Some(mut data_column_recv)) => {
let _column_recv_timer =
metrics::start_timer(&metrics::BLOCK_PROCESSING_DATA_COLUMNS_WAIT);
// Unable to receive data columns from sender, sender is either dropped or
// failed to compute data columns from blobs. We restore fork choice here and
// return to avoid inconsistency in database.
if let Some(columns) = data_column_recv.blocking_recv() {
Some(columns)
} else {
let err_msg = "Did not receive data columns from sender";
error!(
self.log,
"Failed to store data columns into the database";
"msg" => "Restoring fork choice from disk",
"error" => err_msg,
);
return Err(self
.handle_import_block_db_write_error(fork_choice)
.err()
.unwrap_or(BlockError::InternalError(err_msg.to_string())));
}
match self.get_blobs_or_columns_store_op(
block_root,
signed_block.epoch(),
blobs,
data_columns,
data_column_recv,
) {
Ok(Some(blobs_or_columns_store_op)) => {
ops.push(blobs_or_columns_store_op);
}
// No data columns present and compute data columns task was not spawned.
// Could either be no blobs in the block or before PeerDAS activation.
(None, None) => None,
};
Ok(None) => {}
Err(e) => {
error!(
self.log,
"Failed to store data columns into the database";
"msg" => "Restoring fork choice from disk",
"error" => &e,
"block_root" => ?block_root
);
return Err(self
.handle_import_block_db_write_error(fork_choice)
.err()
.unwrap_or(BlockError::InternalError(e)));
}
}
let block = signed_block.message();
let db_write_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_WRITE);
@@ -4069,30 +4128,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
ops.push(StoreOp::PutBlock(block_root, signed_block.clone()));
ops.push(StoreOp::PutState(block.state_root(), &state));
if let Some(blobs) = blobs {
if !blobs.is_empty() {
debug!(
self.log, "Writing blobs to store";
"block_root" => %block_root,
"count" => blobs.len(),
);
ops.push(StoreOp::PutBlobs(block_root, blobs));
}
}
if let Some(data_columns) = data_columns {
// TODO(das): `available_block includes all sampled columns, but we only need to store
// custody columns. To be clarified in spec.
if !data_columns.is_empty() {
debug!(
self.log, "Writing data_columns to store";
"block_root" => %block_root,
"count" => data_columns.len(),
);
ops.push(StoreOp::PutDataColumns(block_root, data_columns));
}
}
let txn_lock = self.store.hot_db.begin_rw_transaction();
if let Err(e) = self.store.do_atomically_with_block_and_blobs_cache(ops) {
@@ -5246,9 +5281,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.start_of(slot)
.unwrap_or_else(|| Duration::from_secs(0)),
);
block_delays.observed.map_or(false, |delay| {
delay >= self.slot_clock.unagg_attestation_production_delay()
})
block_delays
.observed
.is_some_and(|delay| delay >= self.slot_clock.unagg_attestation_production_delay())
}
/// Produce a block for some `slot` upon the given `state`.
@@ -5445,23 +5480,19 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// If required, start the process of loading an execution payload from the EL early. This
// allows it to run concurrently with things like attestation packing.
let prepare_payload_handle = match &state {
BeaconState::Base(_) | BeaconState::Altair(_) => None,
BeaconState::Bellatrix(_)
| BeaconState::Capella(_)
| BeaconState::Deneb(_)
| BeaconState::Electra(_) => {
let prepare_payload_handle = get_execution_payload(
self.clone(),
&state,
parent_root,
proposer_index,
builder_params,
builder_boost_factor,
block_production_version,
)?;
Some(prepare_payload_handle)
}
let prepare_payload_handle = if state.fork_name_unchecked().bellatrix_enabled() {
let prepare_payload_handle = get_execution_payload(
self.clone(),
&state,
parent_root,
proposer_index,
builder_params,
builder_boost_factor,
block_production_version,
)?;
Some(prepare_payload_handle)
} else {
None
};
let (mut proposer_slashings, mut attester_slashings, mut voluntary_exits) =
@@ -5879,6 +5910,48 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
execution_payload_value,
)
}
BeaconState::Fulu(_) => {
let (
payload,
kzg_commitments,
maybe_blobs_and_proofs,
maybe_requests,
execution_payload_value,
) = block_contents
.ok_or(BlockProductionError::MissingExecutionPayload)?
.deconstruct();
(
BeaconBlock::Fulu(BeaconBlockFulu {
slot,
proposer_index,
parent_root,
state_root: Hash256::zero(),
body: BeaconBlockBodyFulu {
randao_reveal,
eth1_data,
graffiti,
proposer_slashings: proposer_slashings.into(),
attester_slashings: attester_slashings_electra.into(),
attestations: attestations_electra.into(),
deposits: deposits.into(),
voluntary_exits: voluntary_exits.into(),
sync_aggregate: sync_aggregate
.ok_or(BlockProductionError::MissingSyncAggregate)?,
execution_payload: payload
.try_into()
.map_err(|_| BlockProductionError::InvalidPayloadFork)?,
bls_to_execution_changes: bls_to_execution_changes.into(),
blob_kzg_commitments: kzg_commitments
.ok_or(BlockProductionError::InvalidPayloadFork)?,
execution_requests: maybe_requests
.ok_or(BlockProductionError::MissingExecutionRequests)?,
},
}),
maybe_blobs_and_proofs,
execution_payload_value,
)
}
};
let block = SignedBeaconBlock::from_block(
@@ -5955,6 +6028,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let kzg = self.kzg.as_ref();
// TODO(fulu): we no longer need blob proofs from PeerDAS and could avoid computing.
kzg_utils::validate_blobs::<T::EthSpec>(
kzg,
expected_kzg_commitments,
@@ -7307,6 +7381,68 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
reqresp_pre_import_cache_len: self.reqresp_pre_import_cache.read().len(),
}
}
fn get_blobs_or_columns_store_op(
&self,
block_root: Hash256,
block_epoch: Epoch,
blobs: Option<BlobSidecarList<T::EthSpec>>,
data_columns: Option<DataColumnSidecarList<T::EthSpec>>,
data_column_recv: Option<oneshot::Receiver<DataColumnSidecarList<T::EthSpec>>>,
) -> Result<Option<StoreOp<T::EthSpec>>, String> {
if self.spec.is_peer_das_enabled_for_epoch(block_epoch) {
// TODO(das) we currently store all subnet sampled columns. Tracking issue to exclude non
// custody columns: https://github.com/sigp/lighthouse/issues/6465
let custody_columns_count = self.data_availability_checker.get_sampling_column_count();
let custody_columns_available = data_columns
.as_ref()
.as_ref()
.is_some_and(|columns| columns.len() == custody_columns_count);
let data_columns_to_persist = if custody_columns_available {
// If the block was made available via custody columns received from gossip / rpc, use them
// since we already have them.
data_columns
} else if let Some(data_column_recv) = data_column_recv {
// Blobs were available from the EL, in this case we wait for the data columns to be computed (blocking).
let _column_recv_timer =
metrics::start_timer(&metrics::BLOCK_PROCESSING_DATA_COLUMNS_WAIT);
// Unable to receive data columns from sender, sender is either dropped or
// failed to compute data columns from blobs. We restore fork choice here and
// return to avoid inconsistency in database.
let computed_data_columns = data_column_recv
.blocking_recv()
.map_err(|e| format!("Did not receive data columns from sender: {e:?}"))?;
Some(computed_data_columns)
} else {
// No blobs in the block.
None
};
if let Some(data_columns) = data_columns_to_persist {
if !data_columns.is_empty() {
debug!(
self.log, "Writing data_columns to store";
"block_root" => %block_root,
"count" => data_columns.len(),
);
return Ok(Some(StoreOp::PutDataColumns(block_root, data_columns)));
}
}
} else if let Some(blobs) = blobs {
if !blobs.is_empty() {
debug!(
self.log, "Writing blobs to store";
"block_root" => %block_root,
"count" => blobs.len(),
);
return Ok(Some(StoreOp::PutBlobs(block_root, blobs)));
}
}
Ok(None)
}
}
impl<T: BeaconChainTypes> Drop for BeaconChain<T> {

2
beacon_node/beacon_chain/src/blob_verification.rs
View File

@@ -400,7 +400,7 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes, O: ObservationStrat
// since we only subscribe to `MaxBlobsPerBlock` subnets over gossip network.
// We include this check only for completeness.
// Getting this error would imply something very wrong with our networking decoding logic.
if blob_index >= T::EthSpec::max_blobs_per_block() as u64 {
if blob_index >= chain.spec.max_blobs_per_block(blob_epoch) {
return Err(GossipBlobError::InvalidSubnet {
expected: subnet,
received: blob_index,

59
beacon_node/beacon_chain/src/block_verification.rs
View File

@@ -208,24 +208,18 @@ pub enum BlockError {
///
/// The block is invalid and the peer is faulty.
IncorrectBlockProposer { block: u64, local_shuffling: u64 },
/// The proposal signature in invalid.
///
/// ## Peer scoring
///
/// The block is invalid and the peer is faulty.
ProposalSignatureInvalid,
/// The `block.proposal_index` is not known.
///
/// ## Peer scoring
///
/// The block is invalid and the peer is faulty.
UnknownValidator(u64),
/// A signature in the block is invalid (exactly which is unknown).
/// A signature in the block is invalid
///
/// ## Peer scoring
///
/// The block is invalid and the peer is faulty.
InvalidSignature,
InvalidSignature(InvalidSignature),
/// The provided block is not from a later slot than its parent.
///
/// ## Peer scoring
@@ -329,6 +323,17 @@ pub enum BlockError {
InternalError(String),
}
/// Which specific signature(s) are invalid in a SignedBeaconBlock
#[derive(Debug)]
pub enum InvalidSignature {
// The outer signature in a SignedBeaconBlock
ProposerSignature,
// One or more signatures in BeaconBlockBody
BlockBodySignatures,
// One or more signatures in SignedBeaconBlock
Unknown,
}
impl From<AvailabilityCheckError> for BlockError {
fn from(e: AvailabilityCheckError) -> Self {
Self::AvailabilityCheck(e)
@@ -523,7 +528,9 @@ pub enum BlockSlashInfo<TErr> {
impl BlockSlashInfo<BlockError> {
pub fn from_early_error_block(header: SignedBeaconBlockHeader, e: BlockError) -> Self {
match e {
BlockError::ProposalSignatureInvalid => BlockSlashInfo::SignatureInvalid(e),
BlockError::InvalidSignature(InvalidSignature::ProposerSignature) => {
BlockSlashInfo::SignatureInvalid(e)
}
// `InvalidSignature` could indicate any signature in the block, so we want
// to recheck the proposer signature alone.
_ => BlockSlashInfo::SignatureNotChecked(header, e),
@@ -652,7 +659,7 @@ pub fn signature_verify_chain_segment<T: BeaconChainTypes>(
}
if signature_verifier.verify().is_err() {
return Err(BlockError::InvalidSignature);
return Err(BlockError::InvalidSignature(InvalidSignature::Unknown));
}
drop(pubkey_cache);
@@ -964,7 +971,9 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
};
if !signature_is_valid {
return Err(BlockError::ProposalSignatureInvalid);
return Err(BlockError::InvalidSignature(
InvalidSignature::ProposerSignature,
));
}
chain
@@ -1098,7 +1107,26 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
parent: Some(parent),
})
} else {
Err(BlockError::InvalidSignature)
// Re-verify the proposer signature in isolation to attribute fault
let pubkey = pubkey_cache
.get(block.message().proposer_index() as usize)
.ok_or_else(|| BlockError::UnknownValidator(block.message().proposer_index()))?;
if block.as_block().verify_signature(
Some(block_root),
pubkey,
&state.fork(),
chain.genesis_validators_root,
&chain.spec,
) {
// Proposer signature is valid, the invalid signature must be in the body
Err(BlockError::InvalidSignature(
InvalidSignature::BlockBodySignatures,
))
} else {
Err(BlockError::InvalidSignature(
InvalidSignature::ProposerSignature,
))
}
}
}
@@ -1153,7 +1181,9 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
consensus_context,
})
} else {
Err(BlockError::InvalidSignature)
Err(BlockError::InvalidSignature(
InvalidSignature::BlockBodySignatures,
))
}
}
@@ -1677,6 +1707,7 @@ impl<T: BeaconChainTypes> ExecutionPendingBlock<T> {
parent_eth1_finalization_data,
confirmed_state_roots,
consensus_context,
data_column_recv: None,
},
payload_verification_handle,
})
@@ -1980,7 +2011,7 @@ impl BlockBlobError for BlockError {
}
fn proposer_signature_invalid() -> Self {
BlockError::ProposalSignatureInvalid
BlockError::InvalidSignature(InvalidSignature::ProposerSignature)
}
}

37
beacon_node/beacon_chain/src/block_verification_types.rs
View File

@@ -4,14 +4,14 @@ use crate::data_column_verification::{CustodyDataColumn, CustodyDataColumnList};
use crate::eth1_finalization_cache::Eth1FinalizationData;
use crate::{get_block_root, PayloadVerificationOutcome};
use derivative::Derivative;
use ssz_types::VariableList;
use state_processing::ConsensusContext;
use std::fmt::{Debug, Formatter};
use std::sync::Arc;
use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList};
use tokio::sync::oneshot;
use types::blob_sidecar::BlobIdentifier;
use types::{
BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, ChainSpec, Epoch, EthSpec,
Hash256, RuntimeVariableList, SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, ChainSpec, DataColumnSidecarList,
Epoch, EthSpec, Hash256, RuntimeVariableList, SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
};
/// A block that has been received over RPC. It has 2 internal variants:
@@ -165,7 +165,7 @@ impl<E: EthSpec> RpcBlock<E> {
let inner = if !custody_columns.is_empty() {
RpcBlockInner::BlockAndCustodyColumns(
block,
RuntimeVariableList::new(custody_columns, spec.number_of_columns)?,
RuntimeVariableList::new(custody_columns, spec.number_of_columns as usize)?,
)
} else {
RpcBlockInner::Block(block)
@@ -176,23 +176,6 @@ impl<E: EthSpec> RpcBlock<E> {
})
}
pub fn new_from_fixed(
block_root: Hash256,
block: Arc<SignedBeaconBlock<E>>,
blobs: FixedBlobSidecarList<E>,
) -> Result<Self, AvailabilityCheckError> {
let filtered = blobs
.into_iter()
.filter_map(|b| b.clone())
.collect::<Vec<_>>();
let blobs = if filtered.is_empty() {
None
} else {
Some(VariableList::from(filtered))
};
Self::new(Some(block_root), block, blobs)
}
#[allow(clippy::type_complexity)]
pub fn deconstruct(
self,
@@ -355,7 +338,8 @@ impl<E: EthSpec> AvailabilityPendingExecutedBlock<E> {
}
}
#[derive(Debug, PartialEq)]
#[derive(Debug, Derivative)]
#[derivative(PartialEq)]
pub struct BlockImportData<E: EthSpec> {
pub block_root: Hash256,
pub state: BeaconState<E>,
@@ -363,6 +347,12 @@ pub struct BlockImportData<E: EthSpec> {
pub parent_eth1_finalization_data: Eth1FinalizationData,
pub confirmed_state_roots: Vec<Hash256>,
pub consensus_context: ConsensusContext<E>,
#[derivative(PartialEq = "ignore")]
/// An optional receiver for `DataColumnSidecarList`.
///
/// This field is `Some` when data columns are being computed asynchronously.
/// The resulting `DataColumnSidecarList` will be sent through this receiver.
pub data_column_recv: Option<oneshot::Receiver<DataColumnSidecarList<E>>>,
}
impl<E: EthSpec> BlockImportData<E> {
@@ -381,6 +371,7 @@ impl<E: EthSpec> BlockImportData<E> {
},
confirmed_state_roots: vec![],
consensus_context: ConsensusContext::new(Slot::new(0)),
data_column_recv: None,
}
}
}

28
beacon_node/beacon_chain/src/builder.rs
View File

@@ -9,6 +9,7 @@ use crate::fork_choice_signal::ForkChoiceSignalTx;
use crate::fork_revert::{reset_fork_choice_to_finalization, revert_to_fork_boundary};
use crate::graffiti_calculator::{GraffitiCalculator, GraffitiOrigin};
use crate::head_tracker::HeadTracker;
use crate::kzg_utils::blobs_to_data_column_sidecars;
use crate::light_client_server_cache::LightClientServerCache;
use crate::migrate::{BackgroundMigrator, MigratorConfig};
use crate::observed_data_sidecars::ObservedDataSidecars;
@@ -562,9 +563,30 @@ where
.put_block(&weak_subj_block_root, weak_subj_block.clone())
.map_err(|e| format!("Failed to store weak subjectivity block: {e:?}"))?;
if let Some(blobs) = weak_subj_blobs {
store
.put_blobs(&weak_subj_block_root, blobs)
.map_err(|e| format!("Failed to store weak subjectivity blobs: {e:?}"))?;
if self
.spec
.is_peer_das_enabled_for_epoch(weak_subj_block.epoch())
{
// After PeerDAS recompute columns from blobs to not force the checkpointz server
// into exposing another route.
let blobs = blobs
.iter()
.map(|blob_sidecar| &blob_sidecar.blob)
.collect::<Vec<_>>();
let data_columns =
blobs_to_data_column_sidecars(&blobs, &weak_subj_block, &self.kzg, &self.spec)
.map_err(|e| {
format!("Failed to compute weak subjectivity data_columns: {e:?}")
})?;
// TODO(das): only persist the columns under custody
store
.put_data_columns(&weak_subj_block_root, data_columns)
.map_err(|e| format!("Failed to store weak subjectivity data_column: {e:?}"))?;
} else {
store
.put_blobs(&weak_subj_block_root, blobs)
.map_err(|e| format!("Failed to store weak subjectivity blobs: {e:?}"))?;
}
}
// Stage the database's metadata fields for atomic storage when `build` is called.

6
beacon_node/beacon_chain/src/canonical_head.rs
View File

@@ -1259,11 +1259,7 @@ pub fn find_reorg_slot<E: EthSpec>(
($state: ident, $block_root: ident) => {
std::iter::once(Ok(($state.slot(), $block_root)))
.chain($state.rev_iter_block_roots(spec))
.skip_while(|result| {
result
.as_ref()
.map_or(false, |(slot, _)| *slot > lowest_slot)
})
.skip_while(|result| result.as_ref().is_ok_and(|(slot, _)| *slot > lowest_slot))
};
}

55
beacon_node/beacon_chain/src/data_availability_checker.rs
View File

@@ -15,6 +15,7 @@ use std::num::NonZeroUsize;
use std::sync::Arc;
use std::time::Duration;
use task_executor::TaskExecutor;
use tokio::sync::oneshot;
use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList};
use types::{
BlobSidecarList, ChainSpec, DataColumnIdentifier, DataColumnSidecar, DataColumnSidecarList,
@@ -116,21 +117,16 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
spec: Arc<ChainSpec>,
log: Logger,
) -> Result<Self, AvailabilityCheckError> {
let custody_subnet_count = if import_all_data_columns {
spec.data_column_sidecar_subnet_count as usize
} else {
spec.custody_requirement as usize
};
let subnet_sampling_size =
std::cmp::max(custody_subnet_count, spec.samples_per_slot as usize);
let sampling_column_count =
subnet_sampling_size.saturating_mul(spec.data_columns_per_subnet());
let custody_group_count = spec.custody_group_count(import_all_data_columns);
// This should only panic if the chain spec contains invalid values.
let sampling_size = spec
.sampling_size(custody_group_count)
.expect("should compute node sampling size from valid chain spec");
let inner = DataAvailabilityCheckerInner::new(
OVERFLOW_LRU_CAPACITY,
store,
sampling_column_count,
sampling_size as usize,
spec.clone(),
)?;
Ok(Self {
@@ -147,7 +143,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
}
pub(crate) fn is_supernode(&self) -> bool {
self.get_sampling_column_count() == self.spec.number_of_columns
self.get_sampling_column_count() == self.spec.number_of_columns as usize
}
/// Checks if the block root is currenlty in the availability cache awaiting import because
@@ -215,12 +211,15 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
// Note: currently not reporting which specific blob is invalid because we fetch all blobs
// from the same peer for both lookup and range sync.
let verified_blobs =
KzgVerifiedBlobList::new(blobs.iter().flatten().cloned(), &self.kzg, seen_timestamp)
.map_err(AvailabilityCheckError::InvalidBlobs)?;
let verified_blobs = KzgVerifiedBlobList::new(
blobs.into_vec().into_iter().flatten(),
&self.kzg,
seen_timestamp,
)
.map_err(AvailabilityCheckError::InvalidBlobs)?;
self.availability_cache
.put_kzg_verified_blobs(block_root, verified_blobs, &self.log)
.put_kzg_verified_blobs(block_root, verified_blobs, None, &self.log)
}
/// Put a list of custody columns received via RPC into the availability cache. This performs KZG
@@ -260,6 +259,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
&self,
block_root: Hash256,
blobs: FixedBlobSidecarList<T::EthSpec>,
data_column_recv: Option<oneshot::Receiver<DataColumnSidecarList<T::EthSpec>>>,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
let seen_timestamp = self
.slot_clock
@@ -269,8 +269,12 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
let verified_blobs =
KzgVerifiedBlobList::from_verified(blobs.iter().flatten().cloned(), seen_timestamp);
self.availability_cache
.put_kzg_verified_blobs(block_root, verified_blobs, &self.log)
self.availability_cache.put_kzg_verified_blobs(
block_root,
verified_blobs,
data_column_recv,
&self.log,
)
}
/// Check if we've cached other blobs for this block. If it completes a set and we also
@@ -285,6 +289,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
self.availability_cache.put_kzg_verified_blobs(
gossip_blob.block_root(),
vec![gossip_blob.into_inner()],
None,
&self.log,
)
}
@@ -400,14 +405,13 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
blocks: Vec<RpcBlock<T::EthSpec>>,
) -> Result<Vec<MaybeAvailableBlock<T::EthSpec>>, AvailabilityCheckError> {
let mut results = Vec::with_capacity(blocks.len());
let all_blobs: BlobSidecarList<T::EthSpec> = blocks
let all_blobs = blocks
.iter()
.filter(|block| self.blobs_required_for_block(block.as_block()))
// this clone is cheap as it's cloning an Arc
.filter_map(|block| block.blobs().cloned())
.flatten()
.collect::<Vec<_>>()
.into();
.collect::<Vec<_>>();
// verify kzg for all blobs at once
if !all_blobs.is_empty() {
@@ -424,7 +428,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
.map(CustodyDataColumn::into_inner)
.collect::<Vec<_>>();
let all_data_columns =
RuntimeVariableList::from_vec(all_data_columns, self.spec.number_of_columns);
RuntimeVariableList::from_vec(all_data_columns, self.spec.number_of_columns as usize);
// verify kzg for all data columns at once
if !all_data_columns.is_empty() {
@@ -519,13 +523,13 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
/// Returns true if the given epoch lies within the da boundary and false otherwise.
pub fn da_check_required_for_epoch(&self, block_epoch: Epoch) -> bool {
self.data_availability_boundary()
.map_or(false, |da_epoch| block_epoch >= da_epoch)
.is_some_and(|da_epoch| block_epoch >= da_epoch)
}
/// Returns `true` if the current epoch is greater than or equal to the `Deneb` epoch.
pub fn is_deneb(&self) -> bool {
self.slot_clock.now().map_or(false, |slot| {
self.spec.deneb_fork_epoch.map_or(false, |deneb_epoch| {
self.slot_clock.now().is_some_and(|slot| {
self.spec.deneb_fork_epoch.is_some_and(|deneb_epoch| {
let now_epoch = slot.epoch(T::EthSpec::slots_per_epoch());
now_epoch >= deneb_epoch
})
@@ -801,7 +805,6 @@ impl<E: EthSpec> AvailableBlock<E> {
block,
blobs,
data_columns,
blobs_available_timestamp: _,
..
} = self;
(block_root, block, blobs, data_columns)

240
beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs
View File

@@ -10,40 +10,55 @@ use crate::BeaconChainTypes;
use lru::LruCache;
use parking_lot::RwLock;
use slog::{debug, Logger};
use ssz_types::FixedVector;
use std::num::NonZeroUsize;
use std::sync::Arc;
use tokio::sync::oneshot;
use types::blob_sidecar::BlobIdentifier;
use types::{
BlobSidecar, ChainSpec, ColumnIndex, DataColumnIdentifier, DataColumnSidecar, Epoch, EthSpec,
Hash256, SignedBeaconBlock,
BlobSidecar, ChainSpec, ColumnIndex, DataColumnIdentifier, DataColumnSidecar,
DataColumnSidecarList, Epoch, EthSpec, Hash256, RuntimeFixedVector, RuntimeVariableList,
SignedBeaconBlock,
};
/// This represents the components of a partially available block
///
/// The blobs are all gossip and kzg verified.
/// The block has completed all verifications except the availability check.
/// TODO(das): this struct can potentially be reafactored as blobs and data columns are mutually
/// exclusive and this could simplify `is_importable`.
#[derive(Clone)]
pub struct PendingComponents<E: EthSpec> {
pub block_root: Hash256,
pub verified_blobs: FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock>,
pub verified_blobs: RuntimeFixedVector<Option<KzgVerifiedBlob<E>>>,
pub verified_data_columns: Vec<KzgVerifiedCustodyDataColumn<E>>,
pub executed_block: Option<DietAvailabilityPendingExecutedBlock<E>>,
pub reconstruction_started: bool,
/// Receiver for data columns that are computed asynchronously;
///
/// If `data_column_recv` is `Some`, it means data column computation or reconstruction has been
/// started. This can happen either via engine blobs fetching or data column reconstruction
/// (triggered when >= 50% columns are received via gossip).
pub data_column_recv: Option<oneshot::Receiver<DataColumnSidecarList<E>>>,
}
impl<E: EthSpec> PendingComponents<E> {
/// Clones the `PendingComponent` without cloning `data_column_recv`, as `Receiver` is not cloneable.
/// This should only be used when the receiver is no longer needed.
pub fn clone_without_column_recv(&self) -> Self {
PendingComponents {
block_root: self.block_root,
verified_blobs: self.verified_blobs.clone(),
verified_data_columns: self.verified_data_columns.clone(),
executed_block: self.executed_block.clone(),
reconstruction_started: self.reconstruction_started,
data_column_recv: None,
}
}
/// Returns an immutable reference to the cached block.
pub fn get_cached_block(&self) -> &Option<DietAvailabilityPendingExecutedBlock<E>> {
&self.executed_block
}
/// Returns an immutable reference to the fixed vector of cached blobs.
pub fn get_cached_blobs(
&self,
) -> &FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock> {
pub fn get_cached_blobs(&self) -> &RuntimeFixedVector<Option<KzgVerifiedBlob<E>>> {
&self.verified_blobs
}
@@ -64,9 +79,7 @@ impl<E: EthSpec> PendingComponents<E> {
}
/// Returns a mutable reference to the fixed vector of cached blobs.
pub fn get_cached_blobs_mut(
&mut self,
) -> &mut FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock> {
pub fn get_cached_blobs_mut(&mut self) -> &mut RuntimeFixedVector<Option<KzgVerifiedBlob<E>>> {
&mut self.verified_blobs
}
@@ -138,10 +151,7 @@ impl<E: EthSpec> PendingComponents<E> {
/// Blobs are only inserted if:
/// 1. The blob entry at the index is empty and no block exists.
/// 2. The block exists and its commitment matches the blob's commitment.
pub fn merge_blobs(
&mut self,
blobs: FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock>,
) {
pub fn merge_blobs(&mut self, blobs: RuntimeFixedVector<Option<KzgVerifiedBlob<E>>>) {
for (index, blob) in blobs.iter().cloned().enumerate() {
let Some(blob) = blob else { continue };
self.merge_single_blob(index, blob);
@@ -185,7 +195,7 @@ impl<E: EthSpec> PendingComponents<E> {
/// Blobs that don't match the new block's commitments are evicted.
pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock<E>) {
self.insert_block(block);
let reinsert = std::mem::take(self.get_cached_blobs_mut());
let reinsert = self.get_cached_blobs_mut().take();
self.merge_blobs(reinsert);
}
@@ -228,25 +238,23 @@ impl<E: EthSpec> PendingComponents<E> {
);
let all_blobs_received = block_kzg_commitments_count_opt
.map_or(false, |num_expected_blobs| {
num_expected_blobs == num_received_blobs
});
.is_some_and(|num_expected_blobs| num_expected_blobs == num_received_blobs);
let all_columns_received = expected_columns_opt.map_or(false, |num_expected_columns| {
num_expected_columns == num_received_columns
});
let all_columns_received = expected_columns_opt
.is_some_and(|num_expected_columns| num_expected_columns == num_received_columns);
all_blobs_received || all_columns_received
}
/// Returns an empty `PendingComponents` object with the given block root.
pub fn empty(block_root: Hash256) -> Self {
pub fn empty(block_root: Hash256, max_len: usize) -> Self {
Self {
block_root,
verified_blobs: FixedVector::default(),
verified_blobs: RuntimeFixedVector::new(vec![None; max_len]),
verified_data_columns: vec![],
executed_block: None,
reconstruction_started: false,
data_column_recv: None,
}
}
@@ -271,6 +279,7 @@ impl<E: EthSpec> PendingComponents<E> {
verified_blobs,
verified_data_columns,
executed_block,
data_column_recv,
..
} = self;
@@ -302,17 +311,22 @@ impl<E: EthSpec> PendingComponents<E> {
else {
return Err(AvailabilityCheckError::Unexpected);
};
(Some(verified_blobs), None)
let max_len = spec.max_blobs_per_block(diet_executed_block.as_block().epoch()) as usize;
(
Some(RuntimeVariableList::new(verified_blobs, max_len)?),
None,
)
};
let executed_block = recover(diet_executed_block)?;
let AvailabilityPendingExecutedBlock {
block,
import_data,
mut import_data,
payload_verification_outcome,
} = executed_block;
import_data.data_column_recv = data_column_recv;
let available_block = AvailableBlock {
block_root,
block,
@@ -344,10 +358,7 @@ impl<E: EthSpec> PendingComponents<E> {
}
if let Some(kzg_verified_data_column) = self.verified_data_columns.first() {
let epoch = kzg_verified_data_column
.as_data_column()
.slot()
.epoch(E::slots_per_epoch());
let epoch = kzg_verified_data_column.as_data_column().epoch();
return Some(epoch);
}
@@ -454,13 +465,31 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
f(self.critical.read().peek(block_root))
}
/// Puts the KZG verified blobs into the availability cache as pending components.
///
/// The `data_column_recv` parameter is an optional `Receiver` for data columns that are
/// computed asynchronously. This method remains **used** after PeerDAS activation, because
/// blocks can be made available if the EL already has the blobs and returns them via the
/// `getBlobsV1` engine method. More details in [fetch_blobs.rs](https://github.com/sigp/lighthouse/blob/44f8add41ea2252769bb967864af95b3c13af8ca/beacon_node/beacon_chain/src/fetch_blobs.rs).
pub fn put_kzg_verified_blobs<I: IntoIterator<Item = KzgVerifiedBlob<T::EthSpec>>>(
&self,
block_root: Hash256,
kzg_verified_blobs: I,
data_column_recv: Option<oneshot::Receiver<DataColumnSidecarList<T::EthSpec>>>,
log: &Logger,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
let mut fixed_blobs = FixedVector::default();
let mut kzg_verified_blobs = kzg_verified_blobs.into_iter().peekable();
let Some(epoch) = kzg_verified_blobs
.peek()
.map(|verified_blob| verified_blob.as_blob().epoch())
else {
// Verified blobs list should be non-empty.
return Err(AvailabilityCheckError::Unexpected);
};
let mut fixed_blobs =
RuntimeFixedVector::new(vec![None; self.spec.max_blobs_per_block(epoch) as usize]);
for blob in kzg_verified_blobs {
if let Some(blob_opt) = fixed_blobs.get_mut(blob.blob_index() as usize) {
@@ -474,14 +503,24 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
let mut pending_components = write_lock
.pop_entry(&block_root)
.map(|(_, v)| v)
.unwrap_or_else(|| PendingComponents::empty(block_root));
.unwrap_or_else(|| {
PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize)
});
// Merge in the blobs.
pending_components.merge_blobs(fixed_blobs);
if data_column_recv.is_some() {
// If `data_column_recv` is `Some`, it means we have all the blobs from engine, and have
// started computing data columns. We store the receiver in `PendingComponents` for
// later use when importing the block.
pending_components.data_column_recv = data_column_recv;
}
if pending_components.is_available(self.sampling_column_count, log) {
write_lock.put(block_root, pending_components.clone());
// No need to hold the write lock anymore
// We keep the pending components in the availability cache during block import (#5845).
// `data_column_recv` is returned as part of the available block and is no longer needed here.
write_lock.put(block_root, pending_components.clone_without_column_recv());
drop(write_lock);
pending_components.make_available(&self.spec, |diet_block| {
self.state_cache.recover_pending_executed_block(diet_block)
@@ -501,20 +540,32 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
kzg_verified_data_columns: I,
log: &Logger,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
let mut kzg_verified_data_columns = kzg_verified_data_columns.into_iter().peekable();
let Some(epoch) = kzg_verified_data_columns
.peek()
.map(|verified_blob| verified_blob.as_data_column().epoch())
else {
// Verified data_columns list should be non-empty.
return Err(AvailabilityCheckError::Unexpected);
};
let mut write_lock = self.critical.write();
// Grab existing entry or create a new entry.
let mut pending_components = write_lock
.pop_entry(&block_root)
.map(|(_, v)| v)
.unwrap_or_else(|| PendingComponents::empty(block_root));
.unwrap_or_else(|| {
PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize)
});
// Merge in the data columns.
pending_components.merge_data_columns(kzg_verified_data_columns)?;
if pending_components.is_available(self.sampling_column_count, log) {
write_lock.put(block_root, pending_components.clone());
// No need to hold the write lock anymore
// We keep the pending components in the availability cache during block import (#5845).
// `data_column_recv` is returned as part of the available block and is no longer needed here.
write_lock.put(block_root, pending_components.clone_without_column_recv());
drop(write_lock);
pending_components.make_available(&self.spec, |diet_block| {
self.state_cache.recover_pending_executed_block(diet_block)
@@ -546,7 +597,7 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
// If we're sampling all columns, it means we must be custodying all columns.
let custody_column_count = self.sampling_column_count();
let total_column_count = self.spec.number_of_columns;
let total_column_count = self.spec.number_of_columns as usize;
let received_column_count = pending_components.verified_data_columns.len();
if pending_components.reconstruction_started {
@@ -555,7 +606,7 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
if custody_column_count != total_column_count {
return ReconstructColumnsDecision::No("not required for full node");
}
if received_column_count == self.spec.number_of_columns {
if received_column_count >= total_column_count {
return ReconstructColumnsDecision::No("all columns received");
}
if received_column_count < total_column_count / 2 {
@@ -563,7 +614,7 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
}
pending_components.reconstruction_started = true;
ReconstructColumnsDecision::Yes(pending_components.clone())
ReconstructColumnsDecision::Yes(pending_components.clone_without_column_recv())
}
/// This could mean some invalid data columns made it through to the `DataAvailabilityChecker`.
@@ -584,6 +635,7 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
log: &Logger,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
let mut write_lock = self.critical.write();
let epoch = executed_block.as_block().epoch();
let block_root = executed_block.import_data.block_root;
// register the block to get the diet block
@@ -595,15 +647,18 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
let mut pending_components = write_lock
.pop_entry(&block_root)
.map(|(_, v)| v)
.unwrap_or_else(|| PendingComponents::empty(block_root));
.unwrap_or_else(|| {
PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize)
});
// Merge in the block.
pending_components.merge_block(diet_executed_block);
// Check if we have all components and entire set is consistent.
if pending_components.is_available(self.sampling_column_count, log) {
write_lock.put(block_root, pending_components.clone());
// No need to hold the write lock anymore
// We keep the pending components in the availability cache during block import (#5845).
// `data_column_recv` is returned as part of the available block and is no longer needed here.
write_lock.put(block_root, pending_components.clone_without_column_recv());
drop(write_lock);
pending_components.make_available(&self.spec, |diet_block| {
self.state_cache.recover_pending_executed_block(diet_block)
@@ -676,7 +731,7 @@ mod test {
use slog::{info, Logger};
use state_processing::ConsensusContext;
use std::collections::VecDeque;
use store::{HotColdDB, ItemStore, LevelDB, StoreConfig};
use store::{database::interface::BeaconNodeBackend, HotColdDB, ItemStore, StoreConfig};
use tempfile::{tempdir, TempDir};
use types::non_zero_usize::new_non_zero_usize;
use types::{ExecPayload, MinimalEthSpec};
@@ -688,7 +743,7 @@ mod test {
db_path: &TempDir,
spec: Arc<ChainSpec>,
log: Logger,
) -> Arc<HotColdDB<E, LevelDB<E>, LevelDB<E>>> {
) -> Arc<HotColdDB<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>> {
let hot_path = db_path.path().join("hot_db");
let cold_path = db_path.path().join("cold_db");
let blobs_path = db_path.path().join("blobs_db");
@@ -815,7 +870,8 @@ mod test {
info!(log, "done printing kzg commitments");
let gossip_verified_blobs = if let Some((kzg_proofs, blobs)) = maybe_blobs {
let sidecars = BlobSidecar::build_sidecars(blobs, &block, kzg_proofs).unwrap();
let sidecars =
BlobSidecar::build_sidecars(blobs, &block, kzg_proofs, &chain.spec).unwrap();
Vec::from(sidecars)
.into_iter()
.map(|sidecar| {
@@ -837,6 +893,7 @@ mod test {
parent_eth1_finalization_data,
confirmed_state_roots: vec![],
consensus_context,
data_column_recv: None,
};
let payload_verification_outcome = PayloadVerificationOutcome {
@@ -862,7 +919,11 @@ mod test {
)
where
E: EthSpec,
T: BeaconChainTypes<HotStore = LevelDB<E>, ColdStore = LevelDB<E>, EthSpec = E>,
T: BeaconChainTypes<
HotStore = BeaconNodeBackend<E>,
ColdStore = BeaconNodeBackend<E>,
EthSpec = E,
>,
{
let log = test_logger();
let chain_db_path = tempdir().expect("should get temp dir");
@@ -939,7 +1000,7 @@ mod test {
for (blob_index, gossip_blob) in blobs.into_iter().enumerate() {
kzg_verified_blobs.push(gossip_blob.into_inner());
let availability = cache
.put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), harness.logger())
.put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), None, harness.logger())
.expect("should put blob");
if blob_index == blobs_expected - 1 {
assert!(matches!(availability, Availability::Available(_)));
@@ -948,6 +1009,8 @@ mod test {
assert_eq!(cache.critical.read().len(), 1);
}
}
// remove the blob to simulate successful import
cache.remove_pending_components(root);
assert!(
cache.critical.read().is_empty(),
"cache should be empty now that all components available"
@@ -965,7 +1028,7 @@ mod test {
for gossip_blob in blobs {
kzg_verified_blobs.push(gossip_blob.into_inner());
let availability = cache
.put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), harness.logger())
.put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), None, harness.logger())
.expect("should put blob");
assert_eq!(
availability,
@@ -1128,7 +1191,7 @@ mod pending_components_tests {
use super::*;
use crate::block_verification_types::BlockImportData;
use crate::eth1_finalization_cache::Eth1FinalizationData;
use crate::test_utils::{generate_rand_block_and_blobs, NumBlobs};
use crate::test_utils::{generate_rand_block_and_blobs, test_spec, NumBlobs};
use crate::PayloadVerificationOutcome;
use fork_choice::PayloadVerificationStatus;
use kzg::KzgCommitment;
@@ -1144,15 +1207,19 @@ mod pending_components_tests {
type Setup<E> = (
SignedBeaconBlock<E>,
FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>>,
RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>>,
usize,
);
pub fn pre_setup() -> Setup<E> {
let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64);
let spec = test_spec::<E>();
let (block, blobs_vec) =
generate_rand_block_and_blobs::<E>(ForkName::Deneb, NumBlobs::Random, &mut rng);
let mut blobs: FixedVector<_, <E as EthSpec>::MaxBlobsPerBlock> = FixedVector::default();
generate_rand_block_and_blobs::<E>(ForkName::Deneb, NumBlobs::Random, &mut rng, &spec);
let max_len = spec.max_blobs_per_block(block.epoch()) as usize;
let mut blobs: RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>> =
RuntimeFixedVector::default(max_len);
for blob in blobs_vec {
if let Some(b) = blobs.get_mut(blob.index as usize) {
@@ -1160,10 +1227,8 @@ mod pending_components_tests {
}
}
let mut invalid_blobs: FixedVector<
Option<Arc<BlobSidecar<E>>>,
<E as EthSpec>::MaxBlobsPerBlock,
> = FixedVector::default();
let mut invalid_blobs: RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>> =
RuntimeFixedVector::default(max_len);
for (index, blob) in blobs.iter().enumerate() {
if let Some(invalid_blob) = blob {
let mut blob_copy = invalid_blob.as_ref().clone();
@@ -1172,21 +1237,21 @@ mod pending_components_tests {
}
}
(block, blobs, invalid_blobs)
(block, blobs, invalid_blobs, max_len)
}
type PendingComponentsSetup<E> = (
DietAvailabilityPendingExecutedBlock<E>,
FixedVector<Option<KzgVerifiedBlob<E>>, <E as EthSpec>::MaxBlobsPerBlock>,
FixedVector<Option<KzgVerifiedBlob<E>>, <E as EthSpec>::MaxBlobsPerBlock>,
RuntimeFixedVector<Option<KzgVerifiedBlob<E>>>,
RuntimeFixedVector<Option<KzgVerifiedBlob<E>>>,
);
pub fn setup_pending_components(
block: SignedBeaconBlock<E>,
valid_blobs: FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
invalid_blobs: FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
valid_blobs: RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>>,
invalid_blobs: RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>>,
) -> PendingComponentsSetup<E> {
let blobs = FixedVector::from(
let blobs = RuntimeFixedVector::new(
valid_blobs
.iter()
.map(|blob_opt| {
@@ -1196,7 +1261,7 @@ mod pending_components_tests {
})
.collect::<Vec<_>>(),
);
let invalid_blobs = FixedVector::from(
let invalid_blobs = RuntimeFixedVector::new(
invalid_blobs
.iter()
.map(|blob_opt| {
@@ -1219,6 +1284,7 @@ mod pending_components_tests {
},
confirmed_state_roots: vec![],
consensus_context: ConsensusContext::new(Slot::new(0)),
data_column_recv: None,
},
payload_verification_outcome: PayloadVerificationOutcome {
payload_verification_status: PayloadVerificationStatus::Verified,
@@ -1228,10 +1294,10 @@ mod pending_components_tests {
(block.into(), blobs, invalid_blobs)
}
pub fn assert_cache_consistent(cache: PendingComponents<E>) {
pub fn assert_cache_consistent(cache: PendingComponents<E>, max_len: usize) {
if let Some(cached_block) = cache.get_cached_block() {
let cached_block_commitments = cached_block.get_commitments();
for index in 0..E::max_blobs_per_block() {
for index in 0..max_len {
let block_commitment = cached_block_commitments.get(index).copied();
let blob_commitment_opt = cache.get_cached_blobs().get(index).unwrap();
let blob_commitment = blob_commitment_opt.as_ref().map(|b| *b.get_commitment());
@@ -1250,40 +1316,40 @@ mod pending_components_tests {
#[test]
fn valid_block_invalid_blobs_valid_blobs() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_block(block_commitments);
cache.merge_blobs(random_blobs);
cache.merge_blobs(blobs);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
#[test]
fn invalid_blobs_block_valid_blobs() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_blobs(random_blobs);
cache.merge_block(block_commitments);
cache.merge_blobs(blobs);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
#[test]
fn invalid_blobs_valid_blobs_block() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_blobs(random_blobs);
cache.merge_blobs(blobs);
cache.merge_block(block_commitments);
@@ -1293,46 +1359,46 @@ mod pending_components_tests {
#[test]
fn block_valid_blobs_invalid_blobs() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_block(block_commitments);
cache.merge_blobs(blobs);
cache.merge_blobs(random_blobs);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
#[test]
fn valid_blobs_block_invalid_blobs() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_blobs(blobs);
cache.merge_block(block_commitments);
cache.merge_blobs(random_blobs);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
#[test]
fn valid_blobs_invalid_blobs_block() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_blobs(blobs);
cache.merge_blobs(random_blobs);
cache.merge_block(block_commitments);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
}

1
beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs
View File

@@ -136,6 +136,7 @@ impl<T: BeaconChainTypes> StateLRUCache<T> {
consensus_context: diet_executed_block
.consensus_context
.into_consensus_context(),
data_column_recv: None,
},
payload_verification_outcome: diet_executed_block.payload_verification_outcome,
})

6
beacon_node/beacon_chain/src/data_column_verification.rs
View File

@@ -423,7 +423,7 @@ fn verify_data_column_sidecar<E: EthSpec>(
data_column: &DataColumnSidecar<E>,
spec: &ChainSpec,
) -> Result<(), GossipDataColumnError> {
if data_column.index >= spec.number_of_columns as u64 {
if data_column.index >= spec.number_of_columns {
return Err(GossipDataColumnError::InvalidColumnIndex(data_column.index));
}
if data_column.kzg_commitments.is_empty() {
@@ -611,7 +611,7 @@ fn verify_index_matches_subnet<E: EthSpec>(
spec: &ChainSpec,
) -> Result<(), GossipDataColumnError> {
let expected_subnet: u64 =
DataColumnSubnetId::from_column_index::<E>(data_column.index as usize, spec).into();
DataColumnSubnetId::from_column_index(data_column.index, spec).into();
if expected_subnet != subnet {
return Err(GossipDataColumnError::InvalidSubnetId {
received: subnet,
@@ -699,7 +699,7 @@ mod test {
#[tokio::test]
async fn empty_data_column_sidecars_fails_validation() {
let spec = ForkName::latest().make_genesis_spec(E::default_spec());
let spec = ForkName::Fulu.make_genesis_spec(E::default_spec());
let harness = BeaconChainHarness::builder(E::default())
.spec(spec.into())
.deterministic_keypairs(64)

2
beacon_node/beacon_chain/src/early_attester_cache.rs
View File

@@ -145,7 +145,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
self.item
.read()
.as_ref()
.map_or(false, |item| item.beacon_block_root == block_root)
.is_some_and(|item| item.beacon_block_root == block_root)
}
/// Returns the block, if `block_root` matches the cached item.

4
beacon_node/beacon_chain/src/errors.rs
View File

@@ -228,6 +228,10 @@ pub enum BeaconChainError {
EmptyRpcCustodyColumns,
AttestationError(AttestationError),
AttestationCommitteeIndexNotSet,
InsufficientColumnsToReconstructBlobs {
columns_found: usize,
},
FailedToReconstructBlobs(String),
}
easy_from_to!(SlotProcessingError, BeaconChainError);

2
beacon_node/beacon_chain/src/eth1_chain.rs
View File

@@ -153,7 +153,7 @@ fn get_sync_status<E: EthSpec>(
// Lighthouse is "cached and ready" when it has cached enough blocks to cover the start of the
// current voting period.
let lighthouse_is_cached_and_ready =
latest_cached_block_timestamp.map_or(false, |t| t >= voting_target_timestamp);
latest_cached_block_timestamp.is_some_and(|t| t >= voting_target_timestamp);
Some(Eth1SyncStatusData {
head_block_number,

15
beacon_node/beacon_chain/src/events.rs
View File

@@ -8,6 +8,7 @@ const DEFAULT_CHANNEL_CAPACITY: usize = 16;
pub struct ServerSentEventHandler<E: EthSpec> {
attestation_tx: Sender<EventKind<E>>,
single_attestation_tx: Sender<EventKind<E>>,
block_tx: Sender<EventKind<E>>,
blob_sidecar_tx: Sender<EventKind<E>>,
finalized_tx: Sender<EventKind<E>>,
@@ -37,6 +38,7 @@ impl<E: EthSpec> ServerSentEventHandler<E> {
pub fn new_with_capacity(log: Logger, capacity: usize) -> Self {
let (attestation_tx, _) = broadcast::channel(capacity);
let (single_attestation_tx, _) = broadcast::channel(capacity);
let (block_tx, _) = broadcast::channel(capacity);
let (blob_sidecar_tx, _) = broadcast::channel(capacity);
let (finalized_tx, _) = broadcast::channel(capacity);
@@ -56,6 +58,7 @@ impl<E: EthSpec> ServerSentEventHandler<E> {
Self {
attestation_tx,
single_attestation_tx,
block_tx,
blob_sidecar_tx,
finalized_tx,
@@ -90,6 +93,10 @@ impl<E: EthSpec> ServerSentEventHandler<E> {
.attestation_tx
.send(kind)
.map(|count| log_count("attestation", count)),
EventKind::SingleAttestation(_) => self
.single_attestation_tx
.send(kind)
.map(|count| log_count("single_attestation", count)),
EventKind::Block(_) => self
.block_tx
.send(kind)
@@ -164,6 +171,10 @@ impl<E: EthSpec> ServerSentEventHandler<E> {
self.attestation_tx.subscribe()
}
pub fn subscribe_single_attestation(&self) -> Receiver<EventKind<E>> {
self.single_attestation_tx.subscribe()
}
pub fn subscribe_block(&self) -> Receiver<EventKind<E>> {
self.block_tx.subscribe()
}
@@ -232,6 +243,10 @@ impl<E: EthSpec> ServerSentEventHandler<E> {
self.attestation_tx.receiver_count() > 0
}
pub fn has_single_attestation_subscribers(&self) -> bool {
self.single_attestation_tx.receiver_count() > 0
}
pub fn has_block_subscribers(&self) -> bool {
self.block_tx.receiver_count() > 0
}

28
beacon_node/beacon_chain/src/execution_payload.rs
View File

@@ -140,9 +140,9 @@ impl<T: BeaconChainTypes> PayloadNotifier<T> {
/// contains a few extra checks by running `partially_verify_execution_payload` first:
///
/// https://github.com/ethereum/consensus-specs/blob/v1.1.9/specs/bellatrix/beacon-chain.md#notify_new_payload
async fn notify_new_payload<'a, T: BeaconChainTypes>(
async fn notify_new_payload<T: BeaconChainTypes>(
chain: &Arc<BeaconChain<T>>,
block: BeaconBlockRef<'a, T::EthSpec>,
block: BeaconBlockRef<'_, T::EthSpec>,
il_transactions: InclusionListTransactions<T::EthSpec>,
) -> Result<PayloadVerificationStatus, BlockError> {
let execution_layer = chain
@@ -258,9 +258,9 @@ async fn notify_new_payload<'a, T: BeaconChainTypes>(
/// Equivalent to the `validate_merge_block` function in the merge Fork Choice Changes:
///
/// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/fork-choice.md#validate_merge_block
pub async fn validate_merge_block<'a, T: BeaconChainTypes>(
pub async fn validate_merge_block<T: BeaconChainTypes>(
chain: &Arc<BeaconChain<T>>,
block: BeaconBlockRef<'a, T::EthSpec>,
block: BeaconBlockRef<'_, T::EthSpec>,
allow_optimistic_import: AllowOptimisticImport,
) -> Result<(), BlockError> {
let spec = &chain.spec;
@@ -402,19 +402,15 @@ pub fn get_execution_payload<T: BeaconChainTypes>(
let latest_execution_payload_header = state.latest_execution_payload_header()?;
let latest_execution_payload_header_block_hash = latest_execution_payload_header.block_hash();
let latest_execution_payload_header_gas_limit = latest_execution_payload_header.gas_limit();
let withdrawals = match state {
&BeaconState::Capella(_) | &BeaconState::Deneb(_) | &BeaconState::Electra(_) => {
Some(get_expected_withdrawals(state, spec)?.0.into())
}
&BeaconState::Bellatrix(_) => None,
// These shouldn't happen but they're here to make the pattern irrefutable
&BeaconState::Base(_) | &BeaconState::Altair(_) => None,
let withdrawals = if state.fork_name_unchecked().capella_enabled() {
Some(get_expected_withdrawals(state, spec)?.0.into())
} else {
None
};
let parent_beacon_block_root = match state {
BeaconState::Deneb(_) | BeaconState::Electra(_) => Some(parent_block_root),
BeaconState::Bellatrix(_) | BeaconState::Capella(_) => None,
// These shouldn't happen but they're here to make the pattern irrefutable
BeaconState::Base(_) | BeaconState::Altair(_) => None,
let parent_beacon_block_root = if state.fork_name_unchecked().deneb_enabled() {
Some(parent_block_root)
} else {
None
};
// Spawn a task to obtain the execution payload from the EL via a series of async calls. The

47
beacon_node/beacon_chain/src/fetch_blobs.rs
View File

@@ -18,11 +18,11 @@ use slog::{debug, error, o, Logger};
use ssz_types::FixedVector;
use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash;
use std::sync::Arc;
use tokio::sync::mpsc::Receiver;
use tokio::sync::oneshot;
use types::blob_sidecar::{BlobSidecarError, FixedBlobSidecarList};
use types::{
BeaconStateError, BlobSidecar, DataColumnSidecar, DataColumnSidecarList, EthSpec, FullPayload,
Hash256, SignedBeaconBlock, SignedBeaconBlockHeader,
BeaconStateError, BlobSidecar, ChainSpec, DataColumnSidecar, DataColumnSidecarList, EthSpec,
FullPayload, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader,
};
pub enum BlobsOrDataColumns<T: BeaconChainTypes> {
@@ -112,6 +112,7 @@ pub async fn fetch_and_process_engine_blobs<T: BeaconChainTypes>(
response,
signed_block_header,
&kzg_commitments_proof,
&chain.spec,
)?;
let num_fetched_blobs = fixed_blob_sidecar_list
@@ -162,6 +163,20 @@ pub async fn fetch_and_process_engine_blobs<T: BeaconChainTypes>(
return Ok(None);
}
if chain
.canonical_head
.fork_choice_read_lock()
.contains_block(&block_root)
{
// Avoid computing columns if block has already been imported.
debug!(
log,
"Ignoring EL blobs response";
"info" => "block has already been imported",
);
return Ok(None);
}
let data_columns_receiver = spawn_compute_and_publish_data_columns_task(
&chain,
block.clone(),
@@ -212,9 +227,9 @@ fn spawn_compute_and_publish_data_columns_task<T: BeaconChainTypes>(
blobs: FixedBlobSidecarList<T::EthSpec>,
publish_fn: impl Fn(BlobsOrDataColumns<T>) + Send + 'static,
log: Logger,
) -> Receiver<Vec<Arc<DataColumnSidecar<T::EthSpec>>>> {
) -> oneshot::Receiver<Vec<Arc<DataColumnSidecar<T::EthSpec>>>> {
let chain_cloned = chain.clone();
let (data_columns_sender, data_columns_receiver) = tokio::sync::mpsc::channel(1);
let (data_columns_sender, data_columns_receiver) = oneshot::channel();
chain.task_executor.spawn_blocking(
move || {
@@ -247,18 +262,21 @@ fn spawn_compute_and_publish_data_columns_task<T: BeaconChainTypes>(
}
};
if let Err(e) = data_columns_sender.try_send(all_data_columns.clone()) {
error!(log, "Failed to send computed data columns"; "error" => ?e);
if data_columns_sender.send(all_data_columns.clone()).is_err() {
// Data column receiver have been dropped - block may have already been imported.
// This race condition exists because gossip columns may arrive and trigger block
// import during the computation. Here we just drop the computed columns.
debug!(
log,
"Failed to send computed data columns";
);
return;
};
// Check indices from cache before sending the columns, to make sure we don't
// publish components already seen on gossip.
let is_supernode = chain_cloned.data_availability_checker.is_supernode();
// At the moment non supernodes are not required to publish any columns.
// TODO(das): we could experiment with having full nodes publish their custodied
// columns here.
if !is_supernode {
if !chain_cloned.data_availability_checker.is_supernode() {
return;
}
@@ -275,8 +293,11 @@ fn build_blob_sidecars<E: EthSpec>(
response: Vec<Option<BlobAndProofV1<E>>>,
signed_block_header: SignedBeaconBlockHeader,
kzg_commitments_inclusion_proof: &FixedVector<Hash256, E::KzgCommitmentsInclusionProofDepth>,
spec: &ChainSpec,
) -> Result<FixedBlobSidecarList<E>, FetchEngineBlobError> {
let mut fixed_blob_sidecar_list = FixedBlobSidecarList::default();
let epoch = block.epoch();
let mut fixed_blob_sidecar_list =
FixedBlobSidecarList::default(spec.max_blobs_per_block(epoch) as usize);
for (index, blob_and_proof) in response
.into_iter()
.enumerate()

115
beacon_node/beacon_chain/src/fulu_readiness.rs Normal file
View File

@@ -0,0 +1,115 @@
//! Provides tools for checking if a node is ready for the Fulu upgrade.
use crate::{BeaconChain, BeaconChainTypes};
use execution_layer::http::{ENGINE_GET_PAYLOAD_V4, ENGINE_NEW_PAYLOAD_V4};
use serde::{Deserialize, Serialize};
use std::fmt;
use std::time::Duration;
use types::*;
/// The time before the Fulu fork when we will start issuing warnings about preparation.
use super::bellatrix_readiness::SECONDS_IN_A_WEEK;
pub const FULU_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2;
pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300;
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[serde(tag = "type")]
pub enum FuluReadiness {
/// The execution engine is fulu-enabled (as far as we can tell)
Ready,
/// We are connected to an execution engine which doesn't support the V5 engine api methods
V5MethodsNotSupported { error: String },
/// The transition configuration with the EL failed, there might be a problem with
/// connectivity, authentication or a difference in configuration.
ExchangeCapabilitiesFailed { error: String },
/// The user has not configured an execution endpoint
NoExecutionEndpoint,
}
impl fmt::Display for FuluReadiness {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
FuluReadiness::Ready => {
write!(f, "This node appears ready for Fulu.")
}
FuluReadiness::ExchangeCapabilitiesFailed { error } => write!(
f,
"Could not exchange capabilities with the \
execution endpoint: {}",
error
),
FuluReadiness::NoExecutionEndpoint => write!(
f,
"The --execution-endpoint flag is not specified, this is a \
requirement post-merge"
),
FuluReadiness::V5MethodsNotSupported { error } => write!(
f,
"Execution endpoint does not support Fulu methods: {}",
error
),
}
}
}
impl<T: BeaconChainTypes> BeaconChain<T> {
/// Returns `true` if fulu epoch is set and Fulu fork has occurred or will
/// occur within `FULU_READINESS_PREPARATION_SECONDS`
pub fn is_time_to_prepare_for_fulu(&self, current_slot: Slot) -> bool {
if let Some(fulu_epoch) = self.spec.fulu_fork_epoch {
let fulu_slot = fulu_epoch.start_slot(T::EthSpec::slots_per_epoch());
let fulu_readiness_preparation_slots =
FULU_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot;
// Return `true` if Fulu has happened or is within the preparation time.
current_slot + fulu_readiness_preparation_slots > fulu_slot
} else {
// The Fulu fork epoch has not been defined yet, no need to prepare.
false
}
}
/// Attempts to connect to the EL and confirm that it is ready for fulu.
pub async fn check_fulu_readiness(&self) -> FuluReadiness {
if let Some(el) = self.execution_layer.as_ref() {
match el
.get_engine_capabilities(Some(Duration::from_secs(
ENGINE_CAPABILITIES_REFRESH_INTERVAL,
)))
.await
{
Err(e) => {
// The EL was either unreachable or responded with an error
FuluReadiness::ExchangeCapabilitiesFailed {
error: format!("{:?}", e),
}
}
Ok(capabilities) => {
let mut missing_methods = String::from("Required Methods Unsupported:");
let mut all_good = true;
// TODO(fulu) switch to v5 when the EL is ready
if !capabilities.get_payload_v4 {
missing_methods.push(' ');
missing_methods.push_str(ENGINE_GET_PAYLOAD_V4);
all_good = false;
}
if !capabilities.new_payload_v4 {
missing_methods.push(' ');
missing_methods.push_str(ENGINE_NEW_PAYLOAD_V4);
all_good = false;
}
if all_good {
FuluReadiness::Ready
} else {
FuluReadiness::V5MethodsNotSupported {
error: missing_methods,
}
}
}
}
} else {
FuluReadiness::NoExecutionEndpoint
}
}
}

7
beacon_node/beacon_chain/src/graffiti_calculator.rs
View File

@@ -293,10 +293,7 @@ mod tests {
.await
.unwrap();
let version_bytes = std::cmp::min(
lighthouse_version::VERSION.as_bytes().len(),
GRAFFITI_BYTES_LEN,
);
let version_bytes = std::cmp::min(lighthouse_version::VERSION.len(), GRAFFITI_BYTES_LEN);
// grab the slice of the graffiti that corresponds to the lighthouse version
let graffiti_slice =
&harness.chain.graffiti_calculator.get_graffiti(None).await.0[..version_bytes];
@@ -361,7 +358,7 @@ mod tests {
let graffiti_str = "nice graffiti bro";
let mut graffiti_bytes = [0u8; GRAFFITI_BYTES_LEN];
graffiti_bytes[..graffiti_str.as_bytes().len()].copy_from_slice(graffiti_str.as_bytes());
graffiti_bytes[..graffiti_str.len()].copy_from_slice(graffiti_str.as_bytes());
let found_graffiti = harness
.chain

11
beacon_node/beacon_chain/src/historical_blocks.rs
View File

@@ -10,10 +10,7 @@ use std::borrow::Cow;
use std::iter;
use std::time::Duration;
use store::metadata::DataColumnInfo;
use store::{
get_key_for_col, AnchorInfo, BlobInfo, DBColumn, Error as StoreError, KeyValueStore,
KeyValueStoreOp,
};
use store::{AnchorInfo, BlobInfo, DBColumn, Error as StoreError, KeyValueStore, KeyValueStoreOp};
use strum::IntoStaticStr;
use types::{FixedBytesExtended, Hash256, Slot};
@@ -153,7 +150,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// Store block roots, including at all skip slots in the freezer DB.
for slot in (block.slot().as_u64()..prev_block_slot.as_u64()).rev() {
cold_batch.push(KeyValueStoreOp::PutKeyValue(
get_key_for_col(DBColumn::BeaconBlockRoots.into(), &slot.to_be_bytes()),
DBColumn::BeaconBlockRoots,
slot.to_be_bytes().to_vec(),
block_root.as_slice().to_vec(),
));
}
@@ -169,7 +167,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let genesis_slot = self.spec.genesis_slot;
for slot in genesis_slot.as_u64()..prev_block_slot.as_u64() {
cold_batch.push(KeyValueStoreOp::PutKeyValue(
get_key_for_col(DBColumn::BeaconBlockRoots.into(), &slot.to_be_bytes()),
DBColumn::BeaconBlockRoots,
slot.to_be_bytes().to_vec(),
self.genesis_block_root.as_slice().to_vec(),
));
}

162
beacon_node/beacon_chain/src/kzg_utils.rs
View File

@@ -7,8 +7,9 @@ use std::sync::Arc;
use types::beacon_block_body::KzgCommitments;
use types::data_column_sidecar::{Cell, DataColumn, DataColumnSidecarError};
use types::{
Blob, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256,
KzgCommitment, KzgProof, KzgProofs, SignedBeaconBlock, SignedBeaconBlockHeader,
Blob, BlobSidecar, BlobSidecarList, ChainSpec, ColumnIndex, DataColumnSidecar,
DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, KzgProofs, SignedBeaconBlock,
SignedBeaconBlockHeader, SignedBlindedBeaconBlock,
};
/// Converts a blob ssz List object to an array to be used with the kzg
@@ -185,17 +186,19 @@ pub fn blobs_to_data_column_sidecars<E: EthSpec>(
.map_err(DataColumnSidecarError::BuildSidecarFailed)
}
fn build_data_column_sidecars<E: EthSpec>(
pub(crate) fn build_data_column_sidecars<E: EthSpec>(
kzg_commitments: KzgCommitments<E>,
kzg_commitments_inclusion_proof: FixedVector<Hash256, E::KzgCommitmentsInclusionProofDepth>,
signed_block_header: SignedBeaconBlockHeader,
blob_cells_and_proofs_vec: Vec<CellsAndKzgProofs>,
spec: &ChainSpec,
) -> Result<DataColumnSidecarList<E>, String> {
let number_of_columns = spec.number_of_columns;
let mut columns = vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns];
let mut column_kzg_proofs =
vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns];
let number_of_columns = spec.number_of_columns as usize;
let max_blobs_per_block = spec
.max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch()))
as usize;
let mut columns = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns];
let mut column_kzg_proofs = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns];
for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec {
// we iterate over each column, and we construct the column from "top to bottom",
@@ -243,6 +246,85 @@ fn build_data_column_sidecars<E: EthSpec>(
Ok(sidecars)
}
/// Reconstruct blobs from a subset of data column sidecars (requires at least 50%).
///
/// If `blob_indices_opt` is `None`, this function attempts to reconstruct all blobs associated
/// with the block.
pub fn reconstruct_blobs<E: EthSpec>(
kzg: &Kzg,
data_columns: &[Arc<DataColumnSidecar<E>>],
blob_indices_opt: Option<Vec<u64>>,
signed_block: &SignedBlindedBeaconBlock<E>,
spec: &ChainSpec,
) -> Result<BlobSidecarList<E>, String> {
// The data columns are from the database, so we assume their correctness.
let first_data_column = data_columns
.first()
.ok_or("data_columns should have at least one element".to_string())?;
let blob_indices: Vec<usize> = match blob_indices_opt {
Some(indices) => indices.into_iter().map(|i| i as usize).collect(),
None => {
let num_of_blobs = first_data_column.kzg_commitments.len();
(0..num_of_blobs).collect()
}
};
let blob_sidecars = blob_indices
.into_par_iter()
.map(|row_index| {
let mut cells: Vec<KzgCellRef> = vec![];
let mut cell_ids: Vec<u64> = vec![];
for data_column in data_columns {
let cell = data_column
.column
.get(row_index)
.ok_or(format!("Missing data column at row index {row_index}"))
.and_then(|cell| {
ssz_cell_to_crypto_cell::<E>(cell).map_err(|e| format!("{e:?}"))
})?;
cells.push(cell);
cell_ids.push(data_column.index);
}
let (cells, _kzg_proofs) = kzg
.recover_cells_and_compute_kzg_proofs(&cell_ids, &cells)
.map_err(|e| format!("Failed to recover cells and compute KZG proofs: {e:?}"))?;
let num_cells_original_blob = cells.len() / 2;
let blob_bytes = cells
.into_iter()
.take(num_cells_original_blob)
.flat_map(|cell| cell.into_iter())
.collect();
let blob = Blob::<E>::new(blob_bytes).map_err(|e| format!("{e:?}"))?;
let kzg_commitment = first_data_column
.kzg_commitments
.get(row_index)
.ok_or(format!("Missing KZG commitment for blob {row_index}"))?;
let kzg_proof = compute_blob_kzg_proof::<E>(kzg, &blob, *kzg_commitment)
.map_err(|e| format!("{e:?}"))?;
BlobSidecar::<E>::new_with_existing_proof(
row_index,
blob,
signed_block,
first_data_column.signed_block_header.clone(),
&first_data_column.kzg_commitments_inclusion_proof,
kzg_proof,
)
.map(Arc::new)
.map_err(|e| format!("{e:?}"))
})
.collect::<Result<Vec<_>, _>>()?;
let max_blobs = spec.max_blobs_per_block(signed_block.epoch()) as usize;
BlobSidecarList::new(blob_sidecars, max_blobs).map_err(|e| format!("{e:?}"))
}
/// Reconstruct all data columns from a subset of data column sidecars (requires at least 50%).
pub fn reconstruct_data_columns<E: EthSpec>(
kzg: &Kzg,
@@ -265,7 +347,7 @@ pub fn reconstruct_data_columns<E: EthSpec>(
for data_column in data_columns {
let cell = data_column.column.get(row_index).ok_or(
KzgError::InconsistentArrayLength(format!(
"Missing data column at index {row_index}"
"Missing data column at row index {row_index}"
)),
)?;
@@ -289,12 +371,16 @@ pub fn reconstruct_data_columns<E: EthSpec>(
#[cfg(test)]
mod test {
use crate::kzg_utils::{blobs_to_data_column_sidecars, reconstruct_data_columns};
use crate::kzg_utils::{
blobs_to_data_column_sidecars, reconstruct_blobs, reconstruct_data_columns,
};
use bls::Signature;
use eth2::types::BlobsBundle;
use execution_layer::test_utils::generate_blobs;
use kzg::{trusted_setup::get_trusted_setup, Kzg, KzgCommitment, TrustedSetup};
use types::{
beacon_block_body::KzgCommitments, BeaconBlock, BeaconBlockDeneb, Blob, BlobsList,
ChainSpec, EmptyBlock, EthSpec, MainnetEthSpec, SignedBeaconBlock,
beacon_block_body::KzgCommitments, BeaconBlock, BeaconBlockDeneb, BlobsList, ChainSpec,
EmptyBlock, EthSpec, MainnetEthSpec, SignedBeaconBlock,
};
type E = MainnetEthSpec;
@@ -308,6 +394,7 @@ mod test {
test_build_data_columns_empty(&kzg, &spec);
test_build_data_columns(&kzg, &spec);
test_reconstruct_data_columns(&kzg, &spec);
test_reconstruct_blobs_from_data_columns(&kzg, &spec);
}
#[track_caller]
@@ -341,7 +428,7 @@ mod test {
.kzg_commitments_merkle_proof()
.unwrap();
assert_eq!(column_sidecars.len(), spec.number_of_columns);
assert_eq!(column_sidecars.len(), spec.number_of_columns as usize);
for (idx, col_sidecar) in column_sidecars.iter().enumerate() {
assert_eq!(col_sidecar.index, idx as u64);
@@ -374,11 +461,42 @@ mod test {
)
.unwrap();
for i in 0..spec.number_of_columns {
for i in 0..spec.number_of_columns as usize {
assert_eq!(reconstructed_columns.get(i), column_sidecars.get(i), "{i}");
}
}
#[track_caller]
fn test_reconstruct_blobs_from_data_columns(kzg: &Kzg, spec: &ChainSpec) {
let num_of_blobs = 6;
let (signed_block, blobs) = create_test_block_and_blobs::<E>(num_of_blobs, spec);
let blob_refs = blobs.iter().collect::<Vec<_>>();
let column_sidecars =
blobs_to_data_column_sidecars(&blob_refs, &signed_block, kzg, spec).unwrap();
// Now reconstruct
let signed_blinded_block = signed_block.into();
let blob_indices = vec![3, 4, 5];
let reconstructed_blobs = reconstruct_blobs(
kzg,
&column_sidecars.iter().as_slice()[0..column_sidecars.len() / 2],
Some(blob_indices.clone()),
&signed_blinded_block,
spec,
)
.unwrap();
for i in blob_indices {
let reconstructed_blob = &reconstructed_blobs
.iter()
.find(|sidecar| sidecar.index == i)
.map(|sidecar| sidecar.blob.clone())
.expect("reconstructed blob should exist");
let original_blob = blobs.get(i as usize).unwrap();
assert_eq!(reconstructed_blob, original_blob, "{i}");
}
}
fn get_kzg() -> Kzg {
let trusted_setup: TrustedSetup = serde_json::from_reader(get_trusted_setup().as_slice())
.map_err(|e| format!("Unable to read trusted setup file: {}", e))
@@ -397,12 +515,20 @@ mod test {
KzgCommitments::<E>::new(vec![KzgCommitment::empty_for_testing(); num_of_blobs])
.unwrap();
let signed_block = SignedBeaconBlock::from_block(block, Signature::empty());
let mut signed_block = SignedBeaconBlock::from_block(block, Signature::empty());
let blobs = (0..num_of_blobs)
.map(|_| Blob::<E>::default())
.collect::<Vec<_>>()
.into();
let (blobs_bundle, _) = generate_blobs::<E>(num_of_blobs).unwrap();
let BlobsBundle {
blobs,
commitments,
proofs: _,
} = blobs_bundle;
*signed_block
.message_mut()
.body_mut()
.blob_kzg_commitments_mut()
.unwrap() = commitments;
(signed_block, blobs)
}

3
beacon_node/beacon_chain/src/lib.rs
View File

@@ -31,6 +31,7 @@ pub mod execution_payload;
pub mod fetch_blobs;
pub mod fork_choice_signal;
pub mod fork_revert;
pub mod fulu_readiness;
pub mod graffiti_calculator;
mod head_tracker;
pub mod historical_blocks;
@@ -78,7 +79,7 @@ pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceSto
pub use block_verification::{
build_blob_data_column_sidecars, get_block_root, BlockError, ExecutionPayloadError,
ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, IntoGossipVerifiedBlock,
PayloadVerificationOutcome, PayloadVerificationStatus,
InvalidSignature, PayloadVerificationOutcome, PayloadVerificationStatus,
};
pub use block_verification_types::AvailabilityPendingExecutedBlock;
pub use block_verification_types::ExecutedBlock;

20
beacon_node/beacon_chain/src/metrics.rs
View File

@@ -1656,7 +1656,7 @@ pub static BLOB_SIDECAR_INCLUSION_PROOF_COMPUTATION: LazyLock<Result<Histogram>>
});
pub static DATA_COLUMN_SIDECAR_COMPUTATION: LazyLock<Result<HistogramVec>> = LazyLock::new(|| {
try_create_histogram_vec_with_buckets(
"data_column_sidecar_computation_seconds",
"beacon_data_column_sidecar_computation_seconds",
"Time taken to compute data column sidecar, including cells, proofs and inclusion proof",
Ok(vec![0.1, 0.15, 0.25, 0.35, 0.5, 0.7, 1.0, 2.5, 5.0, 10.0]),
&["blob_count"],
@@ -1665,7 +1665,7 @@ pub static DATA_COLUMN_SIDECAR_COMPUTATION: LazyLock<Result<HistogramVec>> = Laz
pub static DATA_COLUMN_SIDECAR_INCLUSION_PROOF_VERIFICATION: LazyLock<Result<Histogram>> =
LazyLock::new(|| {
try_create_histogram(
"data_column_sidecar_inclusion_proof_verification_seconds",
"beacon_data_column_sidecar_inclusion_proof_verification_seconds",
"Time taken to verify data_column sidecar inclusion proof",
)
});
@@ -1693,7 +1693,7 @@ pub static DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: LazyLock<Result<Histog
pub static DATA_COLUMNS_SIDECAR_PROCESSING_SUCCESSES: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"beacon_blobs_column_sidecar_processing_successes_total",
"beacon_data_column_sidecar_processing_successes_total",
"Number of data column sidecars verified for gossip",
)
});
@@ -1847,7 +1847,7 @@ pub static KZG_VERIFICATION_BATCH_TIMES: LazyLock<Result<Histogram>> = LazyLock:
pub static KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES: LazyLock<Result<Histogram>> =
LazyLock::new(|| {
try_create_histogram_with_buckets(
"kzg_verification_data_column_single_seconds",
"beacon_kzg_verification_data_column_single_seconds",
"Runtime of single data column kzg verification",
Ok(vec![
0.0005, 0.001, 0.0015, 0.002, 0.003, 0.004, 0.005, 0.007, 0.01, 0.02, 0.05,
@@ -1857,7 +1857,7 @@ pub static KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES: LazyLock<Result<Histogram>
pub static KZG_VERIFICATION_DATA_COLUMN_BATCH_TIMES: LazyLock<Result<Histogram>> =
LazyLock::new(|| {
try_create_histogram_with_buckets(
"kzg_verification_data_column_batch_seconds",
"beacon_kzg_verification_data_column_batch_seconds",
"Runtime of batched data column kzg verification",
Ok(vec![
0.002, 0.004, 0.006, 0.008, 0.01, 0.012, 0.015, 0.02, 0.03, 0.05, 0.07,
@@ -1910,14 +1910,14 @@ pub static DATA_AVAILABILITY_OVERFLOW_STORE_CACHE_SIZE: LazyLock<Result<IntGauge
pub static DATA_AVAILABILITY_RECONSTRUCTION_TIME: LazyLock<Result<Histogram>> =
LazyLock::new(|| {
try_create_histogram(
"data_availability_reconstruction_time_seconds",
"beacon_data_availability_reconstruction_time_seconds",
"Time taken to reconstruct columns",
)
});
pub static DATA_AVAILABILITY_RECONSTRUCTED_COLUMNS: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"data_availability_reconstructed_columns_total",
"beacon_data_availability_reconstructed_columns_total",
"Total count of reconstructed columns",
)
});
@@ -1925,7 +1925,7 @@ pub static DATA_AVAILABILITY_RECONSTRUCTED_COLUMNS: LazyLock<Result<IntCounter>>
pub static KZG_DATA_COLUMN_RECONSTRUCTION_ATTEMPTS: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"kzg_data_column_reconstruction_attempts",
"beacon_kzg_data_column_reconstruction_attempts",
"Count of times data column reconstruction has been attempted",
)
});
@@ -1933,7 +1933,7 @@ pub static KZG_DATA_COLUMN_RECONSTRUCTION_ATTEMPTS: LazyLock<Result<IntCounter>>
pub static KZG_DATA_COLUMN_RECONSTRUCTION_FAILURES: LazyLock<Result<IntCounter>> =
LazyLock::new(|| {
try_create_int_counter(
"kzg_data_column_reconstruction_failures",
"beacon_kzg_data_column_reconstruction_failures",
"Count of times data column reconstruction has failed",
)
});
@@ -1941,7 +1941,7 @@ pub static KZG_DATA_COLUMN_RECONSTRUCTION_FAILURES: LazyLock<Result<IntCounter>>
pub static KZG_DATA_COLUMN_RECONSTRUCTION_INCOMPLETE_TOTAL: LazyLock<Result<IntCounterVec>> =
LazyLock::new(|| {
try_create_int_counter_vec(
"kzg_data_column_reconstruction_incomplete_total",
"beacon_kzg_data_column_reconstruction_incomplete_total",
"Count of times data column reconstruction attempts did not result in an import",
&["reason"],
)

2
beacon_node/beacon_chain/src/observed_aggregates.rs
View File

@@ -293,7 +293,7 @@ impl<I> SlotHashSet<I> {
Ok(self
.map
.get(&root)
.map_or(false, |agg| agg.iter().any(|val| item.is_subset(val))))
.is_some_and(|agg| agg.iter().any(|val| item.is_subset(val))))
}
/// The number of observed items in `self`.

4
beacon_node/beacon_chain/src/observed_attesters.rs
View File

@@ -130,7 +130,7 @@ impl Item<()> for EpochBitfield {
fn get(&self, validator_index: usize) -> Option<()> {
self.bitfield
.get(validator_index)
.map_or(false, |bit| *bit)
.is_some_and(|bit| *bit)
.then_some(())
}
}
@@ -336,7 +336,7 @@ impl<T: Item<()>, E: EthSpec> AutoPruningEpochContainer<T, E> {
let exists = self
.items
.get(&epoch)
.map_or(false, |item| item.get(validator_index).is_some());
.is_some_and(|item| item.get(validator_index).is_some());
Ok(exists)
}

24
beacon_node/beacon_chain/src/observed_data_sidecars.rs
View File

@@ -24,7 +24,7 @@ pub trait ObservableDataSidecar {
fn slot(&self) -> Slot;
fn block_proposer_index(&self) -> u64;
fn index(&self) -> u64;
fn max_num_of_items(spec: &ChainSpec) -> usize;
fn max_num_of_items(spec: &ChainSpec, slot: Slot) -> usize;
}
impl<E: EthSpec> ObservableDataSidecar for BlobSidecar<E> {
@@ -40,8 +40,8 @@ impl<E: EthSpec> ObservableDataSidecar for BlobSidecar<E> {
self.index
}
fn max_num_of_items(_spec: &ChainSpec) -> usize {
E::max_blobs_per_block()
fn max_num_of_items(spec: &ChainSpec, slot: Slot) -> usize {
spec.max_blobs_per_block(slot.epoch(E::slots_per_epoch())) as usize
}
}
@@ -58,8 +58,8 @@ impl<E: EthSpec> ObservableDataSidecar for DataColumnSidecar<E> {
self.index
}
fn max_num_of_items(spec: &ChainSpec) -> usize {
spec.number_of_columns
fn max_num_of_items(spec: &ChainSpec, _slot: Slot) -> usize {
spec.number_of_columns as usize
}
}
@@ -103,7 +103,9 @@ impl<T: ObservableDataSidecar> ObservedDataSidecars<T> {
slot: data_sidecar.slot(),
proposer: data_sidecar.block_proposer_index(),
})
.or_insert_with(|| HashSet::with_capacity(T::max_num_of_items(&self.spec)));
.or_insert_with(|| {
HashSet::with_capacity(T::max_num_of_items(&self.spec, data_sidecar.slot()))
});
let did_not_exist = data_indices.insert(data_sidecar.index());
Ok(!did_not_exist)
@@ -118,12 +120,12 @@ impl<T: ObservableDataSidecar> ObservedDataSidecars<T> {
slot: data_sidecar.slot(),
proposer: data_sidecar.block_proposer_index(),
})
.map_or(false, |indices| indices.contains(&data_sidecar.index()));
.is_some_and(|indices| indices.contains(&data_sidecar.index()));
Ok(is_known)
}
fn sanitize_data_sidecar(&self, data_sidecar: &T) -> Result<(), Error> {
if data_sidecar.index() >= T::max_num_of_items(&self.spec) as u64 {
if data_sidecar.index() >= T::max_num_of_items(&self.spec, data_sidecar.slot()) as u64 {
return Err(Error::InvalidDataIndex(data_sidecar.index()));
}
let finalized_slot = self.finalized_slot;
@@ -179,7 +181,7 @@ mod tests {
use crate::test_utils::test_spec;
use bls::Hash256;
use std::sync::Arc;
use types::MainnetEthSpec;
use types::{Epoch, MainnetEthSpec};
type E = MainnetEthSpec;
@@ -333,7 +335,7 @@ mod tests {
#[test]
fn simple_observations() {
let spec = Arc::new(test_spec::<E>());
let mut cache = ObservedDataSidecars::<BlobSidecar<E>>::new(spec);
let mut cache = ObservedDataSidecars::<BlobSidecar<E>>::new(spec.clone());
// Slot 0, index 0
let proposer_index_a = 420;
@@ -489,7 +491,7 @@ mod tests {
);
// Try adding an out of bounds index
let invalid_index = E::max_blobs_per_block() as u64;
let invalid_index = spec.max_blobs_per_block(Epoch::new(0));
let sidecar_d = get_blob_sidecar(0, proposer_index_a, invalid_index);
assert_eq!(
cache.observe_sidecar(&sidecar_d),

8
beacon_node/beacon_chain/src/schema_change/migration_schema_v21.rs
View File

@@ -3,9 +3,7 @@ use crate::validator_pubkey_cache::DatabasePubkey;
use slog::{info, Logger};
use ssz::{Decode, Encode};
use std::sync::Arc;
use store::{
get_key_for_col, DBColumn, Error, HotColdDB, KeyValueStore, KeyValueStoreOp, StoreItem,
};
use store::{DBColumn, Error, HotColdDB, KeyValueStore, KeyValueStoreOp, StoreItem};
use types::{Hash256, PublicKey};
const LOG_EVERY: usize = 200_000;
@@ -62,9 +60,9 @@ pub fn downgrade_from_v21<T: BeaconChainTypes>(
message: format!("{e:?}"),
})?;
let db_key = get_key_for_col(DBColumn::PubkeyCache.into(), key.as_slice());
ops.push(KeyValueStoreOp::PutKeyValue(
db_key,
DBColumn::PubkeyCache,
key.as_slice().to_vec(),
pubkey_bytes.as_ssz_bytes(),
));

17
beacon_node/beacon_chain/src/schema_change/migration_schema_v22.rs
View File

@@ -4,7 +4,6 @@ use std::sync::Arc;
use store::chunked_iter::ChunkedVectorIter;
use store::{
chunked_vector::BlockRootsChunked,
get_key_for_col,
metadata::{
SchemaVersion, ANCHOR_FOR_ARCHIVE_NODE, ANCHOR_UNINITIALIZED, STATE_UPPER_LIMIT_NO_RETAIN,
},
@@ -21,7 +20,7 @@ fn load_old_schema_frozen_state<T: BeaconChainTypes>(
) -> Result<Option<BeaconState<T::EthSpec>>, Error> {
let Some(partial_state_bytes) = db
.cold_db
.get_bytes(DBColumn::BeaconState.into(), state_root.as_slice())?
.get_bytes(DBColumn::BeaconState, state_root.as_slice())?
else {
return Ok(None);
};
@@ -136,10 +135,7 @@ pub fn delete_old_schema_freezer_data<T: BeaconChainTypes>(
for column in columns {
for res in db.cold_db.iter_column_keys::<Vec<u8>>(column) {
let key = res?;
cold_ops.push(KeyValueStoreOp::DeleteKey(get_key_for_col(
column.as_str(),
&key,
)));
cold_ops.push(KeyValueStoreOp::DeleteKey(column, key));
}
}
let delete_ops = cold_ops.len();
@@ -175,7 +171,8 @@ pub fn write_new_schema_block_roots<T: BeaconChainTypes>(
// Store the genesis block root if it would otherwise not be stored.
if oldest_block_slot != 0 {
cold_ops.push(KeyValueStoreOp::PutKeyValue(
get_key_for_col(DBColumn::BeaconBlockRoots.into(), &0u64.to_be_bytes()),
DBColumn::BeaconBlockRoots,
0u64.to_be_bytes().to_vec(),
genesis_block_root.as_slice().to_vec(),
));
}
@@ -192,10 +189,8 @@ pub fn write_new_schema_block_roots<T: BeaconChainTypes>(
// OK to hold these in memory (10M slots * 43 bytes per KV ~= 430 MB).
for (i, (slot, block_root)) in block_root_iter.enumerate() {
cold_ops.push(KeyValueStoreOp::PutKeyValue(
get_key_for_col(
DBColumn::BeaconBlockRoots.into(),
&(slot as u64).to_be_bytes(),
),
DBColumn::BeaconBlockRoots,
slot.to_be_bytes().to_vec(),
block_root.as_slice().to_vec(),
));

2
beacon_node/beacon_chain/src/shuffling_cache.rs
View File

@@ -253,7 +253,7 @@ impl BlockShufflingIds {
} else if self
.previous
.as_ref()
.map_or(false, |id| id.shuffling_epoch == epoch)
.is_some_and(|id| id.shuffling_epoch == epoch)
{
self.previous.clone()
} else if epoch == self.next.shuffling_epoch {

548
beacon_node/beacon_chain/src/test_utils.rs
View File

@@ -1,8 +1,9 @@
use crate::blob_verification::GossipVerifiedBlob;
use crate::block_verification_types::{AsBlock, RpcBlock};
use crate::kzg_utils::blobs_to_data_column_sidecars;
use crate::data_column_verification::CustodyDataColumn;
use crate::kzg_utils::build_data_column_sidecars;
use crate::observed_operations::ObservationOutcome;
pub use crate::persisted_beacon_chain::PersistedBeaconChain;
use crate::BeaconBlockResponseWrapper;
pub use crate::{
beacon_chain::{BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, FORK_CHOICE_DB_KEY, OP_POOL_DB_KEY},
migrate::MigratorConfig,
@@ -16,6 +17,7 @@ use crate::{
BeaconChain, BeaconChainTypes, BlockError, ChainConfig, ServerSentEventHandler,
StateSkipConfig,
};
use crate::{get_block_root, BeaconBlockResponseWrapper};
use bls::get_withdrawal_credentials;
use eth2::types::SignedBlockContentsTuple;
use execution_layer::test_utils::generate_genesis_header;
@@ -56,7 +58,8 @@ use std::str::FromStr;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, LazyLock};
use std::time::Duration;
use store::{config::StoreConfig, HotColdDB, ItemStore, LevelDB, MemoryStore};
use store::database::interface::BeaconNodeBackend;
use store::{config::StoreConfig, HotColdDB, ItemStore, MemoryStore};
use task_executor::TaskExecutor;
use task_executor::{test_utils::TestRuntime, ShutdownReason};
use tree_hash::TreeHash;
@@ -73,6 +76,11 @@ pub const FORK_NAME_ENV_VAR: &str = "FORK_NAME";
// Environment variable to read if `ci_logger` feature is enabled.
pub const CI_LOGGER_DIR_ENV_VAR: &str = "CI_LOGGER_DIR";
// Pre-computed data column sidecar using a single static blob from:
// `beacon_node/execution_layer/src/test_utils/fixtures/mainnet/test_blobs_bundle.ssz`
const TEST_DATA_COLUMN_SIDECARS_SSZ: &[u8] =
include_bytes!("test_utils/fixtures/test_data_column_sidecars.ssz");
// Default target aggregators to set during testing, this ensures an aggregator at each slot.
//
// You should mutate the `ChainSpec` prior to initialising the harness if you would like to use
@@ -104,7 +112,7 @@ static KZG_NO_PRECOMP: LazyLock<Arc<Kzg>> = LazyLock::new(|| {
});
pub fn get_kzg(spec: &ChainSpec) -> Arc<Kzg> {
if spec.eip7594_fork_epoch.is_some() {
if spec.fulu_fork_epoch.is_some() {
KZG_PEERDAS.clone()
} else if spec.deneb_fork_epoch.is_some() {
KZG.clone()
@@ -116,7 +124,7 @@ pub fn get_kzg(spec: &ChainSpec) -> Arc<Kzg> {
pub type BaseHarnessType<E, THotStore, TColdStore> =
Witness<TestingSlotClock, CachingEth1Backend<E>, E, THotStore, TColdStore>;
pub type DiskHarnessType<E> = BaseHarnessType<E, LevelDB<E>, LevelDB<E>>;
pub type DiskHarnessType<E> = BaseHarnessType<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>;
pub type EphemeralHarnessType<E> = BaseHarnessType<E, MemoryStore<E>, MemoryStore<E>>;
pub type BoxedMutator<E, Hot, Cold> = Box<
@@ -223,6 +231,7 @@ pub struct Builder<T: BeaconChainTypes> {
mock_execution_layer: Option<MockExecutionLayer<T::EthSpec>>,
testing_slot_clock: Option<TestingSlotClock>,
validator_monitor_config: Option<ValidatorMonitorConfig>,
import_all_data_columns: bool,
runtime: TestRuntime,
log: Logger,
}
@@ -299,7 +308,10 @@ impl<E: EthSpec> Builder<EphemeralHarnessType<E>> {
impl<E: EthSpec> Builder<DiskHarnessType<E>> {
/// Disk store, start from genesis.
pub fn fresh_disk_store(mut self, store: Arc<HotColdDB<E, LevelDB<E>, LevelDB<E>>>) -> Self {
pub fn fresh_disk_store(
mut self,
store: Arc<HotColdDB<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>>,
) -> Self {
let validator_keypairs = self
.validator_keypairs
.clone()
@@ -324,7 +336,10 @@ impl<E: EthSpec> Builder<DiskHarnessType<E>> {
}
/// Disk store, resume.
pub fn resumed_disk_store(mut self, store: Arc<HotColdDB<E, LevelDB<E>, LevelDB<E>>>) -> Self {
pub fn resumed_disk_store(
mut self,
store: Arc<HotColdDB<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>>,
) -> Self {
let mutator = move |builder: BeaconChainBuilder<_>| {
builder
.resume_from_db()
@@ -359,6 +374,7 @@ where
mock_execution_layer: None,
testing_slot_clock: None,
validator_monitor_config: None,
import_all_data_columns: false,
runtime,
log,
}
@@ -451,6 +467,11 @@ where
self
}
pub fn import_all_data_columns(mut self, import_all_data_columns: bool) -> Self {
self.import_all_data_columns = import_all_data_columns;
self
}
pub fn execution_layer_from_url(mut self, url: &str) -> Self {
assert!(
self.execution_layer.is_none(),
@@ -501,6 +522,9 @@ where
spec.electra_fork_epoch.map(|epoch| {
genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64()
});
mock.server.execution_block_generator().osaka_time = spec.fulu_fork_epoch.map(|epoch| {
genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64()
});
self
}
@@ -511,7 +535,7 @@ where
pub fn mock_execution_layer_with_config(mut self) -> Self {
let mock = mock_execution_layer_from_parts::<E>(
self.spec.as_ref().expect("cannot build without spec"),
self.spec.clone().expect("cannot build without spec"),
self.runtime.task_executor.clone(),
);
self.execution_layer = Some(mock.el.clone());
@@ -565,6 +589,7 @@ where
.expect("should build dummy backend")
.shutdown_sender(shutdown_tx)
.chain_config(chain_config)
.import_all_data_columns(self.import_all_data_columns)
.event_handler(Some(ServerSentEventHandler::new_with_capacity(
log.clone(),
5,
@@ -611,7 +636,7 @@ where
}
pub fn mock_execution_layer_from_parts<E: EthSpec>(
spec: &ChainSpec,
spec: Arc<ChainSpec>,
task_executor: TaskExecutor,
) -> MockExecutionLayer<E> {
let shanghai_time = spec.capella_fork_epoch.map(|epoch| {
@@ -623,8 +648,11 @@ pub fn mock_execution_layer_from_parts<E: EthSpec>(
let prague_time = spec.electra_fork_epoch.map(|epoch| {
HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64()
});
let osaka_time = spec.fulu_fork_epoch.map(|epoch| {
HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64()
});
let kzg = get_kzg(spec);
let kzg = get_kzg(&spec);
MockExecutionLayer::new(
task_executor,
@@ -632,8 +660,9 @@ pub fn mock_execution_layer_from_parts<E: EthSpec>(
shanghai_time,
cancun_time,
prague_time,
osaka_time,
Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
spec.clone(),
spec,
Some(kzg),
)
}
@@ -662,10 +691,16 @@ pub struct BeaconChainHarness<T: BeaconChainTypes> {
pub rng: Mutex<StdRng>,
}
pub type CommitteeSingleAttestations = Vec<(SingleAttestation, SubnetId)>;
pub type CommitteeAttestations<E> = Vec<(Attestation<E>, SubnetId)>;
pub type HarnessAttestations<E> =
Vec<(CommitteeAttestations<E>, Option<SignedAggregateAndProof<E>>)>;
pub type HarnessSingleAttestations<E> = Vec<(
CommitteeSingleAttestations,
Option<SignedAggregateAndProof<E>>,
)>;
pub type HarnessSyncContributions<E> = Vec<(
Vec<(SyncCommitteeMessage, usize)>,
Option<SignedContributionAndProof<E>>,
@@ -742,15 +777,13 @@ where
pub fn get_head_block(&self) -> RpcBlock<E> {
let block = self.chain.head_beacon_block();
let block_root = block.canonical_root();
let blobs = self.chain.get_blobs(&block_root).unwrap();
RpcBlock::new(Some(block_root), block, Some(blobs)).unwrap()
self.build_rpc_block_from_store_blobs(Some(block_root), block)
}
pub fn get_full_block(&self, block_root: &Hash256) -> RpcBlock<E> {
let block = self.chain.get_blinded_block(block_root).unwrap().unwrap();
let full_block = self.chain.store.make_full_block(block_root, block).unwrap();
let blobs = self.chain.get_blobs(block_root).unwrap();
RpcBlock::new(Some(*block_root), Arc::new(full_block), Some(blobs)).unwrap()
self.build_rpc_block_from_store_blobs(Some(*block_root), Arc::new(full_block))
}
pub fn get_all_validators(&self) -> Vec<usize> {
@@ -913,15 +946,12 @@ where
&self.spec,
));
let block_contents: SignedBlockContentsTuple<E> = match *signed_block {
SignedBeaconBlock::Base(_)
| SignedBeaconBlock::Altair(_)
| SignedBeaconBlock::Bellatrix(_)
| SignedBeaconBlock::Capella(_) => (signed_block, None),
SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) => {
let block_contents: SignedBlockContentsTuple<E> =
if signed_block.fork_name_unchecked().deneb_enabled() {
(signed_block, block_response.blob_items)
}
};
} else {
(signed_block, None)
};
(block_contents, block_response.state)
}
@@ -977,15 +1007,12 @@ where
&self.spec,
));
let block_contents: SignedBlockContentsTuple<E> = match *signed_block {
SignedBeaconBlock::Base(_)
| SignedBeaconBlock::Altair(_)
| SignedBeaconBlock::Bellatrix(_)
| SignedBeaconBlock::Capella(_) => (signed_block, None),
SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) => {
let block_contents: SignedBlockContentsTuple<E> =
if signed_block.fork_name_unchecked().deneb_enabled() {
(signed_block, block_response.blob_items)
}
};
} else {
(signed_block, None)
};
(block_contents, pre_state)
}
@@ -1023,6 +1050,99 @@ where
)
}
#[allow(clippy::too_many_arguments)]
pub fn produce_single_attestation_for_block(
&self,
slot: Slot,
index: CommitteeIndex,
beacon_block_root: Hash256,
mut state: Cow<BeaconState<E>>,
state_root: Hash256,
aggregation_bit_index: usize,
validator_index: usize,
) -> Result<SingleAttestation, BeaconChainError> {
let epoch = slot.epoch(E::slots_per_epoch());
if state.slot() > slot {
return Err(BeaconChainError::CannotAttestToFutureState);
} else if state.current_epoch() < epoch {
let mut_state = state.to_mut();
complete_state_advance(
mut_state,
Some(state_root),
epoch.start_slot(E::slots_per_epoch()),
&self.spec,
)?;
mut_state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
}
let committee_len = state.get_beacon_committee(slot, index)?.committee.len();
let target_slot = epoch.start_slot(E::slots_per_epoch());
let target_root = if state.slot() <= target_slot {
beacon_block_root
} else {
*state.get_block_root(target_slot)?
};
let attestation: Attestation<E> = Attestation::empty_for_signing(
index,
committee_len,
slot,
beacon_block_root,
state.current_justified_checkpoint(),
Checkpoint {
epoch,
root: target_root,
},
&self.spec,
)?;
let attestation = match attestation {
Attestation::Electra(mut attn) => {
attn.aggregation_bits
.set(aggregation_bit_index, true)
.unwrap();
attn
}
Attestation::Base(_) => panic!("Must be an Electra attestation"),
};
let aggregation_bits = attestation.get_aggregation_bits();
if aggregation_bits.len() != 1 {
panic!("Must be an unaggregated attestation")
}
let aggregation_bit = *aggregation_bits.first().unwrap();
let committee = state.get_beacon_committee(slot, index).unwrap();
let attester_index = committee
.committee
.iter()
.enumerate()
.find_map(|(i, &index)| {
if aggregation_bit as usize == i {
return Some(index);
}
None
})
.unwrap();
let single_attestation =
attestation.to_single_attestation_with_attester_index(attester_index as u64)?;
let attestation: Attestation<E> = single_attestation.to_attestation(committee.committee)?;
assert_eq!(
single_attestation.committee_index,
attestation.committee_index().unwrap()
);
assert_eq!(single_attestation.attester_index, validator_index as u64);
Ok(single_attestation)
}
/// Produces an "unaggregated" attestation for the given `slot` and `index` that attests to
/// `beacon_block_root`. The provided `state` should match the `block.state_root` for the
/// `block` identified by `beacon_block_root`.
@@ -1080,6 +1200,33 @@ where
)?)
}
/// A list of attestations for each committee for the given slot.
///
/// The first layer of the Vec is organised per committee. For example, if the return value is
/// called `all_attestations`, then all attestations in `all_attestations[0]` will be for
/// committee 0, whilst all in `all_attestations[1]` will be for committee 1.
pub fn make_single_attestations(
&self,
attesting_validators: &[usize],
state: &BeaconState<E>,
state_root: Hash256,
head_block_root: SignedBeaconBlockHash,
attestation_slot: Slot,
) -> Vec<CommitteeSingleAttestations> {
let fork = self
.spec
.fork_at_epoch(attestation_slot.epoch(E::slots_per_epoch()));
self.make_single_attestations_with_opts(
attesting_validators,
state,
state_root,
head_block_root,
attestation_slot,
MakeAttestationOptions { limit: None, fork },
)
.0
}
/// A list of attestations for each committee for the given slot.
///
/// The first layer of the Vec is organised per committee. For example, if the return value is
@@ -1107,6 +1254,99 @@ where
.0
}
pub fn make_single_attestations_with_opts(
&self,
attesting_validators: &[usize],
state: &BeaconState<E>,
state_root: Hash256,
head_block_root: SignedBeaconBlockHash,
attestation_slot: Slot,
opts: MakeAttestationOptions,
) -> (Vec<CommitteeSingleAttestations>, Vec<usize>) {
let MakeAttestationOptions { limit, fork } = opts;
let committee_count = state.get_committee_count_at_slot(state.slot()).unwrap();
let num_attesters = AtomicUsize::new(0);
let (attestations, split_attesters) = state
.get_beacon_committees_at_slot(attestation_slot)
.expect("should get committees")
.iter()
.map(|bc| {
bc.committee
.par_iter()
.enumerate()
.filter_map(|(i, validator_index)| {
if !attesting_validators.contains(validator_index) {
return None;
}
if let Some(limit) = limit {
// This atomics stuff is necessary because we're under a par_iter,
// and Rayon will deadlock if we use a mutex.
if num_attesters.fetch_add(1, Ordering::Relaxed) >= limit {
num_attesters.fetch_sub(1, Ordering::Relaxed);
return None;
}
}
let mut attestation = self
.produce_single_attestation_for_block(
attestation_slot,
bc.index,
head_block_root.into(),
Cow::Borrowed(state),
state_root,
i,
*validator_index,
)
.unwrap();
attestation.signature = {
let domain = self.spec.get_domain(
attestation.data.target.epoch,
Domain::BeaconAttester,
&fork,
state.genesis_validators_root(),
);
let message = attestation.data.signing_root(domain);
let mut agg_sig = AggregateSignature::infinity();
agg_sig.add_assign(
&self.validator_keypairs[*validator_index].sk.sign(message),
);
agg_sig
};
let subnet_id = SubnetId::compute_subnet_for_single_attestation::<E>(
&attestation,
committee_count,
&self.chain.spec,
)
.unwrap();
Some(((attestation, subnet_id), validator_index))
})
.unzip::<_, _, Vec<_>, Vec<_>>()
})
.unzip::<_, _, Vec<_>, Vec<_>>();
// Flatten attesters.
let attesters = split_attesters.into_iter().flatten().collect::<Vec<_>>();
if let Some(limit) = limit {
assert_eq!(limit, num_attesters.load(Ordering::Relaxed));
assert_eq!(
limit,
attesters.len(),
"failed to generate `limit` attestations"
);
}
(attestations, attesters)
}
pub fn make_unaggregated_attestations_with_opts(
&self,
attesting_validators: &[usize],
@@ -1287,6 +1527,32 @@ where
)
}
/// A list of attestations for each committee for the given slot.
///
/// The first layer of the Vec is organised per committee. For example, if the return value is
/// called `all_attestations`, then all attestations in `all_attestations[0]` will be for
/// committee 0, whilst all in `all_attestations[1]` will be for committee 1.
pub fn get_single_attestations(
&self,
attestation_strategy: &AttestationStrategy,
state: &BeaconState<E>,
state_root: Hash256,
head_block_root: Hash256,
attestation_slot: Slot,
) -> Vec<Vec<(SingleAttestation, SubnetId)>> {
let validators: Vec<usize> = match attestation_strategy {
AttestationStrategy::AllValidators => self.get_all_validators(),
AttestationStrategy::SomeValidators(vals) => vals.clone(),
};
self.make_single_attestations(
&validators,
state,
state_root,
head_block_root.into(),
attestation_slot,
)
}
pub fn make_attestations(
&self,
attesting_validators: &[usize],
@@ -2018,22 +2284,19 @@ where
self.set_current_slot(slot);
let (block, blob_items) = block_contents;
let sidecars = blob_items
.map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs))
.transpose()
.unwrap();
let rpc_block = self.build_rpc_block_from_blobs(block_root, block, blob_items)?;
let block_hash: SignedBeaconBlockHash = self
.chain
.process_block(
block_root,
RpcBlock::new(Some(block_root), block, sidecars).unwrap(),
rpc_block,
NotifyExecutionLayer::Yes,
BlockImportSource::RangeSync,
|| Ok(()),
)
.await?
.try_into()
.unwrap();
.expect("block blobs are available");
self.chain.recompute_head_at_current_slot().await;
Ok(block_hash)
}
@@ -2044,16 +2307,13 @@ where
) -> Result<SignedBeaconBlockHash, BlockError> {
let (block, blob_items) = block_contents;
let sidecars = blob_items
.map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs))
.transpose()
.unwrap();
let block_root = block.canonical_root();
let rpc_block = self.build_rpc_block_from_blobs(block_root, block, blob_items)?;
let block_hash: SignedBeaconBlockHash = self
.chain
.process_block(
block_root,
RpcBlock::new(Some(block_root), block, sidecars).unwrap(),
rpc_block,
NotifyExecutionLayer::Yes,
BlockImportSource::RangeSync,
|| Ok(()),
@@ -2065,6 +2325,75 @@ where
Ok(block_hash)
}
/// Builds an `Rpc` block from a `SignedBeaconBlock` and blobs or data columns retrieved from
/// the database.
pub fn build_rpc_block_from_store_blobs(
&self,
block_root: Option<Hash256>,
block: Arc<SignedBeaconBlock<E>>,
) -> RpcBlock<E> {
let block_root = block_root.unwrap_or_else(|| get_block_root(&block));
let has_blobs = block
.message()
.body()
.blob_kzg_commitments()
.is_ok_and(|c| !c.is_empty());
if !has_blobs {
return RpcBlock::new_without_blobs(Some(block_root), block);
}
// Blobs are stored as data columns from Fulu (PeerDAS)
if self.spec.is_peer_das_enabled_for_epoch(block.epoch()) {
let columns = self.chain.get_data_columns(&block_root).unwrap().unwrap();
let custody_columns = columns
.into_iter()
.map(CustodyDataColumn::from_asserted_custody)
.collect::<Vec<_>>();
RpcBlock::new_with_custody_columns(Some(block_root), block, custody_columns, &self.spec)
.unwrap()
} else {
let blobs = self.chain.get_blobs(&block_root).unwrap().blobs();
RpcBlock::new(Some(block_root), block, blobs).unwrap()
}
}
/// Builds an `RpcBlock` from a `SignedBeaconBlock` and `BlobsList`.
fn build_rpc_block_from_blobs(
&self,
block_root: Hash256,
block: Arc<SignedBeaconBlock<E, FullPayload<E>>>,
blob_items: Option<(KzgProofs<E>, BlobsList<E>)>,
) -> Result<RpcBlock<E>, BlockError> {
Ok(if self.spec.is_peer_das_enabled_for_epoch(block.epoch()) {
let sampling_column_count = self
.chain
.data_availability_checker
.get_sampling_column_count();
if blob_items.is_some_and(|(_, blobs)| !blobs.is_empty()) {
// Note: this method ignores the actual custody columns and just take the first
// `sampling_column_count` for testing purpose only, because the chain does not
// currently have any knowledge of the columns being custodied.
let columns = generate_data_column_sidecars_from_block(&block, &self.spec)
.into_iter()
.take(sampling_column_count)
.map(CustodyDataColumn::from_asserted_custody)
.collect::<Vec<_>>();
RpcBlock::new_with_custody_columns(Some(block_root), block, columns, &self.spec)?
} else {
RpcBlock::new_without_blobs(Some(block_root), block)
}
} else {
let blobs = blob_items
.map(|(proofs, blobs)| {
BlobSidecar::build_sidecars(blobs, &block, proofs, &self.spec)
})
.transpose()
.unwrap();
RpcBlock::new(Some(block_root), block, blobs)?
})
}
pub fn process_attestations(&self, attestations: HarnessAttestations<E>) {
let num_validators = self.validator_keypairs.len();
let mut unaggregated = Vec::with_capacity(num_validators);
@@ -2738,6 +3067,56 @@ where
Ok(())
}
/// Simulate some of the blobs / data columns being seen on gossip.
/// Converts the blobs to data columns if the slot is Fulu or later.
pub async fn process_gossip_blobs_or_columns<'a>(
&self,
block: &SignedBeaconBlock<E>,
blobs: impl Iterator<Item = &'a Blob<E>>,
proofs: impl Iterator<Item = &'a KzgProof>,
custody_columns_opt: Option<HashSet<ColumnIndex>>,
) {
let is_peerdas_enabled = self.chain.spec.is_peer_das_enabled_for_epoch(block.epoch());
if is_peerdas_enabled {
let custody_columns = custody_columns_opt.unwrap_or_else(|| {
let sampling_column_count = self
.chain
.data_availability_checker
.get_sampling_column_count() as u64;
(0..sampling_column_count).collect()
});
let verified_columns = generate_data_column_sidecars_from_block(block, &self.spec)
.into_iter()
.filter(|c| custody_columns.contains(&c.index))
.map(|sidecar| {
let column_index = sidecar.index;
self.chain
.verify_data_column_sidecar_for_gossip(sidecar, column_index)
})
.collect::<Result<Vec<_>, _>>()
.unwrap();
if !verified_columns.is_empty() {
self.chain
.process_gossip_data_columns(verified_columns, || Ok(()))
.await
.unwrap();
}
} else {
for (i, (kzg_proof, blob)) in proofs.into_iter().zip(blobs).enumerate() {
let sidecar =
Arc::new(BlobSidecar::new(i, blob.clone(), block, *kzg_proof).unwrap());
let gossip_blob = GossipVerifiedBlob::new(sidecar, i as u64, &self.chain)
.expect("should obtain gossip verified blob");
self.chain
.process_gossip_blob(gossip_blob)
.await
.expect("should import valid gossip verified blob");
}
}
}
}
// Junk `Debug` impl to satistfy certain trait bounds during testing.
@@ -2816,11 +3195,12 @@ pub fn generate_rand_block_and_blobs<E: EthSpec>(
fork_name: ForkName,
num_blobs: NumBlobs,
rng: &mut impl Rng,
spec: &ChainSpec,
) -> (SignedBeaconBlock<E, FullPayload<E>>, Vec<BlobSidecar<E>>) {
let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(rng));
let mut block = SignedBeaconBlock::from_block(inner, types::Signature::random_for_test(rng));
let max_blobs = spec.max_blobs_per_block(block.epoch()) as usize;
let mut blob_sidecars = vec![];
let bundle = match block {
@@ -2830,7 +3210,7 @@ pub fn generate_rand_block_and_blobs<E: EthSpec>(
// Get either zero blobs or a random number of blobs between 1 and Max Blobs.
let payload: &mut FullPayloadDeneb<E> = &mut message.body.execution_payload;
let num_blobs = match num_blobs {
NumBlobs::Random => rng.gen_range(1..=E::max_blobs_per_block()),
NumBlobs::Random => rng.gen_range(1..=max_blobs),
NumBlobs::Number(n) => n,
NumBlobs::None => 0,
};
@@ -2850,7 +3230,26 @@ pub fn generate_rand_block_and_blobs<E: EthSpec>(
// Get either zero blobs or a random number of blobs between 1 and Max Blobs.
let payload: &mut FullPayloadElectra<E> = &mut message.body.execution_payload;
let num_blobs = match num_blobs {
NumBlobs::Random => rng.gen_range(1..=E::max_blobs_per_block()),
NumBlobs::Random => rng.gen_range(1..=max_blobs),
NumBlobs::Number(n) => n,
NumBlobs::None => 0,
};
let (bundle, transactions) =
execution_layer::test_utils::generate_blobs::<E>(num_blobs).unwrap();
payload.execution_payload.transactions = <_>::default();
for tx in Vec::from(transactions) {
payload.execution_payload.transactions.push(tx).unwrap();
}
message.body.blob_kzg_commitments = bundle.commitments.clone();
bundle
}
SignedBeaconBlock::Fulu(SignedBeaconBlockFulu {
ref mut message, ..
}) => {
// Get either zero blobs or a random number of blobs between 1 and Max Blobs.
let payload: &mut FullPayloadFulu<E> = &mut message.body.execution_payload;
let num_blobs = match num_blobs {
NumBlobs::Random => rng.gen_range(1..=max_blobs),
NumBlobs::Number(n) => n,
NumBlobs::None => 0,
};
@@ -2903,10 +3302,59 @@ pub fn generate_rand_block_and_data_columns<E: EthSpec>(
SignedBeaconBlock<E, FullPayload<E>>,
DataColumnSidecarList<E>,
) {
let kzg = get_kzg(spec);
let (block, blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, rng);
let blob_refs = blobs.iter().map(|b| &b.blob).collect::<Vec<_>>();
let data_columns = blobs_to_data_column_sidecars(&blob_refs, &block, &kzg, spec).unwrap();
let (block, _blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, rng, spec);
let data_columns = generate_data_column_sidecars_from_block(&block, spec);
(block, data_columns)
}
/// Generate data column sidecars from pre-computed cells and proofs.
fn generate_data_column_sidecars_from_block<E: EthSpec>(
block: &SignedBeaconBlock<E>,
spec: &ChainSpec,
) -> DataColumnSidecarList<E> {
let kzg_commitments = block.message().body().blob_kzg_commitments().unwrap();
if kzg_commitments.is_empty() {
return vec![];
}
let kzg_commitments_inclusion_proof = block
.message()
.body()
.kzg_commitments_merkle_proof()
.unwrap();
let signed_block_header = block.signed_block_header();
// load the precomputed column sidecar to avoid computing them for every block in the tests.
let template_data_columns = RuntimeVariableList::<DataColumnSidecar<E>>::from_ssz_bytes(
TEST_DATA_COLUMN_SIDECARS_SSZ,
spec.number_of_columns as usize,
)
.unwrap();
let (cells, proofs) = template_data_columns
.into_iter()
.map(|sidecar| {
let DataColumnSidecar {
column, kzg_proofs, ..
} = sidecar;
// There's only one cell per column for a single blob
let cell_bytes: Vec<u8> = column.into_iter().next().unwrap().into();
let kzg_cell = cell_bytes.try_into().unwrap();
let kzg_proof = kzg_proofs.into_iter().next().unwrap();
(kzg_cell, kzg_proof)
})
.collect::<(Vec<_>, Vec<_>)>();
// Repeat the cells and proofs for every blob
let blob_cells_and_proofs_vec =
vec![(cells.try_into().unwrap(), proofs.try_into().unwrap()); kzg_commitments.len()];
build_data_column_sidecars(
kzg_commitments.clone(),
kzg_commitments_inclusion_proof,
signed_block_header,
blob_cells_and_proofs_vec,
spec,
)
.unwrap()
}

BIN
beacon_node/beacon_chain/src/test_utils/fixtures/test_data_column_sidecars.ssz Normal file
View File

Binary file not shown.