gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-07-05 13:54:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

First pass

Browse Source
This commit is contained in:
Pawan Dhananjay
2024-08-29 16:11:19 -07:00
parent 653126f42e
commit 25feedfde3
29 changed files with 262 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@@ -1249,7 +1249,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
pub fn get_blobs(&self, block_root: &Hash256) -> Result<BlobSidecarList<T::EthSpec>, Error> {
match self.store.get_blobs(block_root)? {
Some(blobs) => Ok(blobs),
None => Ok(BlobSidecarList::default()),
None => Ok(BlobSidecarList::empty(
// TODO(pawan): fix this
self.spec.max_blobs_per_block(Epoch::new(0)) as usize,
)),
}
}

10
beacon_node/beacon_chain/src/blob_verification.rs
View File

@@ -17,7 +17,8 @@ use std::time::Duration;
use tree_hash::TreeHash;
use types::blob_sidecar::BlobIdentifier;
use types::{
BeaconStateError, BlobSidecar, Epoch, EthSpec, Hash256, SignedBeaconBlockHeader, Slot,
BeaconStateError, BlobSidecar, Epoch, EthSpec, Hash256, RuntimeVariableList,
SignedBeaconBlockHeader, Slot,
};
/// An error occurred while validating a gossip blob.
@@ -172,10 +173,7 @@ impl<E: EthSpec> From<BeaconStateError> for GossipBlobError<E> {
}
}
pub type GossipVerifiedBlobList<T> = VariableList<
GossipVerifiedBlob<T>,
<<T as BeaconChainTypes>::EthSpec as EthSpec>::MaxBlobsPerBlock,
>;
pub type GossipVerifiedBlobList<T> = RuntimeVariableList<GossipVerifiedBlob<T>>;
/// A wrapper around a `BlobSidecar` that indicates it has been approved for re-gossiping on
/// the p2p network.
@@ -399,7 +397,7 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes>(
// since we only subscribe to `MaxBlobsPerBlock` subnets over gossip network.
// We include this check only for completeness.
// Getting this error would imply something very wrong with our networking decoding logic.
if blob_index >= T::EthSpec::max_blobs_per_block() as u64 {
if blob_index >= chain.spec.max_blobs_per_block(blob_epoch) {
return Err(GossipBlobError::InvalidSubnet {
expected: subnet,
received: blob_index,

4
beacon_node/beacon_chain/src/block_verification.rs
View File

@@ -779,7 +779,9 @@ fn build_gossip_verified_blobs<T: BeaconChainTypes>(
GossipVerifiedBlob::new(Arc::new(blob), i as u64, chain)?;
gossip_verified_blobs.push(gossip_verified_blob);
}
let gossip_verified_blobs = VariableList::from(gossip_verified_blobs);
let max_len = chain.spec.max_blobs_per_block(block.epoch()) as usize;
let gossip_verified_blobs =
RuntimeVariableList::from_vec(gossip_verified_blobs, max_len);
Ok::<_, BlockContentsError<T::EthSpec>>(gossip_verified_blobs)
})
.transpose()

17
beacon_node/beacon_chain/src/block_verification_types.rs
View File

@@ -181,23 +181,6 @@ impl<E: EthSpec> RpcBlock<E> {
})
}
pub fn new_from_fixed(
block_root: Hash256,
block: Arc<SignedBeaconBlock<E>>,
blobs: FixedBlobSidecarList<E>,
) -> Result<Self, AvailabilityCheckError> {
let filtered = blobs
.into_iter()
.filter_map(|b| b.clone())
.collect::<Vec<_>>();
let blobs = if filtered.is_empty() {
None
} else {
Some(VariableList::from(filtered))
};
Self::new(Some(block_root), block, blobs)
}
#[allow(clippy::type_complexity)]
pub fn deconstruct(
self,

7
beacon_node/beacon_chain/src/data_availability_checker.rs
View File

@@ -200,7 +200,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
.ok_or(AvailabilityCheckError::SlotClockError)?;
let verified_blobs =
KzgVerifiedBlobList::new(Vec::from(blobs).into_iter().flatten(), kzg, seen_timestamp)
KzgVerifiedBlobList::new(blobs.into_vec().into_iter().flatten(), kzg, seen_timestamp)
.map_err(AvailabilityCheckError::Kzg)?;
self.availability_cache
@@ -384,14 +384,13 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
blocks: Vec<RpcBlock<T::EthSpec>>,
) -> Result<Vec<MaybeAvailableBlock<T::EthSpec>>, AvailabilityCheckError> {
let mut results = Vec::with_capacity(blocks.len());
let all_blobs: BlobSidecarList<T::EthSpec> = blocks
let all_blobs = blocks
.iter()
.filter(|block| self.blobs_required_for_block(block.as_block()))
// this clone is cheap as it's cloning an Arc
.filter_map(|block| block.blobs().cloned())
.flatten()
.collect::<Vec<_>>()
.into();
.collect::<Vec<_>>();
// verify kzg for all blobs at once
if !all_blobs.is_empty() {

47
beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs
View File

@@ -16,9 +16,10 @@ use std::collections::HashSet;
use std::num::NonZeroUsize;
use std::sync::Arc;
use types::blob_sidecar::BlobIdentifier;
use types::runtime_var_list::RuntimeFixedList;
use types::{
BlobSidecar, ChainSpec, ColumnIndex, DataColumnIdentifier, DataColumnSidecar,
DataColumnSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock,
DataColumnSidecarList, Epoch, EthSpec, Hash256, RuntimeVariableList, SignedBeaconBlock,
};
pub type DataColumnsToPublish<E> = Option<DataColumnSidecarList<E>>;
@@ -32,7 +33,7 @@ pub type DataColumnsToPublish<E> = Option<DataColumnSidecarList<E>>;
#[derive(Clone)]
pub struct PendingComponents<E: EthSpec> {
pub block_root: Hash256,
pub verified_blobs: FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock>,
pub verified_blobs: RuntimeFixedList<Option<KzgVerifiedBlob<E>>>,
pub verified_data_columns: Vec<KzgVerifiedCustodyDataColumn<E>>,
pub executed_block: Option<DietAvailabilityPendingExecutedBlock<E>>,
pub reconstruction_started: bool,
@@ -50,9 +51,7 @@ impl<E: EthSpec> PendingComponents<E> {
}
/// Returns an immutable reference to the fixed vector of cached blobs.
pub fn get_cached_blobs(
&self,
) -> &FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock> {
pub fn get_cached_blobs(&self) -> &RuntimeFixedList<Option<KzgVerifiedBlob<E>>> {
&self.verified_blobs
}
@@ -73,9 +72,7 @@ impl<E: EthSpec> PendingComponents<E> {
}
/// Returns a mutable reference to the fixed vector of cached blobs.
pub fn get_cached_blobs_mut(
&mut self,
) -> &mut FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock> {
pub fn get_cached_blobs_mut(&mut self) -> &mut RuntimeFixedList<Option<KzgVerifiedBlob<E>>> {
&mut self.verified_blobs
}
@@ -147,10 +144,7 @@ impl<E: EthSpec> PendingComponents<E> {
/// Blobs are only inserted if:
/// 1. The blob entry at the index is empty and no block exists.
/// 2. The block exists and its commitment matches the blob's commitment.
pub fn merge_blobs(
&mut self,
blobs: FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock>,
) {
pub fn merge_blobs(&mut self, blobs: RuntimeFixedList<Option<KzgVerifiedBlob<E>>>) {
for (index, blob) in blobs.iter().cloned().enumerate() {
let Some(blob) = blob else { continue };
self.merge_single_blob(index, blob);
@@ -194,7 +188,7 @@ impl<E: EthSpec> PendingComponents<E> {
/// Blobs that don't match the new block's commitments are evicted.
pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock<E>) {
self.insert_block(block);
let reinsert = std::mem::take(self.get_cached_blobs_mut());
let reinsert = self.get_cached_blobs_mut().take();
self.merge_blobs(reinsert);
}
@@ -223,10 +217,11 @@ impl<E: EthSpec> PendingComponents<E> {
}
/// Returns an empty `PendingComponents` object with the given block root.
pub fn empty(block_root: Hash256) -> Self {
pub fn empty(block_root: Hash256, max_len: usize) -> Self {
Self {
block_root,
verified_blobs: FixedVector::default(),
// TODO(pawan): just make this a vec potentially
verified_blobs: RuntimeFixedList::new(vec![None; max_len]),
verified_data_columns: vec![],
executed_block: None,
reconstruction_started: false,
@@ -280,7 +275,12 @@ impl<E: EthSpec> PendingComponents<E> {
else {
return Err(AvailabilityCheckError::Unexpected);
};
(Some(VariableList::new(verified_blobs)?), None)
let max_len =
spec.max_blobs_per_block(diet_executed_block.as_block().epoch()) as usize;
(
Some(RuntimeVariableList::new(verified_blobs, max_len)?),
None,
)
}
BlockImportRequirement::CustodyColumns(_) => {
let verified_data_columns = verified_data_columns
@@ -477,7 +477,8 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
epoch: Epoch,
kzg_verified_blobs: I,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
let mut fixed_blobs = FixedVector::default();
let mut fixed_blobs =
RuntimeFixedList::new(vec![None; self.spec.max_blobs_per_block(epoch) as usize]);
for blob in kzg_verified_blobs {
if let Some(blob_opt) = fixed_blobs.get_mut(blob.blob_index() as usize) {
@@ -491,7 +492,9 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
let mut pending_components = write_lock
.pop_entry(&block_root)
.map(|(_, v)| v)
.unwrap_or_else(|| PendingComponents::empty(block_root));
.unwrap_or_else(|| {
PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize)
});
// Merge in the blobs.
pending_components.merge_blobs(fixed_blobs);
@@ -527,7 +530,9 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
let mut pending_components = write_lock
.pop_entry(&block_root)
.map(|(_, v)| v)
.unwrap_or_else(|| PendingComponents::empty(block_root));
.unwrap_or_else(|| {
PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize)
});
// Merge in the data columns.
pending_components.merge_data_columns(kzg_verified_data_columns)?;
@@ -614,7 +619,9 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
let mut pending_components = write_lock
.pop_entry(&block_root)
.map(|(_, v)| v)
.unwrap_or_else(|| PendingComponents::empty(block_root));
.unwrap_or_else(|| {
PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize)
});
// Merge in the block.
pending_components.merge_block(diet_executed_block);

7
beacon_node/beacon_chain/src/kzg_utils.rs
View File

@@ -188,9 +188,10 @@ fn build_data_column_sidecars<E: EthSpec>(
spec: &ChainSpec,
) -> Result<DataColumnSidecarList<E>, String> {
let number_of_columns = spec.number_of_columns;
let mut columns = vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns];
let mut column_kzg_proofs =
vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns];
let max_blobs_per_block =
spec.max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) as usize;
let mut columns = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns];
let mut column_kzg_proofs = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns];
for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec {
// we iterate over each column, and we construct the column from "top to bottom",

14
beacon_node/beacon_chain/src/observed_data_sidecars.rs
View File

@@ -23,7 +23,7 @@ pub trait ObservableDataSidecar {
fn slot(&self) -> Slot;
fn block_proposer_index(&self) -> u64;
fn index(&self) -> u64;
fn max_num_of_items(spec: &ChainSpec) -> usize;
fn max_num_of_items(spec: &ChainSpec, slot: Slot) -> usize;
}
impl<E: EthSpec> ObservableDataSidecar for BlobSidecar<E> {
@@ -39,8 +39,8 @@ impl<E: EthSpec> ObservableDataSidecar for BlobSidecar<E> {
self.index
}
fn max_num_of_items(_spec: &ChainSpec) -> usize {
E::max_blobs_per_block()
fn max_num_of_items(spec: &ChainSpec, slot: Slot) -> usize {
spec.max_blobs_per_block(slot.epoch(E::slots_per_epoch())) as usize
}
}
@@ -57,7 +57,7 @@ impl<E: EthSpec> ObservableDataSidecar for DataColumnSidecar<E> {
self.index
}
fn max_num_of_items(spec: &ChainSpec) -> usize {
fn max_num_of_items(spec: &ChainSpec, _slot: Slot) -> usize {
spec.number_of_columns
}
}
@@ -102,7 +102,9 @@ impl<T: ObservableDataSidecar> ObservedDataSidecars<T> {
slot: data_sidecar.slot(),
proposer: data_sidecar.block_proposer_index(),
})
.or_insert_with(|| HashSet::with_capacity(T::max_num_of_items(&self.spec)));
.or_insert_with(|| {
HashSet::with_capacity(T::max_num_of_items(&self.spec, data_sidecar.slot()))
});
let did_not_exist = data_indices.insert(data_sidecar.index());
Ok(!did_not_exist)
@@ -122,7 +124,7 @@ impl<T: ObservableDataSidecar> ObservedDataSidecars<T> {
}
fn sanitize_data_sidecar(&self, data_sidecar: &T) -> Result<(), Error> {
if data_sidecar.index() >= T::max_num_of_items(&self.spec) as u64 {
if data_sidecar.index() >= T::max_num_of_items(&self.spec, data_sidecar.slot()) as u64 {
return Err(Error::InvalidDataIndex(data_sidecar.index()));
}
let finalized_slot = self.finalized_slot;

10
beacon_node/beacon_chain/src/test_utils.rs
View File

@@ -1991,7 +1991,7 @@ where
let (block, blob_items) = block_contents;
let sidecars = blob_items
.map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs))
.map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs, &self.spec))
.transpose()
.unwrap();
let block_hash: SignedBeaconBlockHash = self
@@ -2017,7 +2017,7 @@ where
let (block, blob_items) = block_contents;
let sidecars = blob_items
.map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs))
.map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs, &self.spec))
.transpose()
.unwrap();
let block_root = block.canonical_root();
@@ -2636,7 +2636,8 @@ pub fn generate_rand_block_and_blobs<E: EthSpec>(
// Get either zero blobs or a random number of blobs between 1 and Max Blobs.
let payload: &mut FullPayloadDeneb<E> = &mut message.body.execution_payload;
let num_blobs = match num_blobs {
NumBlobs::Random => rng.gen_range(1..=E::max_blobs_per_block()),
// TODO(pawan): thread the chainspec value here
NumBlobs::Random => rng.gen_range(1..=6),
NumBlobs::Number(n) => n,
NumBlobs::None => 0,
};
@@ -2656,7 +2657,8 @@ pub fn generate_rand_block_and_blobs<E: EthSpec>(
// Get either zero blobs or a random number of blobs between 1 and Max Blobs.
let payload: &mut FullPayloadElectra<E> = &mut message.body.execution_payload;
let num_blobs = match num_blobs {
NumBlobs::Random => rng.gen_range(1..=E::max_blobs_per_block()),
// TODO(pawan): thread the chainspec value here
NumBlobs::Random => rng.gen_range(1..=6),
NumBlobs::Number(n) => n,
NumBlobs::None => 0,
};

3
beacon_node/client/src/builder.rs
View File

@@ -352,10 +352,11 @@ where
let anchor_block = SignedBeaconBlock::from_ssz_bytes(&anchor_block_bytes, &spec)
.map_err(|e| format!("Unable to parse weak subj block SSZ: {:?}", e))?;
let anchor_blobs = if anchor_block.message().body().has_blobs() {
let max_blobs_len = spec.max_blobs_per_block(anchor_block.epoch()) as usize;
let anchor_blobs_bytes = anchor_blobs_bytes
.ok_or("Blobs for checkpoint must be provided using --checkpoint-blobs")?;
Some(
BlobSidecarList::from_ssz_bytes(&anchor_blobs_bytes)
BlobSidecarList::from_ssz_bytes(&anchor_blobs_bytes, max_blobs_len)
.map_err(|e| format!("Unable to parse weak subj blobs SSZ: {e:?}"))?,
)
} else {

3
beacon_node/execution_layer/src/test_utils/execution_block_generator.rs
View File

@@ -673,7 +673,8 @@ impl<E: EthSpec> ExecutionBlockGenerator<E> {
ForkName::Deneb | ForkName::Electra => {
// get random number between 0 and Max Blobs
let mut rng = self.rng.lock();
let num_blobs = rng.gen::<usize>() % (E::max_blobs_per_block() + 1);
// TODO(pawan): thread the chainspec value here somehow
let num_blobs = rng.gen::<usize>() % 6;
let (bundle, transactions) = generate_blobs(num_blobs)?;
for tx in Vec::from(transactions) {
execution_payload

3
beacon_node/http_api/src/block_id.rs
View File

@@ -279,13 +279,14 @@ impl BlockId {
.get_blobs(&root)
.map_err(warp_utils::reject::beacon_chain_error)?;
let max_len = blob_sidecar_list.max_len();
let blob_sidecar_list_filtered = match indices.indices {
Some(vec) => {
let list = blob_sidecar_list
.into_iter()
.filter(|blob_sidecar| vec.contains(&blob_sidecar.index))
.collect();
BlobSidecarList::new(list)
BlobSidecarList::new(list, max_len)
.map_err(|e| warp_utils::reject::custom_server_error(format!("{:?}", e)))?
}
None => blob_sidecar_list,

8
beacon_node/http_api/src/publish_blocks.rs
View File

@@ -22,7 +22,8 @@ use tree_hash::TreeHash;
use types::{
AbstractExecPayload, BeaconBlockRef, BlobSidecarList, BlockImportSource, DataColumnSidecarList,
DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload,
FullPayloadBellatrix, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, VariableList,
FullPayloadBellatrix, Hash256, RuntimeVariableList, SignedBeaconBlock,
SignedBlindedBeaconBlock, VariableList,
};
use warp::http::StatusCode;
use warp::{reply::Response, Rejection, Reply};
@@ -198,7 +199,10 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlockConten
.into_iter()
.map(|b| b.clone_blob())
.collect::<Vec<_>>();
VariableList::from(blobs)
RuntimeVariableList::from_vec(
blobs,
chain.spec.max_blobs_per_block(block.epoch()) as usize,
)
});
let data_cols_opt = gossip_verified_data_columns
.as_ref()

14
beacon_node/lighthouse_network/src/rpc/methods.rs
View File

@@ -24,6 +24,13 @@ use types::{
pub type MaxErrorLen = U256;
pub const MAX_ERROR_LEN: u64 = 256;
/// The max number of blobs we expect in the configs to set for compile time params.
/// Note: This value is an estimate that we should use only for rate limiting,
/// bounds checking and other non-consensus critical operations.
///
/// For exact value, we should always check the chainspec.
pub const MAX_BLOBS_PER_BLOCK_CEILING: u64 = 16;
/// Wrapper over SSZ List to represent error message in rpc responses.
#[derive(Debug, Clone)]
pub struct ErrorType(pub VariableList<u8, MaxErrorLen>);
@@ -326,8 +333,13 @@ pub struct BlobsByRangeRequest {
}
impl BlobsByRangeRequest {
/// This function provides an upper bound on number of blobs expected in
/// a certain slot range.
///
/// Note: **must not** use for anything consensus critical, only for
/// bounds checking and rate limiting.
pub fn max_blobs_requested<E: EthSpec>(&self) -> u64 {
self.count.saturating_mul(E::max_blobs_per_block() as u64)
self.count.saturating_mul(MAX_BLOBS_PER_BLOCK_CEILING as u64)
}
}

6
beacon_node/lighthouse_network/src/rpc/protocol.rs
View File

@@ -92,7 +92,7 @@ pub static SIGNED_BEACON_BLOCK_DENEB_MAX: LazyLock<usize> = LazyLock::new(|| {
*SIGNED_BEACON_BLOCK_CAPELLA_MAX_WITHOUT_PAYLOAD
+ types::ExecutionPayload::<MainnetEthSpec>::max_execution_payload_deneb_size() // adding max size of execution payload (~16gb)
+ ssz::BYTES_PER_LENGTH_OFFSET // Adding the additional offsets for the `ExecutionPayload`
+ (<types::KzgCommitment as Encode>::ssz_fixed_len() * <MainnetEthSpec>::max_blobs_per_block())
+ (<types::KzgCommitment as Encode>::ssz_fixed_len() * MAX_BLOBS_PER_BLOCK_CEILING as usize)
+ ssz::BYTES_PER_LENGTH_OFFSET
}); // Length offset for the blob commitments field.
//
@@ -100,7 +100,7 @@ pub static SIGNED_BEACON_BLOCK_ELECTRA_MAX: LazyLock<usize> = LazyLock::new(|| {
*SIGNED_BEACON_BLOCK_ELECTRA_MAX_WITHOUT_PAYLOAD
+ types::ExecutionPayload::<MainnetEthSpec>::max_execution_payload_electra_size() // adding max size of execution payload (~16gb)
+ ssz::BYTES_PER_LENGTH_OFFSET // Adding the additional ssz offset for the `ExecutionPayload` field
+ (<types::KzgCommitment as Encode>::ssz_fixed_len() * <MainnetEthSpec>::max_blobs_per_block())
+ (<types::KzgCommitment as Encode>::ssz_fixed_len() * MAX_BLOBS_PER_BLOCK_CEILING as usize)
+ ssz::BYTES_PER_LENGTH_OFFSET
}); // Length offset for the blob commitments field.
@@ -636,7 +636,7 @@ pub fn rpc_blob_limits<E: EthSpec>() -> RpcLimits {
pub fn rpc_data_column_limits<E: EthSpec>() -> RpcLimits {
RpcLimits::new(
DataColumnSidecar::<E>::empty().as_ssz_bytes().len(),
DataColumnSidecar::<E>::max_size(),
DataColumnSidecar::<E>::max_size(MAX_BLOBS_PER_BLOCK_CEILING as usize),
)
}

16
beacon_node/network/src/sync/block_sidecar_coupling.rs
View File

@@ -8,7 +8,8 @@ use std::{
sync::Arc,
};
use types::{
BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlock,
BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, EthSpec, Hash256, RuntimeVariableList,
SignedBeaconBlock,
};
#[derive(Debug)]
@@ -31,6 +32,7 @@ pub struct RangeBlockComponentsRequest<E: EthSpec> {
num_custody_column_requests: Option<usize>,
/// The peers the request was made to.
pub(crate) peer_ids: Vec<PeerId>,
max_blobs_per_block: usize,
}
impl<E: EthSpec> RangeBlockComponentsRequest<E> {
@@ -39,6 +41,7 @@ impl<E: EthSpec> RangeBlockComponentsRequest<E> {
expects_custody_columns: Option<Vec<ColumnIndex>>,
num_custody_column_requests: Option<usize>,
peer_ids: Vec<PeerId>,
max_blobs_per_block: usize,
) -> Self {
Self {
blocks: <_>::default(),
@@ -51,6 +54,7 @@ impl<E: EthSpec> RangeBlockComponentsRequest<E> {
expects_custody_columns,
num_custody_column_requests,
peer_ids,
max_blobs_per_block,
}
}
@@ -100,7 +104,7 @@ impl<E: EthSpec> RangeBlockComponentsRequest<E> {
let mut responses = Vec::with_capacity(blocks.len());
let mut blob_iter = blobs.into_iter().peekable();
for block in blocks.into_iter() {
let mut blob_list = Vec::with_capacity(E::max_blobs_per_block());
let mut blob_list = Vec::with_capacity(self.max_blobs_per_block);
while {
let pair_next_blob = blob_iter
.peek()
@@ -111,7 +115,7 @@ impl<E: EthSpec> RangeBlockComponentsRequest<E> {
blob_list.push(blob_iter.next().ok_or("Missing next blob".to_string())?);
}
let mut blobs_buffer = vec![None; E::max_blobs_per_block()];
let mut blobs_buffer = vec![None; self.max_blobs_per_block];
for blob in blob_list {
let blob_index = blob.index as usize;
let Some(blob_opt) = blobs_buffer.get_mut(blob_index) else {
@@ -123,7 +127,11 @@ impl<E: EthSpec> RangeBlockComponentsRequest<E> {
*blob_opt = Some(blob);
}
}
let blobs = VariableList::from(blobs_buffer.into_iter().flatten().collect::<Vec<_>>());
let blobs = RuntimeVariableList::new(
blobs_buffer.into_iter().flatten().collect::<Vec<_>>(),
self.max_blobs_per_block,
)
.map_err(|_| "Blobs returned exceeds max length".to_string())?;
responses.push(RpcBlock::new(None, block, Some(blobs)).map_err(|e| format!("{e:?}"))?)
}

2
beacon_node/network/src/sync/manager.rs
View File

@@ -1120,6 +1120,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
.network
.range_block_and_blob_response(id, block_or_blob)
{
let epoch = resp.sender_id.batch_id();
match resp.responses {
Ok(blocks) => {
match resp.sender_id {
@@ -1163,6 +1164,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
resp.expects_custody_columns,
None,
vec![],
self.chain.spec.max_blobs_per_block(epoch) as usize,
),
);
// inform range that the request needs to be treated as failed

34
beacon_node/network/src/sync/network_context.rs
View File

@@ -36,8 +36,8 @@ use std::time::Duration;
use tokio::sync::mpsc;
use types::blob_sidecar::FixedBlobSidecarList;
use types::{
BlobSidecar, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256,
SignedBeaconBlock, Slot,
chain_spec, BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList,
EthSpec, Hash256, SignedBeaconBlock, Slot,
};
pub mod custody;
@@ -61,6 +61,15 @@ pub enum RangeRequestId {
},
}
impl RangeRequestId {
pub fn batch_id(&self) -> BatchId {
match self {
RangeRequestId::RangeSync { batch_id, .. } => *batch_id,
RangeRequestId::BackfillSync { batch_id, .. } => *batch_id,
}
}
}
#[derive(Debug)]
pub enum RpcEvent<T> {
StreamTermination,
@@ -422,11 +431,14 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
(None, None)
};
// TODO(pawan): this would break if a batch contains multiple epochs
let max_blobs_len = self.chain.spec.max_blobs_per_block(epoch);
let info = RangeBlockComponentsRequest::new(
expected_blobs,
expects_custody_columns,
num_of_custody_column_req,
requested_peers,
max_blobs_len as usize,
);
self.range_block_components_requests
.insert(id, (sender_id, info));
@@ -977,9 +989,16 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
RpcEvent::Response(blob, seen_timestamp) => {
let request = request.get_mut();
match request.add_response(blob) {
Ok(Some(blobs)) => to_fixed_blob_sidecar_list(blobs)
.map(|blobs| (blobs, seen_timestamp))
.map_err(|e| (e.into(), request.resolve())),
Ok(Some(blobs)) => {
let max_len = if let Some(blob) = blobs.first() {
self.chain.spec.max_blobs_per_block(blob.epoch()) as usize
} else {
6
};
to_fixed_blob_sidecar_list(blobs, max_len)
.map(|blobs| (blobs, seen_timestamp))
.map_err(|e| (e.into(), request.resolve()))
}
Ok(None) => return None,
Err(e) => Err((e.into(), request.resolve())),
}
@@ -1218,8 +1237,11 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
fn to_fixed_blob_sidecar_list<E: EthSpec>(
blobs: Vec<Arc<BlobSidecar<E>>>,
max_len: usize,
) -> Result<FixedBlobSidecarList<E>, LookupVerifyError> {
let mut fixed_list = FixedBlobSidecarList::default();
// TODO(pawan): have a method on fixed runtime vector that just initializes a raw vec with max_len = None
// to signify an empty fixed runtime vector
let mut fixed_list = FixedBlobSidecarList::new(vec![None; max_len]);
for blob in blobs.into_iter() {
let index = blob.index as usize;
*fixed_list

18
beacon_node/store/src/hot_cold_store.rs
View File

@@ -1669,7 +1669,23 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
.get_bytes(DBColumn::BeaconBlob.into(), block_root.as_bytes())?
{
Some(ref blobs_bytes) => {
let blobs = BlobSidecarList::from_ssz_bytes(blobs_bytes)?;
// We insert a VariableList of BlobSidecars into the db, but retrieve
// a plain vec since we don't know the length limit of the list without
// knowing the slot.
// The encoding of a VariableList is same as a regular vec.
let blobs = BlobSidecarVec::from_ssz_bytes(blobs_bytes)?;
let max_blobs_per_block = blobs
.first()
.map(|blob| {
self.spec
.max_blobs_per_block(blob.slot().epoch(E::slots_per_epoch()))
})
// This is the case where we have no blobs for the slot, doesn't matter what value we keep for max here
// TODO(pawan): double check that this is the case
// we could also potentially deal with just vecs in the db since we only add length validated sidecar
// lists to the db
.unwrap_or(6);
let blobs = BlobSidecarList::from_vec(blobs, max_blobs_per_block as usize);
self.block_cache
.lock()
.put_blobs(*block_root, blobs.clone());

3
beacon_node/store/src/impls/execution_payload.rs
View File

@@ -1,7 +1,7 @@
use crate::{DBColumn, Error, StoreItem};
use ssz::{Decode, Encode};
use types::{
BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella,
EthSpec, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella,
ExecutionPayloadDeneb, ExecutionPayloadElectra,
};
@@ -26,7 +26,6 @@ impl_store_item!(ExecutionPayloadBellatrix);
impl_store_item!(ExecutionPayloadCapella);
impl_store_item!(ExecutionPayloadDeneb);
impl_store_item!(ExecutionPayloadElectra);
impl_store_item!(BlobSidecarList);
/// This fork-agnostic implementation should be only used for writing.
///

6
consensus/state_processing/src/per_block_processing.rs
View File

@@ -391,10 +391,12 @@ pub fn partially_verify_execution_payload<E: EthSpec, Payload: AbstractExecPaylo
if let Ok(blob_commitments) = body.blob_kzg_commitments() {
// Verify commitments are under the limit.
let max_blobs_per_block =
spec.max_blobs_per_block(block_slot.epoch(E::slots_per_epoch())) as usize;
block_verify!(
blob_commitments.len() <= E::max_blobs_per_block(),
blob_commitments.len() <= max_blobs_per_block,
BlockProcessingError::ExecutionInvalidBlobsLen {
max: E::max_blobs_per_block(),
max: max_blobs_per_block,
actual: blob_commitments.len(),
}
);

2
consensus/types/src/beacon_block_body.rs
View File

@@ -13,8 +13,6 @@ use tree_hash_derive::TreeHash;
pub type KzgCommitments<E> =
VariableList<KzgCommitment, <E as EthSpec>::MaxBlobCommitmentsPerBlock>;
pub type KzgCommitmentOpts<E> =
FixedVector<Option<KzgCommitment>, <E as EthSpec>::MaxBlobsPerBlock>;
/// The number of leaves (including padding) on the `BeaconBlockBody` Merkle tree.
///

33
consensus/types/src/blob_sidecar.rs
View File

@@ -1,10 +1,13 @@
use crate::test_utils::TestRandom;
use crate::ForkName;
use crate::{
beacon_block_body::BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, BeaconStateError, Blob,
Epoch, EthSpec, FixedVector, Hash256, SignedBeaconBlockHeader, Slot, VariableList,
};
use crate::{ForkVersionDeserialize, KzgProofs, SignedBeaconBlock};
use crate::{
runtime_var_list::RuntimeFixedList, ForkVersionDeserialize, KzgProofs, RuntimeVariableList,
SignedBeaconBlock,
};
use crate::{ChainSpec, ForkName};
use bls::Signature;
use derivative::Derivative;
use kzg::{Blob as KzgBlob, Kzg, KzgCommitment, KzgProof, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT};
@@ -30,19 +33,6 @@ pub struct BlobIdentifier {
pub index: u64,
}
impl BlobIdentifier {
pub fn get_all_blob_ids<E: EthSpec>(block_root: Hash256) -> Vec<BlobIdentifier> {
let mut blob_ids = Vec::with_capacity(E::max_blobs_per_block());
for i in 0..E::max_blobs_per_block() {
blob_ids.push(BlobIdentifier {
block_root,
index: i as u64,
});
}
blob_ids
}
}
impl PartialOrd for BlobIdentifier {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
@@ -260,19 +250,24 @@ impl<E: EthSpec> BlobSidecar<E> {
blobs: BlobsList<E>,
block: &SignedBeaconBlock<E>,
kzg_proofs: KzgProofs<E>,
spec: &ChainSpec,
) -> Result<BlobSidecarList<E>, BlobSidecarError> {
let mut blob_sidecars = vec![];
for (i, (kzg_proof, blob)) in kzg_proofs.iter().zip(blobs).enumerate() {
let blob_sidecar = BlobSidecar::new(i, blob, block, *kzg_proof)?;
blob_sidecars.push(Arc::new(blob_sidecar));
}
Ok(VariableList::from(blob_sidecars))
Ok(RuntimeVariableList::from_vec(
blob_sidecars,
spec.max_blobs_per_block(block.epoch()) as usize,
))
}
}
pub type BlobSidecarList<E> = VariableList<Arc<BlobSidecar<E>>, <E as EthSpec>::MaxBlobsPerBlock>;
pub type FixedBlobSidecarList<E> =
FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>;
pub type BlobSidecarList<E> = RuntimeVariableList<Arc<BlobSidecar<E>>>;
/// Alias for a non length-constrained list of `BlobSidecar`s.
pub type BlobSidecarVec<E> = Vec<Arc<BlobSidecar<E>>>;
pub type FixedBlobSidecarList<E> = RuntimeFixedList<Option<Arc<BlobSidecar<E>>>>;
pub type BlobsList<E> = VariableList<Blob<E>, <E as EthSpec>::MaxBlobCommitmentsPerBlock>;
impl<E: EthSpec> ForkVersionDeserialize for BlobSidecarList<E> {

26
consensus/types/src/chain_spec.rs
View File

@@ -230,6 +230,7 @@ pub struct ChainSpec {
pub max_request_data_column_sidecars: u64,
pub min_epochs_for_blob_sidecars_requests: u64,
pub blob_sidecar_subnet_count: u64,
max_blobs_per_block: u64,
/*
* Networking Derived
@@ -607,6 +608,16 @@ impl ChainSpec {
}
}
/// Returns the deneb preset value if peerdas epoch hasn't hit.
/// Otherwise, returns the value obtained from the config.yaml.
pub fn max_blobs_per_block(&self, epoch: Epoch) -> u64 {
if self.is_peer_das_enabled_for_epoch(epoch) {
self.max_blobs_per_block
} else {
default_max_blobs_per_block()
}
}
pub fn data_columns_per_subnet(&self) -> usize {
self.number_of_columns
.safe_div(self.data_column_sidecar_subnet_count as usize)
@@ -843,6 +854,7 @@ impl ChainSpec {
max_request_data_column_sidecars: default_max_request_data_column_sidecars(),
min_epochs_for_blob_sidecars_requests: default_min_epochs_for_blob_sidecars_requests(),
blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(),
max_blobs_per_block: default_max_blobs_per_block(),
/*
* Derived Deneb Specific
@@ -1164,6 +1176,8 @@ impl ChainSpec {
max_request_data_column_sidecars: default_max_request_data_column_sidecars(),
min_epochs_for_blob_sidecars_requests: 16384,
blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(),
// TODO(pawan): check if gnosis preset values match
max_blobs_per_block: default_max_blobs_per_block(),
/*
* Derived Deneb Specific
@@ -1364,6 +1378,9 @@ pub struct Config {
#[serde(default = "default_blob_sidecar_subnet_count")]
#[serde(with = "serde_utils::quoted_u64")]
blob_sidecar_subnet_count: u64,
#[serde(default = "default_max_blobs_per_block")]
#[serde(with = "serde_utils::quoted_u64")]
max_blobs_per_block: u64,
#[serde(default = "default_min_per_epoch_churn_limit_electra")]
#[serde(with = "serde_utils::quoted_u64")]
@@ -1499,6 +1516,12 @@ const fn default_blob_sidecar_subnet_count() -> u64 {
6
}
/// Its important to keep this consistent with the deneb preset value for
/// `MAX_BLOBS_PER_BLOCK` else we might run into consensus issues.
const fn default_max_blobs_per_block() -> u64 {
6
}
const fn default_min_per_epoch_churn_limit_electra() -> u64 {
128_000_000_000
}
@@ -1718,6 +1741,7 @@ impl Config {
max_request_data_column_sidecars: spec.max_request_data_column_sidecars,
min_epochs_for_blob_sidecars_requests: spec.min_epochs_for_blob_sidecars_requests,
blob_sidecar_subnet_count: spec.blob_sidecar_subnet_count,
max_blobs_per_block: spec.max_blobs_per_block,
min_per_epoch_churn_limit_electra: spec.min_per_epoch_churn_limit_electra,
max_per_epoch_activation_exit_churn_limit: spec
@@ -1795,6 +1819,7 @@ impl Config {
max_request_data_column_sidecars,
min_epochs_for_blob_sidecars_requests,
blob_sidecar_subnet_count,
max_blobs_per_block,
min_per_epoch_churn_limit_electra,
max_per_epoch_activation_exit_churn_limit,
@@ -1863,6 +1888,7 @@ impl Config {
max_request_data_column_sidecars,
min_epochs_for_blob_sidecars_requests,
blob_sidecar_subnet_count,
max_blobs_per_block,
min_per_epoch_churn_limit_electra,
max_per_epoch_activation_exit_churn_limit,

14
consensus/types/src/data_column_sidecar.rs
View File

@@ -1,7 +1,7 @@
use crate::beacon_block_body::{KzgCommitments, BLOB_KZG_COMMITMENTS_INDEX};
use crate::test_utils::TestRandom;
use crate::BeaconStateError;
use crate::{BeaconBlockHeader, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot};
use crate::{BeaconBlockHeader, Epoch, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot};
use crate::{BeaconStateError, ChainSpec};
use bls::Signature;
use derivative::Derivative;
use kzg::Error as KzgError;
@@ -110,18 +110,16 @@ impl<E: EthSpec> DataColumnSidecar<E> {
.len()
}
pub fn max_size() -> usize {
pub fn max_size(max_blobs_per_block: usize) -> usize {
Self {
index: 0,
column: VariableList::new(vec![Cell::<E>::default(); E::MaxBlobsPerBlock::to_usize()])
.unwrap(),
column: VariableList::new(vec![Cell::<E>::default(); max_blobs_per_block]).unwrap(),
kzg_commitments: VariableList::new(vec![
KzgCommitment::empty_for_testing();
E::MaxBlobsPerBlock::to_usize()
max_blobs_per_block
])
.unwrap(),
kzg_proofs: VariableList::new(vec![KzgProof::empty(); E::MaxBlobsPerBlock::to_usize()])
.unwrap(),
kzg_proofs: VariableList::new(vec![KzgProof::empty(); max_blobs_per_block]).unwrap(),
signed_block_header: SignedBeaconBlockHeader {
message: BeaconBlockHeader::empty(),
signature: Signature::empty(),

12
consensus/types/src/eth_spec.rs
View File

@@ -4,8 +4,7 @@ use safe_arith::SafeArith;
use serde::{Deserialize, Serialize};
use ssz_types::typenum::{
bit::B0, UInt, U0, U1, U1024, U1048576, U1073741824, U1099511627776, U128, U131072, U134217728,
U16, U16777216, U2, U2048, U256, U262144, U32, U4, U4096, U512, U6, U625, U64, U65536, U8,
U8192,
U16, U16777216, U2, U2048, U256, U262144, U32, U4, U4096, U512, U625, U64, U65536, U8, U8192,
};
use ssz_types::typenum::{U17, U9};
use std::fmt::{self, Debug};
@@ -109,7 +108,6 @@ pub trait EthSpec:
/*
* New in Deneb
*/
type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq + Unpin;
type MaxBlobCommitmentsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq + Unpin;
type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq;
type BytesPerFieldElement: Unsigned + Clone + Sync + Send + Debug + PartialEq;
@@ -280,11 +278,6 @@ pub trait EthSpec:
Self::MaxWithdrawalsPerPayload::to_usize()
}
/// Returns the `MAX_BLOBS_PER_BLOCK` constant for this specification.
fn max_blobs_per_block() -> usize {
Self::MaxBlobsPerBlock::to_usize()
}
/// Returns the `MAX_BLOB_COMMITMENTS_PER_BLOCK` constant for this specification.
fn max_blob_commitments_per_block() -> usize {
Self::MaxBlobCommitmentsPerBlock::to_usize()
@@ -415,7 +408,6 @@ impl EthSpec for MainnetEthSpec {
type GasLimitDenominator = U1024;
type MinGasLimit = U5000;
type MaxExtraDataBytes = U32;
type MaxBlobsPerBlock = U6;
type MaxBlobCommitmentsPerBlock = U4096;
type BytesPerFieldElement = U32;
type FieldElementsPerBlob = U4096;
@@ -498,7 +490,6 @@ impl EthSpec for MinimalEthSpec {
MinGasLimit,
MaxExtraDataBytes,
MaxBlsToExecutionChanges,
MaxBlobsPerBlock,
BytesPerFieldElement,
PendingBalanceDepositsLimit,
MaxConsolidationRequestsPerPayload,
@@ -551,7 +542,6 @@ impl EthSpec for GnosisEthSpec {
type SlotsPerEth1VotingPeriod = U1024; // 64 epochs * 16 slots per epoch
type MaxBlsToExecutionChanges = U16;
type MaxWithdrawalsPerPayload = U8;
type MaxBlobsPerBlock = U6;
type MaxBlobCommitmentsPerBlock = U4096;
type FieldElementsPerBlob = U4096;
type BytesPerFieldElement = U32;

4
consensus/types/src/lib.rs
View File

@@ -138,7 +138,9 @@ pub use crate::beacon_block_body::{
pub use crate::beacon_block_header::BeaconBlockHeader;
pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee};
pub use crate::beacon_state::{Error as BeaconStateError, *};
pub use crate::blob_sidecar::{BlobIdentifier, BlobSidecar, BlobSidecarList, BlobsList};
pub use crate::blob_sidecar::{
BlobIdentifier, BlobSidecar, BlobSidecarList, BlobSidecarVec, BlobsList,
};
pub use crate::bls_to_execution_change::BlsToExecutionChange;
pub use crate::chain_spec::{ChainSpec, Config, Domain};
pub use crate::checkpoint::Checkpoint;

3
consensus/types/src/preset.rs
View File

@@ -208,8 +208,6 @@ impl CapellaPreset {
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
#[serde(rename_all = "UPPERCASE")]
pub struct DenebPreset {
#[serde(with = "serde_utils::quoted_u64")]
pub max_blobs_per_block: u64,
#[serde(with = "serde_utils::quoted_u64")]
pub max_blob_commitments_per_block: u64,
#[serde(with = "serde_utils::quoted_u64")]
@@ -219,7 +217,6 @@ pub struct DenebPreset {
impl DenebPreset {
pub fn from_chain_spec<E: EthSpec>(_spec: &ChainSpec) -> Self {
Self {
max_blobs_per_block: E::max_blobs_per_block() as u64,
max_blob_commitments_per_block: E::max_blob_commitments_per_block() as u64,
field_elements_per_blob: E::field_elements_per_blob() as u64,
}

61
consensus/types/src/runtime_var_list.rs
View File

@@ -214,6 +214,67 @@ where
}
}
#[derive(Clone, Debug)]
pub struct RuntimeFixedList<T> {
vec: Vec<T>,
len: usize,
}
impl<T: Clone> RuntimeFixedList<T> {
// TODO(pawan): no need to take len
pub fn new(vec: Vec<T>) -> Self {
let len = vec.len();
Self { vec, len }
}
pub fn to_vec(&self) -> Vec<T> {
self.vec.clone()
}
pub fn as_slice(&self) -> &[T] {
self.vec.as_slice()
}
pub fn len(&self) -> usize {
self.len
}
pub fn into_vec(self) -> Vec<T> {
self.vec
}
pub fn take(&mut self) -> Self {
let new = std::mem::take(&mut self.vec);
Self {
vec: new,
len: self.len,
}
}
}
impl<T> std::ops::Deref for RuntimeFixedList<T> {
type Target = [T];
fn deref(&self) -> &[T] {
&self.vec[..]
}
}
impl<T> std::ops::DerefMut for RuntimeFixedList<T> {
fn deref_mut(&mut self) -> &mut [T] {
&mut self.vec[..]
}
}
impl<'a, T> IntoIterator for &'a RuntimeFixedList<T> {
type Item = &'a T;
type IntoIter = std::slice::Iter<'a, T>;
fn into_iter(self) -> Self::IntoIter {
self.vec.iter()
}
}
#[cfg(test)]
mod test {
use super::*;