gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-11 18:04:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make max_blobs_per_block a config parameter (#6329)

Browse Source 
* First pass

* Add restrictions to RuntimeVariableList api

* Use empty_uninitialized and fix warnings

* Fix some todos

* Merge branch 'unstable' into max-blobs-preset

* Fix take impl on RuntimeFixedList

* cleanup

* Fix test compilations

* Fix some more tests

* Fix test from unstable

* Merge branch 'unstable' into max-blobs-preset

* Merge remote-tracking branch 'origin/unstable' into max-blobs-preset

* Remove footgun function

* Minor simplifications

* Move from preset to config

* Fix typo

* Revert "Remove footgun function"

This reverts commit de01f923c7.

* Try fixing tests

* Thread through ChainSpec

* Fix release tests

* Move RuntimeFixedVector into module and rename

* Add test

* Remove empty RuntimeVarList awefullness

* Fix tests

* Simplify BlobSidecarListFromRoot

* Merge remote-tracking branch 'origin/unstable' into max-blobs-preset

* Bump quota to account for new target (6)

* Remove clone

* Fix issue from review

* Try to remove ugliness

* Merge branch 'unstable' into max-blobs-preset

* Fix max value

* Fix doctest

* Fix formatting

* Fix max check

* Delete hardcoded max_blobs_per_block in RPC limits

* Merge remote-tracking branch 'origin/unstable' into max-blobs-preset
This commit is contained in:
Pawan Dhananjay
2025-01-10 12:04:58 +05:30
committed by GitHub
parent ecdf2d891f
commit 05727290fb
61 changed files with 655 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs
View File

@@ -10,13 +10,12 @@ use crate::BeaconChainTypes;
use lru::LruCache;
use parking_lot::RwLock;
use slog::{debug, Logger};
use ssz_types::FixedVector;
use std::num::NonZeroUsize;
use std::sync::Arc;
use types::blob_sidecar::BlobIdentifier;
use types::{
BlobSidecar, ChainSpec, ColumnIndex, DataColumnIdentifier, DataColumnSidecar, Epoch, EthSpec,
Hash256, SignedBeaconBlock,
Hash256, RuntimeFixedVector, RuntimeVariableList, SignedBeaconBlock,
};
/// This represents the components of a partially available block
@@ -28,7 +27,7 @@ use types::{
#[derive(Clone)]
pub struct PendingComponents<E: EthSpec> {
pub block_root: Hash256,
pub verified_blobs: FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock>,
pub verified_blobs: RuntimeFixedVector<Option<KzgVerifiedBlob<E>>>,
pub verified_data_columns: Vec<KzgVerifiedCustodyDataColumn<E>>,
pub executed_block: Option<DietAvailabilityPendingExecutedBlock<E>>,
pub reconstruction_started: bool,
@@ -41,9 +40,7 @@ impl<E: EthSpec> PendingComponents<E> {
}
/// Returns an immutable reference to the fixed vector of cached blobs.
pub fn get_cached_blobs(
&self,
) -> &FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock> {
pub fn get_cached_blobs(&self) -> &RuntimeFixedVector<Option<KzgVerifiedBlob<E>>> {
&self.verified_blobs
}
@@ -64,9 +61,7 @@ impl<E: EthSpec> PendingComponents<E> {
}
/// Returns a mutable reference to the fixed vector of cached blobs.
pub fn get_cached_blobs_mut(
&mut self,
) -> &mut FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock> {
pub fn get_cached_blobs_mut(&mut self) -> &mut RuntimeFixedVector<Option<KzgVerifiedBlob<E>>> {
&mut self.verified_blobs
}
@@ -138,10 +133,7 @@ impl<E: EthSpec> PendingComponents<E> {
/// Blobs are only inserted if:
/// 1. The blob entry at the index is empty and no block exists.
/// 2. The block exists and its commitment matches the blob's commitment.
pub fn merge_blobs(
&mut self,
blobs: FixedVector<Option<KzgVerifiedBlob<E>>, E::MaxBlobsPerBlock>,
) {
pub fn merge_blobs(&mut self, blobs: RuntimeFixedVector<Option<KzgVerifiedBlob<E>>>) {
for (index, blob) in blobs.iter().cloned().enumerate() {
let Some(blob) = blob else { continue };
self.merge_single_blob(index, blob);
@@ -185,7 +177,7 @@ impl<E: EthSpec> PendingComponents<E> {
/// Blobs that don't match the new block's commitments are evicted.
pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock<E>) {
self.insert_block(block);
let reinsert = std::mem::take(self.get_cached_blobs_mut());
let reinsert = self.get_cached_blobs_mut().take();
self.merge_blobs(reinsert);
}
@@ -237,10 +229,10 @@ impl<E: EthSpec> PendingComponents<E> {
}
/// Returns an empty `PendingComponents` object with the given block root.
pub fn empty(block_root: Hash256) -> Self {
pub fn empty(block_root: Hash256, max_len: usize) -> Self {
Self {
block_root,
verified_blobs: FixedVector::default(),
verified_blobs: RuntimeFixedVector::new(vec![None; max_len]),
verified_data_columns: vec![],
executed_block: None,
reconstruction_started: false,
@@ -299,7 +291,11 @@ impl<E: EthSpec> PendingComponents<E> {
else {
return Err(AvailabilityCheckError::Unexpected);
};
(Some(verified_blobs), None)
let max_len = spec.max_blobs_per_block(diet_executed_block.as_block().epoch()) as usize;
(
Some(RuntimeVariableList::new(verified_blobs, max_len)?),
None,
)
};
let executed_block = recover(diet_executed_block)?;
@@ -341,10 +337,7 @@ impl<E: EthSpec> PendingComponents<E> {
}
if let Some(kzg_verified_data_column) = self.verified_data_columns.first() {
let epoch = kzg_verified_data_column
.as_data_column()
.slot()
.epoch(E::slots_per_epoch());
let epoch = kzg_verified_data_column.as_data_column().epoch();
return Some(epoch);
}
@@ -457,7 +450,18 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
kzg_verified_blobs: I,
log: &Logger,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
let mut fixed_blobs = FixedVector::default();
let mut kzg_verified_blobs = kzg_verified_blobs.into_iter().peekable();
let Some(epoch) = kzg_verified_blobs
.peek()
.map(|verified_blob| verified_blob.as_blob().epoch())
else {
// Verified blobs list should be non-empty.
return Err(AvailabilityCheckError::Unexpected);
};
let mut fixed_blobs =
RuntimeFixedVector::new(vec![None; self.spec.max_blobs_per_block(epoch) as usize]);
for blob in kzg_verified_blobs {
if let Some(blob_opt) = fixed_blobs.get_mut(blob.blob_index() as usize) {
@@ -471,7 +475,9 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
let mut pending_components = write_lock
.pop_entry(&block_root)
.map(|(_, v)| v)
.unwrap_or_else(|| PendingComponents::empty(block_root));
.unwrap_or_else(|| {
PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize)
});
// Merge in the blobs.
pending_components.merge_blobs(fixed_blobs);
@@ -498,13 +504,24 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
kzg_verified_data_columns: I,
log: &Logger,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
let mut kzg_verified_data_columns = kzg_verified_data_columns.into_iter().peekable();
let Some(epoch) = kzg_verified_data_columns
.peek()
.map(|verified_blob| verified_blob.as_data_column().epoch())
else {
// Verified data_columns list should be non-empty.
return Err(AvailabilityCheckError::Unexpected);
};
let mut write_lock = self.critical.write();
// Grab existing entry or create a new entry.
let mut pending_components = write_lock
.pop_entry(&block_root)
.map(|(_, v)| v)
.unwrap_or_else(|| PendingComponents::empty(block_root));
.unwrap_or_else(|| {
PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize)
});
// Merge in the data columns.
pending_components.merge_data_columns(kzg_verified_data_columns)?;
@@ -581,6 +598,7 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
log: &Logger,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
let mut write_lock = self.critical.write();
let epoch = executed_block.as_block().epoch();
let block_root = executed_block.import_data.block_root;
// register the block to get the diet block
@@ -592,7 +610,9 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
let mut pending_components = write_lock
.pop_entry(&block_root)
.map(|(_, v)| v)
.unwrap_or_else(|| PendingComponents::empty(block_root));
.unwrap_or_else(|| {
PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize)
});
// Merge in the block.
pending_components.merge_block(diet_executed_block);
@@ -812,7 +832,8 @@ mod test {
info!(log, "done printing kzg commitments");
let gossip_verified_blobs = if let Some((kzg_proofs, blobs)) = maybe_blobs {
let sidecars = BlobSidecar::build_sidecars(blobs, &block, kzg_proofs).unwrap();
let sidecars =
BlobSidecar::build_sidecars(blobs, &block, kzg_proofs, &chain.spec).unwrap();
Vec::from(sidecars)
.into_iter()
.map(|sidecar| {
@@ -945,6 +966,8 @@ mod test {
assert_eq!(cache.critical.read().len(), 1);
}
}
// remove the blob to simulate successful import
cache.remove_pending_components(root);
assert!(
cache.critical.read().is_empty(),
"cache should be empty now that all components available"
@@ -1125,7 +1148,7 @@ mod pending_components_tests {
use super::*;
use crate::block_verification_types::BlockImportData;
use crate::eth1_finalization_cache::Eth1FinalizationData;
use crate::test_utils::{generate_rand_block_and_blobs, NumBlobs};
use crate::test_utils::{generate_rand_block_and_blobs, test_spec, NumBlobs};
use crate::PayloadVerificationOutcome;
use fork_choice::PayloadVerificationStatus;
use kzg::KzgCommitment;
@@ -1141,15 +1164,19 @@ mod pending_components_tests {
type Setup<E> = (
SignedBeaconBlock<E>,
FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>>,
RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>>,
usize,
);
pub fn pre_setup() -> Setup<E> {
let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64);
let spec = test_spec::<E>();
let (block, blobs_vec) =
generate_rand_block_and_blobs::<E>(ForkName::Deneb, NumBlobs::Random, &mut rng);
let mut blobs: FixedVector<_, <E as EthSpec>::MaxBlobsPerBlock> = FixedVector::default();
generate_rand_block_and_blobs::<E>(ForkName::Deneb, NumBlobs::Random, &mut rng, &spec);
let max_len = spec.max_blobs_per_block(block.epoch()) as usize;
let mut blobs: RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>> =
RuntimeFixedVector::default(max_len);
for blob in blobs_vec {
if let Some(b) = blobs.get_mut(blob.index as usize) {
@@ -1157,10 +1184,8 @@ mod pending_components_tests {
}
}
let mut invalid_blobs: FixedVector<
Option<Arc<BlobSidecar<E>>>,
<E as EthSpec>::MaxBlobsPerBlock,
> = FixedVector::default();
let mut invalid_blobs: RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>> =
RuntimeFixedVector::default(max_len);
for (index, blob) in blobs.iter().enumerate() {
if let Some(invalid_blob) = blob {
let mut blob_copy = invalid_blob.as_ref().clone();
@@ -1169,21 +1194,21 @@ mod pending_components_tests {
}
}
(block, blobs, invalid_blobs)
(block, blobs, invalid_blobs, max_len)
}
type PendingComponentsSetup<E> = (
DietAvailabilityPendingExecutedBlock<E>,
FixedVector<Option<KzgVerifiedBlob<E>>, <E as EthSpec>::MaxBlobsPerBlock>,
FixedVector<Option<KzgVerifiedBlob<E>>, <E as EthSpec>::MaxBlobsPerBlock>,
RuntimeFixedVector<Option<KzgVerifiedBlob<E>>>,
RuntimeFixedVector<Option<KzgVerifiedBlob<E>>>,
);
pub fn setup_pending_components(
block: SignedBeaconBlock<E>,
valid_blobs: FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
invalid_blobs: FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
valid_blobs: RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>>,
invalid_blobs: RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>>,
) -> PendingComponentsSetup<E> {
let blobs = FixedVector::from(
let blobs = RuntimeFixedVector::new(
valid_blobs
.iter()
.map(|blob_opt| {
@@ -1193,7 +1218,7 @@ mod pending_components_tests {
})
.collect::<Vec<_>>(),
);
let invalid_blobs = FixedVector::from(
let invalid_blobs = RuntimeFixedVector::new(
invalid_blobs
.iter()
.map(|blob_opt| {
@@ -1225,10 +1250,10 @@ mod pending_components_tests {
(block.into(), blobs, invalid_blobs)
}
pub fn assert_cache_consistent(cache: PendingComponents<E>) {
pub fn assert_cache_consistent(cache: PendingComponents<E>, max_len: usize) {
if let Some(cached_block) = cache.get_cached_block() {
let cached_block_commitments = cached_block.get_commitments();
for index in 0..E::max_blobs_per_block() {
for index in 0..max_len {
let block_commitment = cached_block_commitments.get(index).copied();
let blob_commitment_opt = cache.get_cached_blobs().get(index).unwrap();
let blob_commitment = blob_commitment_opt.as_ref().map(|b| *b.get_commitment());
@@ -1247,40 +1272,40 @@ mod pending_components_tests {
#[test]
fn valid_block_invalid_blobs_valid_blobs() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_block(block_commitments);
cache.merge_blobs(random_blobs);
cache.merge_blobs(blobs);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
#[test]
fn invalid_blobs_block_valid_blobs() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_blobs(random_blobs);
cache.merge_block(block_commitments);
cache.merge_blobs(blobs);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
#[test]
fn invalid_blobs_valid_blobs_block() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_blobs(random_blobs);
cache.merge_blobs(blobs);
cache.merge_block(block_commitments);
@@ -1290,46 +1315,46 @@ mod pending_components_tests {
#[test]
fn block_valid_blobs_invalid_blobs() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_block(block_commitments);
cache.merge_blobs(blobs);
cache.merge_blobs(random_blobs);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
#[test]
fn valid_blobs_block_invalid_blobs() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_blobs(blobs);
cache.merge_block(block_commitments);
cache.merge_blobs(random_blobs);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
#[test]
fn valid_blobs_invalid_blobs_block() {
let (block_commitments, blobs, random_blobs) = pre_setup();
let (block_commitments, blobs, random_blobs, max_len) = pre_setup();
let (block_commitments, blobs, random_blobs) =
setup_pending_components(block_commitments, blobs, random_blobs);
let block_root = Hash256::zero();
let mut cache = <PendingComponents<E>>::empty(block_root);
let mut cache = <PendingComponents<E>>::empty(block_root, max_len);
cache.merge_blobs(blobs);
cache.merge_blobs(random_blobs);
cache.merge_block(block_commitments);
assert_cache_consistent(cache);
assert_cache_consistent(cache, max_len);
}
}