gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-11 18:04:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improving blob propagation post-PeerDAS with Decentralized Blob Building (#6268)

Browse Source 
* Get blobs from EL.

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

* Avoid cloning blobs after fetching blobs.

* Address review comments and refactor code.

* Fix lint.

* Move blob computation metric to the right spot.

* Merge branch 'unstable' into das-fetch-blobs

* Merge branch 'unstable' into das-fetch-blobs

# Conflicts:
#	beacon_node/beacon_chain/src/beacon_chain.rs
#	beacon_node/beacon_chain/src/block_verification.rs
#	beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs

* Merge branch 'unstable' into das-fetch-blobs

# Conflicts:
#	beacon_node/beacon_chain/src/beacon_chain.rs

* Gradual publication of data columns for supernodes.

* Recompute head after importing block with blobs from the EL.

* Fix lint

* Merge branch 'unstable' into das-fetch-blobs

* Use blocking task instead of async when computing cells.

* Merge branch 'das-fetch-blobs' of github.com:jimmygchen/lighthouse into das-fetch-blobs

* Merge remote-tracking branch 'origin/unstable' into das-fetch-blobs

* Fix semantic conflicts

* Downgrade error log.

* Merge branch 'unstable' into das-fetch-blobs

# Conflicts:
#	beacon_node/beacon_chain/src/data_availability_checker.rs
#	beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs
#	beacon_node/execution_layer/src/engine_api.rs
#	beacon_node/execution_layer/src/engine_api/json_structures.rs
#	beacon_node/network/src/network_beacon_processor/gossip_methods.rs
#	beacon_node/network/src/network_beacon_processor/mod.rs
#	beacon_node/network/src/network_beacon_processor/sync_methods.rs

* Merge branch 'unstable' into das-fetch-blobs

* Publish block without waiting for blob and column proof computation.

* Address review comments and refactor.

* Merge branch 'unstable' into das-fetch-blobs

* Fix test and docs.

* Comment cleanups.

* Merge branch 'unstable' into das-fetch-blobs

* Address review comments and cleanup

* Address review comments and cleanup

* Refactor to de-duplicate gradual publication logic.

* Add more logging.

* Merge remote-tracking branch 'origin/unstable' into das-fetch-blobs

# Conflicts:
#	Cargo.lock

* Fix incorrect comparison on `num_fetched_blobs`.

* Implement gradual blob publication.

* Merge branch 'unstable' into das-fetch-blobs

* Inline `publish_fn`.

* Merge branch 'das-fetch-blobs' of github.com:jimmygchen/lighthouse into das-fetch-blobs

* Gossip verify blobs before publishing

* Avoid queries for 0 blobs and error for duplicates

* Gossip verified engine blob before processing them, and use observe cache to detect duplicates before publishing.

* Merge branch 'das-fetch-blobs' of github.com:jimmygchen/lighthouse into das-fetch-blobs

# Conflicts:
#	beacon_node/network/src/network_beacon_processor/mod.rs

* Merge branch 'unstable' into das-fetch-blobs

* Fix invalid commitment inclusion proofs in blob sidecars created from EL blobs.

* Only publish EL blobs triggered from gossip block, and not RPC block.

* Downgrade gossip blob log to `debug`.

* Merge branch 'unstable' into das-fetch-blobs

* Merge branch 'unstable' into das-fetch-blobs

* Grammar
This commit is contained in:
Jimmy Chen
2024-11-15 10:34:13 +07:00
committed by GitHub
parent 8e95024945
commit 5f053b0b6d
36 changed files with 1660 additions and 613 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
beacon_node/beacon_chain/src/data_availability_checker/error.rs
View File

@@ -10,7 +10,6 @@ pub enum Error {
blob_commitment: KzgCommitment,
block_commitment: KzgCommitment,
},
UnableToDetermineImportRequirement,
Unexpected,
SszTypes(ssz_types::Error),
MissingBlobs,
@@ -44,7 +43,6 @@ impl Error {
| Error::Unexpected
| Error::ParentStateMissing(_)
| Error::BlockReplayError(_)
| Error::UnableToDetermineImportRequirement
| Error::RebuildingStateCaches(_)
| Error::SlotClockError => ErrorCategory::Internal,
Error::InvalidBlobs { .. }

174
beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs
View File

@@ -10,7 +10,7 @@ use crate::BeaconChainTypes;
use lru::LruCache;
use parking_lot::RwLock;
use slog::{debug, Logger};
use ssz_types::{FixedVector, VariableList};
use ssz_types::FixedVector;
use std::num::NonZeroUsize;
use std::sync::Arc;
use types::blob_sidecar::BlobIdentifier;
@@ -34,11 +34,6 @@ pub struct PendingComponents<E: EthSpec> {
pub reconstruction_started: bool,
}
pub enum BlockImportRequirement {
AllBlobs,
ColumnSampling(usize),
}
impl<E: EthSpec> PendingComponents<E> {
/// Returns an immutable reference to the cached block.
pub fn get_cached_block(&self) -> &Option<DietAvailabilityPendingExecutedBlock<E>> {
@@ -199,63 +194,49 @@ impl<E: EthSpec> PendingComponents<E> {
///
/// Returns `true` if both the block exists and the number of received blobs / custody columns
/// matches the number of expected blobs / custody columns.
pub fn is_available(
&self,
block_import_requirement: &BlockImportRequirement,
log: &Logger,
) -> bool {
pub fn is_available(&self, custody_column_count: usize, log: &Logger) -> bool {
let block_kzg_commitments_count_opt = self.block_kzg_commitments_count();
let expected_blobs_msg = block_kzg_commitments_count_opt
.as_ref()
.map(|num| num.to_string())
.unwrap_or("unknown".to_string());
match block_import_requirement {
BlockImportRequirement::AllBlobs => {
let received_blobs = self.num_received_blobs();
let expected_blobs_msg = block_kzg_commitments_count_opt
.as_ref()
.map(|num| num.to_string())
.unwrap_or("unknown".to_string());
debug!(log,
"Component(s) added to data availability checker";
"block_root" => ?self.block_root,
"received_block" => block_kzg_commitments_count_opt.is_some(),
"received_blobs" => received_blobs,
"expected_blobs" => expected_blobs_msg,
);
block_kzg_commitments_count_opt.map_or(false, |num_expected_blobs| {
num_expected_blobs == received_blobs
})
// No data columns when there are 0 blobs
let expected_columns_opt = block_kzg_commitments_count_opt.map(|blob_count| {
if blob_count > 0 {
custody_column_count
} else {
0
}
BlockImportRequirement::ColumnSampling(num_expected_columns) => {
// No data columns when there are 0 blobs
let expected_columns_opt = block_kzg_commitments_count_opt.map(|blob_count| {
if blob_count > 0 {
*num_expected_columns
} else {
0
}
});
});
let expected_columns_msg = expected_columns_opt
.as_ref()
.map(|num| num.to_string())
.unwrap_or("unknown".to_string());
let expected_columns_msg = expected_columns_opt
.as_ref()
.map(|num| num.to_string())
.unwrap_or("unknown".to_string());
let num_received_blobs = self.num_received_blobs();
let num_received_columns = self.num_received_data_columns();
let num_received_columns = self.num_received_data_columns();
debug!(
log,
"Component(s) added to data availability checker";
"block_root" => ?self.block_root,
"received_blobs" => num_received_blobs,
"expected_blobs" => expected_blobs_msg,
"received_columns" => num_received_columns,
"expected_columns" => expected_columns_msg,
);
debug!(log,
"Component(s) added to data availability checker";
"block_root" => ?self.block_root,
"received_block" => block_kzg_commitments_count_opt.is_some(),
"received_columns" => num_received_columns,
"expected_columns" => expected_columns_msg,
);
let all_blobs_received = block_kzg_commitments_count_opt
.map_or(false, |num_expected_blobs| {
num_expected_blobs == num_received_blobs
});
expected_columns_opt.map_or(false, |num_expected_columns| {
num_expected_columns == num_received_columns
})
}
}
let all_columns_received = expected_columns_opt.map_or(false, |num_expected_columns| {
num_expected_columns == num_received_columns
});
all_blobs_received || all_columns_received
}
/// Returns an empty `PendingComponents` object with the given block root.
@@ -277,7 +258,6 @@ impl<E: EthSpec> PendingComponents<E> {
/// reconstructed from disk. Ensure you are not holding any write locks while calling this.
pub fn make_available<R>(
self,
block_import_requirement: BlockImportRequirement,
spec: &Arc<ChainSpec>,
recover: R,
) -> Result<Availability<E>, AvailabilityCheckError>
@@ -304,26 +284,25 @@ impl<E: EthSpec> PendingComponents<E> {
return Err(AvailabilityCheckError::Unexpected);
};
let (blobs, data_columns) = match block_import_requirement {
BlockImportRequirement::AllBlobs => {
let num_blobs_expected = diet_executed_block.num_blobs_expected();
let Some(verified_blobs) = verified_blobs
.into_iter()
.map(|b| b.map(|b| b.to_blob()))
.take(num_blobs_expected)
.collect::<Option<Vec<_>>>()
else {
return Err(AvailabilityCheckError::Unexpected);
};
(Some(VariableList::new(verified_blobs)?), None)
}
BlockImportRequirement::ColumnSampling(_) => {
let verified_data_columns = verified_data_columns
.into_iter()
.map(|d| d.into_inner())
.collect();
(None, Some(verified_data_columns))
}
let is_peer_das_enabled = spec.is_peer_das_enabled_for_epoch(diet_executed_block.epoch());
let (blobs, data_columns) = if is_peer_das_enabled {
let data_columns = verified_data_columns
.into_iter()
.map(|d| d.into_inner())
.collect::<Vec<_>>();
(None, Some(data_columns))
} else {
let num_blobs_expected = diet_executed_block.num_blobs_expected();
let Some(verified_blobs) = verified_blobs
.into_iter()
.map(|b| b.map(|b| b.to_blob()))
.take(num_blobs_expected)
.collect::<Option<Vec<_>>>()
.map(Into::into)
else {
return Err(AvailabilityCheckError::Unexpected);
};
(Some(verified_blobs), None)
};
let executed_block = recover(diet_executed_block)?;
@@ -475,24 +454,9 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
f(self.critical.read().peek(block_root))
}
fn block_import_requirement(
&self,
epoch: Epoch,
) -> Result<BlockImportRequirement, AvailabilityCheckError> {
let peer_das_enabled = self.spec.is_peer_das_enabled_for_epoch(epoch);
if peer_das_enabled {
Ok(BlockImportRequirement::ColumnSampling(
self.sampling_column_count,
))
} else {
Ok(BlockImportRequirement::AllBlobs)
}
}
pub fn put_kzg_verified_blobs<I: IntoIterator<Item = KzgVerifiedBlob<T::EthSpec>>>(
&self,
block_root: Hash256,
epoch: Epoch,
kzg_verified_blobs: I,
log: &Logger,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
@@ -515,12 +479,11 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
// Merge in the blobs.
pending_components.merge_blobs(fixed_blobs);
let block_import_requirement = self.block_import_requirement(epoch)?;
if pending_components.is_available(&block_import_requirement, log) {
if pending_components.is_available(self.sampling_column_count, log) {
write_lock.put(block_root, pending_components.clone());
// No need to hold the write lock anymore
drop(write_lock);
pending_components.make_available(block_import_requirement, &self.spec, |diet_block| {
pending_components.make_available(&self.spec, |diet_block| {
self.state_cache.recover_pending_executed_block(diet_block)
})
} else {
@@ -535,7 +498,6 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
>(
&self,
block_root: Hash256,
epoch: Epoch,
kzg_verified_data_columns: I,
log: &Logger,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
@@ -550,13 +512,11 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
// Merge in the data columns.
pending_components.merge_data_columns(kzg_verified_data_columns)?;
let block_import_requirement = self.block_import_requirement(epoch)?;
if pending_components.is_available(&block_import_requirement, log) {
if pending_components.is_available(self.sampling_column_count, log) {
write_lock.put(block_root, pending_components.clone());
// No need to hold the write lock anymore
drop(write_lock);
pending_components.make_available(block_import_requirement, &self.spec, |diet_block| {
pending_components.make_available(&self.spec, |diet_block| {
self.state_cache.recover_pending_executed_block(diet_block)
})
} else {
@@ -625,7 +585,6 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
let mut write_lock = self.critical.write();
let block_root = executed_block.import_data.block_root;
let epoch = executed_block.block.epoch();
// register the block to get the diet block
let diet_executed_block = self
@@ -642,12 +601,11 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
pending_components.merge_block(diet_executed_block);
// Check if we have all components and entire set is consistent.
let block_import_requirement = self.block_import_requirement(epoch)?;
if pending_components.is_available(&block_import_requirement, log) {
if pending_components.is_available(self.sampling_column_count, log) {
write_lock.put(block_root, pending_components.clone());
// No need to hold the write lock anymore
drop(write_lock);
pending_components.make_available(block_import_requirement, &self.spec, |diet_block| {
pending_components.make_available(&self.spec, |diet_block| {
self.state_cache.recover_pending_executed_block(diet_block)
})
} else {
@@ -703,6 +661,7 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
#[cfg(test)]
mod test {
use super::*;
use crate::{
blob_verification::GossipVerifiedBlob,
block_verification::PayloadVerificationOutcome,
@@ -712,6 +671,7 @@ mod test {
test_utils::{BaseHarnessType, BeaconChainHarness, DiskHarnessType},
};
use fork_choice::PayloadVerificationStatus;
use logging::test_logger;
use slog::{info, Logger};
use state_processing::ConsensusContext;
@@ -931,7 +891,6 @@ mod test {
let (pending_block, blobs) = availability_pending_block(&harness).await;
let root = pending_block.import_data.block_root;
let epoch = pending_block.block.epoch();
let blobs_expected = pending_block.num_blobs_expected();
assert_eq!(
@@ -980,7 +939,7 @@ mod test {
for (blob_index, gossip_blob) in blobs.into_iter().enumerate() {
kzg_verified_blobs.push(gossip_blob.into_inner());
let availability = cache
.put_kzg_verified_blobs(root, epoch, kzg_verified_blobs.clone(), harness.logger())
.put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), harness.logger())
.expect("should put blob");
if blob_index == blobs_expected - 1 {
assert!(matches!(availability, Availability::Available(_)));
@@ -1002,12 +961,11 @@ mod test {
"should have expected number of blobs"
);
let root = pending_block.import_data.block_root;
let epoch = pending_block.block.epoch();
let mut kzg_verified_blobs = vec![];
for gossip_blob in blobs {
kzg_verified_blobs.push(gossip_blob.into_inner());
let availability = cache
.put_kzg_verified_blobs(root, epoch, kzg_verified_blobs.clone(), harness.logger())
.put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), harness.logger())
.expect("should put blob");
assert_eq!(
availability,

5
beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs
View File

@@ -57,6 +57,11 @@ impl<E: EthSpec> DietAvailabilityPendingExecutedBlock<E> {
.cloned()
.unwrap_or_default()
}
/// Returns the epoch corresponding to `self.slot()`.
pub fn epoch(&self) -> Epoch {
self.block.slot().epoch(E::slots_per_epoch())
}
}
/// This LRU cache holds BeaconStates used for block import. If the cache overflows,