gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-30 20:57:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

DA cache updated

Browse Source
This commit is contained in:
Eitan Seri- Levi
2026-01-30 10:59:43 -08:00
parent d047ace41f
commit 78c61a0621
8 changed files with 139 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
beacon_node/beacon_chain/src/data_availability_checker.rs
View File

@@ -5,7 +5,7 @@ use crate::block_verification_types::{AvailabilityPendingExecutedBlock, Availabl
use crate::data_availability_checker::overflow_lru_cache::{
DataAvailabilityCheckerInner, ReconstructColumnsDecision,
};
use crate::data_availability_router::DataColumnCache;
use crate::data_availability_router::AvailabilityCache;
use crate::{
BeaconChain, BeaconChainTypes, BeaconStore, BlockProcessStatus, CustodyContext, metrics,
};
@@ -366,7 +366,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
}
}
impl<T: BeaconChainTypes> DataColumnCache<T> for DataAvailabilityChecker<T> {
impl<T: BeaconChainTypes> AvailabilityCache<T> for DataAvailabilityChecker<T> {
type Availability = Availability<T::EthSpec>;
type ReconstructionResult = DataColumnReconstructionResult<T::EthSpec>;
@@ -559,6 +559,18 @@ impl<T: BeaconChainTypes> DataColumnCache<T> for DataAvailabilityChecker<T> {
))
})
}
/// Verifies KZG commitments for data columns.
fn verify_kzg_for_data_columns(
&self,
data_columns: &DataColumnSidecarList<T::EthSpec>,
) -> Result<(), AvailabilityCheckError> {
if !data_columns.is_empty() {
verify_kzg_for_data_column_list(data_columns.iter(), &self.kzg)
.map_err(AvailabilityCheckError::InvalidColumn)?;
}
Ok(())
}
}
/// Helper struct to group data availability checker metrics.
@@ -587,6 +599,7 @@ pub fn start_availability_cache_maintenance_service<T: BeaconChainTypes>(
}
}
// TODO(gloas) we can shut down this service once we reach the gloas fork epoch
async fn availability_cache_maintenance_service<T: BeaconChainTypes>(
chain: Arc<BeaconChain<T>>,
overflow_cache: Arc<DataAvailabilityCheckerInner<T>>,

28
beacon_node/beacon_chain/src/data_availability_checker_v2.rs
View File

@@ -3,7 +3,7 @@ use crate::data_availability_checker_v2::overflow_lru_cache::{
};
use crate::data_availability_checker::AvailabilityCheckError;
use crate::data_availability_router::DataColumnCache;
use crate::data_availability_router::AvailabilityCache;
use crate::{BeaconChain, BeaconChainTypes, CustodyContext, metrics};
use kzg::Kzg;
use slot_clock::SlotClock;
@@ -86,7 +86,7 @@ pub struct DataAvailabilityChecker<T: BeaconChainTypes> {
spec: Arc<ChainSpec>,
}
impl<T: BeaconChainTypes> DataColumnCache<T> for DataAvailabilityChecker<T> {
impl<T: BeaconChainTypes> AvailabilityCache<T> for DataAvailabilityChecker<T> {
type Availability = Availability<T::EthSpec>;
type ReconstructionResult = DataColumnReconstructionResult<T::EthSpec>;
@@ -275,6 +275,18 @@ impl<T: BeaconChainTypes> DataColumnCache<T> for DataAvailabilityChecker<T> {
))
})
}
/// Verifies KZG commitments for data columns.
fn verify_kzg_for_data_columns(
&self,
data_columns: &DataColumnSidecarList<T::EthSpec>,
) -> Result<(), AvailabilityCheckError> {
if !data_columns.is_empty() {
verify_kzg_for_data_column_list(data_columns.iter(), &self.kzg)
.map_err(AvailabilityCheckError::InvalidColumn)?;
}
Ok(())
}
}
impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
@@ -311,18 +323,6 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
self.availability_cache.put_block(block_root, block)
}
/// Verifies kzg commitments for data columns.
pub fn verify_kzg_for_data_columns(
&self,
data_columns: &DataColumnSidecarList<T::EthSpec>,
) -> Result<(), AvailabilityCheckError> {
if !data_columns.is_empty() {
verify_kzg_for_data_column_list(data_columns.iter(), &self.kzg)
.map_err(AvailabilityCheckError::InvalidColumn)?;
}
Ok(())
}
/// Collects metrics from the data availability checker.
pub fn metrics(&self) -> DataAvailabilityCheckerMetrics {
DataAvailabilityCheckerMetrics {

62
beacon_node/beacon_chain/src/data_availability_checker_v2/overflow_lru_cache.rs
View File

@@ -70,9 +70,10 @@ impl<E: EthSpec> PendingComponents<E> {
/// Returns the number of blobs expected for this block by reading the bid's kzg commitments.
/// Returns an error if the block is not cached or not a Gloas block.
pub fn num_blobs_expected(&self) -> Result<usize, AvailabilityCheckError> {
let block = self.block.as_ref().ok_or_else(|| {
AvailabilityCheckError::Unexpected("No block available".to_string())
})?;
let block = self
.block
.as_ref()
.ok_or_else(|| AvailabilityCheckError::Unexpected("No block available".to_string()))?;
let bid = block
.message()
@@ -167,10 +168,8 @@ impl<E: EthSpec> PendingComponents<E> {
}
pub fn status_str(&self, num_expected_columns: usize) -> String {
let block_status = if self.block.is_some() { "yes" } else { "no" };
format!(
"block {} data_columns {}/{}",
block_status,
"data_columns {}/{}",
self.verified_data_columns.len(),
num_expected_columns
)
@@ -475,7 +474,7 @@ mod pending_components_tests {
let components = PendingComponents::<E>::empty(block_root);
let status = components.status_str(10);
assert_eq!(status, "block no data_columns 0/10");
assert_eq!(status, "data_columns 0/10");
}
#[test]
@@ -510,20 +509,20 @@ mod data_availability_checker_tests {
use crate::data_column_verification::{KzgVerifiedCustodyDataColumn, KzgVerifiedDataColumn};
use crate::test_utils::{
generate_data_column_indices_rand_order, test_spec, NumBlobs,
generate_rand_block_and_data_columns,
NumBlobs, generate_data_column_indices_rand_order, generate_rand_block_and_data_columns,
test_spec,
};
use crate::{
custody_context::NodeCustodyType,
test_utils::{BeaconChainHarness, DiskHarnessType},
};
use logging::create_test_tracing_subscriber;
use store::{HotColdDB, StoreConfig, database::interface::BeaconNodeBackend};
use tempfile::{TempDir, tempdir};
use types::{ForkName, MinimalEthSpec, Slot};
use types::new_non_zero_usize;
use rand::SeedableRng;
use rand::rngs::StdRng;
use store::{HotColdDB, StoreConfig, database::interface::BeaconNodeBackend};
use tempfile::{TempDir, tempdir};
use types::new_non_zero_usize;
use types::{ForkName, MinimalEthSpec, Slot};
type E = MinimalEthSpec;
@@ -569,17 +568,12 @@ mod data_availability_checker_tests {
let chain_store = get_store_with_spec::<E>(db_path, spec.clone());
let validators_keypairs =
types::test_utils::generate_deterministic_keypairs(LOW_VALIDATOR_COUNT);
let harness = BeaconChainHarness::builder(E::default())
BeaconChainHarness::builder(E::default())
.spec(spec.clone())
.keypairs(validators_keypairs)
.fresh_disk_store(chain_store)
.mock_execution_layer()
.build();
// go to gloas slot
let gloas_fork_slot = Slot::new(0);
harness.extend_to_slot(gloas_fork_slot).await;
harness
.build()
}
async fn setup_harness_and_cache<T>(
@@ -607,8 +601,12 @@ mod data_availability_checker_tests {
&spec,
));
let cache = Arc::new(
DataAvailabilityCheckerInner::<T>::new(capacity_non_zero, custody_context, spec.clone())
.expect("should create cache"),
DataAvailabilityCheckerInner::<T>::new(
capacity_non_zero,
custody_context,
spec.clone(),
)
.expect("should create cache"),
);
(harness, cache, chain_db_path)
}
@@ -643,9 +641,8 @@ mod data_availability_checker_tests {
let mut rng = StdRng::seed_from_u64(0xDEADBEEF);
let spec = harness.spec.clone();
// Generate a block with data columns
let (_block, data_columns) = generate_rand_block_and_data_columns::<E>(
ForkName::Fulu, // Use Fulu for now as Gloas generation may not be ready
ForkName::Gloas,
NumBlobs::Number(1),
&mut rng,
&spec,
@@ -653,7 +650,6 @@ mod data_availability_checker_tests {
let block_root = Hash256::random();
// Convert to KzgVerifiedCustodyDataColumn
let verified_columns: Vec<_> = data_columns
.into_iter()
.take(1) // Just take one column for the test
@@ -693,7 +689,7 @@ mod data_availability_checker_tests {
let spec = harness.spec.clone();
let (_block, data_columns) = generate_rand_block_and_data_columns::<E>(
ForkName::Fulu,
ForkName::Gloas,
NumBlobs::Number(1),
&mut rng,
&spec,
@@ -742,7 +738,7 @@ mod data_availability_checker_tests {
let spec = harness.spec.clone();
let (_block, data_columns) = generate_rand_block_and_data_columns::<E>(
ForkName::Fulu,
ForkName::Gloas,
NumBlobs::Number(1),
&mut rng,
&spec,
@@ -782,7 +778,7 @@ mod data_availability_checker_tests {
let spec = harness.spec.clone();
let (_block, data_columns) = generate_rand_block_and_data_columns::<E>(
ForkName::Fulu,
ForkName::Gloas,
NumBlobs::Number(1),
&mut rng,
&spec,
@@ -824,7 +820,7 @@ mod data_availability_checker_tests {
let spec = harness.spec.clone();
let (_block, data_columns) = generate_rand_block_and_data_columns::<E>(
ForkName::Fulu,
ForkName::Gloas,
NumBlobs::Number(1),
&mut rng,
&spec,
@@ -876,7 +872,7 @@ mod data_availability_checker_tests {
let block_root = Hash256::random();
// Create an empty entry in the cache
let _ = cache.peek_pending_components(&block_root, |_| {});
cache.peek_pending_components(&block_root, |_| {});
// Manually insert a pending component by putting empty columns
// This will create an entry but it won't have an epoch
@@ -884,7 +880,9 @@ mod data_availability_checker_tests {
// Run maintenance with a future cutoff epoch
let cutoff_epoch = Epoch::new(100);
cache.do_maintenance(cutoff_epoch).expect("maintenance should succeed");
cache
.do_maintenance(cutoff_epoch)
.expect("maintenance should succeed");
// Cache should still be empty since we didn't add anything with an epoch
assert_eq!(cache.block_cache_size(), 0);
@@ -904,7 +902,7 @@ mod data_availability_checker_tests {
let spec = harness.spec.clone();
let (_block, data_columns) = generate_rand_block_and_data_columns::<E>(
ForkName::Fulu,
ForkName::Gloas,
NumBlobs::Number(1),
&mut rng,
&spec,

40
beacon_node/beacon_chain/src/data_availability_router.rs
View File

@@ -1,14 +1,14 @@
//! Abstraction layer for data column storage across different DA checkers.
//! Abstraction layer for data availability operations across different DA checkers.
//!
//! This module provides a unified interface for data column operations that are shared
//! This module provides a unified interface for availability operations that are shared
//! between the legacy `DataAvailabilityChecker` (v1, for blocks) and
//! `DataAvailabilityChecker` v2 (for payload envelopes after Gloas).
//!
//! ## Design
//!
//! - **Read operations**: Unified via the `DataColumnCache` trait
//! - **Write operations**: Return `AvailabilityOutcome` enum that wraps both checker types
//! - **Processing**: `BeaconChain::process_availability_outcome()` handles both cases
//! - **Unified operations**: Via the `AvailabilityCache` trait (blocks, columns, availability checks)
//! - **Fork-aware routing**: `DataAvailabilityRouter` dispatches to v1 or v2 based on slot
//! - **Processing**: `BeaconChain::process_availability_outcome()` handles both result types
//!
//! After Gloas is fully activated and v1 is deprecated, this can be deleted and we can
//! use the Gloas DA checker directly.
@@ -122,13 +122,13 @@ impl<E: EthSpec> ReconstructionOutcome<E> {
}
}
/// Trait for data column operations on availability checkers.
/// Trait for data availability operations on availability checkers.
///
/// Both `DataAvailabilityChecker` (v1) and `DataAvailabilityChecker` (v2) implement
/// this trait. The associated types differ:
/// - V1: Returns `Availability<E>` containing `AvailableExecutedBlock<E>`
/// - V2: Returns `Availability<E>` containing `(Hash256, DataColumnSidecarList<E>)` (block root + columns)
pub trait DataColumnCache<T: BeaconChainTypes>: Send + Sync {
pub trait AvailabilityCache<T: BeaconChainTypes>: Send + Sync {
/// The availability type returned by write operations.
/// V1 returns block availability, V2 returns payload availability.
type Availability;
@@ -182,24 +182,30 @@ pub trait DataColumnCache<T: BeaconChainTypes>: Send + Sync {
&self,
block_root: &Hash256,
) -> Result<Self::ReconstructionResult, AvailabilityCheckError>;
/// Verifies KZG commitments for a list of data columns.
fn verify_kzg_for_data_columns(
&self,
data_columns: &DataColumnSidecarList<T::EthSpec>,
) -> Result<(), AvailabilityCheckError>;
}
/// Router that directs data availability checker operations to the appropriate version based on fork.
///
/// This wraps both the legacy (v1) and Gloas (v2) DA checkers, providing:
/// - Unified read operations that query both checkers
/// - Unified operations that dispatch to the correct checker based on fork
/// - Fork-aware routing for write operations that return `AvailabilityOutcome`
///
/// After Gloas is fully activated and v1 is deprecated, this router can be deleted and
/// we can use the Gloas DA checker directly.
pub struct DataAvailabilityRouter<T: BeaconChainTypes, V1, V2>
where
V1: DataColumnCache<
V1: AvailabilityCache<
T,
Availability = BlockAvailability<T::EthSpec>,
ReconstructionResult = BlockReconstructionResult<T::EthSpec>,
>,
V2: DataColumnCache<
V2: AvailabilityCache<
T,
Availability = PayloadAvailability<T::EthSpec>,
ReconstructionResult = PayloadReconstructionResult<T::EthSpec>,
@@ -215,12 +221,12 @@ where
impl<T: BeaconChainTypes, V1, V2> DataAvailabilityRouter<T, V1, V2>
where
V1: DataColumnCache<
V1: AvailabilityCache<
T,
Availability = BlockAvailability<T::EthSpec>,
ReconstructionResult = BlockReconstructionResult<T::EthSpec>,
>,
V2: DataColumnCache<
V2: AvailabilityCache<
T,
Availability = PayloadAvailability<T::EthSpec>,
ReconstructionResult = PayloadReconstructionResult<T::EthSpec>,
@@ -371,18 +377,16 @@ where
}
}
/// Direct access to v1 checker (for block-specific operations).
/// Direct access to v1 checker for block execution/availability checks.
///
/// Use this for operations that are specific to the legacy block-based DA checker,
/// such as `put_executed_block`, `get_cached_block`, blob operations, etc.
/// Use this for operations that are specific to the legacy DA checker,
pub fn v1(&self) -> Arc<V1> {
self.v1.clone()
}
/// Direct access to v2 checker (for payload-specific operations).
/// Direct access to v2 checker for payload availability checks.
///
/// Use this for operations that are specific to the Gloas payload-based DA checker,
/// such as `put_executed_payload`, `get_cached_payload`, etc.
/// Use this for operations that are specific to the Gloas DA checker,
pub fn v2(&self) -> Arc<V2> {
self.v2.clone()
}

1
beacon_node/beacon_chain/src/metrics.rs
View File

@@ -1978,6 +1978,7 @@ pub fn scrape_for_metrics<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
beacon_chain.store.state_cache_len(),
);
// TODO(gloas) configure v2 metrics
let da_checker_metrics = beacon_chain.data_availability_checker.v1().metrics();
set_gauge_by_usize(

164
beacon_node/beacon_chain/src/test_utils.rs
View File

@@ -3416,45 +3416,6 @@ macro_rules! add_blob_transactions {
}};
}
macro_rules! add_blob_transactions_gloas {
($message:expr, $num_blobs:expr, $rng:expr, $fork_name:expr) => {{
let num_blobs = match $num_blobs {
NumBlobs::Random => $rng.random_range(DEFAULT_MIN_BLOBS..=DEFAULT_MAX_BLOBS),
NumBlobs::Number(n) => n,
NumBlobs::None => 0,
};
let (bundle, transactions) =
execution_layer::test_utils::generate_blobs::<E>(num_blobs, $fork_name).unwrap();
let payload = &mut $message.payload;
payload.transactions = <_>::default();
for tx in Vec::from(transactions) {
payload.transactions.push(tx).unwrap();
}
// Note: In Gloas, blob_kzg_commitments are in the bid (block body), not the payload envelope.
// The commitments are returned via the bundle for the caller to use.
bundle
}};
}
pub fn generate_rand_payload_and_columns<E: EthSpec>(
fork_name: ForkName,
num_blobs: NumBlobs,
rng: &mut impl Rng,
spec: &ChainSpec,
) -> (SignedExecutionPayloadEnvelope<E>, DataColumnSidecarList<E>) {
let mut payload = SignedExecutionPayloadEnvelope::random_for_test(rng);
let bundle = add_blob_transactions_gloas!(payload.message, num_blobs, rng, fork_name);
// In Gloas, blob_kzg_commitments are in the bid (block body), not the payload envelope.
// We pass them from the bundle to generate the data columns.
let kzg_commitments = bundle.commitments;
let data_columns = generate_data_column_sidecars_from_payload(&payload, kzg_commitments, spec);
(payload, data_columns)
}
pub fn generate_rand_block_and_blobs<E: EthSpec>(
fork_name: ForkName,
num_blobs: NumBlobs,
@@ -3475,7 +3436,26 @@ pub fn generate_rand_block_and_blobs<E: EthSpec>(
SignedBeaconBlock::Fulu(SignedBeaconBlockFulu {
ref mut message, ..
}) => add_blob_transactions!(message, FullPayloadFulu<E>, num_blobs, rng, fork_name),
// TODO(EIP-7732) Add `SignedBeaconBlock::Gloas` variant
SignedBeaconBlock::Gloas(SignedBeaconBlockGloas {
ref mut message, ..
}) => {
// For Gloas, commitments are in the bid, not directly in the body.
// BlobSidecars cannot be created for Gloas because there's no merkle proof
// from the block body to the commitments. Return early with empty blob_sidecars.
let num_blobs = match num_blobs {
NumBlobs::Random => rng.random_range(DEFAULT_MIN_BLOBS..=DEFAULT_MAX_BLOBS),
NumBlobs::Number(n) => n,
NumBlobs::None => 0,
};
let (bundle, _transactions) =
execution_layer::test_utils::generate_blobs::<E>(num_blobs, fork_name).unwrap();
message
.body
.signed_execution_payload_bid
.message
.blob_kzg_commitments = bundle.commitments.clone();
return (block, blob_sidecars);
}
_ => return (block, blob_sidecars),
};
@@ -3526,32 +3506,37 @@ pub fn generate_data_column_sidecars_from_block<E: EthSpec>(
block: &SignedBeaconBlock<E>,
spec: &ChainSpec,
) -> DataColumnSidecarList<E> {
let kzg_commitments = block.message().body().blob_kzg_commitments().unwrap();
if kzg_commitments.is_empty() {
return vec![];
}
let kzg_commitments_inclusion_proof = block
.message()
.body()
.kzg_commitments_merkle_proof()
.unwrap();
let signed_block_header = block.signed_block_header();
// load the precomputed column sidecar to avoid computing them for every block in the tests.
let template_data_columns = RuntimeVariableList::<DataColumnSidecarFulu<E>>::from_ssz_bytes(
TEST_DATA_COLUMN_SIDECARS_SSZ,
E::number_of_columns(),
)
.unwrap();
// Load the precomputed column sidecar to avoid computing them for every block in the tests.
// Then repeat the cells and proofs for every blob
if block.fork_name_unchecked().gloas_enabled() {
let template_data_columns =
RuntimeVariableList::<DataColumnSidecarGloas<E>>::from_ssz_bytes(
TEST_DATA_COLUMN_SIDECARS_SSZ,
E::number_of_columns(),
)
.unwrap();
// For Gloas, commitments are in the bid, not the block body
let kzg_commitments = block
.message()
.body()
.signed_execution_payload_bid()
.unwrap()
.message
.blob_kzg_commitments
.clone();
if kzg_commitments.is_empty() {
return vec![];
}
// TODO(gloas): The fixture is Fulu format. Generate Gloas-specific fixture once format
// is finalized, or compute columns dynamically for Gloas tests.
let (cells, proofs) = template_data_columns
.into_iter()
.map(|sidecar| {
let DataColumnSidecarGloas {
let DataColumnSidecarFulu {
column, kzg_proofs, ..
} = sidecar;
// There's only one cell per column for a single blob
@@ -3574,12 +3559,15 @@ pub fn generate_data_column_sidecars_from_block<E: EthSpec>(
)
.unwrap()
} else {
// load the precomputed column sidecar to avoid computing them for every block in the tests.
let template_data_columns =
RuntimeVariableList::<DataColumnSidecarFulu<E>>::from_ssz_bytes(
TEST_DATA_COLUMN_SIDECARS_SSZ,
E::number_of_columns(),
)
// For pre-Gloas forks, commitments are in the block body
let kzg_commitments = block.message().body().blob_kzg_commitments().unwrap();
if kzg_commitments.is_empty() {
return vec![];
}
let kzg_commitments_inclusion_proof = block
.message()
.body()
.kzg_commitments_merkle_proof()
.unwrap();
let (cells, proofs) = template_data_columns
@@ -3610,56 +3598,6 @@ pub fn generate_data_column_sidecars_from_block<E: EthSpec>(
}
}
/// Generate data column sidecars from pre-computed cells and proofs for gloas payloads.
///
/// Note: In Gloas, `blob_kzg_commitments` are in the bid (block body), not the payload envelope.
/// The caller must provide the commitments separately.
pub fn generate_data_column_sidecars_from_payload<E: EthSpec>(
payload: &SignedExecutionPayloadEnvelope<E>,
kzg_commitments: KzgCommitments<E>,
spec: &ChainSpec,
) -> DataColumnSidecarList<E> {
if kzg_commitments.is_empty() {
return vec![];
}
// Load the precomputed column sidecar to avoid computing them for every block in the tests.
// TODO(gloas): The fixture is currently in Fulu format. We should generate a Gloas-specific
// fixture once the format is finalized, or compute columns dynamically for Gloas tests.
let template_data_columns = RuntimeVariableList::<DataColumnSidecarFulu<E>>::from_ssz_bytes(
TEST_DATA_COLUMN_SIDECARS_SSZ,
E::number_of_columns(),
)
.unwrap();
let (cells, proofs) = template_data_columns
.into_iter()
.map(|sidecar| {
let DataColumnSidecarFulu {
column, kzg_proofs, ..
} = sidecar;
// There's only one cell per column for a single blob
let cell_bytes: Vec<u8> = column.into_iter().next().unwrap().into();
let kzg_cell = cell_bytes.try_into().unwrap();
let kzg_proof = kzg_proofs.into_iter().next().unwrap();
(kzg_cell, kzg_proof)
})
.collect::<(Vec<_>, Vec<_>)>();
// Repeat the cells and proofs for every blob
let blob_cells_and_proofs_vec =
vec![(cells.try_into().unwrap(), proofs.try_into().unwrap()); kzg_commitments.len()];
build_data_column_sidecars_gloas(
kzg_commitments,
payload.message.beacon_block_root,
payload.message.slot,
blob_cells_and_proofs_vec,
spec,
)
.unwrap()
}
pub fn generate_data_column_indices_rand_order<E: EthSpec>() -> Vec<CustodyIndex> {
let mut indices = (0..E::number_of_columns() as u64).collect::<Vec<_>>();
indices.shuffle(&mut StdRng::seed_from_u64(42));