gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-18 13:28:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Gloas data column support (#8682)

Browse Source 
Co-Authored-By: Eitan Seri-Levi <eserilev@ucsc.edu>

Co-Authored-By: Eitan Seri- Levi <eserilev@gmail.com>
This commit is contained in:
Eitan Seri-Levi
2026-01-27 20:52:12 -08:00
committed by GitHub
parent 0f57fc9d8e
commit 9bec8df37a
44 changed files with 1507 additions and 680 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
beacon_node/beacon_chain/src/kzg_utils.rs
View File

@@ -6,12 +6,13 @@ use rayon::prelude::*;
use ssz_types::{FixedVector, VariableList};
use std::sync::Arc;
use tracing::instrument;
use tree_hash::TreeHash;
use types::data::{Cell, DataColumn, DataColumnSidecarError};
use types::kzg_ext::KzgCommitments;
use types::{
Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarList,
EthSpec, Hash256, KzgCommitment, KzgProof, SignedBeaconBlock, SignedBeaconBlockHeader,
SignedBlindedBeaconBlock,
Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarFulu,
DataColumnSidecarGloas, DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof,
SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlindedBeaconBlock, Slot,
};
/// Converts a blob ssz List object to an array to be used with the kzg
@@ -59,22 +60,22 @@ where
let mut commitments = Vec::new();
for data_column in data_column_iter {
let col_index = data_column.index;
let col_index = *data_column.index();
if data_column.column.is_empty() {
if data_column.column().is_empty() {
return Err((Some(col_index), KzgError::KzgVerificationFailed));
}
for cell in &data_column.column {
for cell in data_column.column() {
cells.push(ssz_cell_to_crypto_cell::<E>(cell).map_err(|e| (Some(col_index), e))?);
column_indices.push(col_index);
}
for &proof in &data_column.kzg_proofs {
for &proof in data_column.kzg_proofs() {
proofs.push(Bytes48::from(proof));
}
for &commitment in &data_column.kzg_commitments {
for &commitment in data_column.kzg_commitments() {
commitments.push(Bytes48::from(commitment));
}
@@ -171,7 +172,6 @@ pub fn blobs_to_data_column_sidecars<E: EthSpec>(
.body()
.blob_kzg_commitments()
.map_err(|_err| DataColumnSidecarError::PreDeneb)?;
let kzg_commitments_inclusion_proof = block.message().body().kzg_commitments_merkle_proof()?;
let signed_block_header = block.signed_block_header();
if cell_proofs.len() != blobs.len() * E::number_of_columns() {
@@ -207,14 +207,27 @@ pub fn blobs_to_data_column_sidecars<E: EthSpec>(
})
.collect::<Result<Vec<_>, KzgError>>()?;
build_data_column_sidecars(
kzg_commitments.clone(),
kzg_commitments_inclusion_proof,
signed_block_header,
blob_cells_and_proofs_vec,
spec,
)
.map_err(DataColumnSidecarError::BuildSidecarFailed)
if block.fork_name_unchecked().gloas_enabled() {
build_data_column_sidecars_gloas(
kzg_commitments.clone(),
signed_block_header.message.tree_hash_root(),
block.slot(),
blob_cells_and_proofs_vec,
spec,
)
.map_err(DataColumnSidecarError::BuildSidecarFailed)
} else {
let kzg_commitments_inclusion_proof =
block.message().body().kzg_commitments_merkle_proof()?;
build_data_column_sidecars_fulu(
kzg_commitments.clone(),
kzg_commitments_inclusion_proof,
signed_block_header,
blob_cells_and_proofs_vec,
spec,
)
.map_err(DataColumnSidecarError::BuildSidecarFailed)
}
}
pub fn compute_cells<E: EthSpec>(blobs: &[&Blob<E>], kzg: &Kzg) -> Result<Vec<KzgCell>, KzgError> {
@@ -235,13 +248,20 @@ pub fn compute_cells<E: EthSpec>(blobs: &[&Blob<E>], kzg: &Kzg) -> Result<Vec<Kz
Ok(cells_flattened)
}
pub(crate) fn build_data_column_sidecars<E: EthSpec>(
pub(crate) fn build_data_column_sidecars_fulu<E: EthSpec>(
kzg_commitments: KzgCommitments<E>,
kzg_commitments_inclusion_proof: FixedVector<Hash256, E::KzgCommitmentsInclusionProofDepth>,
signed_block_header: SignedBeaconBlockHeader,
blob_cells_and_proofs_vec: Vec<CellsAndKzgProofs>,
spec: &ChainSpec,
) -> Result<DataColumnSidecarList<E>, String> {
if spec
.fork_name_at_slot::<E>(signed_block_header.message.slot)
.gloas_enabled()
{
return Err("Attempting to construct Fulu data columns post-Gloas".to_owned());
}
let number_of_columns = E::number_of_columns();
let max_blobs_per_block = spec
.max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch()))
@@ -283,7 +303,7 @@ pub(crate) fn build_data_column_sidecars<E: EthSpec>(
.enumerate()
.map(
|(index, (col, proofs))| -> Result<Arc<DataColumnSidecar<E>>, String> {
Ok(Arc::new(DataColumnSidecar {
Ok(Arc::new(DataColumnSidecar::Fulu(DataColumnSidecarFulu {
index: index as u64,
column: DataColumn::<E>::try_from(col)
.map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?,
@@ -292,7 +312,7 @@ pub(crate) fn build_data_column_sidecars<E: EthSpec>(
.map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?,
signed_block_header: signed_block_header.clone(),
kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(),
}))
})))
},
)
.collect();
@@ -300,6 +320,75 @@ pub(crate) fn build_data_column_sidecars<E: EthSpec>(
sidecars
}
pub(crate) fn build_data_column_sidecars_gloas<E: EthSpec>(
kzg_commitments: KzgCommitments<E>,
beacon_block_root: Hash256,
slot: Slot,
blob_cells_and_proofs_vec: Vec<CellsAndKzgProofs>,
spec: &ChainSpec,
) -> Result<DataColumnSidecarList<E>, String> {
if !spec.fork_name_at_slot::<E>(slot).gloas_enabled() {
return Err("Attempting to construct Gloas data columns pre-Gloas".to_owned());
}
let number_of_columns = E::number_of_columns();
let max_blobs_per_block = spec.max_blobs_per_block(slot.epoch(E::slots_per_epoch())) as usize;
let mut columns = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns];
let mut column_kzg_proofs = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns];
for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec {
// we iterate over each column, and we construct the column from "top to bottom",
// pushing on the cell and the corresponding proof at each column index. we do this for
// each blob (i.e. the outer loop).
for col in 0..number_of_columns {
let cell = blob_cells
.get(col)
.ok_or(format!("Missing blob cell at index {col}"))?;
let cell: Vec<u8> = cell.to_vec();
let cell =
Cell::<E>::try_from(cell).map_err(|e| format!("BytesPerCell exceeded: {e:?}"))?;
let proof = blob_cell_proofs
.get(col)
.ok_or(format!("Missing blob cell KZG proof at index {col}"))?;
let column = columns
.get_mut(col)
.ok_or(format!("Missing data column at index {col}"))?;
let column_proofs = column_kzg_proofs
.get_mut(col)
.ok_or(format!("Missing data column proofs at index {col}"))?;
column.push(cell);
column_proofs.push(*proof);
}
}
let sidecars: Result<Vec<Arc<DataColumnSidecar<E>>>, String> = columns
.into_iter()
.zip(column_kzg_proofs)
.enumerate()
.map(
|(index, (col, proofs))| -> Result<Arc<DataColumnSidecar<E>>, String> {
Ok(Arc::new(DataColumnSidecar::Gloas(DataColumnSidecarGloas {
index: index as u64,
column: DataColumn::<E>::try_from(col)
.map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?,
kzg_commitments: kzg_commitments.clone(),
kzg_proofs: VariableList::try_from(proofs)
.map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?,
beacon_block_root,
slot,
})))
},
)
.collect();
sidecars
}
// TODO(gloas) blob reconstruction will fail post gloas. We should just return `Blob`s
// instead of a `BlobSidecar`. This might require a beacon api spec change as well.
/// Reconstruct blobs from a subset of data column sidecars (requires at least 50%).
///
/// If `blob_indices_opt` is `None`, this function attempts to reconstruct all blobs associated
@@ -314,7 +403,7 @@ pub fn reconstruct_blobs<E: EthSpec>(
spec: &ChainSpec,
) -> Result<BlobSidecarList<E>, String> {
// Sort data columns by index to ensure ascending order for KZG operations
data_columns.sort_unstable_by_key(|dc| dc.index);
data_columns.sort_unstable_by_key(|dc| *dc.index());
let first_data_column = data_columns
.first()
@@ -323,7 +412,7 @@ pub fn reconstruct_blobs<E: EthSpec>(
let blob_indices: Vec<usize> = match blob_indices_opt {
Some(indices) => indices.into_iter().map(|i| i as usize).collect(),
None => {
let num_of_blobs = first_data_column.kzg_commitments.len();
let num_of_blobs = first_data_column.kzg_commitments().len();
(0..num_of_blobs).collect()
}
};
@@ -335,7 +424,7 @@ pub fn reconstruct_blobs<E: EthSpec>(
let mut cell_ids: Vec<u64> = vec![];
for data_column in &data_columns {
let cell = data_column
.column
.column()
.get(row_index)
.ok_or(format!("Missing data column at row index {row_index}"))
.and_then(|cell| {
@@ -343,7 +432,7 @@ pub fn reconstruct_blobs<E: EthSpec>(
})?;
cells.push(cell);
cell_ids.push(data_column.index);
cell_ids.push(*data_column.index());
}
let num_cells_original_blob = E::number_of_columns() / 2;
@@ -374,8 +463,13 @@ pub fn reconstruct_blobs<E: EthSpec>(
row_index,
blob,
signed_block,
first_data_column.signed_block_header.clone(),
&first_data_column.kzg_commitments_inclusion_proof,
first_data_column
.signed_block_header()
.map_err(|e| format!("{e:?}"))?
.clone(),
first_data_column
.kzg_commitments_inclusion_proof()
.map_err(|e| format!("{e:?}"))?,
kzg_proof,
)
.map(Arc::new)
@@ -395,7 +489,7 @@ pub fn reconstruct_data_columns<E: EthSpec>(
spec: &ChainSpec,
) -> Result<DataColumnSidecarList<E>, KzgError> {
// Sort data columns by index to ensure ascending order for KZG operations
data_columns.sort_unstable_by_key(|dc| dc.index);
data_columns.sort_unstable_by_key(|dc| *dc.index());
let first_data_column = data_columns
.first()
@@ -403,37 +497,47 @@ pub fn reconstruct_data_columns<E: EthSpec>(
"data_columns should have at least one element".to_string(),
))?;
let num_of_blobs = first_data_column.kzg_commitments.len();
let num_of_blobs = first_data_column.kzg_commitments().len();
let blob_cells_and_proofs_vec =
(0..num_of_blobs)
.into_par_iter()
.map(|row_index| {
let mut cells: Vec<KzgCellRef> = vec![];
let mut cell_ids: Vec<u64> = vec![];
for data_column in &data_columns {
let cell = data_column.column.get(row_index).ok_or(
KzgError::InconsistentArrayLength(format!(
"Missing data column at row index {row_index}"
)),
)?;
let blob_cells_and_proofs_vec = (0..num_of_blobs)
.into_par_iter()
.map(|row_index| {
let mut cells: Vec<KzgCellRef> = vec![];
let mut cell_ids: Vec<u64> = vec![];
for data_column in &data_columns {
let cell = data_column.column().get(row_index).ok_or(
KzgError::InconsistentArrayLength(format!(
"Missing data column at row index {row_index}"
)),
)?;
cells.push(ssz_cell_to_crypto_cell::<E>(cell)?);
cell_ids.push(data_column.index);
}
kzg.recover_cells_and_compute_kzg_proofs(&cell_ids, &cells)
})
.collect::<Result<Vec<_>, KzgError>>()?;
// Clone sidecar elements from existing data column, no need to re-compute
build_data_column_sidecars(
first_data_column.kzg_commitments.clone(),
first_data_column.kzg_commitments_inclusion_proof.clone(),
first_data_column.signed_block_header.clone(),
blob_cells_and_proofs_vec,
spec,
)
.map_err(KzgError::ReconstructFailed)
cells.push(ssz_cell_to_crypto_cell::<E>(cell)?);
cell_ids.push(*data_column.index());
}
kzg.recover_cells_and_compute_kzg_proofs(&cell_ids, &cells)
})
.collect::<Result<Vec<_>, KzgError>>()?;
match first_data_column.as_ref() {
DataColumnSidecar::Fulu(first_column) => {
// Clone sidecar elements from existing data column, no need to re-compute
build_data_column_sidecars_fulu(
first_column.kzg_commitments.clone(),
first_column.kzg_commitments_inclusion_proof.clone(),
first_column.signed_block_header.clone(),
blob_cells_and_proofs_vec,
spec,
)
.map_err(KzgError::ReconstructFailed)
}
DataColumnSidecar::Gloas(first_column) => build_data_column_sidecars_gloas(
first_column.kzg_commitments.clone(),
first_column.beacon_block_root,
first_column.slot,
blob_cells_and_proofs_vec,
spec,
)
.map_err(KzgError::ReconstructFailed),
}
}
#[cfg(test)]
@@ -455,12 +559,13 @@ mod test {
// Loading and initializing PeerDAS KZG is expensive and slow, so we group the tests together
// only load it once.
// TODO(Gloas) make this generic over fulu/gloas, or write a separate function for Gloas
#[test]
fn test_build_data_columns_sidecars() {
let spec = ForkName::Fulu.make_genesis_spec(E::default_spec());
let kzg = get_kzg();
test_build_data_columns_empty(&kzg, &spec);
test_build_data_columns(&kzg, &spec);
test_build_data_columns_fulu(&kzg, &spec);
test_reconstruct_data_columns(&kzg, &spec);
test_reconstruct_data_columns_unordered(&kzg, &spec);
test_reconstruct_blobs_from_data_columns(&kzg, &spec);
@@ -494,8 +599,10 @@ mod test {
assert!(column_sidecars.is_empty());
}
// TODO(gloas) create `test_build_data_columns_gloas` and make sure its called
// in the relevant places
#[track_caller]
fn test_build_data_columns(kzg: &Kzg, spec: &ChainSpec) {
fn test_build_data_columns_fulu(kzg: &Kzg, spec: &ChainSpec) {
// Using at least 2 blobs to make sure we're arranging the data columns correctly.
let num_of_blobs = 2;
let (signed_block, blobs, proofs) =
@@ -520,18 +627,21 @@ mod test {
assert_eq!(column_sidecars.len(), E::number_of_columns());
for (idx, col_sidecar) in column_sidecars.iter().enumerate() {
assert_eq!(col_sidecar.index, idx as u64);
assert_eq!(*col_sidecar.index(), idx as u64);
assert_eq!(col_sidecar.kzg_commitments.len(), num_of_blobs);
assert_eq!(col_sidecar.column.len(), num_of_blobs);
assert_eq!(col_sidecar.kzg_proofs.len(), num_of_blobs);
assert_eq!(col_sidecar.kzg_commitments().len(), num_of_blobs);
assert_eq!(col_sidecar.column().len(), num_of_blobs);
assert_eq!(col_sidecar.kzg_proofs().len(), num_of_blobs);
assert_eq!(col_sidecar.kzg_commitments, block_kzg_commitments);
assert_eq!(col_sidecar.kzg_commitments().clone(), block_kzg_commitments);
assert_eq!(
col_sidecar.kzg_commitments_inclusion_proof,
col_sidecar
.kzg_commitments_inclusion_proof()
.unwrap()
.clone(),
block_kzg_commitments_inclusion_proof
);
assert!(col_sidecar.verify_inclusion_proof());
assert!(col_sidecar.as_fulu().unwrap().verify_inclusion_proof());
}
}