gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-22 06:14:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add PeerDAS KZG lib integration (construction & KZG verification) (#6212)

Browse Source 
* Add peerdas KZG library and use it for data column construction and cell kzg verification (#5701, #5941, #6118, #6179)

Co-authored-by: kevaundray <kevtheappdev@gmail.com>

* Update `rust_eth_kzg` crate to published version.

* Update kzg metrics buckets.

* Merge branch 'unstable' into peerdas-kzg

* Update KZG version to fix windows mem allocation.

* Refactor common logic from build sidecar and reconstruction. Remove unnecessary `needless_lifetimes`.

Co-authored-by: realbigsean <sean@sigmaprime.io>

* Copy existing trusted setup into `PeerDASTrustedSetup` for consistency and maintain `--trusted-setup` functionality.

* Merge branch 'unstable' into peerdas-kzg

* Merge branch 'peerdas-kzg' of github.com:jimmygchen/lighthouse into peerdas-kzg

* Merge branch 'unstable' into peerdas-kzg

* Merge branch 'unstable' into peerdas-kzg

* Load PeerDAS KZG only if PeerDAS is enabled.
This commit is contained in:
Jimmy Chen
2024-08-13 10:16:17 +10:00
committed by GitHub
parent ff15c78ced
commit 6dc614fede
11 changed files with 627 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
consensus/types/src/data_column_sidecar.rs
View File

@@ -1,17 +1,12 @@
use crate::beacon_block_body::{KzgCommitments, BLOB_KZG_COMMITMENTS_INDEX};
use crate::test_utils::TestRandom;
use crate::{
BeaconBlockHeader, ChainSpec, EthSpec, Hash256, KzgProofs, SignedBeaconBlock,
SignedBeaconBlockHeader, Slot,
};
use crate::{BeaconStateError, BlobsList};
use crate::BeaconStateError;
use crate::{BeaconBlockHeader, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot};
use bls::Signature;
use derivative::Derivative;
use kzg::Kzg;
use kzg::{Blob as KzgBlob, Cell as KzgCell, Error as KzgError};
use kzg::Error as KzgError;
use kzg::{KzgCommitment, KzgProof};
use merkle_proof::verify_merkle_proof;
use rayon::prelude::*;
use safe_arith::ArithError;
use serde::{Deserialize, Serialize};
use ssz::Encode;
@@ -60,7 +55,7 @@ pub struct DataColumnSidecar<E: EthSpec> {
pub index: ColumnIndex,
#[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")]
pub column: DataColumn<E>,
/// All of the KZG commitments and proofs associated with the block, used for verifying sample cells.
/// All the KZG commitments and proofs associated with the block, used for verifying sample cells.
pub kzg_commitments: KzgCommitments<E>,
pub kzg_proofs: KzgProofs<E>,
pub signed_block_header: SignedBeaconBlockHeader,
@@ -98,197 +93,6 @@ impl<E: EthSpec> DataColumnSidecar<E> {
)
}
pub fn build_sidecars(
blobs: &BlobsList<E>,
block: &SignedBeaconBlock<E>,
kzg: &Kzg,
spec: &ChainSpec,
) -> Result<DataColumnSidecarList<E>, DataColumnSidecarError> {
let number_of_columns = spec.number_of_columns;
if blobs.is_empty() {
return Ok(vec![]);
}
let kzg_commitments = block
.message()
.body()
.blob_kzg_commitments()
.map_err(|_err| DataColumnSidecarError::PreDeneb)?;
let kzg_commitments_inclusion_proof =
block.message().body().kzg_commitments_merkle_proof()?;
let signed_block_header = block.signed_block_header();
let mut columns = vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns];
let mut column_kzg_proofs =
vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns];
// NOTE: assumes blob sidecars are ordered by index
let blob_cells_and_proofs_vec = blobs
.into_par_iter()
.map(|blob| {
let blob = KzgBlob::from_bytes(blob).map_err(KzgError::from)?;
kzg.compute_cells_and_proofs(&blob)
})
.collect::<Result<Vec<_>, KzgError>>()?;
for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec {
// we iterate over each column, and we construct the column from "top to bottom",
// pushing on the cell and the corresponding proof at each column index. we do this for
// each blob (i.e. the outer loop).
for col in 0..number_of_columns {
let cell =
blob_cells
.get(col)
.ok_or(DataColumnSidecarError::InconsistentArrayLength(format!(
"Missing blob cell at index {col}"
)))?;
let cell: Vec<u8> = cell.into_inner().into_iter().collect();
let cell = Cell::<E>::from(cell);
let proof = blob_cell_proofs.get(col).ok_or(
DataColumnSidecarError::InconsistentArrayLength(format!(
"Missing blob cell KZG proof at index {col}"
)),
)?;
let column =
columns
.get_mut(col)
.ok_or(DataColumnSidecarError::InconsistentArrayLength(format!(
"Missing data column at index {col}"
)))?;
let column_proofs = column_kzg_proofs.get_mut(col).ok_or(
DataColumnSidecarError::InconsistentArrayLength(format!(
"Missing data column proofs at index {col}"
)),
)?;
column.push(cell);
column_proofs.push(*proof);
}
}
let sidecars: Vec<Arc<DataColumnSidecar<E>>> = columns
.into_iter()
.zip(column_kzg_proofs)
.enumerate()
.map(|(index, (col, proofs))| {
Arc::new(DataColumnSidecar {
index: index as u64,
column: DataColumn::<E>::from(col),
kzg_commitments: kzg_commitments.clone(),
kzg_proofs: KzgProofs::<E>::from(proofs),
signed_block_header: signed_block_header.clone(),
kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(),
})
})
.collect();
Ok(sidecars)
}
pub fn reconstruct(
kzg: &Kzg,
data_columns: &[Arc<Self>],
spec: &ChainSpec,
) -> Result<Vec<Arc<Self>>, KzgError> {
let number_of_columns = spec.number_of_columns;
let mut columns = vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns];
let mut column_kzg_proofs =
vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns];
let first_data_column = data_columns
.first()
.ok_or(KzgError::InconsistentArrayLength(
"data_columns should have at least one element".to_string(),
))?;
let num_of_blobs = first_data_column.kzg_commitments.len();
let blob_cells_and_proofs_vec = (0..num_of_blobs)
.into_par_iter()
.map(|row_index| {
let mut cells: Vec<KzgCell> = vec![];
let mut cell_ids: Vec<u64> = vec![];
for data_column in data_columns {
let cell = data_column.column.get(row_index).ok_or(
KzgError::InconsistentArrayLength(format!(
"Missing data column at index {row_index}"
)),
)?;
cells.push(ssz_cell_to_crypto_cell::<E>(cell)?);
cell_ids.push(data_column.index);
}
// recover_all_cells does not expect sorted
let all_cells = kzg.recover_all_cells(&cell_ids, &cells)?;
let blob = kzg.cells_to_blob(&all_cells)?;
// Note: This function computes all cells and proofs. According to Justin this is okay,
// computing a partial set may be more expensive and requires code paths that don't exist.
// Computing the blobs cells is technically unnecessary but very cheap. It's done here again
// for simplicity.
kzg.compute_cells_and_proofs(&blob)
})
.collect::<Result<Vec<_>, KzgError>>()?;
for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec {
// we iterate over each column, and we construct the column from "top to bottom",
// pushing on the cell and the corresponding proof at each column index. we do this for
// each blob (i.e. the outer loop).
for col in 0..number_of_columns {
let cell = blob_cells
.get(col)
.ok_or(KzgError::InconsistentArrayLength(format!(
"Missing blob cell at index {col}"
)))?;
let cell: Vec<u8> = cell.into_inner().into_iter().collect();
let cell = Cell::<E>::from(cell);
let proof = blob_cell_proofs
.get(col)
.ok_or(KzgError::InconsistentArrayLength(format!(
"Missing blob cell KZG proof at index {col}"
)))?;
let column = columns
.get_mut(col)
.ok_or(KzgError::InconsistentArrayLength(format!(
"Missing data column at index {col}"
)))?;
let column_proofs =
column_kzg_proofs
.get_mut(col)
.ok_or(KzgError::InconsistentArrayLength(format!(
"Missing data column proofs at index {col}"
)))?;
column.push(cell);
column_proofs.push(*proof);
}
}
// Clone sidecar elements from existing data column, no need to re-compute
let kzg_commitments = &first_data_column.kzg_commitments;
let signed_block_header = &first_data_column.signed_block_header;
let kzg_commitments_inclusion_proof = &first_data_column.kzg_commitments_inclusion_proof;
let sidecars: Vec<Arc<DataColumnSidecar<E>>> = columns
.into_iter()
.zip(column_kzg_proofs)
.enumerate()
.map(|(index, (col, proofs))| {
Arc::new(DataColumnSidecar {
index: index as u64,
column: DataColumn::<E>::from(col),
kzg_commitments: kzg_commitments.clone(),
kzg_proofs: KzgProofs::<E>::from(proofs),
signed_block_header: signed_block_header.clone(),
kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(),
})
})
.collect();
Ok(sidecars)
}
pub fn min_size() -> usize {
// min size is one cell
Self {
@@ -360,7 +164,7 @@ pub enum DataColumnSidecarError {
MissingBlobSidecars,
PreDeneb,
SszError(SszError),
InconsistentArrayLength(String),
BuildSidecarFailed(String),
}
impl From<ArithError> for DataColumnSidecarError {
@@ -386,9 +190,3 @@ impl From<SszError> for DataColumnSidecarError {
Self::SszError(e)
}
}
/// Converts a cell ssz List object to an array to be used with the kzg
/// crypto library.
fn ssz_cell_to_crypto_cell<E: EthSpec>(cell: &Cell<E>) -> Result<KzgCell, KzgError> {
KzgCell::from_bytes(cell.as_ref()).map_err(Into::into)
}