Gloas payload cache (#9209)

In Gloas, beacon blocks are imported into fork choice immediately - the payload envelope and data columns arrive
separately. KZG commitments moved from the column sidecar into the execution payload bid, so the existing
`DataAvailabilityChecker` (which assumes block and data are coupled) can't be used for Gloas.


  * Introduced `PendingPayloadCache` to keep track of payload and data columns per block root.
* Added gossip column verification
* Added support for Gloas data column reconstruction
* Payload envelope verification simplified: removed `MaybeAvailableEnvelope`, `ExecutedEnvelope`, `EnvelopeImportData`

Not yet implemented (tracked with TODOs):
- Proper lookup sync for Gloas columns arriving before blocks
- Partial column merging for Gloas
- Moving `load_gloas_payload_bid` disk reads off the async runtime
- Backfill/range sync for Gloas

Based on @eserilev's PR and work in progress. See also #9202 for verification.


Co-Authored-By: Eitan Seri-Levi <eserilev@ucsc.edu>

Co-Authored-By: Eitan Seri- Levi <eserilev@gmail.com>

Co-Authored-By: Daniel Knopik <daniel@dknopik.de>

Co-Authored-By: Daniel Knopik <107140945+dknopik@users.noreply.github.com>

Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com>

Co-Authored-By: Jimmy Chen <jchen.tc@gmail.com>

beacon_node/beacon_chain/src/kzg_utils.rs

@@ -111,6 +111,57 @@ pub fn validate_full_data_columns<'a, E: EthSpec>(
     kzg.verify_cell_proof_batch(&cells, &proofs, column_indices, &commitments)
 }
 
+/// Validate a batch of full `DataColumnSidecar`s against commitments supplied out-of-band.
+///
+/// Gloas sidecars do not carry commitments. Their commitments come from the block's
+/// `ExecutionPayloadBid`.
+pub fn validate_data_columns_with_commitments<'a, E: EthSpec>(
+    kzg: &Kzg,
+    data_column_iter: impl Iterator<Item = &'a Arc<DataColumnSidecar<E>>>,
+    kzg_commitments: &[KzgCommitment],
+) -> Result<(), (Option<u64>, KzgError)> {
+    let mut cells = Vec::new();
+    let mut proofs = Vec::new();
+    let mut column_indices = Vec::new();
+    let mut commitments = Vec::new();
+
+    for data_column in data_column_iter {
+        let col_index = *data_column.index();
+
+        if data_column.column().is_empty() {
+            return Err((Some(col_index), KzgError::KzgVerificationFailed));
+        }
+
+        for cell in data_column.column() {
+            cells.push(ssz_cell_to_crypto_cell::<E>(cell).map_err(|e| (Some(col_index), e))?);
+            column_indices.push(col_index);
+        }
+
+        for &proof in data_column.kzg_proofs() {
+            proofs.push(proof.0);
+        }
+
+        for &commitment in kzg_commitments {
+            commitments.push(commitment.0);
+        }
+
+        let expected_len = column_indices.len();
+
+        // We make this check at each iteration so that the error is attributable to a specific column.
+        if cells.len() != expected_len
+            || proofs.len() != expected_len
+            || commitments.len() != expected_len
+        {
+            return Err((
+                Some(col_index),
+                KzgError::InconsistentArrayLength("Invalid data column".to_string()),
+            ));
+        }
+    }
+
+    kzg.verify_cell_proof_batch(&cells, &proofs, column_indices, &commitments)
+}
+
 /// Validate a batch of partial `VerifiablePartialDataColumn`s.
 ///
 /// Partial columns may have missing cells, indicated by a bitmap. We only verify present cells.
@@ -618,19 +669,17 @@ pub fn reconstruct_blobs<E: EthSpec>(
     // Sort data columns by index to ensure ascending order for KZG operations
     data_columns.sort_unstable_by_key(|dc| *dc.index());
 
-    let first_data_column = data_columns
-        .first()
-        .ok_or("data_columns should have at least one element".to_string())?;
+    if data_columns.is_empty() {
+        return Err("data_columns should have at least one element".to_string());
+    }
 
     let blob_indices: Vec<usize> = match blob_indices_opt {
         Some(indices) => indices.into_iter().map(|i| i as usize).collect(),
         None => {
-            // TODO(gloas): support blob reconstruction for Gloas
-            // https://github.com/sigp/lighthouse/issues/7413
-            let num_of_blobs = first_data_column
-                .kzg_commitments()
-                .map_err(|_| "Gloas blob reconstruction not yet supported".to_string())?
-                .len();
+            let num_of_blobs = signed_block
+                .message()
+                .blob_kzg_commitments_len()
+                .ok_or_else(|| "Block does not have blob KZG commitments".to_string())?;
             (0..num_of_blobs).collect()
         }
     };
@@ -689,9 +738,14 @@ pub fn reconstruct_blobs<E: EthSpec>(
 }
 
 /// Reconstruct all data columns from a subset of data column sidecars (requires at least 50%).
+///
+/// `kzg_commitments` are the commitments for the underlying blobs. For Fulu they live in the
+/// column itself; for Gloas they live in the bid. We take them as a parameter so this function
+/// works for both forks (mirroring `validate_data_columns_with_commitments`).
 pub fn reconstruct_data_columns<E: EthSpec>(
     kzg: &Kzg,
     mut data_columns: Vec<Arc<DataColumnSidecar<E>>>,
+    kzg_commitments: &[KzgCommitment],
     spec: &ChainSpec,
 ) -> Result<DataColumnSidecarList<E>, KzgError> {
     // Sort data columns by index to ensure ascending order for KZG operations
@@ -703,16 +757,7 @@ pub fn reconstruct_data_columns<E: EthSpec>(
             "data_columns should have at least one element".to_string(),
         ))?;
 
-    // TODO(gloas): support data column reconstruction for Gloas
-    // https://github.com/sigp/lighthouse/issues/7413
-    let num_of_blobs = first_data_column
-        .kzg_commitments()
-        .map_err(|_| {
-            KzgError::InconsistentArrayLength(
-                "Gloas data column reconstruction not yet supported".to_string(),
-            )
-        })?
-        .len();
+    let num_of_blobs = kzg_commitments.len();
 
     let blob_cells_and_proofs_vec = (0..num_of_blobs)
         .into_par_iter()
@@ -757,8 +802,9 @@ pub fn reconstruct_data_columns<E: EthSpec>(
 #[cfg(test)]
 mod test {
     use crate::kzg_utils::{
-        blobs_to_data_column_sidecars, blobs_to_data_column_sidecars_gloas, reconstruct_blobs,
-        reconstruct_data_columns, validate_full_data_columns,
+        blob_to_kzg_commitment, blobs_to_data_column_sidecars, blobs_to_data_column_sidecars_gloas,
+        reconstruct_blobs, reconstruct_data_columns, validate_data_columns_with_commitments,
+        validate_full_data_columns,
     };
     use bls::Signature;
     use eth2::types::BlobsBundle;
@@ -787,9 +833,13 @@ mod test {
         test_reconstruct_blobs_from_data_columns_unordered(&kzg, &fulu_spec);
         test_validate_data_columns(&kzg, &fulu_spec);
 
+        test_validate_data_columns_with_commitments(&kzg, &fulu_spec);
+
         let gloas_spec = ForkName::Gloas.make_genesis_spec(E::default_spec());
         test_build_data_columns_gloas(&kzg, &gloas_spec);
         test_build_data_columns_gloas_empty(&kzg, &gloas_spec);
+        test_reconstruct_data_columns_gloas(&kzg, &gloas_spec);
+        test_validate_data_columns_with_commitments_gloas(&kzg, &gloas_spec);
     }
 
     #[track_caller]
@@ -806,6 +856,63 @@ mod test {
         assert!(result.is_ok());
     }
 
+    #[track_caller]
+    fn test_validate_data_columns_with_commitments(kzg: &Kzg, spec: &ChainSpec) {
+        let num_of_blobs = 2;
+        let (signed_block, blobs, proofs) =
+            create_test_fulu_block_and_blobs::<E>(num_of_blobs, spec);
+        let blob_refs = blobs.iter().collect::<Vec<_>>();
+        let column_sidecars =
+            blobs_to_data_column_sidecars(&blob_refs, proofs.to_vec(), &signed_block, kzg, spec)
+                .unwrap();
+
+        let commitments = signed_block
+            .message()
+            .body()
+            .blob_kzg_commitments()
+            .unwrap();
+
+        let result =
+            validate_data_columns_with_commitments(kzg, column_sidecars.iter(), commitments);
+        assert!(result.is_ok());
+
+        // Verify that wrong commitments cause a failure
+        let bad_commitments = vec![KzgCommitment::empty_for_testing(); num_of_blobs];
+        let result =
+            validate_data_columns_with_commitments(kzg, column_sidecars.iter(), &bad_commitments);
+        assert!(result.is_err());
+    }
+
+    #[track_caller]
+    fn test_validate_data_columns_with_commitments_gloas(kzg: &Kzg, spec: &ChainSpec) {
+        let num_of_blobs = 2;
+        let (blobs, _proofs) = create_test_gloas_blobs::<E>(num_of_blobs);
+        let blob_refs: Vec<_> = blobs.iter().collect();
+        let column_sidecars = blobs_to_data_column_sidecars_gloas::<E>(
+            &blob_refs,
+            Hash256::random(),
+            Slot::new(0),
+            kzg,
+            spec,
+        )
+        .unwrap();
+
+        let commitments: Vec<KzgCommitment> = blobs
+            .iter()
+            .map(|blob| blob_to_kzg_commitment::<E>(kzg, blob).unwrap())
+            .collect();
+
+        let result =
+            validate_data_columns_with_commitments(kzg, column_sidecars.iter(), &commitments);
+        assert!(result.is_ok());
+
+        // Verify that wrong commitments cause a failure
+        let bad_commitments = vec![KzgCommitment::empty_for_testing(); num_of_blobs];
+        let result =
+            validate_data_columns_with_commitments(kzg, column_sidecars.iter(), &bad_commitments);
+        assert!(result.is_err());
+    }
+
     #[track_caller]
     fn test_build_data_columns_empty(kzg: &Kzg, spec: &ChainSpec) {
         let num_of_blobs = 0;
@@ -918,11 +1025,18 @@ mod test {
         let column_sidecars =
             blobs_to_data_column_sidecars(&blob_refs, proofs.to_vec(), &signed_block, kzg, spec)
                 .unwrap();
+        let commitments = signed_block
+            .message()
+            .body()
+            .blob_kzg_commitments()
+            .unwrap()
+            .clone();
 
         // Now reconstruct
         let reconstructed_columns = reconstruct_data_columns(
             kzg,
             column_sidecars.iter().as_slice()[0..column_sidecars.len() / 2].to_vec(),
+            &commitments,
             spec,
         )
         .unwrap();
@@ -942,12 +1056,49 @@ mod test {
         let column_sidecars =
             blobs_to_data_column_sidecars(&blob_refs, proofs.to_vec(), &signed_block, kzg, spec)
                 .unwrap();
+        let commitments = signed_block
+            .message()
+            .body()
+            .blob_kzg_commitments()
+            .unwrap()
+            .clone();
 
         // Test reconstruction with columns in reverse order (non-ascending)
         let mut subset_columns: Vec<_> =
             column_sidecars.iter().as_slice()[0..column_sidecars.len() / 2].to_vec();
         subset_columns.reverse(); // This would fail without proper sorting in reconstruct_data_columns
-        let reconstructed_columns = reconstruct_data_columns(kzg, subset_columns, spec).unwrap();
+        let reconstructed_columns =
+            reconstruct_data_columns(kzg, subset_columns, &commitments, spec).unwrap();
+
+        for i in 0..E::number_of_columns() {
+            assert_eq!(reconstructed_columns.get(i), column_sidecars.get(i), "{i}");
+        }
+    }
+
+    /// Reconstruct a full Gloas column set from a 50% subset and assert the recovered sidecars
+    /// match the originals. Commitments come from the bid (here mocked via the same
+    /// `KzgCommitments` used to build the columns) since Gloas columns don't carry them.
+    #[track_caller]
+    fn test_reconstruct_data_columns_gloas(kzg: &Kzg, spec: &ChainSpec) {
+        let num_of_blobs = 2;
+        let (blobs, _proofs) = create_test_gloas_blobs::<E>(num_of_blobs);
+        let blob_refs: Vec<_> = blobs.iter().collect();
+        let column_sidecars = blobs_to_data_column_sidecars_gloas::<E>(
+            &blob_refs,
+            Hash256::random(),
+            Slot::new(0),
+            kzg,
+            spec,
+        )
+        .unwrap();
+
+        let commitments =
+            KzgCommitments::<E>::new(vec![KzgCommitment::empty_for_testing(); num_of_blobs])
+                .unwrap();
+
+        let subset = column_sidecars[..column_sidecars.len() / 2].to_vec();
+        let reconstructed_columns =
+            reconstruct_data_columns(kzg, subset, &commitments, spec).unwrap();
 
         for i in 0..E::number_of_columns() {
             assert_eq!(reconstructed_columns.get(i), column_sidecars.get(i), "{i}");