Add Gloas data column support (#8682)

Co-Authored-By: Eitan Seri-Levi <eserilev@ucsc.edu> Co-Authored-By: Eitan Seri- Levi <eserilev@gmail.com>
2026-03-15 02:42:38 +00:00 · 2026-01-27 20:52:12 -08:00
parent 0f57fc9d8e
commit 9bec8df37a
44 changed files with 1507 additions and 680 deletions
--- a/beacon_node/beacon_chain/src/data_column_verification.rs
+++ b/beacon_node/beacon_chain/src/data_column_verification.rs
@@ -2,14 +2,16 @@ use crate::block_verification::{
    BlockSlashInfo, get_validator_pubkey_cache, process_block_slash_info,
 };
 use crate::kzg_utils::{reconstruct_data_columns, validate_data_columns};
-use crate::observed_data_sidecars::{ObservationStrategy, Observe};
+use crate::observed_data_sidecars::{
+    Error as ObservedDataSidecarsError, ObservationKey, ObservationStrategy, Observe,
+};
 use crate::{BeaconChain, BeaconChainError, BeaconChainTypes, metrics};
 use educe::Educe;
 use fork_choice::ProtoBlock;
 use kzg::{Error as KzgError, Kzg};
 use proto_array::Block;
 use slot_clock::SlotClock;
-use ssz_derive::{Decode, Encode};
+use ssz_derive::Encode;
 use ssz_types::VariableList;
 use std::iter;
 use std::marker::PhantomData;
@@ -17,13 +19,14 @@ use std::sync::Arc;
 use tracing::{debug, instrument};
 use types::data::ColumnIndex;
 use types::{
-    BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256,
-    SignedBeaconBlockHeader, Slot,
+    BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId,
+    EthSpec, Hash256, Slot,
 };

 /// An error occurred while validating a gossip data column.
 #[derive(Debug)]
 pub enum GossipDataColumnError {
+    InvalidVariant,
    /// There was an error whilst processing the data column. It is not known if it is
    /// valid or invalid.
    ///
@@ -64,7 +67,10 @@ pub enum GossipDataColumnError {
    /// ## Peer scoring
    ///
    /// The column is invalid or the peer is faulty.
-    InvalidSubnetId { received: u64, expected: u64 },
+    InvalidSubnetId {
+        received: u64,
+        expected: u64,
+    },
    /// The column sidecar is from a slot that is later than the current slot (with respect to the
    /// gossip clock disparity).
    ///
@@ -97,35 +103,40 @@ pub enum GossipDataColumnError {
    /// ## Peer scoring
    ///
    /// The column is invalid and the peer is faulty.
-    ProposerIndexMismatch { sidecar: usize, local: usize },
+    ProposerIndexMismatch {
+        sidecar: usize,
+        local: usize,
+    },
    /// The provided columns's parent block is unknown.
    ///
    /// ## Peer scoring
    ///
    /// We cannot process the columns without validating its parent, the peer isn't necessarily faulty.
-    ParentUnknown { parent_root: Hash256 },
+    ParentUnknown {
+        parent_root: Hash256,
+    },
    /// The column conflicts with finalization, no need to propagate.
    ///
    /// ## Peer scoring
    ///
    /// It's unclear if this column is valid, but it conflicts with finality and shouldn't be
    /// imported.
-    NotFinalizedDescendant { block_parent_root: Hash256 },
+    NotFinalizedDescendant {
+        block_parent_root: Hash256,
+    },
    /// Invalid kzg commitment inclusion proof
    ///
    /// ## Peer scoring
    ///
    /// The column sidecar is invalid and the peer is faulty
    InvalidInclusionProof,
-    /// A column has already been seen for the given `(sidecar.block_root, sidecar.index)` tuple
-    /// over gossip or no gossip sources.
+    /// A column has already been seen for the given observation key and index.
    ///
    /// ## Peer scoring
    ///
    /// The peer isn't faulty, but we do not forward it over gossip.
    PriorKnown {
-        proposer: u64,
-        slot: Slot,
+        observation_key: ObservationKey,
        index: ColumnIndex,
    },
    /// A column has already been processed from non-gossip source and have not yet been seen on
@@ -160,7 +171,10 @@ pub enum GossipDataColumnError {
    /// ## Peer scoring
    ///
    /// The column sidecar is invalid and the peer is faulty
-    InconsistentProofsLength { cells_len: usize, proofs_len: usize },
+    InconsistentProofsLength {
+        cells_len: usize,
+        proofs_len: usize,
+    },
    /// The number of KZG commitments exceeds the maximum number of blobs allowed for the fork. The
    /// sidecar is invalid.
    ///
@@ -209,17 +223,26 @@ impl<T: BeaconChainTypes, O: ObservationStrategy> GossipVerifiedDataColumn<T, O>
        subnet_id: DataColumnSubnetId,
        chain: &BeaconChain<T>,
    ) -> Result<Self, GossipDataColumnError> {
-        let header = column_sidecar.signed_block_header.clone();
-        // We only process slashing info if the gossip verification failed
-        // since we do not process the data column any further in that case.
-        validate_data_column_sidecar_for_gossip::<T, O>(column_sidecar, subnet_id, chain).map_err(
-            |e| {
-                process_block_slash_info::<_, GossipDataColumnError>(
+        match column_sidecar.as_ref() {
+            DataColumnSidecar::Fulu(c) => {
+                let header = c.signed_block_header.clone();
+                // We only process slashing info if the gossip verification failed
+                // since we do not process the data column any further in that case.
+                validate_data_column_sidecar_for_gossip_fulu::<T, O>(
+                    column_sidecar,
+                    subnet_id,
                    chain,
-                    BlockSlashInfo::from_early_error_data_column(header, e),
                )
-            },
-        )
+                .map_err(|e| {
+                    process_block_slash_info::<_, GossipDataColumnError>(
+                        chain,
+                        BlockSlashInfo::from_early_error_data_column(header, e),
+                    )
+                })
+            }
+            // TODO(gloas) support gloas data column variant
+            DataColumnSidecar::Gloas(_) => Err(GossipDataColumnError::InvalidVariant),
+        }
    }

    /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for block production ONLY.
@@ -283,11 +306,7 @@ impl<T: BeaconChainTypes, O: ObservationStrategy> GossipVerifiedDataColumn<T, O>
    }

    pub fn index(&self) -> ColumnIndex {
-        self.data_column.data.index
-    }
-
-    pub fn signed_block_header(&self) -> SignedBeaconBlockHeader {
-        self.data_column.data.signed_block_header.clone()
+        *self.data_column.data.index()
    }

    pub fn into_inner(self) -> KzgVerifiedDataColumn<T::EthSpec> {
@@ -296,7 +315,7 @@ impl<T: BeaconChainTypes, O: ObservationStrategy> GossipVerifiedDataColumn<T, O>
 }

 /// Wrapper over a `DataColumnSidecar` for which we have completed kzg verification.
-#[derive(Debug, Educe, Clone, Encode, Decode)]
+#[derive(Debug, Educe, Clone, Encode)]
 #[educe(PartialEq, Eq)]
 #[ssz(struct_behaviour = "transparent")]
 pub struct KzgVerifiedDataColumn<E: EthSpec> {
@@ -345,7 +364,7 @@ impl<E: EthSpec> KzgVerifiedDataColumn<E> {
    }

    pub fn index(&self) -> ColumnIndex {
-        self.data.index
+        *self.data.index()
    }
 }

@@ -353,7 +372,7 @@ pub type CustodyDataColumnList<E> =
    VariableList<CustodyDataColumn<E>, <E as EthSpec>::NumberOfColumns>;

 /// Data column that we must custody
-#[derive(Debug, Educe, Clone, Encode, Decode)]
+#[derive(Debug, Educe, Clone, Encode)]
 #[educe(PartialEq, Eq, Hash(bound(E: EthSpec)))]
 #[ssz(struct_behaviour = "transparent")]
 pub struct CustodyDataColumn<E: EthSpec> {
@@ -378,12 +397,12 @@ impl<E: EthSpec> CustodyDataColumn<E> {
        self.data.clone()
    }
    pub fn index(&self) -> u64 {
-        self.data.index
+        *self.data.index()
    }
 }

 /// Data column that we must custody and has completed kzg verification
-#[derive(Debug, Educe, Clone, Encode, Decode)]
+#[derive(Debug, Educe, Clone, Encode)]
 #[educe(PartialEq, Eq)]
 #[ssz(struct_behaviour = "transparent")]
 pub struct KzgVerifiedCustodyDataColumn<E: EthSpec> {
@@ -443,7 +462,7 @@ impl<E: EthSpec> KzgVerifiedCustodyDataColumn<E> {
        self.data.clone()
    }
    pub fn index(&self) -> ColumnIndex {
-        self.data.index
+        *self.data.index()
    }
 }

@@ -477,12 +496,21 @@ where
    Ok(())
 }

-#[instrument(skip_all, level = "debug")]
-pub fn validate_data_column_sidecar_for_gossip<T: BeaconChainTypes, O: ObservationStrategy>(
+// TODO(gloas) make sure the gloas variant uses the same span name
+#[instrument(
+    skip_all,
+    name = "validate_data_column_sidecar_for_gossip",
+    level = "debug"
+)]
+pub fn validate_data_column_sidecar_for_gossip_fulu<T: BeaconChainTypes, O: ObservationStrategy>(
    data_column: Arc<DataColumnSidecar<T::EthSpec>>,
    subnet: DataColumnSubnetId,
    chain: &BeaconChain<T>,
 ) -> Result<GossipVerifiedDataColumn<T, O>, GossipDataColumnError> {
+    let DataColumnSidecar::Fulu(data_column_fulu) = data_column.as_ref() else {
+        return Err(GossipDataColumnError::InvalidVariant);
+    };
+
    let column_slot = data_column.slot();
    verify_data_column_sidecar(&data_column, &chain.spec)?;
    verify_index_matches_subnet(&data_column, subnet, &chain.spec)?;
@@ -506,10 +534,10 @@ pub fn validate_data_column_sidecar_for_gossip<T: BeaconChainTypes, O: Observati
        return Err(GossipDataColumnError::PriorKnownUnpublished);
    }

-    verify_column_inclusion_proof(&data_column)?;
-    let parent_block = verify_parent_block_and_finalized_descendant(data_column.clone(), chain)?;
+    verify_column_inclusion_proof(data_column_fulu)?;
+    let parent_block = verify_parent_block_and_finalized_descendant(data_column_fulu, chain)?;
    verify_slot_higher_than_parent(&parent_block, column_slot)?;
-    verify_proposer_and_signature(&data_column, &parent_block, chain)?;
+    verify_proposer_and_signature(data_column_fulu, &parent_block, chain)?;
    let kzg = &chain.kzg;
    let kzg_verified_data_column = verify_kzg_for_data_column(data_column.clone(), kzg)
        .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?;
@@ -519,7 +547,7 @@ pub fn validate_data_column_sidecar_for_gossip<T: BeaconChainTypes, O: Observati
        .write()
        .observe_slashable(
            column_slot,
-            data_column.block_proposer_index(),
+            data_column_fulu.block_proposer_index(),
            data_column.block_root(),
        )
        .map_err(|e| GossipDataColumnError::BeaconChainError(Box::new(e.into())))?;
@@ -540,16 +568,18 @@ fn verify_data_column_sidecar<E: EthSpec>(
    data_column: &DataColumnSidecar<E>,
    spec: &ChainSpec,
 ) -> Result<(), GossipDataColumnError> {
-    if data_column.index >= E::number_of_columns() as u64 {
-        return Err(GossipDataColumnError::InvalidColumnIndex(data_column.index));
+    if *data_column.index() >= E::number_of_columns() as u64 {
+        return Err(GossipDataColumnError::InvalidColumnIndex(
+            *data_column.index(),
+        ));
    }
-    if data_column.kzg_commitments.is_empty() {
+    if data_column.kzg_commitments().is_empty() {
        return Err(GossipDataColumnError::UnexpectedDataColumn);
    }

-    let cells_len = data_column.column.len();
-    let commitments_len = data_column.kzg_commitments.len();
-    let proofs_len = data_column.kzg_proofs.len();
+    let cells_len = data_column.column().len();
+    let commitments_len = data_column.kzg_commitments().len();
+    let proofs_len = data_column.kzg_proofs().len();
    let max_blobs_per_block = spec.max_blobs_per_block(data_column.epoch()) as usize;

    if commitments_len > max_blobs_per_block {
@@ -582,23 +612,24 @@ fn verify_is_unknown_sidecar<T: BeaconChainTypes>(
    chain: &BeaconChain<T>,
    column_sidecar: &DataColumnSidecar<T::EthSpec>,
 ) -> Result<(), GossipDataColumnError> {
-    if chain
+    if let Some(observation_key) = chain
        .observed_column_sidecars
        .read()
-        .proposer_is_known(column_sidecar)
-        .map_err(|e| GossipDataColumnError::BeaconChainError(Box::new(e.into())))?
+        .observation_key_is_known(column_sidecar)
+        .map_err(|e: ObservedDataSidecarsError| {
+            GossipDataColumnError::BeaconChainError(Box::new(e.into()))
+        })?
    {
        return Err(GossipDataColumnError::PriorKnown {
-            proposer: column_sidecar.block_proposer_index(),
-            slot: column_sidecar.slot(),
-            index: column_sidecar.index,
+            observation_key,
+            index: *column_sidecar.index(),
        });
    }
    Ok(())
 }

 fn verify_column_inclusion_proof<E: EthSpec>(
-    data_column: &DataColumnSidecar<E>,
+    data_column: &DataColumnSidecarFulu<E>,
 ) -> Result<(), GossipDataColumnError> {
    let _timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_INCLUSION_PROOF_VERIFICATION);
    if !data_column.verify_inclusion_proof() {
@@ -622,7 +653,7 @@ fn verify_slot_higher_than_parent(
 }

 fn verify_parent_block_and_finalized_descendant<T: BeaconChainTypes>(
-    data_column: Arc<DataColumnSidecar<T::EthSpec>>,
+    data_column: &DataColumnSidecarFulu<T::EthSpec>,
    chain: &BeaconChain<T>,
 ) -> Result<ProtoBlock, GossipDataColumnError> {
    let fork_choice = chain.canonical_head.fork_choice_read_lock();
@@ -646,7 +677,7 @@ fn verify_parent_block_and_finalized_descendant<T: BeaconChainTypes>(
 }

 fn verify_proposer_and_signature<T: BeaconChainTypes>(
-    data_column: &DataColumnSidecar<T::EthSpec>,
+    data_column: &DataColumnSidecarFulu<T::EthSpec>,
    parent_block: &ProtoBlock,
    chain: &BeaconChain<T>,
 ) -> Result<(), GossipDataColumnError> {
@@ -723,7 +754,7 @@ fn verify_index_matches_subnet<E: EthSpec>(
    subnet: DataColumnSubnetId,
    spec: &ChainSpec,
 ) -> Result<(), GossipDataColumnError> {
-    let expected_subnet = DataColumnSubnetId::from_column_index(data_column.index, spec);
+    let expected_subnet = DataColumnSubnetId::from_column_index(*data_column.index(), spec);
    if expected_subnet != subnet {
        return Err(GossipDataColumnError::InvalidSubnetId {
            received: subnet.into(),
@@ -772,27 +803,31 @@ pub fn observe_gossip_data_column<T: BeaconChainTypes>(
    data_column_sidecar: &DataColumnSidecar<T::EthSpec>,
    chain: &BeaconChain<T>,
 ) -> Result<(), GossipDataColumnError> {
-    // Now the signature is valid, store the proposal so we don't accept another data column sidecar
-    // with the same `ColumnIndex`.  It's important to double-check that the proposer still
-    // hasn't been observed so we don't have a race-condition when verifying two blocks
+    // Pre-gloas: Now the signature is valid, store the proposal so we don't accept another data column sidecar
+    // with the same `ColumnIndex`.
+    // Post-gloas: The block associated with the sidecar has already been imported into fork choice. Store the
+    // columns `beacon_block_root` so we don't accept another data column sidecar with the same `ColumnIndex`.
+    // It's important to double-check that the `Observationkey` still
+    // hasn't been observed so we don't have a race-condition when verifying two sidecars
    // simultaneously.
    //
    // Note: If this DataColumnSidecar goes on to fail full verification, we do not evict it from the
    // seen_cache as alternate data_column_sidecars for the same identifier can still be retrieved over
    // rpc. Evicting them from this cache would allow faster propagation over gossip. So we
-    // allow retrieval of potentially valid blocks over rpc, but try to punish the proposer for
+    // allow retrieval of potentially valid sidecars over rpc, but try to punish the proposer for
    // signing invalid messages. Issue for more background
    // https://github.com/ethereum/consensus-specs/issues/3261
-    if chain
+    if let Some(observation_key) = chain
        .observed_column_sidecars
        .write()
        .observe_sidecar(data_column_sidecar)
-        .map_err(|e| GossipDataColumnError::BeaconChainError(Box::new(e.into())))?
+        .map_err(|e: ObservedDataSidecarsError| {
+            GossipDataColumnError::BeaconChainError(Box::new(e.into()))
+        })?
    {
        return Err(GossipDataColumnError::PriorKnown {
-            proposer: data_column_sidecar.block_proposer_index(),
-            slot: data_column_sidecar.slot(),
-            index: data_column_sidecar.index,
+            observation_key,
+            index: *data_column_sidecar.index(),
        });
    }
    Ok(())
@@ -801,7 +836,8 @@ pub fn observe_gossip_data_column<T: BeaconChainTypes>(
 #[cfg(test)]
 mod test {
    use crate::data_column_verification::{
-        GossipDataColumnError, GossipVerifiedDataColumn, validate_data_column_sidecar_for_gossip,
+        GossipDataColumnError, GossipVerifiedDataColumn,
+        validate_data_column_sidecar_for_gossip_fulu,
    };
    use crate::observed_data_sidecars::Observe;
    use crate::test_utils::{
@@ -810,12 +846,16 @@ mod test {
    use eth2::types::BlobsBundle;
    use execution_layer::test_utils::generate_blobs;
    use std::sync::Arc;
-    use types::{DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkName, MainnetEthSpec};
+    use types::{
+        DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, EthSpec, ForkName,
+        MainnetEthSpec,
+    };

    type E = MainnetEthSpec;

+    // TODO(gloas) make this generic over gloas/fulu
    #[tokio::test]
-    async fn test_validate_data_column_sidecar_for_gossip() {
+    async fn test_validate_data_column_sidecar_for_gossip_fulu() {
        // Setting up harness is slow, we initialise once and use it for all gossip validation tests.
        let spec = ForkName::Fulu.make_genesis_spec(E::default_spec());
        let harness = BeaconChainHarness::builder(E::default())
@@ -827,19 +867,20 @@ mod test {
        harness.advance_slot();

        let verify_fn = |column_sidecar: DataColumnSidecar<E>| {
-            let col_index = column_sidecar.index;
-            validate_data_column_sidecar_for_gossip::<_, Observe>(
+            let col_index = *column_sidecar.index();
+            validate_data_column_sidecar_for_gossip_fulu::<_, Observe>(
                column_sidecar.into(),
                DataColumnSubnetId::from_column_index(col_index, &harness.spec),
                &harness.chain,
            )
        };
-        empty_data_column_sidecars_fails_validation(&harness, &verify_fn).await;
+        empty_data_column_sidecars_fails_validation_fulu(&harness, &verify_fn).await;
        data_column_sidecar_commitments_exceed_max_blobs_per_block(&harness, &verify_fn).await;
    }

+    // TODO(gloas) make this generic over gloas/fulu
    #[tokio::test]
-    async fn test_new_for_block_publishing() {
+    async fn test_new_for_block_publishing_fulu() {
        // Setting up harness is slow, we initialise once and use it for all gossip validation tests.
        let spec = ForkName::Fulu.make_genesis_spec(E::default_spec());
        let harness = BeaconChainHarness::builder(E::default())
@@ -856,11 +897,12 @@ mod test {
                &harness.chain,
            )
        };
-        empty_data_column_sidecars_fails_validation(&harness, &verify_fn).await;
+        empty_data_column_sidecars_fails_validation_fulu(&harness, &verify_fn).await;
        data_column_sidecar_commitments_exceed_max_blobs_per_block(&harness, &verify_fn).await;
    }

-    async fn empty_data_column_sidecars_fails_validation<D>(
+    // TODO(gloas) make this generic over gloas/fulu
+    async fn empty_data_column_sidecars_fails_validation_fulu<D>(
        harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
        verify_fn: &impl Fn(DataColumnSidecar<E>) -> Result<D, GossipDataColumnError>,
    ) {
@@ -873,7 +915,7 @@ mod test {
            .await;

        let index = 0;
-        let column_sidecar = DataColumnSidecar::<E> {
+        let column_sidecar: DataColumnSidecar<E> = DataColumnSidecar::Fulu(DataColumnSidecarFulu {
            index,
            column: vec![].try_into().unwrap(),
            kzg_commitments: vec![].try_into().unwrap(),
@@ -884,7 +926,7 @@ mod test {
                .body()
                .kzg_commitments_merkle_proof()
                .unwrap(),
-        };
+        });

        let result = verify_fn(column_sidecar);
        assert!(matches!(