Gloas payload cache (#9209)

In Gloas, beacon blocks are imported into fork choice immediately - the payload envelope and data columns arrive
separately. KZG commitments moved from the column sidecar into the execution payload bid, so the existing
`DataAvailabilityChecker` (which assumes block and data are coupled) can't be used for Gloas.


  * Introduced `PendingPayloadCache` to keep track of payload and data columns per block root.
* Added gossip column verification
* Added support for Gloas data column reconstruction
* Payload envelope verification simplified: removed `MaybeAvailableEnvelope`, `ExecutedEnvelope`, `EnvelopeImportData`

Not yet implemented (tracked with TODOs):
- Proper lookup sync for Gloas columns arriving before blocks
- Partial column merging for Gloas
- Moving `load_gloas_payload_bid` disk reads off the async runtime
- Backfill/range sync for Gloas

Based on @eserilev's PR and work in progress. See also #9202 for verification.


Co-Authored-By: Eitan Seri-Levi <eserilev@ucsc.edu>

Co-Authored-By: Eitan Seri- Levi <eserilev@gmail.com>

Co-Authored-By: Daniel Knopik <daniel@dknopik.de>

Co-Authored-By: Daniel Knopik <107140945+dknopik@users.noreply.github.com>

Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com>

Co-Authored-By: Jimmy Chen <jchen.tc@gmail.com>

beacon_node/beacon_chain/src/payload_envelope_verification/execution_pending_envelope.rs

@@ -1,23 +1,22 @@
-use std::sync::Arc;
-
+use bls::Hash256;
 use slot_clock::SlotClock;
 use state_processing::{VerifySignatures, envelope_processing::verify_execution_payload_envelope};
-use types::EthSpec;
+use std::sync::Arc;
+use types::{EthSpec, SignedExecutionPayloadEnvelope};
 
 use crate::{
     BeaconChain, BeaconChainError, BeaconChainTypes, NotifyExecutionLayer,
     PayloadVerificationOutcome,
     block_verification::PayloadVerificationHandle,
     payload_envelope_verification::{
-        EnvelopeError, EnvelopeImportData, MaybeAvailableEnvelope,
-        gossip_verified_envelope::GossipVerifiedEnvelope, load_snapshot_from_state_root,
-        payload_notifier::PayloadNotifier,
+        EnvelopeError, gossip_verified_envelope::GossipVerifiedEnvelope,
+        load_snapshot_from_state_root, payload_notifier::PayloadNotifier,
     },
 };
 
 pub struct ExecutionPendingEnvelope<E: EthSpec> {
-    pub signed_envelope: MaybeAvailableEnvelope<E>,
-    pub import_data: EnvelopeImportData<E>,
+    pub signed_envelope: Arc<SignedExecutionPayloadEnvelope<E>>,
+    pub block_root: Hash256,
     pub payload_verification_handle: PayloadVerificationHandle,
 }
 
@@ -29,7 +28,6 @@ impl<T: BeaconChainTypes> GossipVerifiedEnvelope<T> {
     ) -> Result<ExecutionPendingEnvelope<T::EthSpec>, EnvelopeError> {
         let signed_envelope = self.signed_envelope;
         let envelope = &signed_envelope.message;
-        let payload = &envelope.payload;
 
         // Define a future that will verify the execution payload with an execution engine.
         //
@@ -87,14 +85,8 @@ impl<T: BeaconChainTypes> GossipVerifiedEnvelope<T> {
         )?;
 
         Ok(ExecutionPendingEnvelope {
-            signed_envelope: MaybeAvailableEnvelope::AvailabilityPending {
-                block_hash: payload.block_hash,
-                envelope: signed_envelope,
-            },
-            import_data: EnvelopeImportData {
-                block_root,
-                _phantom: Default::default(),
-            },
+            signed_envelope,
+            block_root,
             payload_verification_handle,
         })
     }

beacon_node/beacon_chain/src/payload_envelope_verification/import.rs

@@ -9,13 +9,18 @@ use tracing::{debug, error, info, info_span, instrument, warn};
 use types::{BlockImportSource, Hash256, SignedExecutionPayloadEnvelope};
 
 use super::{
-    AvailableEnvelope, AvailableExecutedEnvelope, EnvelopeError, EnvelopeImportData,
-    ExecutedEnvelope, gossip_verified_envelope::GossipVerifiedEnvelope,
+    AvailableEnvelope, AvailableExecutedEnvelope, EnvelopeError,
+    gossip_verified_envelope::GossipVerifiedEnvelope,
 };
 use crate::{
-    AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes,
-    NotifyExecutionLayer, block_verification_types::AvailableBlockData, metrics,
-    payload_envelope_verification::ExecutionPendingEnvelope, validator_monitor::get_slot_delay_ms,
+    AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError,
+    NotifyExecutionLayer,
+    block_verification_types::AvailableBlockData,
+    metrics,
+    payload_envelope_verification::{
+        AvailabilityPendingExecutedEnvelope, ExecutionPendingEnvelope,
+    },
+    validator_monitor::get_slot_delay_ms,
 };
 
 const ENVELOPE_METRICS_CACHE_SLOT_LIMIT: u32 = 64;
@@ -28,13 +33,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     ///
     /// Returns an `Err` if the given payload envelope was invalid, or an error was encountered during
     /// verification.
-    #[instrument(skip_all, fields(block_root = ?block_root, block_source = %block_source))]
+    #[instrument(skip_all, fields(block_root = ?block_root, envelope_source = %envelope_source))]
     pub async fn process_execution_payload_envelope(
         self: &Arc<Self>,
         block_root: Hash256,
         unverified_envelope: GossipVerifiedEnvelope<T>,
         notify_execution_layer: NotifyExecutionLayer,
-        block_source: BlockImportSource,
+        envelope_source: BlockImportSource,
         publish_fn: impl FnOnce() -> Result<(), EnvelopeError>,
     ) -> Result<AvailabilityProcessingStatus, EnvelopeError> {
         let block_slot = unverified_envelope.signed_envelope.slot();
@@ -50,7 +55,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             );
         }
 
-        // TODO(gloas) insert the pre-executed envelope into some type of cache.
+        // TODO(gloas) insert the pre-executed envelope into some type of cache?
 
         let _full_timer = metrics::start_timer(&metrics::ENVELOPE_PROCESSING_TIMES);
 
@@ -79,12 +84,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             let executed_envelope = chain
                 .into_executed_payload_envelope(execution_pending)
                 .await
-                .inspect_err(|_| {
-                    // TODO(gloas) If the envelope fails execution for whatever reason (e.g. engine offline),
-                    // and we keep it in the cache, then the node will NOT perform lookup and
-                    // reprocess this block until the block is evicted from DA checker, causing the
-                    // chain to get stuck temporarily if the block is canonical. Therefore we remove
-                    // it from the cache if execution fails.
+                .map_err(|error| match error {
+                    BlockError::ExecutionPayloadError(error) => {
+                        EnvelopeError::ExecutionPayloadError(error)
+                    }
+                    error => EnvelopeError::ImportError(error),
                 })?;
 
             // Record the time it took to wait for execution layer verification.
@@ -94,15 +98,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     .set_time_executed(block_root, block_slot, timestamp);
             }
 
-            match executed_envelope {
-                ExecutedEnvelope::Available(envelope) => {
-                    self.import_available_execution_payload_envelope(Box::new(envelope))
-                        .await
-                }
-                ExecutedEnvelope::AvailabilityPending() => Err(EnvelopeError::InternalError(
-                    "Pending payload envelope not yet implemented".to_owned(),
-                )),
-            }
+            self.check_envelope_availability_and_import(executed_envelope)
+                .await
+                .map_err(EnvelopeError::ImportError)
         };
 
         // Verify and import the payload envelope.
@@ -112,7 +110,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 info!(
                     ?block_root,
                     %block_slot,
-                    source = %block_source,
+                    source = %envelope_source,
                     "Execution payload envelope imported"
                 );
 
@@ -138,6 +136,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 }
                 Err(EnvelopeError::BeaconChainError(e))
             }
+            Err(EnvelopeError::ImportError(BlockError::BeaconChainError(e))) => {
+                if matches!(e.as_ref(), BeaconChainError::TokioJoin(_)) {
+                    debug!(error = ?e, "Envelope processing cancelled");
+                } else {
+                    warn!(error = ?e, "Execution payload envelope rejected");
+                }
+                Err(EnvelopeError::ImportError(BlockError::BeaconChainError(e)))
+            }
             Err(other) => {
                 warn!(
                     reason = other.to_string(),
@@ -148,6 +154,19 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
     }
 
+    #[instrument(skip_all)]
+    async fn check_envelope_availability_and_import(
+        self: &Arc<Self>,
+        envelope: AvailabilityPendingExecutedEnvelope<T::EthSpec>,
+    ) -> Result<AvailabilityProcessingStatus, BlockError> {
+        let slot = envelope.envelope.slot();
+        let availability = self
+            .pending_payload_cache
+            .put_executed_payload_envelope(envelope)?;
+        self.process_payload_envelope_availability(slot, availability, || Ok(()))
+            .await
+    }
+
     /// Accepts a fully-verified payload envelope and awaits on its payload verification handle to
     /// get a fully `ExecutedEnvelope`.
     ///
@@ -156,10 +175,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     async fn into_executed_payload_envelope(
         self: Arc<Self>,
         pending_envelope: ExecutionPendingEnvelope<T::EthSpec>,
-    ) -> Result<ExecutedEnvelope<T::EthSpec>, EnvelopeError> {
+    ) -> Result<AvailabilityPendingExecutedEnvelope<T::EthSpec>, BlockError> {
         let ExecutionPendingEnvelope {
             signed_envelope,
-            import_data,
+            block_root,
             payload_verification_handle,
         } = pending_envelope;
 
@@ -173,16 +192,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .payload_verification_status
             .is_optimistic()
         {
-            return Err(EnvelopeError::OptimisticSyncNotSupported {
-                block_root: import_data.block_root,
-            });
+            return Err(BlockError::OptimisticSyncNotSupported { block_root });
         }
 
-        Ok(ExecutedEnvelope::new(
+        Ok(AvailabilityPendingExecutedEnvelope::new(
             signed_envelope,
-            import_data,
+            block_root,
             payload_verification_outcome,
-            self.spec.clone(),
         ))
     }
 
@@ -190,18 +206,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     pub async fn import_available_execution_payload_envelope(
         self: &Arc<Self>,
         envelope: Box<AvailableExecutedEnvelope<T::EthSpec>>,
-    ) -> Result<AvailabilityProcessingStatus, EnvelopeError> {
+    ) -> Result<AvailabilityProcessingStatus, BlockError> {
         let AvailableExecutedEnvelope {
             envelope,
-            import_data,
+            block_root,
             payload_verification_outcome,
         } = *envelope;
 
-        let EnvelopeImportData {
-            block_root,
-            _phantom,
-        } = import_data;
-
         let block_root = {
             let chain = self.clone();
             self.spawn_blocking_handle(
@@ -232,13 +243,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         signed_envelope: AvailableEnvelope<T::EthSpec>,
         block_root: Hash256,
         payload_verification_status: PayloadVerificationStatus,
-    ) -> Result<Hash256, EnvelopeError> {
+    ) -> Result<Hash256, BlockError> {
         // Everything in this initial section is on the hot path for processing the envelope.
         // Take an upgradable read lock on fork choice so we can check if this block has already
         // been imported. We don't want to repeat work importing a block that is already imported.
         let fork_choice_reader = self.canonical_head.fork_choice_upgradable_read_lock();
         if !fork_choice_reader.contains_block(&block_root) {
-            return Err(EnvelopeError::BlockRootUnknown { block_root });
+            return Err(BlockError::EnvelopeBlockRootUnknown(block_root));
         }
 
         // TODO(gloas) add defensive check to see if payload envelope is already in fork choice
@@ -253,7 +264,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         // node which can be eligible for head.
         fork_choice
             .on_valid_payload_envelope_received(block_root)
-            .map_err(|e| EnvelopeError::InternalError(format!("{e:?}")))?;
+            .map_err(|e| BlockError::InternalError(format!("{e:?}")))?;
 
         // TODO(gloas) emit SSE event if the payload became the new head payload
 

beacon_node/beacon_chain/src/payload_envelope_verification/mod.rs

@@ -18,14 +18,13 @@
 //!
 //! ```
 
-use std::marker::PhantomData;
+use state_processing::envelope_processing::EnvelopeProcessingError;
 use std::sync::Arc;
-
-use state_processing::{BlockProcessingError, envelope_processing::EnvelopeProcessingError};
 use store::Error as DBError;
+use strum::AsRefStr;
 use tracing::instrument;
 use types::{
-    BeaconState, BeaconStateError, ChainSpec, DataColumnSidecarList, EthSpec, ExecutionBlockHash,
+    BeaconState, BeaconStateError, DataColumnSidecarList, EthSpec, ExecutionBlockHash,
     ExecutionPayloadEnvelope, Hash256, SignedExecutionPayloadEnvelope, Slot,
 };
 
@@ -41,39 +40,18 @@ mod payload_notifier;
 
 pub use execution_pending_envelope::ExecutionPendingEnvelope;
 
-// TODO(gloas): could remove this type completely, or remove the generic
-#[derive(PartialEq)]
-pub struct EnvelopeImportData<E: EthSpec> {
-    pub block_root: Hash256,
-    _phantom: PhantomData<E>,
-}
-
 #[derive(Debug)]
-#[allow(dead_code)]
 pub struct AvailableEnvelope<E: EthSpec> {
-    execution_block_hash: ExecutionBlockHash,
     envelope: Arc<SignedExecutionPayloadEnvelope<E>>,
-    columns: DataColumnSidecarList<E>,
-    /// Timestamp at which this envelope first became available (UNIX timestamp, time since 1970).
-    columns_available_timestamp: Option<std::time::Duration>,
-    pub spec: Arc<ChainSpec>,
+    pub columns: DataColumnSidecarList<E>,
 }
 
 impl<E: EthSpec> AvailableEnvelope<E> {
     pub fn new(
-        execution_block_hash: ExecutionBlockHash,
         envelope: Arc<SignedExecutionPayloadEnvelope<E>>,
         columns: DataColumnSidecarList<E>,
-        columns_available_timestamp: Option<std::time::Duration>,
-        spec: Arc<ChainSpec>,
     ) -> Self {
-        Self {
-            execution_block_hash,
-            envelope,
-            columns,
-            columns_available_timestamp,
-            spec,
-        }
+        Self { envelope, columns }
     }
 
     pub fn message(&self) -> &ExecutionPayloadEnvelope<E> {
@@ -94,14 +72,6 @@ impl<E: EthSpec> AvailableEnvelope<E> {
     }
 }
 
-pub enum MaybeAvailableEnvelope<E: EthSpec> {
-    Available(AvailableEnvelope<E>),
-    AvailabilityPending {
-        block_hash: ExecutionBlockHash,
-        envelope: Arc<SignedExecutionPayloadEnvelope<E>>,
-    },
-}
-
 /// This snapshot is to be used for verifying a payload envelope.
 #[derive(Debug, Clone)]
 pub struct EnvelopeProcessingSnapshot<E: EthSpec> {
@@ -111,46 +81,25 @@ pub struct EnvelopeProcessingSnapshot<E: EthSpec> {
     pub beacon_block_root: Hash256,
 }
 
-/// A payload envelope that has gone through processing checks and execution by an EL client.
-/// This envelope hasn't necessarily completed data availability checks.
-///
-///
-/// It contains 2 variants:
-/// 1. `Available`: This envelope has been executed and also contains all data to consider it
-///    fully available.
-/// 2. `AvailabilityPending`: This envelope hasn't received all required blobs to consider it
-///    fully available.
-#[allow(dead_code)]
-pub enum ExecutedEnvelope<E: EthSpec> {
-    Available(AvailableExecutedEnvelope<E>),
-    // TODO(gloas): check data column availability via DA checker
-    AvailabilityPending(),
+/// A payload envelope that has completed all envelope processing checks, verification
+/// by an EL client but does not have all requisite columns to get imported into
+/// fork choice.
+pub struct AvailabilityPendingExecutedEnvelope<E: EthSpec> {
+    pub envelope: Arc<SignedExecutionPayloadEnvelope<E>>,
+    pub block_root: Hash256,
+    pub payload_verification_outcome: PayloadVerificationOutcome,
 }
 
-impl<E: EthSpec> ExecutedEnvelope<E> {
+impl<E: EthSpec> AvailabilityPendingExecutedEnvelope<E> {
     pub fn new(
-        envelope: MaybeAvailableEnvelope<E>,
-        import_data: EnvelopeImportData<E>,
+        envelope: Arc<SignedExecutionPayloadEnvelope<E>>,
+        block_root: Hash256,
         payload_verification_outcome: PayloadVerificationOutcome,
-        spec: Arc<ChainSpec>,
     ) -> Self {
-        match envelope {
-            MaybeAvailableEnvelope::Available(available_envelope) => {
-                Self::Available(AvailableExecutedEnvelope::new(
-                    available_envelope,
-                    import_data,
-                    payload_verification_outcome,
-                ))
-            }
-            // TODO(gloas): check data column availability via DA checker
-            MaybeAvailableEnvelope::AvailabilityPending {
-                block_hash,
-                envelope,
-            } => Self::Available(AvailableExecutedEnvelope::new(
-                AvailableEnvelope::new(block_hash, envelope, vec![], None, spec),
-                import_data,
-                payload_verification_outcome,
-            )),
+        Self {
+            envelope,
+            block_root,
+            payload_verification_outcome,
         }
     }
 }
@@ -159,25 +108,25 @@ impl<E: EthSpec> ExecutedEnvelope<E> {
 /// by an EL client **and** has all requisite blob data to be imported into fork choice.
 pub struct AvailableExecutedEnvelope<E: EthSpec> {
     pub envelope: AvailableEnvelope<E>,
-    pub import_data: EnvelopeImportData<E>,
+    pub block_root: Hash256,
     pub payload_verification_outcome: PayloadVerificationOutcome,
 }
 
 impl<E: EthSpec> AvailableExecutedEnvelope<E> {
     pub fn new(
         envelope: AvailableEnvelope<E>,
-        import_data: EnvelopeImportData<E>,
+        block_root: Hash256,
         payload_verification_outcome: PayloadVerificationOutcome,
     ) -> Self {
         Self {
             envelope,
-            import_data,
+            block_root,
             payload_verification_outcome,
         }
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, AsRefStr)]
 pub enum EnvelopeError {
     /// The envelope's block root is unknown.
     BlockRootUnknown { block_root: Hash256 },
@@ -205,22 +154,16 @@ pub enum EnvelopeError {
         payload_slot: Slot,
         latest_finalized_slot: Slot,
     },
-    /// Optimistic sync is not supported for Gloas payload envelopes.
-    OptimisticSyncNotSupported { block_root: Hash256 },
     /// Some Beacon Chain Error
     BeaconChainError(Arc<BeaconChainError>),
     /// Some Beacon State error
     BeaconStateError(BeaconStateError),
-    /// Some BlockProcessingError (for electra operations)
-    BlockProcessingError(BlockProcessingError),
     /// Some EnvelopeProcessingError
     EnvelopeProcessingError(EnvelopeProcessingError),
     /// Error verifying the execution payload
     ExecutionPayloadError(ExecutionPayloadError),
-    /// An error from block-level checks reused during envelope import
-    BlockError(BlockError),
-    /// Internal error
-    InternalError(String),
+    /// An error from importing the envelope.
+    ImportError(BlockError),
 }
 
 impl std::fmt::Display for EnvelopeError {
@@ -253,13 +196,6 @@ impl From<DBError> for EnvelopeError {
     }
 }
 
-impl From<BlockError> for EnvelopeError {
-    fn from(e: BlockError) -> Self {
-        EnvelopeError::BlockError(e)
-    }
-}
-
-/// Pull errors up from EnvelopeProcessingError to EnvelopeError
 impl From<EnvelopeProcessingError> for EnvelopeError {
     fn from(e: EnvelopeProcessingError) -> Self {
         match e {

beacon_node/beacon_chain/src/payload_envelope_verification/payload_notifier.rs

@@ -31,7 +31,8 @@ impl<T: BeaconChainTypes> PayloadNotifier<T> {
 
             match notify_execution_layer {
                 NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => {
-                    let new_payload_request = Self::build_new_payload_request(&envelope, &block)?;
+                    let new_payload_request = Self::build_new_payload_request(&envelope, &block)
+                        .map_err(EnvelopeError::ImportError)?;
                     // TODO(gloas): check and test RLP block hash calculation post-Gloas
                     if let Err(e) = new_payload_request.perform_optimistic_sync_verifications() {
                         warn!(