Merge branch 'gloas-fc-proto' of github.com:hopinheimer/lighthouse into gloas-fc-proto

consensus/state_processing/src/block_replayer.rs

@@ -1,6 +1,11 @@
 use crate::{
     BlockProcessingError, BlockSignatureStrategy, ConsensusContext, SlotProcessingError,
-    VerifyBlockRoot, per_block_processing, per_epoch_processing::EpochProcessingSummary,
+    VerifyBlockRoot, VerifySignatures,
+    envelope_processing::{
+        EnvelopeProcessingError, VerifyStateRoot, process_execution_payload_envelope,
+    },
+    per_block_processing,
+    per_epoch_processing::EpochProcessingSummary,
     per_slot_processing,
 };
 use itertools::Itertools;
@@ -8,7 +13,7 @@ use std::iter::Peekable;
 use std::marker::PhantomData;
 use types::{
     BeaconState, BeaconStateError, BlindedPayload, ChainSpec, EthSpec, Hash256, SignedBeaconBlock,
-    Slot,
+    SignedExecutionPayloadEnvelope, Slot, execution::StatePayloadStatus,
 };
 
 pub type PreBlockHook<'a, E, Error> = Box<
@@ -24,7 +29,7 @@ pub type PostSlotHook<'a, E, Error> = Box<
 >;
 pub type StateRootIterDefault<Error> = std::iter::Empty<Result<(Hash256, Slot), Error>>;
 
-/// Efficiently apply blocks to a state while configuring various parameters.
+/// Efficiently apply blocks and payloads to a state while configuring various parameters.
 ///
 /// Usage follows a builder pattern.
 pub struct BlockReplayer<
@@ -41,8 +46,21 @@ pub struct BlockReplayer<
     post_block_hook: Option<PostBlockHook<'a, Spec, Error>>,
     pre_slot_hook: Option<PreSlotHook<'a, Spec, Error>>,
     post_slot_hook: Option<PostSlotHook<'a, Spec, Error>>,
+    /// Iterator over state roots for all *block* states.
+    ///
+    /// Pre-Gloas, this is all states. Post-Gloas, this is *just* the states corresponding to beacon
+    /// blocks. For states corresponding to payloads, we read the state root from the payload
+    /// envelope.
+    // TODO(gloas): this concept might need adjusting when we implement the cold DB.
     pub(crate) state_root_iter: Option<Peekable<StateRootIter>>,
     state_root_miss: bool,
+    /// The payload status of the state desired as the end result of block replay.
+    ///
+    /// This dictates whether a payload should be applied after applying the last block.
+    ///
+    /// Prior to Gloas, this should always be set to `StatePayloadStatus::Pending` to indicate
+    /// that no envelope needs to be applied.
+    desired_state_payload_status: StatePayloadStatus,
     _phantom: PhantomData<Error>,
 }
 
@@ -50,7 +68,12 @@ pub struct BlockReplayer<
 pub enum BlockReplayError {
     SlotProcessing(SlotProcessingError),
     BlockProcessing(BlockProcessingError),
+    EnvelopeProcessing(EnvelopeProcessingError),
     BeaconState(BeaconStateError),
+    /// A payload envelope for this `slot` was required but not provided.
+    MissingPayloadEnvelope {
+        slot: Slot,
+    },
 }
 
 impl From<SlotProcessingError> for BlockReplayError {
@@ -65,6 +88,12 @@ impl From<BlockProcessingError> for BlockReplayError {
     }
 }
 
+impl From<EnvelopeProcessingError> for BlockReplayError {
+    fn from(e: EnvelopeProcessingError) -> Self {
+        Self::EnvelopeProcessing(e)
+    }
+}
+
 impl From<BeaconStateError> for BlockReplayError {
     fn from(e: BeaconStateError) -> Self {
         Self::BeaconState(e)
@@ -96,6 +125,7 @@ where
             post_slot_hook: None,
             state_root_iter: None,
             state_root_miss: false,
+            desired_state_payload_status: StatePayloadStatus::Pending,
             _phantom: PhantomData,
         }
     }
@@ -161,6 +191,14 @@ where
         self
     }
 
+    /// Set the desired payload status of the state reached by replay.
+    ///
+    /// This determines whether to apply a payload after applying the last block.
+    pub fn desired_state_payload_status(mut self, payload_status: StatePayloadStatus) -> Self {
+        self.desired_state_payload_status = payload_status;
+        self
+    }
+
     /// Compute the state root for `self.state` as efficiently as possible.
     ///
     /// This function MUST only be called when `self.state` is a post-state, i.e. it MUST not be
@@ -208,6 +246,38 @@ where
         Ok(state_root)
     }
 
+    /// Apply an execution payload envelope to `self.state`.
+    ///
+    /// The `block_state_root` MUST be the `state_root` of the most recently applied block.
+    ///
+    /// Returns the `state_root` of `self.state` after payload application.
+    fn apply_payload_envelope(
+        &mut self,
+        envelope: &SignedExecutionPayloadEnvelope<E>,
+        block_state_root: Hash256,
+    ) -> Result<Hash256, Error> {
+        // TODO(gloas): bulk signature verification could be relevant here?
+        let verify_payload_signatures =
+            if let BlockSignatureStrategy::NoVerification = self.block_sig_strategy {
+                VerifySignatures::False
+            } else {
+                VerifySignatures::True
+            };
+        // TODO(gloas): state root verif enabled during initial prototyping
+        let verify_state_root = VerifyStateRoot::True;
+        process_execution_payload_envelope(
+            &mut self.state,
+            Some(block_state_root),
+            envelope,
+            verify_payload_signatures,
+            verify_state_root,
+            self.spec,
+        )
+        .map_err(BlockReplayError::from)?;
+
+        Ok(envelope.message.state_root)
+    }
+
     /// Apply `blocks` atop `self.state`, taking care of slot processing.
     ///
     /// If `target_slot` is provided then the state will be advanced through to `target_slot`
@@ -215,8 +285,21 @@ where
     pub fn apply_blocks(
         mut self,
         blocks: Vec<SignedBeaconBlock<E, BlindedPayload<E>>>,
+        payload_envelopes: Vec<SignedExecutionPayloadEnvelope<E>>,
         target_slot: Option<Slot>,
     ) -> Result<Self, Error> {
+        let mut envelopes_iter = payload_envelopes.into_iter();
+
+        let mut next_envelope_at_slot = |slot| {
+            if let Some(envelope) = envelopes_iter.next()
+                && envelope.message.slot == slot
+            {
+                Ok(envelope)
+            } else {
+                Err(BlockReplayError::MissingPayloadEnvelope { slot })
+            }
+        };
+
         for (i, block) in blocks.iter().enumerate() {
             // Allow one additional block at the start which is only used for its state root.
             if i == 0 && block.slot() <= self.state.slot() {
@@ -224,7 +307,35 @@ where
             }
 
             while self.state.slot() < block.slot() {
-                let state_root = self.get_state_root(&blocks, i)?;
+                let mut state_root = self.get_state_root(&blocks, i)?;
+
+                // Apply the payload for the *previous* block if the bid in the current block
+                // indicates that the parent is full (and it hasn't already been applied).
+                state_root = if block.fork_name_unchecked().gloas_enabled()
+                    && self.state.slot() == self.state.latest_block_header().slot
+                {
+                    let latest_bid_block_hash = self
+                        .state
+                        .latest_execution_payload_bid()
+                        .map_err(BlockReplayError::from)?
+                        .block_hash;
+
+                    // Similar to `is_parent_block_full`, but reading the block hash from the
+                    // not-yet-applied `block`. The slot 0 case covers genesis (no block replay reqd).
+                    if self.state.slot() != 0 && block.is_parent_block_full(latest_bid_block_hash) {
+                        let envelope = next_envelope_at_slot(self.state.slot())?;
+                        // State root for the next slot processing is now the envelope's state root.
+                        self.apply_payload_envelope(&envelope, state_root)?
+                    } else {
+                        // Empty payload at this slot, the state root is unchanged from when the
+                        // beacon block was applied.
+                        state_root
+                    }
+                } else {
+                    // Pre-Gloas or at skipped slots post-Gloas, the state root of the parent state
+                    // is always the output from `self.get_state_root`.
+                    state_root
+                };
 
                 if let Some(ref mut pre_slot_hook) = self.pre_slot_hook {
                     pre_slot_hook(state_root, &mut self.state)?;
@@ -268,9 +379,24 @@ where
             }
         }
 
+        // Apply the last payload if desired.
+        let mut opt_state_root = if let StatePayloadStatus::Full = self.desired_state_payload_status
+            && let Some(last_block) = blocks.last()
+        {
+            let envelope = next_envelope_at_slot(self.state.slot())?;
+            Some(self.apply_payload_envelope(&envelope, last_block.state_root())?)
+        } else {
+            None
+        };
+
         if let Some(target_slot) = target_slot {
             while self.state.slot() < target_slot {
-                let state_root = self.get_state_root(&blocks, blocks.len())?;
+                // Read state root from `opt_state_root` if a payload was just applied.
+                let state_root = if let Some(root) = opt_state_root.take() {
+                    root
+                } else {
+                    self.get_state_root(&blocks, blocks.len())?
+                };
 
                 if let Some(ref mut pre_slot_hook) = self.pre_slot_hook {
                     pre_slot_hook(state_root, &mut self.state)?;

consensus/state_processing/src/envelope_processing.rs

@@ -241,8 +241,6 @@ pub fn process_execution_payload_envelope<E: EthSpec>(
     // TODO(gloas): newPayload happens here in the spec, ensure we wire that up correctly
 
     process_deposit_requests_post_gloas(state, &execution_requests.deposits, spec)?;
-
-    // TODO(gloas): gotta update these
     process_withdrawal_requests(state, &execution_requests.withdrawals, spec)?;
     process_consolidation_requests(state, &execution_requests.consolidations, spec)?;
 

consensus/state_processing/src/per_block_processing/tests.rs

@@ -1014,7 +1014,7 @@ async fn block_replayer_peeking_state_roots() {
     let block_replayer = BlockReplayer::new(parent_state, &harness.chain.spec)
         .state_root_iter(state_root_iter.into_iter())
         .no_signature_verification()
-        .apply_blocks(vec![target_block], None)
+        .apply_blocks(vec![target_block], vec![], None)
         .unwrap();
 
     assert_eq!(

consensus/state_processing/src/state_advance.rs

@@ -77,6 +77,11 @@ pub fn partial_state_advance<E: EthSpec>(
     // (all-zeros) state root.
     let mut initial_state_root = Some(if state.slot() > state.latest_block_header().slot {
         state_root_opt.unwrap_or_else(Hash256::zero)
+    } else if state.slot() == state.latest_block_header().slot
+        && !state.latest_block_header().state_root.is_zero()
+    {
+        // Post-Gloas Full state case.
+        state.latest_block_header().state_root
     } else {
         state_root_opt.ok_or(Error::StateRootNotProvided)?
     });

consensus/types/src/block/signed_beacon_block.rs

@@ -14,6 +14,7 @@ use tree_hash::TreeHash;
 use tree_hash_derive::TreeHash;
 
 use crate::{
+    ExecutionBlockHash,
     block::{
         BLOB_KZG_COMMITMENTS_INDEX, BeaconBlock, BeaconBlockAltair, BeaconBlockBase,
         BeaconBlockBellatrix, BeaconBlockBodyBellatrix, BeaconBlockBodyCapella,
@@ -365,6 +366,32 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> SignedBeaconBlock<E, Payload>
 
         format_kzg_commitments(commitments.as_ref())
     }
+
+    /// Convenience accessor for the block's bid's `block_hash`.
+    ///
+    /// This method returns an error prior to Gloas.
+    pub fn payload_bid_block_hash(&self) -> Result<ExecutionBlockHash, BeaconStateError> {
+        self.message()
+            .body()
+            .signed_execution_payload_bid()
+            .map(|bid| bid.message.block_hash)
+    }
+
+    /// Check if the `parent_hash` in this block's `signed_payload_bid` matches `parent_block_hash`.
+    ///
+    /// This function is useful post-Gloas for determining if the parent block is full, *without*
+    /// necessarily needing access to a beacon state. The passed in `parent_block_hash` MUST be the
+    /// `block_hash` from the parent beacon block's bid. If the parent beacon state is available
+    /// this can alternatively be fetched from `state.latest_payload_bid`.
+    ///
+    /// This function returns `false` for all blocks prior to Gloas.
+    pub fn is_parent_block_full(&self, parent_block_hash: ExecutionBlockHash) -> bool {
+        let Ok(signed_payload_bid) = self.message().body().signed_execution_payload_bid() else {
+            // Prior to Gloas.
+            return false;
+        };
+        signed_payload_bid.message.parent_block_hash == parent_block_hash
+    }
 }
 
 // We can convert pre-Bellatrix blocks without payloads into blocks with payloads.

consensus/types/src/data/blob_sidecar.rs

@@ -3,7 +3,7 @@ use std::{fmt::Debug, hash::Hash, sync::Arc};
 use bls::Signature;
 use context_deserialize::context_deserialize;
 use educe::Educe;
-use kzg::{BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT, Blob as KzgBlob, Kzg, KzgCommitment, KzgProof};
+use kzg::{BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT, Kzg, KzgCommitment, KzgProof};
 use merkle_proof::{MerkleTreeError, merkle_root_from_branch, verify_merkle_proof};
 use rand::Rng;
 use safe_arith::ArithError;
@@ -253,14 +253,17 @@ impl<E: EthSpec> BlobSidecar<E> {
 
         let blob = Blob::<E>::new(blob_bytes)
             .map_err(|e| format!("error constructing random blob: {:?}", e))?;
-        let kzg_blob = KzgBlob::from_bytes(&blob).unwrap();
+        let kzg_blob: &[u8; BYTES_PER_BLOB] = blob
+            .as_ref()
+            .try_into()
+            .map_err(|e| format!("error converting blob to kzg blob ref: {:?}", e))?;
 
         let commitment = kzg
-            .blob_to_kzg_commitment(&kzg_blob)
+            .blob_to_kzg_commitment(kzg_blob)
             .map_err(|e| format!("error computing kzg commitment: {:?}", e))?;
 
         let proof = kzg
-            .compute_blob_kzg_proof(&kzg_blob, commitment)
+            .compute_blob_kzg_proof(kzg_blob, commitment)
             .map_err(|e| format!("error computing kzg proof: {:?}", e))?;
 
         Ok(Self {

consensus/types/src/execution/mod.rs

@@ -12,6 +12,7 @@ mod payload;
 mod signed_bls_to_execution_change;
 mod signed_execution_payload_bid;
 mod signed_execution_payload_envelope;
+mod state_payload_status;
 
 pub use bls_to_execution_change::BlsToExecutionChange;
 pub use eth1_data::Eth1Data;
@@ -41,3 +42,4 @@ pub use payload::{
 pub use signed_bls_to_execution_change::SignedBlsToExecutionChange;
 pub use signed_execution_payload_bid::SignedExecutionPayloadBid;
 pub use signed_execution_payload_envelope::SignedExecutionPayloadEnvelope;
+pub use state_payload_status::StatePayloadStatus;

consensus/types/src/execution/state_payload_status.rs

@@ -0,0 +1,18 @@
+use serde::{Deserialize, Serialize};
+
+/// Payload status as it applies to a `BeaconState` post-Gloas.
+///
+/// A state can either be a post-state for a block (in which case we call it `Pending`) or a
+/// payload envelope (`Full`). When handling states it is often necessary to know which of these
+/// two variants is required.
+///
+/// Note that states at skipped slots could be either `Pending` or `Full`, depending on whether
+/// the payload for the most-recently applied block was also applied.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum StatePayloadStatus {
+    /// For states produced by `process_block` executed on a `BeaconBlock`.
+    Pending,
+    /// For states produced by `process_execution_payload` on a `ExecutionPayloadEnvelope`.
+    Full,
+}

consensus/types/src/kzg_ext/mod.rs

@@ -1,6 +1,6 @@
 pub mod consts;
 
-pub use kzg::{Blob as KzgBlob, Error as KzgError, Kzg, KzgCommitment, KzgProof};
+pub use kzg::{Error as KzgError, Kzg, KzgCommitment, KzgProof};
 
 use ssz_types::VariableList;
 

consensus/types/src/state/beacon_state.rs

@@ -36,7 +36,7 @@ use crate::{
     execution::{
         Eth1Data, ExecutionPayloadHeaderBellatrix, ExecutionPayloadHeaderCapella,
         ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderFulu,
-        ExecutionPayloadHeaderRef, ExecutionPayloadHeaderRefMut,
+        ExecutionPayloadHeaderRef, ExecutionPayloadHeaderRefMut, StatePayloadStatus,
     },
     fork::{Fork, ForkName, ForkVersionDecode, InconsistentFork, map_fork_name},
     light_client::consts::{
@@ -1266,6 +1266,24 @@ impl<E: EthSpec> BeaconState<E> {
         }
     }
 
+    /// Determine the payload status of this state.
+    ///
+    /// Prior to Gloas this is always `Pending`.
+    ///
+    /// Post-Gloas, the definition of the `StatePayloadStatus` is:
+    ///
+    /// - `Full` if this state is the result of envelope processing.
+    /// - `Pending` if this state is the result of block processing.
+    pub fn payload_status(&self) -> StatePayloadStatus {
+        if !self.fork_name_unchecked().gloas_enabled() {
+            StatePayloadStatus::Pending
+        } else if self.is_parent_block_full() {
+            StatePayloadStatus::Full
+        } else {
+            StatePayloadStatus::Pending
+        }
+    }
+
     /// Return `true` if the validator who produced `slot_signature` is eligible to aggregate.
     ///
     /// Spec v0.12.1