Gloas consensus: epoch processing, block signature verification, more tests (#8808)

- [x] Implement `process_builder_pending_payments` in epoch processing for Gloas. Enable the new EF tests for this sub-component as well.
- [x] Update `include_all_signatures_except_proposal` for Gloas to safely include the execution payload bid signature (this was an omission in the previous bid PR).
- [x] Enable Gloas for _all_ remaining EF tests by default. They all pass with the exception of the finality tests (see below).


Co-Authored-By: Michael Sproul <michael@sigmaprime.io>

Co-Authored-By: Eitan Seri- Levi <eserilev@gmail.com>

consensus/state_processing/src/per_block_processing.rs

@@ -181,7 +181,7 @@ pub fn per_block_processing<E: EthSpec, Payload: AbstractExecPayload<E>>(
         let body = block.body();
         if state.fork_name_unchecked().gloas_enabled() {
             withdrawals::gloas::process_withdrawals::<E>(state, spec)?;
-            // TODO(EIP-7732): process execution payload bid
+            process_execution_payload_bid(state, block, verify_signatures, spec)?;
         } else {
             if state.fork_name_unchecked().capella_enabled() {
                 withdrawals::capella_electra::process_withdrawals::<E, Payload>(

consensus/state_processing/src/per_block_processing/block_signature_verifier.rs

@@ -170,6 +170,7 @@ where
         self.include_exits(block)?;
         self.include_sync_aggregate(block)?;
         self.include_bls_to_execution_changes(block)?;
+        self.include_execution_payload_bid(block)?;
 
         Ok(())
     }
@@ -357,6 +358,27 @@ where
         Ok(())
     }
 
+    /// Include the signature of the block's execution payload bid.
+    pub fn include_execution_payload_bid<Payload: AbstractExecPayload<E>>(
+        &mut self,
+        block: &'a SignedBeaconBlock<E, Payload>,
+    ) -> Result<()> {
+        if let Ok(signed_execution_payload_bid) =
+            block.message().body().signed_execution_payload_bid()
+        {
+            // TODO(gloas): if we implement a global builder pubkey cache we need to inject it here
+            if let Some(signature_set) = execution_payload_bid_signature_set(
+                self.state,
+                |builder_index| get_builder_pubkey_from_state(self.state, builder_index),
+                signed_execution_payload_bid,
+                self.spec,
+            )? {
+                self.sets.push(signature_set);
+            }
+        }
+        Ok(())
+    }
+
     /// Verify all the signatures that have been included in `self`, returning `true` if and only if
     /// all the signatures are valid.
     ///

consensus/state_processing/src/per_epoch_processing/single_pass.rs

@@ -15,9 +15,9 @@ use std::collections::{BTreeSet, HashMap};
 use tracing::instrument;
 use typenum::Unsigned;
 use types::{
-    ActivationQueue, BeaconState, BeaconStateError, ChainSpec, Checkpoint, DepositData, Epoch,
-    EthSpec, ExitCache, ForkName, ParticipationFlags, PendingDeposit, ProgressiveBalancesCache,
-    RelativeEpoch, Validator,
+    ActivationQueue, BeaconState, BeaconStateError, BuilderPendingPayment, ChainSpec, Checkpoint,
+    DepositData, Epoch, EthSpec, ExitCache, ForkName, ParticipationFlags, PendingDeposit,
+    ProgressiveBalancesCache, RelativeEpoch, Validator,
     consts::altair::{
         NUM_FLAG_INDICES, PARTICIPATION_FLAG_WEIGHTS, TIMELY_HEAD_FLAG_INDEX,
         TIMELY_TARGET_FLAG_INDEX, WEIGHT_DENOMINATOR,
@@ -33,6 +33,7 @@ pub struct SinglePassConfig {
     pub pending_consolidations: bool,
     pub effective_balance_updates: bool,
     pub proposer_lookahead: bool,
+    pub builder_pending_payments: bool,
 }
 
 impl Default for SinglePassConfig {
@@ -52,6 +53,7 @@ impl SinglePassConfig {
             pending_consolidations: true,
             effective_balance_updates: true,
             proposer_lookahead: true,
+            builder_pending_payments: true,
         }
     }
 
@@ -65,6 +67,7 @@ impl SinglePassConfig {
             pending_consolidations: false,
             effective_balance_updates: false,
             proposer_lookahead: false,
+            builder_pending_payments: false,
         }
     }
 }
@@ -455,6 +458,12 @@ pub fn process_epoch_single_pass<E: EthSpec>(
         )?;
     }
 
+    // Process builder pending payments outside the single-pass loop, as they depend on balances for
+    // multiple validators and cannot be computed accurately inside the loop.
+    if fork_name.gloas_enabled() && conf.builder_pending_payments {
+        process_builder_pending_payments(state, state_ctxt, spec)?;
+    }
+
     // Finally, finish updating effective balance caches. We need this to happen *after* processing
     // of pending consolidations, which recomputes some effective balances.
     if conf.effective_balance_updates {
@@ -503,6 +512,58 @@ pub fn process_proposer_lookahead<E: EthSpec>(
     Ok(())
 }
 
+/// Calculate the quorum threshold for builder payments based on total active balance.
+fn get_builder_payment_quorum_threshold<E: EthSpec>(
+    state_ctxt: &StateContext,
+    spec: &ChainSpec,
+) -> Result<u64, Error> {
+    let per_slot_balance = state_ctxt
+        .total_active_balance
+        .safe_div(E::slots_per_epoch())?;
+    let quorum = per_slot_balance.safe_mul(spec.builder_payment_threshold_numerator)?;
+    quorum
+        .safe_div(spec.builder_payment_threshold_denominator)
+        .map_err(Error::from)
+}
+
+/// Processes the builder pending payments from the previous epoch.
+fn process_builder_pending_payments<E: EthSpec>(
+    state: &mut BeaconState<E>,
+    state_ctxt: &StateContext,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
+    let quorum = get_builder_payment_quorum_threshold::<E>(state_ctxt, spec)?;
+
+    // Collect qualifying payments and append to `builder_pending_withdrawals`.
+    // We use this pattern rather than a loop to avoid multiple borrows of the state's fields.
+    let new_pending_builder_withdrawals = state
+        .builder_pending_payments()?
+        .iter()
+        .take(E::SlotsPerEpoch::to_usize())
+        .filter(|payment| payment.weight >= quorum)
+        .map(|payment| payment.withdrawal.clone())
+        .collect::<Vec<_>>();
+    for payment_withdrawal in new_pending_builder_withdrawals {
+        state
+            .builder_pending_withdrawals_mut()?
+            .push(payment_withdrawal)?;
+    }
+
+    // NOTE: this could be a little more memory-efficient with some juggling to reuse parts
+    // of the persistent tree (could convert to list, use pop_front, convert back).
+    let updated_payments = state
+        .builder_pending_payments()?
+        .iter()
+        .skip(E::SlotsPerEpoch::to_usize())
+        .cloned()
+        .chain((0..E::SlotsPerEpoch::to_usize()).map(|_| BuilderPendingPayment::default()))
+        .collect::<Vec<_>>();
+
+    *state.builder_pending_payments_mut()? = Vector::new(updated_payments)?;
+
+    Ok(())
+}
+
 fn process_single_inactivity_update(
     inactivity_score: &mut Cow<u64>,
     validator_info: &ValidatorInfo,

consensus/state_processing/src/per_slot_processing.rs

@@ -14,6 +14,7 @@ pub enum Error {
     EpochProcessingError(EpochProcessingError),
     ArithError(ArithError),
     InconsistentStateFork(InconsistentFork),
+    BitfieldError(ssz::BitfieldError),
 }
 
 impl From<ArithError> for Error {
@@ -22,6 +23,12 @@ impl From<ArithError> for Error {
     }
 }
 
+impl From<ssz::BitfieldError> for Error {
+    fn from(e: ssz::BitfieldError) -> Self {
+        Self::BitfieldError(e)
+    }
+}
+
 /// Advances a state forward by one slot, performing per-epoch processing if required.
 ///
 /// If the root of the supplied `state` is known, then it can be passed as `state_root`. If
@@ -48,6 +55,18 @@ pub fn per_slot_processing<E: EthSpec>(
         None
     };
 
+    // Unset the next payload availability
+    if state.fork_name_unchecked().gloas_enabled() {
+        let next_slot_index = state
+            .slot()
+            .as_usize()
+            .safe_add(1)?
+            .safe_rem(E::slots_per_historical_root())?;
+        state
+            .execution_payload_availability_mut()?
+            .set(next_slot_index, false)?;
+    }
+
     state.slot_mut().safe_add_assign(1)?;
 
     // Process fork upgrades here. Note that multiple upgrades can potentially run

testing/ef_tests/check_all_files_accessed.py

@@ -49,15 +49,9 @@ excluded_paths = [
     "tests/.*/eip7805",
     # TODO(gloas): remove these ignores as more Gloas operations are implemented
     "tests/.*/gloas/operations/payload_attestation/.*",
-    # TODO(EIP-7732): remove these ignores as Gloas consensus is implemented
-    "tests/.*/gloas/epoch_processing/.*",
-    "tests/.*/gloas/finality/.*",
+    # TODO(gloas): remove these ignores as Gloas consensus is implemented
     "tests/.*/gloas/fork/.*",
     "tests/.*/gloas/fork_choice/.*",
-    "tests/.*/gloas/networking/.*",
-    "tests/.*/gloas/rewards/.*",
-    "tests/.*/gloas/sanity/.*",
-    "tests/.*/gloas/transition/.*",
     # Ignore MatrixEntry SSZ tests for now.
     "tests/.*/.*/ssz_static/MatrixEntry/.*",
     # TODO(gloas): Ignore Gloas light client stuff for now

testing/ef_tests/src/cases/epoch_processing.rs

@@ -79,6 +79,8 @@ pub struct InactivityUpdates;
 pub struct ParticipationFlagUpdates;
 #[derive(Debug)]
 pub struct ProposerLookahead;
+#[derive(Debug)]
+pub struct BuilderPendingPayments;
 
 type_name!(
     JustificationAndFinalization,
@@ -100,6 +102,7 @@ type_name!(SyncCommitteeUpdates, "sync_committee_updates");
 type_name!(InactivityUpdates, "inactivity_updates");
 type_name!(ParticipationFlagUpdates, "participation_flag_updates");
 type_name!(ProposerLookahead, "proposer_lookahead");
+type_name!(BuilderPendingPayments, "builder_pending_payments");
 
 impl<E: EthSpec> EpochTransition<E> for JustificationAndFinalization {
     fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
@@ -293,6 +296,20 @@ impl<E: EthSpec> EpochTransition<E> for ProposerLookahead {
     }
 }
 
+impl<E: EthSpec> EpochTransition<E> for BuilderPendingPayments {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        process_epoch_single_pass(
+            state,
+            spec,
+            SinglePassConfig {
+                builder_pending_payments: true,
+                ..SinglePassConfig::disable_all()
+            },
+        )
+        .map(|_| ())
+    }
+}
+
 impl<E: EthSpec, T: EpochTransition<E>> LoadCase for EpochProcessing<E, T> {
     fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
         let spec = &testing_spec::<E>(fork_name);
@@ -356,6 +373,10 @@ impl<E: EthSpec, T: EpochTransition<E>> Case for EpochProcessing<E, T> {
             return false;
         }
 
+        if !fork_name.gloas_enabled() && T::name() == "builder_pending_payments" {
+            return false;
+        }
+
         true
     }
 

testing/ef_tests/src/handler.rs

@@ -22,7 +22,7 @@ pub trait Handler {
     // Add forks here to exclude them from EF spec testing. Helpful for adding future or
     // unspecified forks.
     fn disabled_forks(&self) -> Vec<ForkName> {
-        vec![ForkName::Gloas]
+        vec![]
     }
 
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
@@ -395,11 +395,6 @@ where
         T::name().into()
     }
 
-    fn disabled_forks(&self) -> Vec<ForkName> {
-        // TODO(gloas): Can be removed once we enable Gloas on all tests
-        vec![]
-    }
-
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
         self.supported_forks.contains(&fork_name)
     }
@@ -422,11 +417,6 @@ where
     fn handler_name(&self) -> String {
         BeaconState::<E>::name().into()
     }
-
-    fn disabled_forks(&self) -> Vec<ForkName> {
-        // TODO(gloas): Can be removed once we enable Gloas on all tests
-        vec![]
-    }
 }
 
 impl<T, E> Handler for SszStaticWithSpecHandler<T, E>
@@ -449,11 +439,6 @@ where
         T::name().into()
     }
 
-    fn disabled_forks(&self) -> Vec<ForkName> {
-        // TODO(gloas): Can be removed once we enable Gloas on all tests
-        vec![]
-    }
-
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
         self.supported_forks.contains(&fork_name)
     }
@@ -552,6 +537,11 @@ impl<E: EthSpec + TypeName> Handler for RandomHandler<E> {
     fn handler_name(&self) -> String {
         "random".into()
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas random tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -622,6 +612,11 @@ impl<E: EthSpec + TypeName> Handler for ForkHandler<E> {
     fn handler_name(&self) -> String {
         "fork".into()
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once onboard_builders_from_pending_deposits is implemented
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -726,6 +721,11 @@ impl<E: EthSpec + TypeName> Handler for ForkChoiceHandler<E> {
         // run them with fake crypto.
         cfg!(not(feature = "fake_crypto"))
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas fork choice tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -755,6 +755,11 @@ impl<E: EthSpec + TypeName> Handler for OptimisticSyncHandler<E> {
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
         fork_name.bellatrix_enabled() && cfg!(not(feature = "fake_crypto"))
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas optimistic sync tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -975,6 +980,11 @@ impl<E: EthSpec> Handler for KZGComputeCellsHandler<E> {
     fn handler_name(&self) -> String {
         "compute_cells".into()
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas KZG tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -995,6 +1005,11 @@ impl<E: EthSpec> Handler for KZGComputeCellsAndKZGProofHandler<E> {
     fn handler_name(&self) -> String {
         "compute_cells_and_kzg_proofs".into()
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas KZG tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -1015,6 +1030,11 @@ impl<E: EthSpec> Handler for KZGVerifyCellKZGProofBatchHandler<E> {
     fn handler_name(&self) -> String {
         "verify_cell_kzg_proof_batch".into()
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas KZG tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -1035,6 +1055,11 @@ impl<E: EthSpec> Handler for KZGRecoverCellsAndKZGProofHandler<E> {
     fn handler_name(&self) -> String {
         "recover_cells_and_kzg_proofs".into()
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas KZG tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -1059,6 +1084,11 @@ impl<E: EthSpec + TypeName> Handler for KzgInclusionMerkleProofValidityHandler<E
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
         fork_name.deneb_enabled()
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas KZG merkle proof tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -1083,6 +1113,11 @@ impl<E: EthSpec + TypeName> Handler for MerkleProofValidityHandler<E> {
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
         fork_name.altair_enabled()
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas light client tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -1108,6 +1143,11 @@ impl<E: EthSpec + TypeName> Handler for LightClientUpdateHandler<E> {
         // Enabled in Altair
         fork_name.altair_enabled()
     }
+
+    fn disabled_forks(&self) -> Vec<ForkName> {
+        // TODO(gloas): remove once we have Gloas light client tests
+        vec![ForkName::Gloas]
+    }
 }
 
 #[derive(Educe)]
@@ -1129,11 +1169,6 @@ impl<E: EthSpec + TypeName, O: Operation<E>> Handler for OperationsHandler<E, O>
         O::handler_name()
     }
 
-    fn disabled_forks(&self) -> Vec<ForkName> {
-        // TODO(gloas): Can be removed once we enable Gloas on all tests
-        vec![]
-    }
-
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
         Self::Case::is_enabled_for_fork(fork_name)
             && (!fork_name.gloas_enabled()

testing/ef_tests/src/lib.rs

@@ -1,7 +1,7 @@
 pub use case_result::CaseResult;
 pub use cases::{
-    Case, EffectiveBalanceUpdates, Eth1DataReset, ExecutionPayloadBidBlock, FeatureName,
-    HistoricalRootsUpdate, HistoricalSummariesUpdate, InactivityUpdates,
+    BuilderPendingPayments, Case, EffectiveBalanceUpdates, Eth1DataReset, ExecutionPayloadBidBlock,
+    FeatureName, HistoricalRootsUpdate, HistoricalSummariesUpdate, InactivityUpdates,
     JustificationAndFinalization, ParticipationFlagUpdates, ParticipationRecordUpdates,
     PendingBalanceDeposits, PendingConsolidations, ProposerLookahead, RandaoMixesReset,
     RegistryUpdates, RewardsAndPenalties, Slashings, SlashingsReset, SyncCommitteeUpdates,

testing/ef_tests/tests/tests.rs

@@ -954,6 +954,12 @@ fn epoch_processing_proposer_lookahead() {
     EpochProcessingHandler::<MainnetEthSpec, ProposerLookahead>::default().run();
 }
 
+#[test]
+fn epoch_processing_builder_pending_payments() {
+    EpochProcessingHandler::<MinimalEthSpec, BuilderPendingPayments>::default().run();
+    EpochProcessingHandler::<MainnetEthSpec, BuilderPendingPayments>::default().run();
+}
+
 #[test]
 fn fork_upgrade() {
     ForkHandler::<MinimalEthSpec>::default().run();