Gloas spec v1.7.0-alpha.5 and beacon_chain tests (#8998)

Fix database pruning post-Gloas


  - Fix DB pruning logic (and state summaries DAG)
- Get the `beacon_chain` tests running with `FORK_NAME=gloas` 🎉


Co-Authored-By: Michael Sproul <michael@sigmaprime.io>

Co-Authored-By: Jimmy Chen <jchen.tc@gmail.com>

Co-Authored-By: Eitan Seri- Levi <eserilev@gmail.com>

Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com>

Co-Authored-By: Eitan Seri-Levi <eserilev@ucsc.edu>

testing/ef_tests/Makefile

@@ -1,6 +1,6 @@
 # To download/extract nightly tests, run:
 # CONSENSUS_SPECS_TEST_VERSION=nightly make
-CONSENSUS_SPECS_TEST_VERSION ?= v1.7.0-alpha.4
+CONSENSUS_SPECS_TEST_VERSION ?= v1.7.0-alpha.5
 REPO_NAME := consensus-spec-tests
 OUTPUT_DIR := ./$(REPO_NAME)
 

testing/ef_tests/check_all_files_accessed.py

@@ -49,8 +49,6 @@ excluded_paths = [
     "tests/.*/eip7805",
     # Heze fork is not implemented
     "tests/.*/heze/.*",
-    # TODO(gloas): remove these ignores as Gloas consensus is implemented
-    "tests/.*/gloas/fork_choice/.*",
     # Ignore MatrixEntry SSZ tests for now.
     "tests/.*/.*/ssz_static/MatrixEntry/.*",
     # TODO: partial data column not implemented yet
@@ -77,7 +75,9 @@ excluded_paths = [
     # We don't need these manifest files at the moment.
     "tests/.*/manifest.yaml",
     # TODO: gossip condition tests not implemented yet
-    "tests/.*/.*/networking/.*"
+    "tests/.*/.*/networking/.*",
+    # TODO: fast confirmation rule not merged yet
+    "tests/.*/.*/fast_confirmation",
 ]
 
 

testing/ef_tests/src/cases/fork_choice.rs

@@ -19,9 +19,13 @@ use beacon_chain::{
     custody_context::NodeCustodyType,
     test_utils::{BeaconChainHarness, EphemeralHarnessType},
 };
-use execution_layer::{PayloadStatusV1, json_structures::JsonPayloadStatusV1Status};
+use execution_layer::{
+    PayloadStatusV1, PayloadStatusV1Status, json_structures::JsonPayloadStatusV1Status,
+};
 use serde::Deserialize;
 use ssz_derive::Decode;
+use state_processing::VerifySignatures;
+use state_processing::envelope_processing::verify_execution_payload_envelope;
 use state_processing::state_advance::complete_state_advance;
 use std::future::Future;
 use std::sync::Arc;
@@ -995,38 +999,95 @@ impl<E: EthSpec> Tester<E> {
         valid: bool,
     ) -> Result<(), Error> {
         let block_root = signed_envelope.message.beacon_block_root;
+        let block_hash = signed_envelope.message.payload.block_hash;
+        let store = &self.harness.chain.store;
+        let spec = &self.harness.chain.spec;
 
-        // Store the envelope in the database so that child blocks extending
-        // the FULL path can load the parent's post-payload state.
+        // Simulate the EL: pre-configure the mock execution engine to return VALID
+        // for envelopes the test expects to be valid. Invalid envelopes are left
+        // unconfigured so the mock EE's default (SYNCING) rejects them.
+        let el = self.harness.mock_execution_layer.as_ref().unwrap();
         if valid {
-            self.harness
-                .chain
-                .store
-                .put_payload_envelope(&block_root, signed_envelope.clone())
+            el.server.set_new_payload_status(
+                block_hash,
+                PayloadStatusV1 {
+                    status: JsonPayloadStatusV1Status::Valid.into(),
+                    latest_valid_hash: Some(block_hash),
+                    validation_error: None,
+                },
+            );
+        }
+
+        // Attempt to verify the envelope against the block's post-state.
+        let verification_result = (|| {
+            let block = store
+                .get_blinded_block(&block_root)
+                .map_err(|e| Error::InternalError(format!("Failed to load block: {e:?}")))?
+                .ok_or_else(|| {
+                    Error::InternalError(format!("Block not found for root {block_root:?}"))
+                })?;
+            let block_state_root = block.state_root();
+
+            let state = store
+                .get_hot_state(&block_state_root, CACHE_STATE_IN_TESTS)
+                .map_err(|e| Error::InternalError(format!("Failed to load state: {e:?}")))?
+                .ok_or_else(|| {
+                    Error::InternalError(format!("State not found for root {block_state_root:?}"))
+                })?;
+
+            verify_execution_payload_envelope(
+                &state,
+                signed_envelope,
+                VerifySignatures::True,
+                block_state_root,
+                spec,
+            )
+            .map_err(|e| {
+                Error::InternalError(format!("Failed to process execution payload: {e:?}"))
+            })?;
+
+            // Check the mock EE's response for this block hash (simulates newPayload).
+            let ee_valid = el
+                .server
+                .ctx
+                .get_new_payload_status(&block_hash)
+                .and_then(|r| r.ok())
+                .is_some_and(|s| s.status == PayloadStatusV1Status::Valid);
+            if !ee_valid {
+                return Err(Error::InternalError(format!(
+                    "Mock EE rejected payload with block hash {block_hash:?}",
+                )));
+            }
+
+            Ok(())
+        })();
+
+        if valid {
+            verification_result?;
+
+            // Store the envelope so that child blocks can load the parent's payload.
+            store
+                .put_payload_envelope(&block_root, signed_envelope)
                 .map_err(|e| {
                     Error::InternalError(format!(
                         "Failed to store payload envelope for {block_root:?}: {e:?}",
                     ))
                 })?;
-        }
 
-        let result = self
-            .harness
-            .chain
-            .canonical_head
-            .fork_choice_write_lock()
-            .on_valid_payload_envelope_received(block_root);
-
-        if valid {
-            result.map_err(|e| {
-                Error::InternalError(format!(
-                    "on_execution_payload for block root {} failed: {:?}",
-                    block_root, e
-                ))
-            })?;
-        } else if result.is_ok() {
+            self.harness
+                .chain
+                .canonical_head
+                .fork_choice_write_lock()
+                .on_valid_payload_envelope_received(block_root)
+                .map_err(|e| {
+                    Error::InternalError(format!(
+                        "on_execution_payload for block root {} failed: {:?}",
+                        block_root, e
+                    ))
+                })?;
+        } else if verification_result.is_ok() {
             return Err(Error::DidntFail(format!(
-                "on_execution_payload for block root {} should have failed",
+                "on_execution_payload envelope for block root {} should have failed",
                 block_root
             )));
         }

testing/ef_tests/src/cases/operations.rs

@@ -5,7 +5,7 @@ use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yam
 use serde::Deserialize;
 use ssz::Decode;
 use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache;
-use state_processing::envelope_processing::VerifyStateRoot;
+use state_processing::envelope_processing::verify_execution_payload_envelope;
 use state_processing::epoch_cache::initialize_epoch_cache;
 use state_processing::per_block_processing::process_operations::{
     process_consolidation_requests, process_deposit_requests_post_gloas,
@@ -13,7 +13,7 @@ use state_processing::per_block_processing::process_operations::{
 };
 use state_processing::{
     ConsensusContext,
-    envelope_processing::{EnvelopeProcessingError, process_execution_payload_envelope},
+    envelope_processing::EnvelopeProcessingError,
     per_block_processing::{
         VerifyBlockRoot, VerifySignatures,
         errors::BlockProcessingError,
@@ -23,7 +23,7 @@ use state_processing::{
             process_bls_to_execution_changes, process_deposits, process_exits,
             process_payload_attestation, process_proposer_slashings,
         },
-        process_sync_aggregate, withdrawals,
+        process_parent_execution_payload, process_sync_aggregate, withdrawals,
     },
 };
 use std::fmt::Debug;
@@ -59,6 +59,12 @@ pub struct ExecutionPayloadBidBlock<E: EthSpec> {
     block: BeaconBlock<E>,
 }
 
+/// Newtype for testing parent execution payload processing.
+#[derive(Debug, Clone, Deserialize)]
+pub struct ParentExecutionPayloadBlock<E: EthSpec> {
+    block: BeaconBlock<E>,
+}
+
 #[derive(Debug, Clone)]
 pub struct Operations<E: EthSpec, O: Operation<E>> {
     metadata: Metadata,
@@ -441,8 +447,10 @@ impl<E: EthSpec> Operation<E> for SignedExecutionPayloadEnvelope<E> {
         "signed_envelope.ssz_snappy".into()
     }
 
-    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
-        fork_name.gloas_enabled()
+    fn is_enabled_for_fork(_fork_name: ForkName) -> bool {
+        // TODO(gloas): re-enable this test when enabled upstream
+        // fork_name.gloas_enabled()
+        false
     }
 
     fn decode(path: &Path, _: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
@@ -460,12 +468,12 @@ impl<E: EthSpec> Operation<E> for SignedExecutionPayloadEnvelope<E> {
             .as_ref()
             .is_some_and(|e| e.execution_valid);
         if valid {
-            process_execution_payload_envelope(
+            let block_state_root = state.update_tree_hash_cache()?;
+            verify_execution_payload_envelope(
                 state,
-                None,
                 self,
                 VerifySignatures::True,
-                VerifyStateRoot::True,
+                block_state_root,
                 spec,
             )
         } else {
@@ -505,6 +513,36 @@ impl<E: EthSpec> Operation<E> for ExecutionPayloadBidBlock<E> {
     }
 }
 
+impl<E: EthSpec> Operation<E> for ParentExecutionPayloadBlock<E> {
+    type Error = BlockProcessingError;
+
+    fn handler_name() -> String {
+        "parent_execution_payload".into()
+    }
+
+    fn filename() -> String {
+        "block.ssz_snappy".into()
+    }
+
+    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
+        fork_name.gloas_enabled()
+    }
+
+    fn decode(path: &Path, _fork_name: ForkName, spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file_with(path, |bytes| BeaconBlock::from_ssz_bytes(bytes, spec))
+            .map(|block| ParentExecutionPayloadBlock { block })
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+        _: &Operations<E, Self>,
+    ) -> Result<(), BlockProcessingError> {
+        process_parent_execution_payload(state, self.block.to_ref(), spec)
+    }
+}
+
 impl<E: EthSpec> Operation<E> for WithdrawalsPayload<E> {
     type Error = BlockProcessingError;
 

testing/ef_tests/src/handler.rs

@@ -723,8 +723,12 @@ impl<E: EthSpec + TypeName> Handler for ForkChoiceHandler<E> {
             return false;
         }
 
-        // on_execution_payload tests exist only for Gloas.
-        if self.handler_name == "on_execution_payload" && !fork_name.gloas_enabled() {
+        // on_execution_payload_envelope and get_parent_payload_status tests exist only for
+        // Gloas and later.
+        if (self.handler_name == "on_execution_payload_envelope"
+            || self.handler_name == "get_parent_payload_status")
+            && !fork_name.gloas_enabled()
+        {
             return false;
         }
 

testing/ef_tests/src/lib.rs

@@ -2,10 +2,10 @@ pub use case_result::CaseResult;
 pub use cases::{
     BuilderPendingPayments, Case, EffectiveBalanceUpdates, Eth1DataReset, ExecutionPayloadBidBlock,
     FeatureName, HistoricalRootsUpdate, HistoricalSummariesUpdate, InactivityUpdates,
-    JustificationAndFinalization, ParticipationFlagUpdates, ParticipationRecordUpdates,
-    PendingBalanceDeposits, PendingConsolidations, ProposerLookahead, PtcWindow, RandaoMixesReset,
-    RegistryUpdates, RewardsAndPenalties, Slashings, SlashingsReset, SyncCommitteeUpdates,
-    WithdrawalsPayload,
+    JustificationAndFinalization, ParentExecutionPayloadBlock, ParticipationFlagUpdates,
+    ParticipationRecordUpdates, PendingBalanceDeposits, PendingConsolidations, ProposerLookahead,
+    PtcWindow, RandaoMixesReset, RegistryUpdates, RewardsAndPenalties, Slashings, SlashingsReset,
+    SyncCommitteeUpdates, WithdrawalsPayload,
 };
 pub use decode::log_file_access;
 pub use error::Error;

testing/ef_tests/tests/tests.rs

@@ -99,6 +99,12 @@ fn operations_execution_payload_bid() {
     OperationsHandler::<MainnetEthSpec, ExecutionPayloadBidBlock<_>>::default().run();
 }
 
+#[test]
+fn operations_parent_execution_payload() {
+    OperationsHandler::<MinimalEthSpec, ParentExecutionPayloadBlock<_>>::default().run();
+    OperationsHandler::<MainnetEthSpec, ParentExecutionPayloadBlock<_>>::default().run();
+}
+
 #[test]
 fn operations_payload_attestation() {
     OperationsHandler::<MinimalEthSpec, PayloadAttestation<_>>::default().run();
@@ -1039,9 +1045,15 @@ fn fork_choice_deposit_with_reorg() {
 }
 
 #[test]
-fn fork_choice_on_execution_payload() {
-    ForkChoiceHandler::<MinimalEthSpec>::new("on_execution_payload").run();
-    ForkChoiceHandler::<MainnetEthSpec>::new("on_execution_payload").run();
+fn fork_choice_on_execution_payload_envelope() {
+    ForkChoiceHandler::<MinimalEthSpec>::new("on_execution_payload_envelope").run();
+    ForkChoiceHandler::<MainnetEthSpec>::new("on_execution_payload_envelope").run();
+}
+
+#[test]
+fn fork_choice_get_parent_payload_status() {
+    ForkChoiceHandler::<MinimalEthSpec>::new("get_parent_payload_status").run();
+    ForkChoiceHandler::<MainnetEthSpec>::new("get_parent_payload_status").run();
 }
 
 #[test]