Merge branch 'unstable' into progressive-list-tests

Cargo.lock

@@ -695,7 +695,7 @@ version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
 dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -706,7 +706,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
 dependencies = [
  "anstyle",
  "once_cell_polyfill",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -2848,6 +2848,7 @@ dependencies = [
  "context_deserialize",
  "educe",
  "eth2_network_config",
+ "ethereum_hashing",
  "ethereum_ssz",
  "ethereum_ssz_derive",
  "execution_layer",
@@ -3116,7 +3117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -3284,8 +3285,7 @@ dependencies = [
 [[package]]
 name = "ethereum_ssz"
 version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e462875ad8693755ea8913d6e905715c76ea4836e2254e18c9cf0f7a8f8c2a13"
+source = "git+https://github.com/sigp/ethereum_ssz?branch=progressive#988c6b430bfb0cf9c5f2ba6e7bbc1956ebb7fe64"
 dependencies = [
  "alloy-primitives",
  "arbitrary",
@@ -3300,9 +3300,8 @@ dependencies = [
 
 [[package]]
 name = "ethereum_ssz_derive"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2cd82c68120c89361e1a457245cf212f7d9f541bffaffed530c8f2d54a160b2"
+version = "0.10.4"
+source = "git+https://github.com/sigp/ethereum_ssz?branch=progressive#988c6b430bfb0cf9c5f2ba6e7bbc1956ebb7fe64"
 dependencies = [
  "darling 0.23.0",
  "proc-macro2",
@@ -5774,8 +5773,7 @@ dependencies = [
 [[package]]
 name = "milhouse"
 version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "259dd9da2ae5e0278b95da0b7ecef9c18c309d0a2d9e6db57ed33b9e8910c5e7"
+source = "git+https://github.com/sigp/milhouse?branch=progressive-list#dca91b533df1bfce563dd105c728a67985e1a848"
 dependencies = [
  "alloy-primitives",
  "arbitrary",
@@ -6194,7 +6192,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -7128,7 +7126,7 @@ dependencies = [
  "once_cell",
  "socket2 0.6.3",
  "tracing",
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -7650,7 +7648,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -8356,7 +8354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
 dependencies = [
  "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -8701,7 +8699,7 @@ dependencies = [
  "getrandom 0.3.4",
  "once_cell",
  "rustix",
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -9262,8 +9260,7 @@ dependencies = [
 [[package]]
 name = "tree_hash"
 version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7fd51aa83d2eb83b04570808430808b5d24fdbf479a4d5ac5dee4a2e2dd2be4"
+source = "git+https://github.com/sigp/tree_hash?branch=progressive#ca07459738c9b1584eec99e37f3a52774fa97dd0"
 dependencies = [
  "alloy-primitives",
  "ethereum_hashing",
@@ -9275,8 +9272,7 @@ dependencies = [
 [[package]]
 name = "tree_hash_derive"
 version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8840ad4d852e325d3afa7fde8a50b2412f89dce47d7eb291c0cc7f87cd040f38"
+source = "git+https://github.com/sigp/tree_hash?branch=progressive#ca07459738c9b1584eec99e37f3a52774fa97dd0"
 dependencies = [
  "darling 0.23.0",
  "proc-macro2",
@@ -10012,7 +10008,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]

Cargo.toml

@@ -275,4 +275,9 @@ debug = true
 
 [patch.crates-io]
 quick-protobuf = { git = "https://github.com/sigp/quick-protobuf.git", rev = "87c4ccb9bb2af494de375f5f6c62850badd26304" }
-
+# FIXME(sproul): REMOVE patch
+milhouse = { git = "https://github.com/sigp/milhouse", branch = "progressive-list" }
+ethereum_ssz = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" }
+ethereum_ssz_derive = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" }
+tree_hash = { git = "https://github.com/sigp/tree_hash", branch = "progressive" }
+tree_hash_derive = { git = "https://github.com/sigp/tree_hash", branch = "progressive" }

consensus/fork_choice/src/fork_choice.rs

@@ -278,8 +278,6 @@ pub struct QueuedAttestation {
     target_epoch: Epoch,
     /// Per Gloas spec: `payload_present = attestation.data.index == 1`.
     payload_present: bool,
-    /// Pre-Gloas latest messages update by target epoch. Gloas updates by slot.
-    update_by_slot: bool,
 }
 
 /// Legacy queued attestation without payload_present (pre-Gloas, schema V28).
@@ -291,18 +289,14 @@ pub struct QueuedAttestationV28 {
     target_epoch: Epoch,
 }
 
-impl QueuedAttestation {
-    fn from_indexed_attestation<E: EthSpec>(
-        a: IndexedAttestationRef<'_, E>,
-        update_by_slot: bool,
-    ) -> Self {
+impl<'a, E: EthSpec> From<IndexedAttestationRef<'a, E>> for QueuedAttestation {
+    fn from(a: IndexedAttestationRef<'a, E>) -> Self {
         Self {
             slot: a.data().slot,
             attesting_indices: a.attesting_indices_to_vec(),
             block_root: a.data().beacon_block_root,
             target_epoch: a.data().target.epoch,
-            payload_present: update_by_slot && a.data().index == 1,
-            update_by_slot,
+            payload_present: a.data().index == 1,
         }
     }
 }
@@ -771,30 +765,18 @@ where
     ) -> Result<(), Error<T::Error>> {
         let _timer = metrics::start_timer(&metrics::FORK_CHOICE_ON_BLOCK_TIMES);
 
+        // If this block has already been processed we do not need to reprocess it.
+        // We check this immediately in case re-processing the block mutates some property of the
+        // global fork choice store, e.g. the justified checkpoints or the proposer boost root.
+        if self.proto_array.contains_block(&block_root) {
+            return Ok(());
+        }
+
         // Provide the slot (as per the system clock) to the `fc_store` and then return its view of
         // the current slot. The `fc_store` will ensure that the `current_slot` is never
         // decreasing, a property which we must maintain.
         let current_slot = self.update_time(system_time_current_slot)?;
 
-        // Check block is later than the finalized epoch slot (optimization to reduce calls to
-        // get_ancestor).
-        let finalized_slot =
-            compute_start_slot_at_epoch::<E>(self.fc_store.finalized_checkpoint().epoch);
-        if block.slot() <= finalized_slot {
-            return Err(Error::InvalidBlock(InvalidBlock::FinalizedSlot {
-                finalized_slot,
-                block_slot: block.slot(),
-            }));
-        }
-
-        // If this block has already been processed we do not need to reprocess it.
-        // We check this before parent lookup and state updates in case re-processing the block
-        // mutates some property of the global fork choice store, e.g. the justified checkpoints or
-        // the proposer boost root. The finalized-slot check above still applies to match the spec.
-        if self.proto_array.contains_block(&block_root) {
-            return Ok(());
-        }
-
         // Parent block must be known.
         let parent_block = self
             .proto_array
@@ -812,6 +794,17 @@ where
             }));
         }
 
+        // Check that block is later than the finalized epoch slot (optimization to reduce calls to
+        // get_ancestor).
+        let finalized_slot =
+            compute_start_slot_at_epoch::<E>(self.fc_store.finalized_checkpoint().epoch);
+        if block.slot() <= finalized_slot {
+            return Err(Error::InvalidBlock(InvalidBlock::FinalizedSlot {
+                finalized_slot,
+                block_slot: block.slot(),
+            }));
+        }
+
         // Check block is a descendant of the finalized block at the checkpoint finalized slot.
         //
         // Note: the specification uses `hash_tree_root(block)` instead of `block.parent_root` for
@@ -830,45 +823,6 @@ where
             }));
         }
 
-        let execution_status = if let Ok(execution_payload) = block.body().execution_payload() {
-            let block_hash = execution_payload.block_hash();
-
-            if block_hash == ExecutionBlockHash::zero() {
-                // The block is post-merge-fork, but pre-terminal-PoW block. We don't need to verify
-                // the payload.
-                ExecutionStatus::irrelevant()
-            } else {
-                match payload_verification_status {
-                    PayloadVerificationStatus::Verified => ExecutionStatus::Valid(block_hash),
-                    PayloadVerificationStatus::Optimistic => {
-                        ExecutionStatus::Optimistic(block_hash)
-                    }
-                    // It would be a logic error to declare a block irrelevant if it has an
-                    // execution payload with a non-zero block hash.
-                    PayloadVerificationStatus::Irrelevant => {
-                        return Err(Error::InvalidPayloadStatus {
-                            block_slot: block.slot(),
-                            block_root,
-                            payload_verification_status,
-                        });
-                    }
-                }
-            }
-        } else {
-            // There is no payload to verify.
-            ExecutionStatus::irrelevant()
-        };
-
-        let (execution_payload_parent_hash, execution_payload_block_hash) =
-            if let Ok(signed_bid) = block.body().signed_execution_payload_bid() {
-                (
-                    Some(signed_bid.message.parent_block_hash),
-                    Some(signed_bid.message.block_hash),
-                )
-            } else {
-                (None, None)
-            };
-
         let attestation_threshold = spec.get_attestation_due::<E>(block.slot());
 
         // Add proposer score boost if the block is the first timely block for this slot and its
@@ -1002,6 +956,45 @@ where
             .on_verified_block(block, block_root, state)
             .map_err(Error::AfterBlockFailed)?;
 
+        let execution_status = if let Ok(execution_payload) = block.body().execution_payload() {
+            let block_hash = execution_payload.block_hash();
+
+            if block_hash == ExecutionBlockHash::zero() {
+                // The block is post-merge-fork, but pre-terminal-PoW block. We don't need to verify
+                // the payload.
+                ExecutionStatus::irrelevant()
+            } else {
+                match payload_verification_status {
+                    PayloadVerificationStatus::Verified => ExecutionStatus::Valid(block_hash),
+                    PayloadVerificationStatus::Optimistic => {
+                        ExecutionStatus::Optimistic(block_hash)
+                    }
+                    // It would be a logic error to declare a block irrelevant if it has an
+                    // execution payload with a non-zero block hash.
+                    PayloadVerificationStatus::Irrelevant => {
+                        return Err(Error::InvalidPayloadStatus {
+                            block_slot: block.slot(),
+                            block_root,
+                            payload_verification_status,
+                        });
+                    }
+                }
+            }
+        } else {
+            // There is no payload to verify.
+            ExecutionStatus::irrelevant()
+        };
+
+        let (execution_payload_parent_hash, execution_payload_block_hash) =
+            if let Ok(signed_bid) = block.body().signed_execution_payload_bid() {
+                (
+                    Some(signed_bid.message.parent_block_hash),
+                    Some(signed_bid.message.block_hash),
+                )
+            } else {
+                (None, None)
+            };
+
         // This does not apply a vote to the block, it just makes fork choice aware of the block so
         // it can still be identified as the head even if it doesn't have any votes.
         self.proto_array.process_block::<E>(
@@ -1290,10 +1283,10 @@ where
         self.validate_on_attestation(attestation, is_from_block, spec)?;
 
         // Per Gloas spec: `payload_present = attestation.data.index == 1`.
-        let is_gloas = spec
+        let payload_present = spec
             .fork_name_at_slot::<E>(attestation.data().slot)
-            .gloas_enabled();
-        let payload_present = is_gloas && attestation.data().index == 1;
+            .gloas_enabled()
+            && attestation.data().index == 1;
 
         if attestation.data().slot < self.fc_store.get_current_slot() {
             for validator_index in attestation.attesting_indices_iter() {
@@ -1302,8 +1295,6 @@ where
                     attestation.data().beacon_block_root,
                     attestation.data().slot,
                     payload_present,
-                    is_gloas,
-                    E::slots_per_epoch(),
                 )?;
             }
         } else {
@@ -1314,10 +1305,7 @@ where
             // Delay consideration in the fork choice until their slot is in the past.
             // ```
             self.queued_attestations
-                .push(QueuedAttestation::from_indexed_attestation(
-                    attestation,
-                    is_gloas,
-                ));
+                .push(QueuedAttestation::from(attestation));
         }
 
         Ok(())
@@ -1519,8 +1507,6 @@ where
                     attestation.block_root,
                     attestation.slot,
                     attestation.payload_present,
-                    attestation.update_by_slot,
-                    E::slots_per_epoch(),
                 )?;
             }
         }
@@ -1935,7 +1921,6 @@ mod tests {
                 block_root: Hash256::zero(),
                 target_epoch: Epoch::new(0),
                 payload_present: false,
-                update_by_slot: false,
             })
             .collect()
     }

consensus/proto_array/src/fork_choice_test_definition.rs

@@ -338,14 +338,7 @@ impl ForkChoiceTestDefinition {
                     attestation_slot,
                 } => {
                     fork_choice
-                        .process_attestation(
-                            validator_index,
-                            block_root,
-                            attestation_slot,
-                            false,
-                            true,
-                            MainnetEthSpec::slots_per_epoch(),
-                        )
+                        .process_attestation(validator_index, block_root, attestation_slot, false)
                         .unwrap_or_else(|_| {
                             panic!(
                                 "process_attestation op at index {} returned error",
@@ -366,8 +359,6 @@ impl ForkChoiceTestDefinition {
                             block_root,
                             attestation_slot,
                             payload_present,
-                            true,
-                            MainnetEthSpec::slots_per_epoch(),
                         )
                         .unwrap_or_else(|_| {
                             panic!(

consensus/proto_array/src/proto_array.rs

@@ -717,7 +717,7 @@ impl ProtoArray {
     /// Returns `true` if the proposer boost should be kept. Returns `false` if the
     /// boost should be subtracted (invalidated) because the parent is weak and there
     /// are no equivocating blocks at the parent's slot.
-    pub(crate) fn should_apply_proposer_boost<E: EthSpec>(
+    fn should_apply_proposer_boost<E: EthSpec>(
         &self,
         proposer_boost_root: Hash256,
         justified_balances: &JustifiedBalances,
@@ -1122,7 +1122,7 @@ impl ProtoArray {
     ///
     /// Returns the set of node indices on viable branches — those with at least
     /// one leaf descendant with correct justified/finalized checkpoints.
-    pub(crate) fn get_filtered_block_tree<E: EthSpec>(
+    fn get_filtered_block_tree<E: EthSpec>(
         &self,
         start_index: usize,
         current_slot: Slot,
@@ -1360,7 +1360,7 @@ impl ProtoArray {
 
     /// Spec: `get_weight`.
     #[allow(clippy::too_many_arguments)]
-    pub(crate) fn get_weight<E: EthSpec>(
+    fn get_weight<E: EthSpec>(
         &self,
         fc_node: &IndexedForkChoiceNode,
         proto_node: &ProtoNode,

consensus/proto_array/src/proto_array_fork_choice.rs

@@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize};
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
 use std::{
-    collections::{BTreeSet, HashMap, HashSet},
+    collections::{BTreeSet, HashMap},
     fmt,
     time::Duration,
 };
@@ -594,18 +594,10 @@ impl ProtoArrayForkChoice {
         block_root: Hash256,
         attestation_slot: Slot,
         payload_present: bool,
-        update_by_slot: bool,
-        slots_per_epoch: u64,
     ) -> Result<(), String> {
         let vote = self.votes.get_mut(validator_index);
 
-        let is_newer_vote = if update_by_slot {
-            attestation_slot > vote.next_slot
-        } else {
-            attestation_slot.epoch(slots_per_epoch) > vote.next_slot.epoch(slots_per_epoch)
-        };
-
-        if is_newer_vote || *vote == VoteTracker::default() {
+        if attestation_slot > vote.next_slot || *vote == VoteTracker::default() {
             vote.next_root = block_root;
             vote.next_slot = attestation_slot;
             vote.next_payload_present = payload_present;
@@ -1118,121 +1110,6 @@ impl ProtoArrayForkChoice {
             .map(|node| node.weight())
     }
 
-    /// Returns the leaves of the filtered block tree (rooted at `justified_root`) along with
-    /// their weights — i.e. roots that are viable for head and have no descendant that is also
-    /// viable for head. Mirrors the spec's `viable_for_head_roots_and_weights` check.
-    #[allow(clippy::too_many_arguments)]
-    pub fn filtered_block_tree_leaves_and_weights<E: EthSpec>(
-        &self,
-        justified_root: &Hash256,
-        current_slot: Slot,
-        justified_checkpoint: Checkpoint,
-        finalized_checkpoint: Checkpoint,
-        proposer_boost_root: Hash256,
-        justified_balances: &JustifiedBalances,
-        spec: &ChainSpec,
-    ) -> Result<Vec<(Hash256, PayloadStatus, u64)>, String> {
-        let start_index = self
-            .proto_array
-            .indices
-            .get(justified_root)
-            .copied()
-            .ok_or_else(|| {
-                format!(
-                    "filtered_block_tree_leaves_and_weights: justified node \
-                     {justified_root:?} unknown"
-                )
-            })?;
-        let viable = self.proto_array.get_filtered_block_tree::<E>(
-            start_index,
-            current_slot,
-            justified_checkpoint,
-            finalized_checkpoint,
-        );
-        let viable_indices = viable.iter().copied().collect::<HashSet<_>>();
-
-        let apply_proposer_boost = self
-            .proto_array
-            .should_apply_proposer_boost::<E>(proposer_boost_root, justified_balances, spec)
-            .map_err(|e| format!("should_apply_proposer_boost failed: {e:?}"))?;
-
-        let mut leaves = Vec::new();
-        let mut stack = vec![IndexedForkChoiceNode {
-            root: *justified_root,
-            proto_node_index: start_index,
-            payload_status: PayloadStatus::Pending,
-        }];
-
-        while let Some(fc_node) = stack.pop() {
-            let proto_node = self
-                .proto_array
-                .nodes
-                .get(fc_node.proto_node_index)
-                .ok_or_else(|| format!("invalid viable node index {}", fc_node.proto_node_index))?;
-
-            let children = if proto_node.payload_received().is_ok() {
-                if fc_node.payload_status == PayloadStatus::Pending {
-                    let mut children = vec![fc_node.with_status(PayloadStatus::Empty)];
-                    if proto_node.payload_received().is_ok_and(|received| received) {
-                        children.push(fc_node.with_status(PayloadStatus::Full));
-                    }
-                    children
-                } else {
-                    self.proto_array
-                        .nodes
-                        .iter()
-                        .enumerate()
-                        .filter(|(child_index, child_node)| {
-                            viable_indices.contains(child_index)
-                                && child_node.parent() == Some(fc_node.proto_node_index)
-                                && child_node.get_parent_payload_status() == fc_node.payload_status
-                        })
-                        .map(|(child_index, child_node)| IndexedForkChoiceNode {
-                            root: child_node.root(),
-                            proto_node_index: child_index,
-                            payload_status: PayloadStatus::Pending,
-                        })
-                        .collect()
-                }
-            } else {
-                self.proto_array
-                    .nodes
-                    .iter()
-                    .enumerate()
-                    .filter(|(child_index, child_node)| {
-                        viable_indices.contains(child_index)
-                            && child_node.parent() == Some(fc_node.proto_node_index)
-                    })
-                    .map(|(child_index, child_node)| IndexedForkChoiceNode {
-                        root: child_node.root(),
-                        proto_node_index: child_index,
-                        payload_status: PayloadStatus::Pending,
-                    })
-                    .collect()
-            };
-
-            if children.is_empty() {
-                let weight = self
-                    .proto_array
-                    .get_weight::<E>(
-                        &fc_node,
-                        proto_node,
-                        apply_proposer_boost,
-                        proposer_boost_root,
-                        current_slot,
-                        justified_balances,
-                        spec,
-                    )
-                    .map_err(|e| format!("get_weight failed: {e:?}"))?;
-                leaves.push((fc_node.root, fc_node.payload_status, weight));
-            } else {
-                stack.extend(children);
-            }
-        }
-
-        Ok(leaves)
-    }
-
     /// Returns the payload status of the head node based on accumulated weights and tiebreaker.
     ///
     /// See `ProtoArray` documentation.

consensus/state_processing/src/per_block_processing/tests.rs

@@ -493,7 +493,10 @@ async fn invalid_attestation_bad_aggregation_bitfield_len() {
         .next()
         .unwrap()
         .aggregation_bits_electra_mut()
-        .unwrap() = Bitfield::with_capacity(spec.target_committee_size).unwrap();
+        .unwrap() = Bitfield::<
+        ssz_types::length::Variable<<MainnetEthSpec as EthSpec>::MaxValidatorsPerSlot>,
+    >::with_capacity(spec.target_committee_size)
+    .unwrap();
 
     let mut ctxt = ConsensusContext::new(state.slot());
     let result = process_operations::process_attestations(

testing/ef_tests/Cargo.toml

@@ -19,6 +19,7 @@ compare_fields = { workspace = true }
 context_deserialize = { workspace = true }
 educe = { workspace = true }
 eth2_network_config = { workspace = true }
+ethereum_hashing = { workspace = true } # FIXME(sproul): remove
 ethereum_ssz = { workspace = true }
 ethereum_ssz_derive = { workspace = true }
 execution_layer = { workspace = true }

testing/ef_tests/Makefile

@@ -10,15 +10,9 @@ BLS_TEST = bls_tests_yaml
 BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME)
 BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_VERSION)
 
-# Fork-choice compliance tests from consensus-specs CI.
-# Pin to a specific workflow run ID for reproducibility. Update to pull newer vectors.
-COMPLIANCE_RUN_ID ?= 26135062633
-COMPLIANCE_PRESET ?= minimal
-COMPLIANCE_OUTPUT_DIR := $(OUTPUT_DIR)/tests/$(COMPLIANCE_PRESET)/fulu/fork_choice_compliance
-
 .PHONY: all clean
 
-all: clean $(OUTPUT_DIR) $(BLS_OUTPUT_DIR) $(COMPLIANCE_OUTPUT_DIR)
+all: clean $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
 
 clean:
 	rm -rf *.tar.gz $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
@@ -37,12 +31,3 @@ $(BLS_OUTPUT_DIR):
 		$(BLS_BASE_URL)/$(BLS_TEST).tar.gz
 	tar -xzf *.tar.gz -C $(BLS_OUTPUT_DIR)
 	rm -f *.tar.gz
-
-$(COMPLIANCE_OUTPUT_DIR):
-	@echo "Fetching fork-choice compliance tests (run $(COMPLIANCE_RUN_ID))..."
-	@curl -L -f -H "Authorization: token $(GITHUB_TOKEN)" \
-		"https://api.github.com/repos/ethereum/consensus-specs/actions/runs/$(COMPLIANCE_RUN_ID)/artifacts" \
-		| python3 -c "import sys,json; arts=json.load(sys.stdin)['artifacts']; url=next(a['archive_download_url'] for a in arts if a['name']=='small.tar.gz'); print(url)" \
-		| xargs curl -L -f --output compliance.tar.gz -H "Authorization: token $(GITHUB_TOKEN)"
-	tar -xzf compliance.tar.gz -C $(OUTPUT_DIR)
-	rm -f compliance.tar.gz

testing/ef_tests/check_all_files_accessed.py

@@ -60,13 +60,6 @@ excluded_paths = [
     "tests/.*/gloas/ssz_static/ExecutionPayloadHeader/.*",
     # ForkChoiceNode is internal to fork choice and probably doesn't need SSZ tests.
     "tests/.*/gloas/ssz_static/ForkChoiceNode/.*",
-    # EIP-7916 is still in draft and hasn't been implemented yet https://eips.ethereum.org/EIPS/eip-7916
-    "tests/general/phase0/ssz_generic/progressive_bitlist",
-    "tests/general/phase0/ssz_generic/basic_progressive_list",
-    "tests/general/phase0/ssz_generic/containers/.*/ProgressiveBitsStruct.*",
-    "tests/general/phase0/ssz_generic/containers/.*/ProgressiveTestStruct.*",
-    "tests/general/phase0/ssz_generic/progressive_containers/.*",
-    "tests/general/phase0/ssz_generic/compatible_unions/.*",
     # Ignore full epoch tests for now (just test the sub-transitions).
     "tests/.*/.*/epoch_processing/.*/pre_epoch.ssz_snappy",
     "tests/.*/.*/epoch_processing/.*/post_epoch.ssz_snappy",
@@ -79,8 +72,6 @@ excluded_paths = [
     "tests/.*/.*/networking/.*",
     # TODO: fast confirmation rule not merged yet
     "tests/.*/.*/fast_confirmation",
-    # TODO: fork choice compliance invalid_message tests not implemented yet
-    "tests/minimal/.*/fork_choice_compliance/invalid_message_test/.*"
 ]
 
 

testing/ef_tests/src/cases/fork_choice.rs

@@ -1,9 +1,6 @@
 use super::*;
 use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
-use ::fork_choice::{
-    AttestationFromBlock, ForkChoiceStore, PayloadStatus as FcPayloadStatus,
-    PayloadVerificationStatus, ProposerHeadError,
-};
+use ::fork_choice::{AttestationFromBlock, PayloadVerificationStatus, ProposerHeadError};
 use beacon_chain::beacon_proposer_cache::compute_proposer_duties_from_head;
 use beacon_chain::blob_verification::GossipBlobError;
 use beacon_chain::block_verification_types::LookupBlock;
@@ -12,7 +9,9 @@ use beacon_chain::data_column_verification::GossipVerifiedDataColumn;
 use beacon_chain::slot_clock::SlotClock;
 use beacon_chain::{
     AvailabilityProcessingStatus, BeaconChainTypes, CachedHead, ChainConfig, NotifyExecutionLayer,
-    attestation_verification::VerifiedAttestation,
+    attestation_verification::{
+        VerifiedAttestation, obtain_indexed_attestation_and_committees_per_slot,
+    },
     blob_verification::GossipVerifiedBlob,
     custody_context::NodeCustodyType,
     test_utils::{BeaconChainHarness, EphemeralHarnessType},
@@ -26,10 +25,8 @@ use serde::Deserialize;
 use ssz_derive::Decode;
 use ssz_types::VariableList;
 use state_processing::VerifySignatures;
-use state_processing::common::{attesting_indices_base, attesting_indices_electra};
 use state_processing::envelope_processing::verify_execution_payload_envelope;
 use state_processing::per_block_processing::is_valid_indexed_payload_attestation;
-use state_processing::per_block_processing::verify_attester_slashing;
 use state_processing::state_advance::complete_state_advance;
 use std::future::Future;
 use std::sync::Arc;
@@ -91,14 +88,6 @@ pub struct Checks {
     head_payload_status: Option<u8>,
     payload_timeliness_vote: Option<PayloadVoteCheck>,
     payload_data_availability_vote: Option<PayloadVoteCheck>,
-    viable_for_head_roots_and_weights: Option<Vec<RootAndWeight>>,
-}
-
-#[derive(Debug, Clone, Deserialize)]
-pub struct RootAndWeight {
-    pub root: Hash256,
-    pub weight: u64,
-    pub payload_status: Option<u8>,
 }
 
 #[derive(Debug, Clone, Deserialize)]
@@ -145,13 +134,9 @@ pub enum Step<
     },
     Attestation {
         attestation: TAttestation,
-        #[serde(default)]
-        valid: Option<bool>,
     },
     AttesterSlashing {
         attester_slashing: TAttesterSlashing,
-        #[serde(default)]
-        valid: Option<bool>,
     },
     PowBlock {
         pow_block: TPowBlock,
@@ -183,10 +168,11 @@ fn default_true() -> bool {
     true
 }
 
-#[derive(Debug, Clone, Default, Deserialize)]
+#[derive(Debug, Clone, Deserialize)]
+#[serde(deny_unknown_fields)]
 pub struct Meta {
-    #[serde(rename(deserialize = "description"), default)]
-    _description: Option<String>,
+    #[serde(rename(deserialize = "description"))]
+    _description: String,
 }
 
 #[derive(Debug)]
@@ -256,38 +242,31 @@ impl<E: EthSpec> LoadCase for ForkChoiceTest<E> {
                         valid,
                     })
                 }
-                Step::Attestation { attestation, valid } => {
+                Step::Attestation { attestation } => {
                     if fork_name.electra_enabled() {
                         ssz_decode_file(&path.join(format!("{}.ssz_snappy", attestation))).map(
                             |attestation| Step::Attestation {
                                 attestation: Attestation::Electra(attestation),
-                                valid,
                             },
                         )
                     } else {
                         ssz_decode_file(&path.join(format!("{}.ssz_snappy", attestation))).map(
                             |attestation| Step::Attestation {
                                 attestation: Attestation::Base(attestation),
-                                valid,
                             },
                         )
                     }
                 }
-                Step::AttesterSlashing {
-                    attester_slashing,
-                    valid,
-                } => {
+                Step::AttesterSlashing { attester_slashing } => {
                     if fork_name.electra_enabled() {
                         ssz_decode_file(&path.join(format!("{}.ssz_snappy", attester_slashing)))
                             .map(|attester_slashing| Step::AttesterSlashing {
                                 attester_slashing: AttesterSlashing::Electra(attester_slashing),
-                                valid,
                             })
                     } else {
                         ssz_decode_file(&path.join(format!("{}.ssz_snappy", attester_slashing)))
                             .map(|attester_slashing| Step::AttesterSlashing {
                                 attester_slashing: AttesterSlashing::Base(attester_slashing),
-                                valid,
                             })
                     }
                 }
@@ -412,51 +391,9 @@ impl<E: EthSpec> Case for ForkChoiceTest<E> {
                     proofs.clone(),
                     *valid,
                 )?,
-                Step::Attestation { attestation, valid } => {
-                    let result = tester.process_attestation(attestation);
-                    // Compliance tests use `valid: false` to indicate the attestation is
-                    // intentionally malformed and should be rejected. In that case, an error
-                    // here is the expected outcome.
-                    match valid {
-                        Some(false) => {
-                            if result.is_ok() {
-                                // We allow acceptance of future slot attestations which the spec
-                                // deems invalid (we just queue them).
-                                let current_slot = tester
-                                    .harness
-                                    .chain
-                                    .canonical_head
-                                    .fork_choice_read_lock()
-                                    .fc_store()
-                                    .get_current_slot();
-                                let future_attestation = attestation.data().slot >= current_slot;
-                                if !future_attestation {
-                                    return Err(Error::DidntFail(
-                                        "attestation marked valid=false should have been rejected"
-                                            .into(),
-                                    ));
-                                }
-                            }
-                        }
-                        Some(true) | None => result?,
-                    }
-                }
-                Step::AttesterSlashing {
-                    attester_slashing,
-                    valid,
-                } => {
-                    let result = tester.process_attester_slashing(attester_slashing.to_ref());
-                    match valid {
-                        Some(false) => {
-                            if result.is_ok() {
-                                return Err(Error::DidntFail(
-                                    "attester slashing marked valid=false should have been rejected"
-                                        .into(),
-                                ));
-                            }
-                        }
-                        Some(true) | None => result?,
-                    }
+                Step::Attestation { attestation } => tester.process_attestation(attestation)?,
+                Step::AttesterSlashing { attester_slashing } => {
+                    tester.process_attester_slashing(attester_slashing.to_ref())
                 }
                 Step::PowBlock { pow_block } => tester.process_pow_block(pow_block),
                 Step::OnPayloadInfo {
@@ -483,7 +420,6 @@ impl<E: EthSpec> Case for ForkChoiceTest<E> {
                         head_payload_status,
                         payload_timeliness_vote,
                         payload_data_availability_vote,
-                        viable_for_head_roots_and_weights,
                     } = checks.as_ref();
 
                     if let Some(expected_head) = head {
@@ -542,10 +478,6 @@ impl<E: EthSpec> Case for ForkChoiceTest<E> {
                     if let Some(expected) = payload_data_availability_vote {
                         tester.check_payload_data_availability_vote(expected)?;
                     }
-
-                    if let Some(expected) = viable_for_head_roots_and_weights {
-                        tester.check_viable_for_head_roots_and_weights(expected)?;
-                    }
                 }
 
                 Step::MaybeValidBlockAndColumns {
@@ -670,6 +602,7 @@ impl<E: EthSpec> Tester<E> {
         // Compute the slot time manually to ensure the slot clock is correct.
         let slot = self.tick_to_slot(tick).unwrap();
         assert_eq!(slot, self.harness.chain.slot().unwrap());
+
         self.harness
             .chain
             .canonical_head
@@ -684,13 +617,6 @@ impl<E: EthSpec> Tester<E> {
         columns: Option<DataColumnSidecarList<E>>,
         valid: bool,
     ) -> Result<(), Error> {
-        // Some fake-crypto EF fixtures contain block attestations that index to different
-        // validators depending on whether they are decoded against the block context or the
-        // attestation target context. Real BLS signatures should make these fixtures impossible.
-        if valid && self.block_attestations_have_divergent_indices(&block)? {
-            return Err(Error::SkippedKnownFailure);
-        }
-
         let block_root = block.canonical_root();
 
         let mut data_column_success = true;
@@ -729,16 +655,14 @@ impl<E: EthSpec> Tester<E> {
                 || Ok(()),
             ))?
             .map(|avail: AvailabilityProcessingStatus| avail.try_into());
-        let is_duplicate = matches!(
-            &result,
-            Err(beacon_chain::BlockError::DuplicateFullyImported(_))
-        );
-        let success = data_column_success
-            && (result.as_ref().is_ok_and(|inner| inner.is_ok()) || is_duplicate);
+        let success = data_column_success && result.as_ref().is_ok_and(|inner| inner.is_ok());
         if success != valid {
             return Err(Error::DidntFail(format!(
                 "block with root {} was valid={} whilst test expects valid={}. result: {:?}",
-                block_root, success, valid, result
+                block_root,
+                result.is_ok(),
+                valid,
+                result
             )));
         }
 
@@ -746,6 +670,18 @@ impl<E: EthSpec> Tester<E> {
             self.apply_invalid_block(&block)?;
         }
 
+        // Per spec test runner: an on_block step implies receiving block's attestations
+        // and attester slashings.
+        if success {
+            for attestation in block.message().body().attestations() {
+                let att = attestation.clone_as_attestation();
+                let _ = self.process_attestation(&att);
+            }
+            for attester_slashing in block.message().body().attester_slashings() {
+                self.process_attester_slashing(attester_slashing);
+            }
+        }
+
         Ok(())
     }
 
@@ -756,13 +692,6 @@ impl<E: EthSpec> Tester<E> {
         kzg_proofs: Option<Vec<KzgProof>>,
         valid: bool,
     ) -> Result<(), Error> {
-        // Some fake-crypto EF fixtures contain block attestations that index to different
-        // validators depending on whether they are decoded against the block context or the
-        // attestation target context. Real BLS signatures should make these fixtures impossible.
-        if valid && self.block_attestations_have_divergent_indices(&block)? {
-            return Err(Error::SkippedKnownFailure);
-        }
-
         let block_root = block.canonical_root();
 
         let mut blob_success = true;
@@ -825,19 +754,14 @@ impl<E: EthSpec> Tester<E> {
                 || Ok(()),
             ))?
             .map(|avail: AvailabilityProcessingStatus| avail.try_into());
-        // Spec `on_block` is idempotent: re-importing an already-known block is a no-op
-        // success. Lighthouse surfaces this as `BlockError::DuplicateFullyImported`; the
-        // compliance suite re-feeds blocks repeatedly, so treat duplicates as success.
-        let is_duplicate = matches!(
-            &result,
-            Err(beacon_chain::BlockError::DuplicateFullyImported(_))
-        );
-        let success =
-            blob_success && (result.as_ref().is_ok_and(|inner| inner.is_ok()) || is_duplicate);
+        let success = blob_success && result.as_ref().is_ok_and(|inner| inner.is_ok());
         if success != valid {
             return Err(Error::DidntFail(format!(
                 "block with root {} was valid={} whilst test expects valid={}. result: {:?}",
-                block_root, success, valid, result
+                block_root,
+                result.is_ok(),
+                valid,
+                result
             )));
         }
 
@@ -845,143 +769,21 @@ impl<E: EthSpec> Tester<E> {
             self.apply_invalid_block(&block)?;
         }
 
+        // Per spec test runner: an on_block step implies receiving block's attestations
+        // and attester slashings.
+        if success {
+            for attestation in block.message().body().attestations() {
+                let att = attestation.clone_as_attestation();
+                let _ = self.process_attestation(&att);
+            }
+            for attester_slashing in block.message().body().attester_slashings() {
+                self.process_attester_slashing(attester_slashing);
+            }
+        }
+
         Ok(())
     }
 
-    fn block_attestations_have_divergent_indices(
-        &self,
-        block: &SignedBeaconBlock<E>,
-    ) -> Result<bool, Error> {
-        let parent_root = block.parent_root();
-        let Some(parent_block) = self
-            .harness
-            .chain
-            .get_blinded_block(&parent_root)
-            .map_err(|e| Error::InternalError(format!("failed to load parent block: {e:?}")))?
-        else {
-            return Ok(false);
-        };
-        let parent_state_root = parent_block.state_root();
-        let Some(mut block_context_state) = self
-            .harness
-            .chain
-            .get_state(
-                &parent_state_root,
-                Some(parent_block.slot()),
-                CACHE_STATE_IN_TESTS,
-            )
-            .map_err(|e| Error::InternalError(format!("failed to load parent state: {e:?}")))?
-        else {
-            return Ok(false);
-        };
-
-        complete_state_advance(
-            &mut block_context_state,
-            Some(parent_state_root),
-            block.slot(),
-            &self.harness.chain.spec,
-        )
-        .map_err(|e| {
-            Error::InternalError(format!("failed to advance block context state: {e:?}"))
-        })?;
-        block_context_state
-            .build_all_committee_caches(&self.harness.chain.spec)
-            .map_err(|e| {
-                Error::InternalError(format!(
-                    "failed to build block context committee caches: {e:?}"
-                ))
-            })?;
-
-        for attestation in block.message().body().attestations() {
-            let attestation = attestation.clone_as_attestation();
-            let Ok(block_context_indexed) =
-                Self::indexed_attestation_from_state(&block_context_state, &attestation)
-            else {
-                continue;
-            };
-            let Some(target_context_indexed) =
-                self.indexed_attestation_from_target_state(&attestation)?
-            else {
-                continue;
-            };
-
-            if block_context_indexed.attesting_indices_to_vec()
-                != target_context_indexed.attesting_indices_to_vec()
-            {
-                return Ok(true);
-            }
-        }
-
-        Ok(false)
-    }
-
-    fn indexed_attestation_from_target_state(
-        &self,
-        attestation: &Attestation<E>,
-    ) -> Result<Option<IndexedAttestation<E>>, Error> {
-        let target_root = attestation.data().target.root;
-        let Some(target_block) = self
-            .harness
-            .chain
-            .canonical_head
-            .fork_choice_read_lock()
-            .get_block(&target_root)
-        else {
-            return Ok(None);
-        };
-        let Some(mut target_state) = self
-            .harness
-            .chain
-            .store
-            .get_hot_state(&target_block.state_root, CACHE_STATE_IN_TESTS)
-            .map_err(|e| Error::InternalError(format!("failed to load target state: {e:?}")))?
-        else {
-            return Ok(None);
-        };
-        let target_epoch_start_slot = attestation
-            .data()
-            .target
-            .epoch
-            .start_slot(E::slots_per_epoch());
-        complete_state_advance(
-            &mut target_state,
-            Some(target_block.state_root),
-            target_epoch_start_slot,
-            &self.harness.chain.spec,
-        )
-        .map_err(|e| {
-            Error::InternalError(format!("failed to advance attestation target state: {e:?}"))
-        })?;
-
-        match Self::indexed_attestation_from_state(&target_state, attestation) {
-            Ok(indexed_attestation) => Ok(Some(indexed_attestation)),
-            Err(_) => Ok(None),
-        }
-    }
-
-    fn indexed_attestation_from_state(
-        state: &BeaconState<E>,
-        attestation: &Attestation<E>,
-    ) -> Result<IndexedAttestation<E>, Error> {
-        match attestation.to_ref() {
-            AttestationRef::Base(att) => {
-                let committee = state
-                    .get_beacon_committee(att.data.slot, att.data.index)
-                    .map_err(|e| {
-                        Error::InternalError(format!("attestation committee lookup failed: {e:?}"))
-                    })?;
-                attesting_indices_base::get_indexed_attestation(committee.committee, att).map_err(
-                    |e| Error::InternalError(format!("attestation indexing failed: {e:?}")),
-                )
-            }
-            AttestationRef::Electra(att) => {
-                attesting_indices_electra::get_indexed_attestation_from_state(state, att).map_err(
-                    |e| Error::InternalError(format!("attestation indexing failed: {e:?}")),
-                )
-            }
-        }
-    }
-
     // Apply invalid blocks directly against the fork choice `on_block` function. This ensures
     // that the block is being rejected by `on_block`, not just some upstream block processing
     // function. When data columns or blobs exist, we don't do this.
@@ -1022,9 +824,6 @@ impl<E: EthSpec> Tester<E> {
                 .seconds_from_current_slot_start()
                 .unwrap();
 
-            // FIXME(sproul): this whole concept is a bit ill-conceived, the blocks just get
-            // rejected here due to passing payload status irrelevant, which is not a real codepath
-            // that should be reached
             let result = self
                 .harness
                 .chain
@@ -1053,14 +852,11 @@ impl<E: EthSpec> Tester<E> {
     }
 
     pub fn process_attestation(&self, attestation: &Attestation<E>) -> Result<(), Error> {
-        let indexed_attestation = self
-            .indexed_attestation_from_target_state(attestation)?
-            .ok_or_else(|| {
-                Error::InternalError(format!(
-                    "attestation target block {:?} unknown or could not be indexed from target state",
-                    attestation.data().target.root
-                ))
-            })?;
+        let (indexed_attestation, _) = obtain_indexed_attestation_and_committees_per_slot(
+            &self.harness.chain,
+            attestation.to_ref(),
+        )
+        .map_err(|e| Error::InternalError(format!("attestation indexing failed with {:?}", e)))?;
         let verified_attestation: ManuallyVerifiedAttestation<EphemeralHarnessType<E>> =
             ManuallyVerifiedAttestation {
                 attestation,
@@ -1073,44 +869,12 @@ impl<E: EthSpec> Tester<E> {
             .map_err(|e| Error::InternalError(format!("attestation import failed with {:?}", e)))
     }
 
-    pub fn process_attester_slashing(
-        &self,
-        attester_slashing: AttesterSlashingRef<E>,
-    ) -> Result<(), Error> {
-        let justified_block = {
-            let fork_choice = self.harness.chain.canonical_head.fork_choice_read_lock();
-            fork_choice
-                .get_block(&fork_choice.justified_checkpoint().root)
-                .ok_or_else(|| Error::InternalError("justified block not found".into()))?
-        };
-        let justified_state = self
-            .harness
-            .chain
-            .store
-            .get_hot_state(&justified_block.state_root, CACHE_STATE_IN_TESTS)
-            .map_err(|e| Error::InternalError(format!("failed to load justified state: {e:?}")))?
-            .ok_or_else(|| {
-                Error::InternalError(format!(
-                    "justified state {:?} not found",
-                    justified_block.state_root
-                ))
-            })?;
-
-        verify_attester_slashing(
-            &justified_state,
-            attester_slashing,
-            VerifySignatures::True,
-            &self.harness.chain.spec,
-        )
-        .map_err(|e| Error::InternalError(format!("invalid attester slashing: {e:?}")))?;
-
+    pub fn process_attester_slashing(&self, attester_slashing: AttesterSlashingRef<E>) {
         self.harness
             .chain
             .canonical_head
             .fork_choice_write_lock()
-            .on_attester_slashing(attester_slashing);
-
-        Ok(())
+            .on_attester_slashing(attester_slashing)
     }
 
     pub fn process_pow_block(&self, pow_block: &PowBlock) {
@@ -1384,61 +1148,6 @@ impl<E: EthSpec> Tester<E> {
         check_equal("head_payload_status", actual, expected_status)
     }
 
-    pub fn check_viable_for_head_roots_and_weights(
-        &self,
-        expected: &[RootAndWeight],
-    ) -> Result<(), Error> {
-        // Apply pending vote deltas so weights reflect the latest store state.
-        let _ = self.find_head()?;
-
-        let fork_choice = self.harness.chain.canonical_head.fork_choice_read_lock();
-        let justified = fork_choice.justified_checkpoint();
-        let finalized = fork_choice.finalized_checkpoint();
-        let current_slot = fork_choice.fc_store().get_current_slot();
-        let proposer_boost_root = fork_choice.proposer_boost_root();
-        let justified_balances = fork_choice.fc_store().justified_balances().clone();
-        let actual = fork_choice
-            .proto_array()
-            .filtered_block_tree_leaves_and_weights::<E>(
-                &justified.root,
-                current_slot,
-                justified,
-                finalized,
-                proposer_boost_root,
-                &justified_balances,
-                &self.spec,
-            )
-            .map_err(|e| {
-                Error::InternalError(format!(
-                    "filtered_block_tree_leaves_and_weights failed: {e}"
-                ))
-            })?;
-        drop(fork_choice);
-
-        let mut actual_sorted: Vec<(Hash256, u8, u64)> = actual
-            .into_iter()
-            .map(|(root, status, weight)| (root, status as u8, weight))
-            .collect();
-        actual_sorted.sort();
-        let mut expected_sorted: Vec<(Hash256, u8, u64)> = expected
-            .iter()
-            .map(|x| {
-                (
-                    x.root,
-                    x.payload_status.unwrap_or(FcPayloadStatus::Pending as u8),
-                    x.weight,
-                )
-            })
-            .collect();
-        expected_sorted.sort();
-
-        check_equal(
-            "viable_for_head_roots_and_weights",
-            actual_sorted,
-            expected_sorted,
-        )
-    }
-
     pub fn check_should_override_fcu(
         &self,
         expected_should_override_fcu: ShouldOverrideFcu,

testing/ef_tests/src/cases/ssz_generic.rs

@@ -5,8 +5,10 @@ use crate::cases::common::{DecimalU128, DecimalU256, SszStaticType};
 use crate::cases::ssz_static::{check_serialization, check_tree_hash};
 use crate::decode::{context_yaml_decode_file, log_file_access, snappy_decode_file};
 use context_deserialize::{ContextDeserialize, context_deserialize};
-use milhouse::Vector;
+use milhouse::{List, ProgressiveList, Vector};
 use serde::{Deserialize, Deserializer, de::Error as SerdeError};
+use serde_json::Value as JsonValue;
+use ssz::ProgressiveBitList;
 use ssz_derive::{Decode, Encode};
 use ssz_types::{BitList, BitVector, FixedVector, VariableList};
 use tree_hash::TreeHash;
@@ -14,6 +16,46 @@ use tree_hash_derive::TreeHash;
 use typenum::*;
 use types::ForkName;
 
+/// Helper struct for deserializing compatible unions from `{selector, data}` YAML format.
+#[derive(Deserialize)]
+struct CompatibleUnionYaml {
+    selector: u8,
+    data: JsonValue,
+}
+
+/// Implements `Deserialize` for compatible union types to handle the EF test YAML format.
+///
+/// Deserialize into `CompatibleUnionYaml` which captures `selector` (`u8`) and
+///    `data` (`JsonValue`).
+/// Match on the selector to determine which variant to construct.
+/// Deserialize the `data` field into the appropriate inner type.
+macro_rules! impl_compatible_union_deserialize {
+    ($type:ty, { $($selector:literal => $variant:ident($inner:ty)),+ $(,)? }) => {
+        impl<'de> Deserialize<'de> for $type {
+            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+            where
+                D: Deserializer<'de>,
+            {
+                let yaml = CompatibleUnionYaml::deserialize(deserializer)?;
+                match yaml.selector {
+                    $(
+                        $selector => {
+                            let inner: $inner = serde_json::from_value(yaml.data).map_err(D::Error::custom)?;
+                            Ok(<$type>::$variant(inner))
+                        }
+                    )+
+                    s => Err(D::Error::custom(format!(
+                        "unknown selector {s} for {}", stringify!($type)
+                    ))),
+                }
+            }
+        }
+    };
+}
+
+type U1280 = op!(U128 * U10);
+type U1281 = op!(U1280 + U1);
+
 #[derive(Debug, Clone, Deserialize)]
 #[context_deserialize(ForkName)]
 struct Metadata {
@@ -113,8 +155,15 @@ macro_rules! type_dispatch {
             "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*),
             "ComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ComplexTestStruct>, $($rest)*),
             "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*),
-            // EIP-7916 is still in draft and hasn't been implemented yet https://eips.ethereum.org/EIPS/eip-7916
-            "ProgressiveTestStruct" | "ProgressiveBitsStruct" => Err(Error::SkippedKnownFailure),
+            "ProgressiveBitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveBitsStruct>, $($rest)*),
+            "ProgressiveSingleFieldContainerTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveSingleFieldContainerTestStruct>, $($rest)*),
+            "ProgressiveSingleListContainerTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveSingleListContainerTestStruct>, $($rest)*),
+            "ProgressiveVarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveVarTestStruct>, $($rest)*),
+            "ProgressiveComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveComplexTestStruct>, $($rest)*),
+            "ProgressiveTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveTestStruct>, $($rest)*),
+            "CompatibleUnionA" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* CompatibleUnionA>, $($rest)*),
+            "CompatibleUnionBC" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* CompatibleUnionBC>, $($rest)*),
+            "CompatibleUnionABCA" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* CompatibleUnionABCA>, $($rest)*),
             _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
         }
     };
@@ -159,6 +208,17 @@ impl Case for SszGeneric {
                     [length => typenum]
                 )?;
             }
+            "basic_progressive_list" => {
+                let elem_ty = parts[1];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    ProgressiveList,
+                    <>,
+                    [elem_ty => primitive_type]
+                )?;
+            }
             "bitlist" => {
                 let mut limit = parts[1];
 
@@ -186,6 +246,14 @@ impl Case for SszGeneric {
                     [length => typenum]
                 )?;
             }
+            "progressive_bitlist" => {
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    ProgressiveBitList,
+                    <>,
+                )?;
+            }
             "boolean" => {
                 ssz_generic_test::<bool>(&self.path, fork_name)?;
             }
@@ -211,6 +279,28 @@ impl Case for SszGeneric {
                     [type_name => test_container]
                 )?;
             }
+            "progressive_containers" => {
+                let type_name = parts[0];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    _,
+                    <>,
+                    [type_name => test_container]
+                )?;
+            }
+            "compatible_unions" => {
+                let type_name = parts[0];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    _,
+                    <>,
+                    [type_name => test_container]
+                )?;
+            }
             _ => panic!("unsupported handler: {}", self.handler_name),
         }
         Ok(())
@@ -302,6 +392,15 @@ struct ComplexTestStruct {
     G: Vector<VarTestStruct, U2>,
 }
 
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[context_deserialize(ForkName)]
+struct ProgressiveTestStruct {
+    A: ProgressiveList<u8>,
+    B: ProgressiveList<u64>,
+    C: ProgressiveList<SmallTestStruct>,
+    D: ProgressiveList<ProgressiveList<VarTestStruct>>,
+}
+
 #[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
 #[context_deserialize(ForkName)]
 struct BitsStruct {
@@ -312,6 +411,120 @@ struct BitsStruct {
     E: BitVector<U8>,
 }
 
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[context_deserialize(ForkName)]
+struct ProgressiveBitsStruct {
+    A: BitVector<U256>,
+    B: BitList<U256>,
+    C: ProgressiveBitList,
+    D: BitVector<U257>,
+    E: BitList<U257>,
+    F: ProgressiveBitList,
+    G: BitVector<U1280>,
+    H: BitList<U1280>,
+    I: ProgressiveBitList,
+    J: BitVector<U1281>,
+    K: BitList<U1281>,
+    L: ProgressiveBitList,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(struct_behaviour = "progressive_container", active_fields(1))]
+#[context_deserialize(ForkName)]
+struct ProgressiveSingleFieldContainerTestStruct {
+    A: u8,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(
+    struct_behaviour = "progressive_container",
+    active_fields(0, 0, 0, 0, 1)
+)]
+#[context_deserialize(ForkName)]
+struct ProgressiveSingleListContainerTestStruct {
+    C: ProgressiveBitList,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(
+    struct_behaviour = "progressive_container",
+    active_fields(1, 0, 1, 0, 1)
+)]
+#[context_deserialize(ForkName)]
+struct ProgressiveVarTestStruct {
+    A: u8,
+    B: List<u16, U123>,
+    C: ProgressiveBitList,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(
+    struct_behaviour = "progressive_container",
+    active_fields(1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1)
+)]
+#[context_deserialize(ForkName)]
+struct ProgressiveComplexTestStruct {
+    A: u8,
+    B: List<u16, U123>,
+    C: ProgressiveBitList,
+    D: ProgressiveList<u64>,
+    E: ProgressiveList<SmallTestStruct>,
+    F: ProgressiveList<ProgressiveList<VarTestStruct>>,
+    G: List<ProgressiveSingleFieldContainerTestStruct, U10>,
+    H: ProgressiveList<ProgressiveVarTestStruct>,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash)]
+#[ssz(enum_behaviour = "compatible_union")]
+#[tree_hash(enum_behaviour = "compatible_union")]
+#[context_deserialize(ForkName)]
+enum CompatibleUnionA {
+    #[ssz(selector = "1")]
+    ProgressiveSingleFieldContainerTestStruct(ProgressiveSingleFieldContainerTestStruct),
+}
+
+impl_compatible_union_deserialize!(CompatibleUnionA, {
+    1 => ProgressiveSingleFieldContainerTestStruct(ProgressiveSingleFieldContainerTestStruct),
+});
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash)]
+#[ssz(enum_behaviour = "compatible_union")]
+#[tree_hash(enum_behaviour = "compatible_union")]
+#[context_deserialize(ForkName)]
+enum CompatibleUnionBC {
+    #[ssz(selector = "2")]
+    ProgressiveSingleListContainerTestStruct(ProgressiveSingleListContainerTestStruct),
+    #[ssz(selector = "3")]
+    ProgressiveVarTestStruct(ProgressiveVarTestStruct),
+}
+
+impl_compatible_union_deserialize!(CompatibleUnionBC, {
+    2 => ProgressiveSingleListContainerTestStruct(ProgressiveSingleListContainerTestStruct),
+    3 => ProgressiveVarTestStruct(ProgressiveVarTestStruct),
+});
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash)]
+#[ssz(enum_behaviour = "compatible_union")]
+#[tree_hash(enum_behaviour = "compatible_union")]
+#[context_deserialize(ForkName)]
+enum CompatibleUnionABCA {
+    #[ssz(selector = "1")]
+    A1(ProgressiveSingleFieldContainerTestStruct),
+    #[ssz(selector = "2")]
+    B1(ProgressiveSingleListContainerTestStruct),
+    #[ssz(selector = "3")]
+    C1(ProgressiveVarTestStruct),
+    #[ssz(selector = "4")]
+    A2(ProgressiveSingleFieldContainerTestStruct),
+}
+
+impl_compatible_union_deserialize!(CompatibleUnionABCA, {
+    1 => A1(ProgressiveSingleFieldContainerTestStruct),
+    2 => B1(ProgressiveSingleListContainerTestStruct),
+    3 => C1(ProgressiveVarTestStruct),
+    4 => A2(ProgressiveSingleFieldContainerTestStruct),
+});
+
 fn byte_list_from_hex_str<'de, D, N: Unsigned>(
     deserializer: D,
 ) -> Result<VariableList<u8, N>, D::Error>

testing/ef_tests/src/handler.rs

@@ -745,48 +745,6 @@ impl<E: EthSpec + TypeName> Handler for ForkChoiceHandler<E> {
     }
 }
 
-pub struct ForkChoiceComplianceHandler<E> {
-    handler_name: String,
-    _phantom: PhantomData<E>,
-}
-
-impl<E: EthSpec> ForkChoiceComplianceHandler<E> {
-    pub fn new(handler_name: &str) -> Self {
-        Self {
-            handler_name: handler_name.into(),
-            _phantom: PhantomData,
-        }
-    }
-}
-
-impl<E: EthSpec + TypeName> Handler for ForkChoiceComplianceHandler<E> {
-    type Case = cases::ForkChoiceTest<E>;
-
-    fn config_name() -> &'static str {
-        E::name()
-    }
-
-    fn runner_name() -> &'static str {
-        "fork_choice_compliance"
-    }
-
-    fn handler_name(&self) -> String {
-        self.handler_name.clone()
-    }
-
-    fn use_rayon() -> bool {
-        false
-    }
-
-    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
-        cfg!(feature = "fake_crypto") && fork_name.fulu_enabled()
-    }
-
-    fn disabled_forks(&self) -> Vec<ForkName> {
-        vec![]
-    }
-}
-
 #[derive(Educe)]
 #[educe(Default)]
 pub struct OptimisticSyncHandler<E>(PhantomData<E>);
@@ -1257,13 +1215,21 @@ impl<H: TypeName> Handler for SszGenericHandler<H> {
 // Supported SSZ generic handlers
 pub struct BasicVector;
 type_name!(BasicVector, "basic_vector");
+pub struct BasicProgressiveList;
+type_name!(BasicProgressiveList, "basic_progressive_list");
 pub struct Bitlist;
 type_name!(Bitlist, "bitlist");
 pub struct Bitvector;
 type_name!(Bitvector, "bitvector");
+pub struct ProgressiveBitlist;
+type_name!(ProgressiveBitlist, "progressive_bitlist");
 pub struct Boolean;
 type_name!(Boolean, "boolean");
 pub struct Uints;
 type_name!(Uints, "uints");
 pub struct Containers;
 type_name!(Containers, "containers");
+pub struct ProgressiveContainers;
+type_name!(ProgressiveContainers, "progressive_containers");
+pub struct CompatibleUnions;
+type_name!(CompatibleUnions, "compatible_unions");

testing/ef_tests/tests/tests.rs

@@ -880,6 +880,14 @@ fn ssz_generic() {
     SszGenericHandler::<Containers>::default().run();
 }
 
+#[test]
+fn ssz_generic_progressive() {
+    SszGenericHandler::<BasicProgressiveList>::default().run();
+    SszGenericHandler::<ProgressiveBitlist>::default().run();
+    SszGenericHandler::<ProgressiveContainers>::default().run();
+    SszGenericHandler::<CompatibleUnions>::default().run();
+}
+
 #[test]
 fn epoch_processing_justification_and_finalization() {
     EpochProcessingHandler::<MinimalEthSpec, JustificationAndFinalization>::default().run();
@@ -1085,37 +1093,6 @@ fn fork_choice_on_payload_attestation_message() {
     ForkChoiceHandler::<MainnetEthSpec>::new("on_payload_attestation_message").run();
 }
 
-#[test]
-fn fork_choice_compliance_attester_slashing_test() {
-    ForkChoiceComplianceHandler::<MinimalEthSpec>::new("attester_slashing_test").run();
-}
-
-#[test]
-fn fork_choice_compliance_block_cover_test() {
-    ForkChoiceComplianceHandler::<MinimalEthSpec>::new("block_cover_test").run();
-}
-
-#[test]
-fn fork_choice_compliance_block_tree_test() {
-    ForkChoiceComplianceHandler::<MinimalEthSpec>::new("block_tree_test").run();
-}
-
-#[test]
-fn fork_choice_compliance_block_weight_test() {
-    ForkChoiceComplianceHandler::<MinimalEthSpec>::new("block_weight_test").run();
-}
-
-#[test]
-#[ignore]
-fn fork_choice_compliance_invalid_message_test() {
-    ForkChoiceComplianceHandler::<MinimalEthSpec>::new("invalid_message_test").run();
-}
-
-#[test]
-fn fork_choice_compliance_shuffling_test() {
-    ForkChoiceComplianceHandler::<MinimalEthSpec>::new("shuffling_test").run();
-}
-
 #[test]
 fn optimistic_sync() {
     OptimisticSyncHandler::<MinimalEthSpec>::default().run();