Fixes to make EF Capella tests pass (#3719)

* Fixes to make EF Capella tests pass

* Clippy for state_processing

testing/ef_tests/src/cases/epoch_processing.rs

@@ -289,8 +289,9 @@ impl<E: EthSpec, T: EpochTransition<E>> Case for EpochProcessing<E, T> {
                     && T::name() != "participation_flag_updates"
             }
             // No phase0 tests for Altair and later.
-            ForkName::Altair | ForkName::Merge => T::name() != "participation_record_updates",
-            ForkName::Capella => false, // TODO: revisit when tests are out
+            ForkName::Altair | ForkName::Merge | ForkName::Capella => {
+                T::name() != "participation_record_updates"
+            }
             ForkName::Eip4844 => false, // TODO: revisit when tests are out
         }
     }

testing/ef_tests/src/cases/fork.rs

@@ -3,7 +3,7 @@ use crate::case_result::compare_beacon_state_results_without_caches;
 use crate::cases::common::previous_fork;
 use crate::decode::{ssz_decode_state, yaml_decode_file};
 use serde_derive::Deserialize;
-use state_processing::upgrade::{upgrade_to_altair, upgrade_to_bellatrix};
+use state_processing::upgrade::{upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella};
 use types::{BeaconState, ForkName};
 
 #[derive(Debug, Clone, Default, Deserialize)]
@@ -61,8 +61,8 @@ impl<E: EthSpec> Case for ForkTest<E> {
             ForkName::Base => panic!("phase0 not supported"),
             ForkName::Altair => upgrade_to_altair(&mut result_state, spec).map(|_| result_state),
             ForkName::Merge => upgrade_to_bellatrix(&mut result_state, spec).map(|_| result_state),
+            ForkName::Capella => upgrade_to_capella(&mut result_state, spec).map(|_| result_state),
             ForkName::Eip4844 => panic!("eip4844 not supported"),
-            ForkName::Capella => panic!("capella not supported"),
         };
 
         compare_beacon_state_results_without_caches(&mut result, &mut expected)

testing/ef_tests/src/cases/genesis_initialization.rs

@@ -1,13 +1,10 @@
 use super::*;
 use crate::case_result::compare_beacon_state_results_without_caches;
-use crate::decode::{ssz_decode_file, ssz_decode_state, yaml_decode_file};
+use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
 use serde_derive::Deserialize;
 use state_processing::initialize_beacon_state_from_eth1;
 use std::path::PathBuf;
-use types::{
-    BeaconState, Deposit, EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderMerge, ForkName,
-    Hash256,
-};
+use types::{BeaconState, Deposit, EthSpec, ExecutionPayloadHeader, ForkName, Hash256};
 
 #[derive(Debug, Clone, Deserialize)]
 struct Metadata {
@@ -41,14 +38,10 @@ impl<E: EthSpec> LoadCase for GenesisInitialization<E> {
         let meta: Metadata = yaml_decode_file(&path.join("meta.yaml"))?;
         let execution_payload_header: Option<ExecutionPayloadHeader<E>> =
             if meta.execution_payload_header.unwrap_or(false) {
-                //FIXME(sean) we could decode based on timestamp - we probably don't do decode a payload
-                // without a block this elsewhere at presetn. But when we support SSZ in the builder api we may need to.
-                // Although that API should include fork info. Hardcoding this for now
-                Some(ExecutionPayloadHeader::Merge(ssz_decode_file::<
-                    ExecutionPayloadHeaderMerge<E>,
-                >(
+                Some(ssz_decode_file_with(
                     &path.join("execution_payload_header.ssz_snappy"),
-                )?))
+                    |bytes| ExecutionPayloadHeader::from_ssz_bytes(bytes, fork_name),
+                )?)
             } else {
                 None
             };

testing/ef_tests/src/cases/operations.rs

@@ -3,17 +3,16 @@ use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
 use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
 use crate::testing_spec;
-use crate::type_name::TypeName;
 use serde_derive::Deserialize;
 use state_processing::{
     per_block_processing::{
         errors::BlockProcessingError,
         process_block_header, process_execution_payload,
         process_operations::{
-            altair, base, process_attester_slashings, process_deposits, process_exits,
-            process_proposer_slashings,
+            altair, base, process_attester_slashings, process_bls_to_execution_changes,
+            process_deposits, process_exits, process_proposer_slashings,
         },
-        process_sync_aggregate, VerifyBlockRoot, VerifySignatures,
+        process_sync_aggregate, process_withdrawals, VerifyBlockRoot, VerifySignatures,
     },
     ConsensusContext,
 };
@@ -21,7 +20,7 @@ use std::fmt::Debug;
 use std::path::Path;
 use types::{
     Attestation, AttesterSlashing, BeaconBlock, BeaconState, BlindedPayload, ChainSpec, Deposit,
-    EthSpec, ExecutionPayload, ExecutionPayloadMerge, ForkName, FullPayload, ProposerSlashing,
+    EthSpec, ExecutionPayload, ForkName, FullPayload, ProposerSlashing, SignedBlsToExecutionChange,
     SignedVoluntaryExit, SyncAggregate,
 };
 
@@ -36,6 +35,12 @@ struct ExecutionMetadata {
     execution_valid: bool,
 }
 
+/// Newtype for testing withdrawals.
+#[derive(Debug, Clone, Deserialize)]
+pub struct WithdrawalsPayload<T: EthSpec> {
+    payload: FullPayload<T>,
+}
+
 #[derive(Debug, Clone)]
 pub struct Operations<E: EthSpec, O: Operation<E>> {
     metadata: Metadata,
@@ -45,10 +50,8 @@ pub struct Operations<E: EthSpec, O: Operation<E>> {
     pub post: Option<BeaconState<E>>,
 }
 
-pub trait Operation<E: EthSpec>: TypeName + Debug + Sync + Sized {
-    fn handler_name() -> String {
-        Self::name().to_lowercase()
-    }
+pub trait Operation<E: EthSpec>: Debug + Sync + Sized {
+    fn handler_name() -> String;
 
     fn filename() -> String {
         format!("{}.ssz_snappy", Self::handler_name())
@@ -58,7 +61,7 @@ pub trait Operation<E: EthSpec>: TypeName + Debug + Sync + Sized {
         true
     }
 
-    fn decode(path: &Path, spec: &ChainSpec) -> Result<Self, Error>;
+    fn decode(path: &Path, fork_name: ForkName, spec: &ChainSpec) -> Result<Self, Error>;
 
     fn apply_to(
         &self,
@@ -69,7 +72,11 @@ pub trait Operation<E: EthSpec>: TypeName + Debug + Sync + Sized {
 }
 
 impl<E: EthSpec> Operation<E> for Attestation<E> {
-    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+    fn handler_name() -> String {
+        "attestation".into()
+    }
+
+    fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
         ssz_decode_file(path)
     }
 
@@ -109,7 +116,7 @@ impl<E: EthSpec> Operation<E> for AttesterSlashing<E> {
         "attester_slashing".into()
     }
 
-    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+    fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
         ssz_decode_file(path)
     }
 
@@ -131,7 +138,11 @@ impl<E: EthSpec> Operation<E> for AttesterSlashing<E> {
 }
 
 impl<E: EthSpec> Operation<E> for Deposit {
-    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+    fn handler_name() -> String {
+        "deposit".into()
+    }
+
+    fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
         ssz_decode_file(path)
     }
 
@@ -155,7 +166,7 @@ impl<E: EthSpec> Operation<E> for ProposerSlashing {
         "proposer_slashing".into()
     }
 
-    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+    fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
         ssz_decode_file(path)
     }
 
@@ -181,7 +192,7 @@ impl<E: EthSpec> Operation<E> for SignedVoluntaryExit {
         "voluntary_exit".into()
     }
 
-    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+    fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
         ssz_decode_file(path)
     }
 
@@ -204,7 +215,7 @@ impl<E: EthSpec> Operation<E> for BeaconBlock<E> {
         "block.ssz_snappy".into()
     }
 
-    fn decode(path: &Path, spec: &ChainSpec) -> Result<Self, Error> {
+    fn decode(path: &Path, _fork_name: ForkName, spec: &ChainSpec) -> Result<Self, Error> {
         ssz_decode_file_with(path, |bytes| BeaconBlock::from_ssz_bytes(bytes, spec))
     }
 
@@ -239,7 +250,7 @@ impl<E: EthSpec> Operation<E> for SyncAggregate<E> {
         fork_name != ForkName::Base
     }
 
-    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+    fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
         ssz_decode_file(path)
     }
 
@@ -267,13 +278,11 @@ impl<E: EthSpec> Operation<E> for FullPayload<E> {
         fork_name != ForkName::Base && fork_name != ForkName::Altair
     }
 
-    //FIXME(sean) we could decode based on timestamp - we probably don't do decode a payload
-    // without a block this elsewhere at presetn. But when we support SSZ in the builder api we may need to.
-    // Although that API should include fork info. Hardcoding this for now
-    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
-        ssz_decode_file::<ExecutionPayloadMerge<E>>(path)
-            .map(ExecutionPayload::Merge)
-            .map(Into::into)
+    fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file_with(path, |bytes| {
+            ExecutionPayload::from_ssz_bytes(bytes, fork_name)
+        })
+        .map(Into::into)
     }
 
     fn apply_to(
@@ -306,13 +315,11 @@ impl<E: EthSpec> Operation<E> for BlindedPayload<E> {
         fork_name != ForkName::Base && fork_name != ForkName::Altair
     }
 
-    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
-        //FIXME(sean) we could decode based on timestamp - we probably don't do decode a payload
-        // without a block this elsewhere at presetn. But when we support SSZ in the builder api we may need to.
-        // Although that API should include fork info. Hardcoding this for now
-        let payload: Result<ExecutionPayload<E>, Error> =
-            ssz_decode_file::<ExecutionPayloadMerge<E>>(path).map(Into::into);
-        payload.map(Into::into)
+    fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file_with(path, |bytes| {
+            ExecutionPayload::from_ssz_bytes(bytes, fork_name)
+        })
+        .map(Into::into)
     }
 
     fn apply_to(
@@ -333,6 +340,65 @@ impl<E: EthSpec> Operation<E> for BlindedPayload<E> {
     }
 }
 
+impl<E: EthSpec> Operation<E> for WithdrawalsPayload<E> {
+    fn handler_name() -> String {
+        "withdrawals".into()
+    }
+
+    fn filename() -> String {
+        "execution_payload.ssz_snappy".into()
+    }
+
+    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
+        fork_name != ForkName::Base && fork_name != ForkName::Altair && fork_name != ForkName::Merge
+    }
+
+    fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file_with(path, |bytes| {
+            ExecutionPayload::from_ssz_bytes(bytes, fork_name)
+        })
+        .map(|payload| WithdrawalsPayload {
+            payload: payload.into(),
+        })
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+        _: &Operations<E, Self>,
+    ) -> Result<(), BlockProcessingError> {
+        process_withdrawals::<_, FullPayload<_>>(state, self.payload.to_ref(), spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for SignedBlsToExecutionChange {
+    fn handler_name() -> String {
+        "bls_to_execution_change".into()
+    }
+
+    fn filename() -> String {
+        "address_change.ssz_snappy".into()
+    }
+
+    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
+        fork_name != ForkName::Base && fork_name != ForkName::Altair && fork_name != ForkName::Merge
+    }
+
+    fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file(path)
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+        _extra: &Operations<E, Self>,
+    ) -> Result<(), BlockProcessingError> {
+        process_bls_to_execution_changes(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
 impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
     fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
         let spec = &testing_spec::<E>(fork_name);
@@ -356,7 +422,7 @@ impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
         // Check BLS setting here before SSZ deserialization, as most types require signatures
         // to be valid.
         let (operation, bls_error) = if metadata.bls_setting.unwrap_or_default().check().is_ok() {
-            match O::decode(&path.join(O::filename()), spec) {
+            match O::decode(&path.join(O::filename()), fork_name, spec) {
                 Ok(op) => (Some(op), None),
                 Err(Error::InvalidBLSInput(error)) => (None, Some(error)),
                 Err(e) => return Err(e),
@@ -399,9 +465,11 @@ impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
         let mut expected = self.post.clone();
 
         // Processing requires the committee caches.
-        state
-            .build_all_committee_caches(spec)
-            .expect("committee caches OK");
+        // NOTE: some of the withdrawals tests have 0 active validators, do not try
+        // to build the commitee cache in this case.
+        if O::handler_name() != "withdrawals" {
+            state.build_all_committee_caches(spec).unwrap();
+        }
 
         let mut result = self
             .operation

testing/ef_tests/src/cases/transition.rs

@@ -42,14 +42,17 @@ impl<E: EthSpec> LoadCase for TransitionTest<E> {
                 spec.altair_fork_epoch = Some(Epoch::new(0));
                 spec.bellatrix_fork_epoch = Some(metadata.fork_epoch);
             }
-            ForkName::Eip4844 => {
-                spec.bellatrix_fork_epoch = Some(Epoch::new(0));
-                spec.eip4844_fork_epoch = Some(metadata.fork_epoch);
-            }
             ForkName::Capella => {
-                spec.capella_fork_epoch = Some(Epoch::new(0));
+                spec.altair_fork_epoch = Some(Epoch::new(0));
+                spec.bellatrix_fork_epoch = Some(Epoch::new(0));
                 spec.capella_fork_epoch = Some(metadata.fork_epoch);
             }
+            ForkName::Eip4844 => {
+                spec.altair_fork_epoch = Some(Epoch::new(0));
+                spec.bellatrix_fork_epoch = Some(Epoch::new(0));
+                spec.capella_fork_epoch = Some(Epoch::new(0));
+                spec.eip4844_fork_epoch = Some(metadata.fork_epoch);
+            }
         }
 
         // Load blocks