Fix consensus, SSZ, tree hash & run merge EF tests (#2622)

* Update to v1.1.0-beta.4 (squash of #2548)

* SSZ, cached tree hash, EF tests

testing/ef_tests/src/cases/fork.rs

@@ -49,7 +49,10 @@ impl<E: EthSpec> Case for ForkTest<E> {
     fn is_enabled_for_fork(fork_name: ForkName) -> bool {
         // Upgrades exist targeting all forks except phase0/base.
         // Fork tests also need BLS.
-        cfg!(not(feature = "fake_crypto")) && fork_name != ForkName::Base
+        // FIXME(merge): enable merge tests once available
+        cfg!(not(feature = "fake_crypto"))
+            && fork_name != ForkName::Base
+            && fork_name != ForkName::Merge
     }
 
     fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/cases/genesis_initialization.rs

@@ -56,7 +56,9 @@ impl<E: EthSpec> LoadCase for GenesisInitialization<E> {
 impl<E: EthSpec> Case for GenesisInitialization<E> {
     fn is_enabled_for_fork(fork_name: ForkName) -> bool {
         // Altair genesis and later requires real crypto.
-        fork_name == ForkName::Base || cfg!(not(feature = "fake_crypto"))
+        // FIXME(merge): enable merge tests once available
+        fork_name == ForkName::Base
+            || cfg!(not(feature = "fake_crypto")) && fork_name != ForkName::Merge
     }
 
     fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/cases/operations.rs

@@ -7,7 +7,7 @@ use crate::type_name::TypeName;
 use serde_derive::Deserialize;
 use state_processing::per_block_processing::{
     errors::BlockProcessingError,
-    process_block_header,
+    process_block_header, process_execution_payload,
     process_operations::{
         altair, base, process_attester_slashings, process_deposits, process_exits,
         process_proposer_slashings,
@@ -17,8 +17,8 @@ use state_processing::per_block_processing::{
 use std::fmt::Debug;
 use std::path::Path;
 use types::{
-    Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, ForkName,
-    ProposerSlashing, SignedVoluntaryExit, SyncAggregate,
+    Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec,
+    ExecutionPayload, ForkName, ProposerSlashing, SignedVoluntaryExit, SyncAggregate,
 };
 
 #[derive(Debug, Clone, Default, Deserialize)]
@@ -27,9 +27,15 @@ struct Metadata {
     bls_setting: Option<BlsSetting>,
 }
 
+#[derive(Debug, Clone, Deserialize)]
+struct ExecutionMetadata {
+    execution_valid: bool,
+}
+
 #[derive(Debug, Clone)]
 pub struct Operations<E: EthSpec, O: Operation<E>> {
     metadata: Metadata,
+    execution_metadata: Option<ExecutionMetadata>,
     pub pre: BeaconState<E>,
     pub operation: Option<O>,
     pub post: Option<BeaconState<E>>,
@@ -54,6 +60,7 @@ pub trait Operation<E: EthSpec>: TypeName + Debug + Sync + Sized {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError>;
 }
 
@@ -66,6 +73,7 @@ impl<E: EthSpec> Operation<E> for Attestation<E> {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         let proposer_index = state.get_beacon_proposer_index(state.slot(), spec)? as u64;
         match state {
@@ -97,6 +105,7 @@ impl<E: EthSpec> Operation<E> for AttesterSlashing<E> {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_attester_slashings(state, &[self.clone()], VerifySignatures::True, spec)
     }
@@ -111,6 +120,7 @@ impl<E: EthSpec> Operation<E> for Deposit {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_deposits(state, &[self.clone()], spec)
     }
@@ -129,6 +139,7 @@ impl<E: EthSpec> Operation<E> for ProposerSlashing {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_proposer_slashings(state, &[self.clone()], VerifySignatures::True, spec)
     }
@@ -147,6 +158,7 @@ impl<E: EthSpec> Operation<E> for SignedVoluntaryExit {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_exits(state, &[self.clone()], VerifySignatures::True, spec)
     }
@@ -169,6 +181,7 @@ impl<E: EthSpec> Operation<E> for BeaconBlock<E> {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_block_header(state, self.to_ref(), spec)?;
         Ok(())
@@ -196,12 +209,49 @@ impl<E: EthSpec> Operation<E> for SyncAggregate<E> {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         let proposer_index = state.get_beacon_proposer_index(state.slot(), spec)? as u64;
         process_sync_aggregate(state, self, proposer_index, VerifySignatures::True, spec)
     }
 }
 
+impl<E: EthSpec> Operation<E> for ExecutionPayload<E> {
+    fn handler_name() -> String {
+        "execution_payload".into()
+    }
+
+    fn filename() -> String {
+        "execution_payload.ssz_snappy".into()
+    }
+
+    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
+        fork_name != ForkName::Base && fork_name != ForkName::Altair
+    }
+
+    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file(path)
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+        extra: &Operations<E, Self>,
+    ) -> Result<(), BlockProcessingError> {
+        // FIXME(merge): we may want to plumb the validity bool into state processing
+        let valid = extra
+            .execution_metadata
+            .as_ref()
+            .map_or(false, |e| e.execution_valid);
+        if valid {
+            process_execution_payload(state, self, spec)
+        } else {
+            Err(BlockProcessingError::ExecutionInvalid)
+        }
+    }
+}
+
 impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
     fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
         let spec = &testing_spec::<E>(fork_name);
@@ -212,6 +262,14 @@ impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
             Metadata::default()
         };
 
+        // For execution payloads only.
+        let execution_yaml_path = path.join("execution.yaml");
+        let execution_metadata = if execution_yaml_path.is_file() {
+            Some(yaml_decode_file(&execution_yaml_path)?)
+        } else {
+            None
+        };
+
         let pre = ssz_decode_state(&path.join("pre.ssz_snappy"), spec)?;
 
         // Check BLS setting here before SSZ deserialization, as most types require signatures
@@ -237,6 +295,7 @@ impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
 
         Ok(Self {
             metadata,
+            execution_metadata,
             pre,
             operation,
             post,
@@ -270,7 +329,7 @@ impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
             .operation
             .as_ref()
             .ok_or(Error::SkippedBls)?
-            .apply_to(&mut state, spec)
+            .apply_to(&mut state, spec, self)
             .map(|()| state);
 
         compare_beacon_state_results_without_caches(&mut result, &mut expected)

testing/ef_tests/src/cases/rewards.rs

@@ -3,6 +3,7 @@ use crate::case_result::compare_result_detailed;
 use crate::decode::{ssz_decode_file, ssz_decode_state, yaml_decode_file};
 use compare_fields_derive::CompareFields;
 use serde_derive::Deserialize;
+use ssz::four_byte_option_impl;
 use ssz_derive::{Decode, Encode};
 use state_processing::{
     per_epoch_processing::{
@@ -26,11 +27,16 @@ pub struct Deltas {
     penalties: Vec<u64>,
 }
 
-#[derive(Debug, Clone, PartialEq, CompareFields)]
+// Define "legacy" implementations of `Option<Epoch>`, `Option<NonZeroUsize>` which use four bytes
+// for encoding the union selector.
+four_byte_option_impl!(four_byte_option_deltas, Deltas);
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, CompareFields)]
 pub struct AllDeltas {
     source_deltas: Deltas,
     target_deltas: Deltas,
     head_deltas: Deltas,
+    #[ssz(with = "four_byte_option_deltas")]
     inclusion_delay_deltas: Option<Deltas>,
     inactivity_penalty_deltas: Deltas,
 }

testing/ef_tests/src/cases/transition.rs

@@ -72,7 +72,10 @@ impl<E: EthSpec> Case for TransitionTest<E> {
     fn is_enabled_for_fork(fork_name: ForkName) -> bool {
         // Upgrades exist targeting all forks except phase0/base.
         // Transition tests also need BLS.
-        cfg!(not(feature = "fake_crypto")) && fork_name != ForkName::Base
+        // FIXME(merge): enable merge tests once available
+        cfg!(not(feature = "fake_crypto"))
+            && fork_name != ForkName::Base
+            && fork_name != ForkName::Merge
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/handler.rs

@@ -34,7 +34,7 @@ pub trait Handler {
         let fork_name_str = match fork_name {
             ForkName::Base => "phase0",
             ForkName::Altair => "altair",
-            ForkName::Merge => "merge", // TODO: check this
+            ForkName::Merge => "merge",
         };
 
         let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
@@ -145,6 +145,18 @@ impl<T, E> SszStaticHandler<T, E> {
     pub fn altair_only() -> Self {
         Self::for_forks(vec![ForkName::Altair])
     }
+
+    pub fn altair_and_later() -> Self {
+        Self::for_forks(ForkName::list_all()[1..].to_vec())
+    }
+
+    pub fn merge_only() -> Self {
+        Self::for_forks(vec![ForkName::Merge])
+    }
+
+    pub fn merge_and_later() -> Self {
+        Self::for_forks(ForkName::list_all()[2..].to_vec())
+    }
 }
 
 /// Handler for SSZ types that implement `CachedTreeHash`.
@@ -298,6 +310,11 @@ pub struct RandomHandler<E>(PhantomData<E>);
 impl<E: EthSpec + TypeName> Handler for RandomHandler<E> {
     type Case = cases::SanityBlocks<E>;
 
+    // FIXME(merge): enable merge tests once available
+    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
+        fork_name != ForkName::Merge
+    }
+
     fn config_name() -> &'static str {
         E::name()
     }
@@ -481,6 +498,11 @@ pub struct GenesisValidityHandler<E>(PhantomData<E>);
 impl<E: EthSpec + TypeName> Handler for GenesisValidityHandler<E> {
     type Case = cases::GenesisValidity<E>;
 
+    // FIXME(merge): enable merge test once available
+    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
+        fork_name != ForkName::Merge
+    }
+
     fn config_name() -> &'static str {
         E::name()
     }

testing/ef_tests/src/type_name.rs

@@ -41,21 +41,20 @@ type_name_generic!(Attestation);
 type_name!(AttestationData);
 type_name_generic!(AttesterSlashing);
 type_name_generic!(BeaconBlock);
-type_name_generic!(BeaconBlockBase, "BeaconBlock");
-type_name_generic!(BeaconBlockAltair, "BeaconBlock");
 type_name_generic!(BeaconBlockBody);
 type_name_generic!(BeaconBlockBodyBase, "BeaconBlockBody");
 type_name_generic!(BeaconBlockBodyAltair, "BeaconBlockBody");
+type_name_generic!(BeaconBlockBodyMerge, "BeaconBlockBody");
 type_name!(BeaconBlockHeader);
 type_name_generic!(BeaconState);
-type_name_generic!(BeaconStateBase, "BeaconState");
-type_name_generic!(BeaconStateAltair, "BeaconState");
 type_name!(Checkpoint);
 type_name_generic!(ContributionAndProof);
 type_name!(Deposit);
 type_name!(DepositData);
 type_name!(DepositMessage);
 type_name!(Eth1Data);
+type_name_generic!(ExecutionPayload);
+type_name_generic!(ExecutionPayloadHeader);
 type_name!(Fork);
 type_name!(ForkData);
 type_name_generic!(HistoricalBatch);