merge with upstream

testing/ef_tests/check_all_files_accessed.py

@@ -39,10 +39,10 @@ excluded_paths = [
     "tests/.*/.*/ssz_static/LightClientOptimistic",
     # LightClientFinalityUpdate
     "tests/.*/.*/ssz_static/LightClientFinalityUpdate",
-    # Merkle-proof tests for light clients
-    "tests/.*/.*/merkle/single_proof",
     # Eip4844 tests are disabled for now.
     "tests/.*/eip4844",
+    # Capella tests are disabled for now.
+    "tests/.*/capella",
     # One of the EF researchers likes to pack the tarballs on a Mac
     ".*\.DS_Store.*",
     # More Mac weirdness.

testing/ef_tests/src/cases.rs

@@ -18,6 +18,7 @@ mod fork;
 mod fork_choice;
 mod genesis_initialization;
 mod genesis_validity;
+mod merkle_proof_validity;
 mod operations;
 mod rewards;
 mod sanity_blocks;
@@ -41,6 +42,7 @@ pub use epoch_processing::*;
 pub use fork::ForkTest;
 pub use genesis_initialization::*;
 pub use genesis_validity::*;
+pub use merkle_proof_validity::*;
 pub use operations::*;
 pub use rewards::RewardsTest;
 pub use sanity_blocks::*;

testing/ef_tests/src/cases/fork_choice.rs

@@ -7,7 +7,7 @@ use beacon_chain::{
         obtain_indexed_attestation_and_committees_per_slot, VerifiedAttestation,
     },
     test_utils::{BeaconChainHarness, EphemeralHarnessType},
-    BeaconChainTypes, CachedHead, CountUnrealized,
+    BeaconChainTypes, CachedHead, CountUnrealized, NotifyExecutionLayer,
 };
 use execution_layer::{json_structures::JsonPayloadStatusV1Status, PayloadStatusV1};
 use serde::Deserialize;
@@ -388,6 +388,7 @@ impl<E: EthSpec> Tester<E> {
             block_root,
             block.clone(),
             CountUnrealized::False,
+            NotifyExecutionLayer::Yes,
         ))?;
         if result.is_ok() != valid {
             return Err(Error::DidntFail(format!(

testing/ef_tests/src/cases/merkle_proof_validity.rs

@@ -0,0 +1,87 @@
+use super::*;
+use crate::decode::{ssz_decode_state, yaml_decode_file};
+use serde_derive::Deserialize;
+use std::path::Path;
+use tree_hash::Hash256;
+use types::{BeaconState, EthSpec, ForkName};
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct Metadata {
+    #[serde(rename(deserialize = "description"))]
+    _description: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct MerkleProof {
+    pub leaf: Hash256,
+    pub leaf_index: usize,
+    pub branch: Vec<Hash256>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct MerkleProofValidity<E: EthSpec> {
+    pub metadata: Option<Metadata>,
+    pub state: BeaconState<E>,
+    pub merkle_proof: MerkleProof,
+}
+
+impl<E: EthSpec> LoadCase for MerkleProofValidity<E> {
+    fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
+        let spec = &testing_spec::<E>(fork_name);
+        let state = ssz_decode_state(&path.join("state.ssz_snappy"), spec)?;
+        let merkle_proof = yaml_decode_file(&path.join("proof.yaml"))?;
+        // Metadata does not exist in these tests but it is left like this just in case.
+        let meta_path = path.join("meta.yaml");
+        let metadata = if meta_path.exists() {
+            Some(yaml_decode_file(&meta_path)?)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            metadata,
+            state,
+            merkle_proof,
+        })
+    }
+}
+
+impl<E: EthSpec> Case for MerkleProofValidity<E> {
+    fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
+        let mut state = self.state.clone();
+        state.initialize_tree_hash_cache();
+        let proof = match state.compute_merkle_proof(self.merkle_proof.leaf_index) {
+            Ok(proof) => proof,
+            Err(_) => {
+                return Err(Error::FailedToParseTest(
+                    "Could not retrieve merkle proof".to_string(),
+                ))
+            }
+        };
+        let proof_len = proof.len();
+        let branch_len = self.merkle_proof.branch.len();
+        if proof_len != branch_len {
+            return Err(Error::NotEqual(format!(
+                "Branches not equal in length computed: {}, expected {}",
+                proof_len, branch_len
+            )));
+        }
+
+        for (i, proof_leaf) in proof.iter().enumerate().take(proof_len) {
+            let expected_leaf = self.merkle_proof.branch[i];
+            if *proof_leaf != expected_leaf {
+                return Err(Error::NotEqual(format!(
+                    "Leaves not equal in merke proof computed: {}, expected: {}",
+                    hex::encode(proof_leaf),
+                    hex::encode(expected_leaf)
+                )));
+            }
+        }
+
+        // Tree hash cache should still be initialized (not dropped).
+        assert!(state.tree_hash_cache().is_initialized());
+
+        Ok(())
+    }
+}

testing/ef_tests/src/cases/operations.rs

@@ -90,7 +90,6 @@ impl<E: EthSpec> Operation<E> for Attestation<E> {
         _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         let mut ctxt = ConsensusContext::new(state.slot());
-        let proposer_index = ctxt.get_proposer_index(state, spec)?;
         match state {
             BeaconState::Base(_) => base::process_attestations(
                 state,
@@ -102,14 +101,9 @@ impl<E: EthSpec> Operation<E> for Attestation<E> {
             BeaconState::Altair(_)
             | BeaconState::Merge(_)
             | BeaconState::Capella(_)
-            | BeaconState::Eip4844(_) => altair::process_attestation(
-                state,
-                self,
-                0,
-                proposer_index,
-                VerifySignatures::True,
-                spec,
-            ),
+            | BeaconState::Eip4844(_) => {
+                altair::process_attestation(state, self, 0, &mut ctxt, VerifySignatures::True, spec)
+            }
         }
     }
 }

testing/ef_tests/src/handler.rs

@@ -627,6 +627,30 @@ impl<E: EthSpec + TypeName> Handler for GenesisInitializationHandler<E> {
     }
 }
 
+#[derive(Derivative)]
+#[derivative(Default(bound = ""))]
+pub struct MerkleProofValidityHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for MerkleProofValidityHandler<E> {
+    type Case = cases::MerkleProofValidity<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "light_client"
+    }
+
+    fn handler_name(&self) -> String {
+        "single_merkle_proof".into()
+    }
+
+    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
+        fork_name != ForkName::Base
+    }
+}
+
 #[derive(Derivative)]
 #[derivative(Default(bound = ""))]
 pub struct OperationsHandler<E, O>(PhantomData<(E, O)>);

testing/ef_tests/tests/tests.rs

@@ -532,6 +532,11 @@ fn genesis_validity() {
     // Note: there are no genesis validity tests for mainnet
 }
 
+#[test]
+fn merkle_proof_validity() {
+    MerkleProofValidityHandler::<MainnetEthSpec>::default().run();
+}
+
 #[test]
 fn rewards() {
     for handler in &["basic", "leak", "random"] {

testing/execution_engine_integration/src/nethermind.rs

@@ -8,7 +8,10 @@ use std::process::{Child, Command, Output};
 use tempfile::TempDir;
 use unused_port::unused_tcp_port;
 
-const NETHERMIND_BRANCH: &str = "master";
+/// We've pinned the Nethermind version since our method of using the `master` branch to
+/// find the latest tag isn't working. It appears Nethermind don't always tag on `master`.
+/// We should fix this so we always pull the latest version of Nethermind.
+const NETHERMIND_BRANCH: &str = "release/1.14.6";
 const NETHERMIND_REPO_URL: &str = "https://github.com/NethermindEth/nethermind";
 
 fn build_result(repo_dir: &Path) -> Output {

testing/execution_engine_integration/src/test_rig.rs

@@ -4,8 +4,7 @@ use crate::execution_engine::{
 use crate::transactions::transactions;
 use ethers_providers::Middleware;
 use execution_layer::{
-    BuilderParams, ChainHealth, ExecutionLayer, PayloadAttributes, PayloadAttributesV1,
-    PayloadStatus,
+    BuilderParams, ChainHealth, ExecutionLayer, PayloadAttributes, PayloadStatus,
 };
 use fork_choice::ForkchoiceUpdateParameters;
 use reqwest::{header::CONTENT_TYPE, Client};
@@ -19,7 +18,7 @@ use types::{
     Address, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, ForkName, FullPayload,
     Hash256, MainnetEthSpec, PublicKeyBytes, Slot, Uint256,
 };
-const EXECUTION_ENGINE_START_TIMEOUT: Duration = Duration::from_secs(20);
+const EXECUTION_ENGINE_START_TIMEOUT: Duration = Duration::from_secs(30);
 
 struct ExecutionPair<E, T: EthSpec> {
     /// The Lighthouse `ExecutionLayer` struct, connected to the `execution_engine` via HTTP.
@@ -279,11 +278,8 @@ impl<E: GenericExecutionEngine> TestRig<E> {
                 Slot::new(1), // Insert proposer for the next slot
                 head_root,
                 proposer_index,
-                PayloadAttributes::V1(PayloadAttributesV1 {
-                    timestamp,
-                    prev_randao,
-                    suggested_fee_recipient: Address::zero(),
-                }),
+                // TODO: think about how to test different forks
+                PayloadAttributes::new(timestamp, prev_randao, Address::zero(), None),
             )
             .await;
 
@@ -316,20 +312,23 @@ impl<E: GenericExecutionEngine> TestRig<E> {
             slot: Slot::new(0),
             chain_health: ChainHealth::Healthy,
         };
+        let suggested_fee_recipient = self
+            .ee_a
+            .execution_layer
+            .get_suggested_fee_recipient(proposer_index)
+            .await;
+        let payload_attributes =
+            PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None);
         let valid_payload = self
             .ee_a
             .execution_layer
             .get_payload::<FullPayload<MainnetEthSpec>>(
                 parent_hash,
-                timestamp,
-                prev_randao,
-                proposer_index,
+                &payload_attributes,
                 forkchoice_update_params,
                 builder_params,
                 // FIXME: think about how to test other forks
                 ForkName::Merge,
-                #[cfg(feature = "withdrawals")]
-                None,
                 &self.spec,
             )
             .await
@@ -444,20 +443,23 @@ impl<E: GenericExecutionEngine> TestRig<E> {
             slot: Slot::new(0),
             chain_health: ChainHealth::Healthy,
         };
+        let suggested_fee_recipient = self
+            .ee_a
+            .execution_layer
+            .get_suggested_fee_recipient(proposer_index)
+            .await;
+        let payload_attributes =
+            PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None);
         let second_payload = self
             .ee_a
             .execution_layer
             .get_payload::<FullPayload<MainnetEthSpec>>(
                 parent_hash,
-                timestamp,
-                prev_randao,
-                proposer_index,
+                &payload_attributes,
                 forkchoice_update_params,
                 builder_params,
                 // FIXME: think about how to test other forks
                 ForkName::Merge,
-                #[cfg(feature = "withdrawals")]
-                None,
                 &self.spec,
             )
             .await
@@ -487,11 +489,9 @@ impl<E: GenericExecutionEngine> TestRig<E> {
          */
         let head_block_hash = valid_payload.block_hash();
         let finalized_block_hash = ExecutionBlockHash::zero();
-        let payload_attributes = PayloadAttributes::V1(PayloadAttributesV1 {
-            timestamp: second_payload.timestamp() + 1,
-            prev_randao: Hash256::zero(),
-            suggested_fee_recipient: Address::zero(),
-        });
+        // TODO: think about how to handle different forks
+        let payload_attributes =
+            PayloadAttributes::new(timestamp, prev_randao, Address::zero(), None);
         let slot = Slot::new(42);
         let head_block_root = Hash256::repeat_byte(100);
         let validator_index = 0;

testing/node_test_rig/src/lib.rs

@@ -48,7 +48,7 @@ impl<E: EthSpec> LocalBeaconNode<E> {
             .tempdir()
             .expect("should create temp directory for client datadir");
 
-        client_config.data_dir = datadir.path().into();
+        client_config.set_data_dir(datadir.path().into());
         client_config.network.network_dir = PathBuf::from(datadir.path()).join("network");
 
         ProductionBeaconNode::new(context, client_config)

testing/simulator/src/eth1_sim.rs

@@ -67,6 +67,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
             max_log_size: 0,
             max_log_number: 0,
             compression: false,
+            is_restricted: true,
         })?
         .multi_threaded_tokio_runtime()?
         .build()?;

testing/simulator/src/no_eth1_sim.rs

@@ -52,6 +52,7 @@ pub fn run_no_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
             max_log_size: 0,
             max_log_number: 0,
             compression: false,
+            is_restricted: true,
         })?
         .multi_threaded_tokio_runtime()?
         .build()?;

testing/simulator/src/sync_sim.rs

@@ -56,6 +56,7 @@ fn syncing_sim(
             max_log_size: 0,
             max_log_number: 0,
             compression: false,
+            is_restricted: true,
         })?
         .multi_threaded_tokio_runtime()?
         .build()?;