merge upstream, fix compile errors

testing/ef_tests/Makefile

@@ -1,4 +1,4 @@
-TESTS_TAG := v1.3.0-alpha.2
+TESTS_TAG := v1.3.0-rc.0
 TESTS = general minimal mainnet
 TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS))
 
@@ -13,6 +13,8 @@ BLS_TARBALL = $(patsubst %,%-$(BLS_TEST_TAG).tar.gz,$(BLS_TEST))
 BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME)
 BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_TAG)
 
+WGET := $(if $(LIGHTHOUSE_GITHUB_TOKEN),wget --header="Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",wget)
+
 all:
 	make $(OUTPUT_DIR)
 	make $(BLS_OUTPUT_DIR)
@@ -25,11 +27,11 @@ $(OUTPUT_DIR): $(TARBALLS)
 
 $(BLS_OUTPUT_DIR):
 	mkdir $(BLS_OUTPUT_DIR)
-	wget $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL)
+	$(WGET) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL)
 	tar -xzf $(BLS_TARBALL) -C $(BLS_OUTPUT_DIR)
 
 %-$(TESTS_TAG).tar.gz:
-	wget $(BASE_URL)/$*.tar.gz -O $@
+	$(WGET) $(BASE_URL)/$*.tar.gz -O $@
 
 clean-test-files:
 	rm -rf $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)

testing/ef_tests/check_all_files_accessed.py

@@ -41,8 +41,6 @@ excluded_paths = [
     "tests/.*/.*/ssz_static/LightClientFinalityUpdate",
     # Eip4844 tests are disabled for now.
     "tests/.*/eip4844",
-    # Capella tests are disabled for now.
-    "tests/.*/capella",
     # One of the EF researchers likes to pack the tarballs on a Mac
     ".*\.DS_Store.*",
     # More Mac weirdness.

testing/ef_tests/src/cases/epoch_processing.rs

@@ -5,6 +5,7 @@ use crate::decode::{ssz_decode_state, yaml_decode_file};
 use crate::type_name;
 use crate::type_name::TypeName;
 use serde_derive::Deserialize;
+use state_processing::per_epoch_processing::capella::process_historical_summaries_update;
 use state_processing::per_epoch_processing::{
     altair, base,
     effective_balance_updates::process_effective_balance_updates,
@@ -57,6 +58,8 @@ pub struct RandaoMixesReset;
 #[derive(Debug)]
 pub struct HistoricalRootsUpdate;
 #[derive(Debug)]
+pub struct HistoricalSummariesUpdate;
+#[derive(Debug)]
 pub struct ParticipationRecordUpdates;
 #[derive(Debug)]
 pub struct SyncCommitteeUpdates;
@@ -77,6 +80,7 @@ type_name!(EffectiveBalanceUpdates, "effective_balance_updates");
 type_name!(SlashingsReset, "slashings_reset");
 type_name!(RandaoMixesReset, "randao_mixes_reset");
 type_name!(HistoricalRootsUpdate, "historical_roots_update");
+type_name!(HistoricalSummariesUpdate, "historical_summaries_update");
 type_name!(ParticipationRecordUpdates, "participation_record_updates");
 type_name!(SyncCommitteeUpdates, "sync_committee_updates");
 type_name!(InactivityUpdates, "inactivity_updates");
@@ -194,7 +198,23 @@ impl<E: EthSpec> EpochTransition<E> for RandaoMixesReset {
 
 impl<E: EthSpec> EpochTransition<E> for HistoricalRootsUpdate {
     fn run(state: &mut BeaconState<E>, _spec: &ChainSpec) -> Result<(), EpochProcessingError> {
-        process_historical_roots_update(state)
+        match state {
+            BeaconState::Base(_) | BeaconState::Altair(_) | BeaconState::Merge(_) => {
+                process_historical_roots_update(state)
+            }
+            _ => Ok(()),
+        }
+    }
+}
+
+impl<E: EthSpec> EpochTransition<E> for HistoricalSummariesUpdate {
+    fn run(state: &mut BeaconState<E>, _spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        match state {
+            BeaconState::Capella(_) | BeaconState::Eip4844(_) => {
+                process_historical_summaries_update(state)
+            }
+            _ => Ok(()),
+        }
     }
 }
 
@@ -287,10 +307,16 @@ impl<E: EthSpec, T: EpochTransition<E>> Case for EpochProcessing<E, T> {
                 T::name() != "sync_committee_updates"
                     && T::name() != "inactivity_updates"
                     && T::name() != "participation_flag_updates"
+                    && T::name() != "historical_summaries_update"
             }
             // No phase0 tests for Altair and later.
-            ForkName::Altair | ForkName::Merge | ForkName::Capella | ForkName::Eip4844 => {
+            ForkName::Altair | ForkName::Merge => {
                 T::name() != "participation_record_updates"
+                    && T::name() != "historical_summaries_update"
+            }
+            ForkName::Capella | ForkName::Eip4844 => {
+                T::name() != "participation_record_updates"
+                    && T::name() != "historical_roots_update"
             }
         }
     }

testing/ef_tests/src/cases/fork_choice.rs

@@ -311,6 +311,7 @@ impl<E: EthSpec> Tester<E> {
             .keypairs(vec![])
             .genesis_state_ephemeral_store(case.anchor_state.clone())
             .mock_execution_layer()
+            .recalculate_fork_times_with_genesis(0)
             .mock_execution_layer_all_payloads_valid()
             .build();
 

testing/ef_tests/src/cases/operations.rs

@@ -4,30 +4,24 @@ use crate::case_result::compare_beacon_state_results_without_caches;
 use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
 use crate::testing_spec;
 use serde_derive::Deserialize;
-#[cfg(feature = "withdrawals-processing")]
-use state_processing::per_block_processing::process_operations::process_bls_to_execution_changes;
-#[cfg(feature = "withdrawals-processing")]
-use state_processing::per_block_processing::process_withdrawals;
 use state_processing::{
     per_block_processing::{
         errors::BlockProcessingError,
         process_block_header, process_execution_payload,
         process_operations::{
-            altair, base, process_attester_slashings, process_deposits, process_exits,
-            process_proposer_slashings,
+            altair, base, process_attester_slashings, process_bls_to_execution_changes,
+            process_deposits, process_exits, process_proposer_slashings,
         },
-        process_sync_aggregate, VerifyBlockRoot, VerifySignatures,
+        process_sync_aggregate, process_withdrawals, VerifyBlockRoot, VerifySignatures,
     },
     ConsensusContext,
 };
 use std::fmt::Debug;
 use std::path::Path;
-#[cfg(feature = "withdrawals-processing")]
-use types::SignedBlsToExecutionChange;
 use types::{
     Attestation, AttesterSlashing, BeaconBlock, BeaconState, BlindedPayload, ChainSpec, Deposit,
-    EthSpec, ExecutionPayload, ForkName, FullPayload, ProposerSlashing, SignedVoluntaryExit,
-    SyncAggregate,
+    EthSpec, ExecutionPayload, ForkName, FullPayload, ProposerSlashing, SignedBlsToExecutionChange,
+    SignedVoluntaryExit, SyncAggregate,
 };
 
 #[derive(Debug, Clone, Default, Deserialize)]
@@ -42,7 +36,6 @@ struct ExecutionMetadata {
 }
 
 /// Newtype for testing withdrawals.
-#[cfg(feature = "withdrawals-processing")]
 #[derive(Debug, Clone, Deserialize)]
 pub struct WithdrawalsPayload<T: EthSpec> {
     payload: FullPayload<T>,
@@ -341,7 +334,6 @@ impl<E: EthSpec> Operation<E> for BlindedPayload<E> {
     }
 }
 
-#[cfg(feature = "withdrawals-processing")]
 impl<E: EthSpec> Operation<E> for WithdrawalsPayload<E> {
     fn handler_name() -> String {
         "withdrawals".into()
@@ -368,7 +360,6 @@ impl<E: EthSpec> Operation<E> for WithdrawalsPayload<E> {
         })
     }
 
-    #[cfg(feature = "withdrawals-processing")]
     fn apply_to(
         &self,
         state: &mut BeaconState<E>,
@@ -384,7 +375,6 @@ impl<E: EthSpec> Operation<E> for WithdrawalsPayload<E> {
     }
 }
 
-#[cfg(feature = "withdrawals-processing")]
 impl<E: EthSpec> Operation<E> for SignedBlsToExecutionChange {
     fn handler_name() -> String {
         "bls_to_execution_change".into()

testing/ef_tests/src/handler.rs

@@ -377,6 +377,11 @@ impl<E: EthSpec + TypeName> Handler for SanitySlotsHandler<E> {
     fn handler_name(&self) -> String {
         "slots".into()
     }
+
+    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
+        // Some sanity tests compute sync committees, which requires real crypto.
+        fork_name == ForkName::Base || cfg!(not(feature = "fake_crypto"))
+    }
 }
 
 #[derive(Derivative)]

testing/ef_tests/src/lib.rs

@@ -1,11 +1,10 @@
 pub use case_result::CaseResult;
-#[cfg(feature = "withdrawals-processing")]
 pub use cases::WithdrawalsPayload;
 pub use cases::{
-    Case, EffectiveBalanceUpdates, Eth1DataReset, HistoricalRootsUpdate, InactivityUpdates,
-    JustificationAndFinalization, ParticipationFlagUpdates, ParticipationRecordUpdates,
-    RandaoMixesReset, RegistryUpdates, RewardsAndPenalties, Slashings, SlashingsReset,
-    SyncCommitteeUpdates,
+    Case, EffectiveBalanceUpdates, Eth1DataReset, HistoricalRootsUpdate, HistoricalSummariesUpdate,
+    InactivityUpdates, JustificationAndFinalization, ParticipationFlagUpdates,
+    ParticipationRecordUpdates, RandaoMixesReset, RegistryUpdates, RewardsAndPenalties, Slashings,
+    SlashingsReset, SyncCommitteeUpdates,
 };
 pub use decode::log_file_access;
 pub use error::Error;

testing/ef_tests/src/type_name.rs

@@ -1,4 +1,5 @@
 //! Mapping from types to canonical string identifiers used in testing.
+use types::historical_summary::HistoricalSummary;
 use types::*;
 
 pub trait TypeName {
@@ -92,3 +93,4 @@ type_name_generic!(
     "SignedBeaconBlockAndBlobsSidecar"
 );
 type_name!(SignedBlsToExecutionChange, "SignedBLSToExecutionChange");
+type_name!(HistoricalSummary);

testing/ef_tests/tests/tests.rs

@@ -82,14 +82,12 @@ fn operations_execution_payload_blinded() {
     OperationsHandler::<MainnetEthSpec, BlindedPayload<_>>::default().run();
 }
 
-#[cfg(feature = "withdrawals-processing")]
 #[test]
 fn operations_withdrawals() {
     OperationsHandler::<MinimalEthSpec, WithdrawalsPayload<_>>::default().run();
     OperationsHandler::<MainnetEthSpec, WithdrawalsPayload<_>>::default().run();
 }
 
-#[cfg(feature = "withdrawals-processing")]
 #[test]
 fn operations_bls_to_execution_change() {
     OperationsHandler::<MinimalEthSpec, SignedBlsToExecutionChange>::default().run();
@@ -217,6 +215,7 @@ macro_rules! ssz_static_test_no_run {
 #[cfg(feature = "fake_crypto")]
 mod ssz_static {
     use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler, SszStaticWithSpecHandler};
+    use types::historical_summary::HistoricalSummary;
     use types::signed_block_and_blobs::SignedBeaconBlockAndBlobsSidecarDecode;
     use types::*;
 
@@ -384,6 +383,12 @@ mod ssz_static {
         SszStaticHandler::<SignedBeaconBlockAndBlobsSidecarDecode<MinimalEthSpec>, MinimalEthSpec>::eip4844_only().run();
         SszStaticHandler::<SignedBeaconBlockAndBlobsSidecarDecode<MainnetEthSpec>, MainnetEthSpec>::eip4844_only().run();
     }
+
+    #[test]
+    fn historical_summary() {
+        SszStaticHandler::<HistoricalSummary, MinimalEthSpec>::capella_only().run();
+        SszStaticHandler::<HistoricalSummary, MainnetEthSpec>::capella_only().run();
+    }
 }
 
 #[test]
@@ -450,6 +455,12 @@ fn epoch_processing_historical_roots_update() {
     EpochProcessingHandler::<MainnetEthSpec, HistoricalRootsUpdate>::default().run();
 }
 
+#[test]
+fn epoch_processing_historical_summaries_update() {
+    EpochProcessingHandler::<MinimalEthSpec, HistoricalSummariesUpdate>::default().run();
+    EpochProcessingHandler::<MainnetEthSpec, HistoricalSummariesUpdate>::default().run();
+}
+
 #[test]
 fn epoch_processing_participation_record_updates() {
     EpochProcessingHandler::<MinimalEthSpec, ParticipationRecordUpdates>::default().run();

testing/execution_engine_integration/src/nethermind.rs

@@ -76,7 +76,7 @@ impl GenericExecutionEngine for NethermindEngine {
     fn init_datadir() -> TempDir {
         let datadir = TempDir::new().unwrap();
         let genesis_json_path = datadir.path().join("genesis.json");
-        let mut file = File::create(&genesis_json_path).unwrap();
+        let mut file = File::create(genesis_json_path).unwrap();
         let json = nethermind_genesis_json();
         serde_json::to_writer(&mut file, &json).unwrap();
         datadir

testing/node_test_rig/src/lib.rs

@@ -231,7 +231,7 @@ impl<E: EthSpec> LocalExecutionNode<E> {
             .tempdir()
             .expect("should create temp directory for client datadir");
         let jwt_file_path = datadir.path().join("jwt.hex");
-        if let Err(e) = std::fs::write(&jwt_file_path, config.jwt_key.hex_string()) {
+        if let Err(e) = std::fs::write(jwt_file_path, config.jwt_key.hex_string()) {
             panic!("Failed to write jwt file {}", e);
         }
         Self {

testing/web3signer_tests/build.rs

@@ -1,7 +1,10 @@
 //! This build script downloads the latest Web3Signer release and places it in the `OUT_DIR` so it
 //! can be used for integration testing.
 
-use reqwest::Client;
+use reqwest::{
+    header::{self, HeaderValue},
+    Client,
+};
 use serde_json::Value;
 use std::env;
 use std::fs;
@@ -15,10 +18,15 @@ const FIXED_VERSION_STRING: Option<&str> = None;
 #[tokio::main]
 async fn main() {
     let out_dir = env::var("OUT_DIR").unwrap();
-    download_binary(out_dir.into()).await;
+
+    // Read a Github API token from the environment. This is intended to prevent rate-limits on CI.
+    // We use a name that is unlikely to accidentally collide with anything the user has configured.
+    let github_token = env::var("LIGHTHOUSE_GITHUB_TOKEN");
+
+    download_binary(out_dir.into(), github_token.as_deref().unwrap_or("")).await;
 }
 
-pub async fn download_binary(dest_dir: PathBuf) {
+pub async fn download_binary(dest_dir: PathBuf, github_token: &str) {
     let version_file = dest_dir.join("version");
 
     let client = Client::builder()
@@ -33,8 +41,11 @@ pub async fn download_binary(dest_dir: PathBuf) {
         env_version
     } else {
         // Get the latest release of the web3 signer repo.
+        let mut token_header_value = HeaderValue::from_str(github_token).unwrap();
+        token_header_value.set_sensitive(true);
         let latest_response: Value = client
             .get("https://api.github.com/repos/ConsenSys/web3signer/releases/latest")
+            .header(header::AUTHORIZATION, token_header_value)
             .send()
             .await
             .unwrap()