Merge remote-tracking branch 'origin/unstable' into tree-states

This commit is contained in:
Michael Sproul
2023-10-19 12:07:35 +11:00
189 changed files with 1475 additions and 951 deletions

View File

@@ -20,7 +20,6 @@ hex = { workspace = true }
kzg = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_derive = "1.0.116"
serde_json = { workspace = true }
serde_repr = { workspace = true }
serde_yaml = { workspace = true }

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use crate::impl_bls_load_case;
use bls::{AggregateSignature, Signature};
use serde_derive::Deserialize;
use serde::Deserialize;
#[derive(Debug, Clone, Deserialize)]
pub struct BlsAggregateSigs {

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use crate::impl_bls_load_case;
use bls::{AggregateSignature, PublicKeyBytes};
use serde_derive::Deserialize;
use serde::Deserialize;
use types::Hash256;
#[derive(Debug, Clone, Deserialize)]

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use crate::impl_bls_load_case;
use bls::{verify_signature_sets, BlsWrappedSignature, PublicKeyBytes, Signature, SignatureSet};
use serde_derive::Deserialize;
use serde::Deserialize;
use std::borrow::Cow;
use std::str::FromStr;
use types::Hash256;

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use crate::impl_bls_load_case;
use bls::{AggregatePublicKey, PublicKeyBytes};
use serde_derive::Deserialize;
use serde::Deserialize;
#[derive(Debug, Clone, Deserialize)]
pub struct BlsEthAggregatePubkeys {

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use crate::impl_bls_load_case;
use bls::{AggregateSignature, PublicKeyBytes};
use serde_derive::Deserialize;
use serde::Deserialize;
use std::convert::TryInto;
use types::Hash256;

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use crate::impl_bls_load_case;
use bls::{AggregateSignature, PublicKeyBytes};
use serde_derive::Deserialize;
use serde::Deserialize;
use std::convert::TryInto;
use types::Hash256;

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use crate::impl_bls_load_case;
use bls::SecretKey;
use serde_derive::Deserialize;
use serde::Deserialize;
use types::Hash256;
#[derive(Debug, Clone, Deserialize)]

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use crate::impl_bls_load_case;
use bls::{PublicKey, PublicKeyBytes, Signature, SignatureBytes};
use serde_derive::Deserialize;
use serde::Deserialize;
use std::convert::TryInto;
use types::Hash256;

View File

@@ -1,4 +1,4 @@
use serde_derive::Deserialize;
use serde::Deserialize;
use ssz::Encode;
use ssz_derive::{Decode, Encode};
use std::convert::TryFrom;

View File

@@ -4,7 +4,7 @@ use crate::case_result::{check_state_diff, compare_beacon_state_results_without_
use crate::decode::{ssz_decode_state, yaml_decode_file};
use crate::type_name;
use crate::type_name::TypeName;
use serde_derive::Deserialize;
use serde::Deserialize;
use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache;
use state_processing::epoch_cache::initialize_epoch_cache;
use state_processing::per_epoch_processing::capella::process_historical_summaries_update;

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_beacon_state_results_without_caches;
use crate::cases::common::previous_fork;
use crate::decode::{ssz_decode_state, yaml_decode_file};
use serde_derive::Deserialize;
use serde::Deserialize;
use state_processing::upgrade::{
upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb,
};

View File

@@ -1,7 +1,7 @@
use super::*;
use crate::case_result::compare_beacon_state_results_without_caches;
use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
use serde_derive::Deserialize;
use serde::Deserialize;
use state_processing::initialize_beacon_state_from_eth1;
use std::path::PathBuf;
use types::{BeaconState, Deposit, EthSpec, ExecutionPayloadHeader, ForkName, Hash256};

View File

@@ -1,6 +1,6 @@
use super::*;
use crate::decode::{ssz_decode_state, yaml_decode_file};
use serde_derive::Deserialize;
use serde::Deserialize;
use state_processing::is_valid_genesis_state;
use std::path::Path;
use types::{BeaconState, EthSpec, ForkName};

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::blob_to_kzg_commitment;
use kzg::KzgCommitment;
use serde_derive::Deserialize;
use serde::Deserialize;
use std::marker::PhantomData;
#[derive(Debug, Clone, Deserialize)]

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::compute_blob_kzg_proof;
use kzg::KzgProof;
use serde_derive::Deserialize;
use serde::Deserialize;
use std::marker::PhantomData;
#[derive(Debug, Clone, Deserialize)]

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::compute_kzg_proof;
use kzg::KzgProof;
use serde_derive::Deserialize;
use serde::Deserialize;
use std::marker::PhantomData;
use std::str::FromStr;
use types::Hash256;

View File

@@ -3,7 +3,7 @@ use crate::case_result::compare_result;
use beacon_chain::kzg_utils::validate_blob;
use eth2_network_config::get_trusted_setup;
use kzg::{Kzg, KzgCommitment, KzgPreset, KzgProof, TrustedSetup};
use serde_derive::Deserialize;
use serde::Deserialize;
use std::convert::TryInto;
use std::marker::PhantomData;
use types::Blob;
@@ -91,7 +91,7 @@ impl<E: EthSpec> Case for KZGVerifyBlobKZGProof<E> {
let kzg = get_kzg::<E::Kzg>()?;
let result = parse_input(&self.input).and_then(|(blob, commitment, proof)| {
validate_blob::<E>(&kzg, blob, commitment, proof)
validate_blob::<E>(&kzg, &blob, commitment, proof)
.map_err(|e| Error::InternalError(format!("Failed to validate blob: {:?}", e)))
});

View File

@@ -1,7 +1,7 @@
use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::validate_blobs;
use serde_derive::Deserialize;
use serde::Deserialize;
use std::marker::PhantomData;
#[derive(Debug, Clone, Deserialize)]
@@ -54,7 +54,7 @@ impl<E: EthSpec> Case for KZGVerifyBlobKZGProofBatch<E> {
let kzg = get_kzg::<E::Kzg>()?;
let result = parse_input(&self.input).and_then(|(commitments, blobs, proofs)| {
validate_blobs::<E>(&kzg, &commitments, &blobs, &proofs)
validate_blobs::<E>(&kzg, &commitments, blobs.iter().collect(), &proofs)
.map_err(|e| Error::InternalError(format!("Failed to validate blobs: {:?}", e)))
});

View File

@@ -1,7 +1,7 @@
use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::verify_kzg_proof;
use serde_derive::Deserialize;
use serde::Deserialize;
use std::marker::PhantomData;
#[derive(Debug, Clone, Deserialize)]

View File

@@ -1,6 +1,6 @@
use super::*;
use crate::decode::{ssz_decode_state, yaml_decode_file};
use serde_derive::Deserialize;
use serde::Deserialize;
use std::path::Path;
use tree_hash::Hash256;
use types::{BeaconState, EthSpec, ForkName};

View File

@@ -3,7 +3,7 @@ use crate::bls_setting::BlsSetting;
use crate::case_result::{check_state_diff, compare_beacon_state_results_without_caches};
use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
use crate::testing_spec;
use serde_derive::Deserialize;
use serde::Deserialize;
use ssz::Decode;
use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache;
use state_processing::epoch_cache::initialize_epoch_cache;

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result_detailed;
use crate::decode::{ssz_decode_file, ssz_decode_state, yaml_decode_file};
use compare_fields_derive::CompareFields;
use serde_derive::Deserialize;
use serde::Deserialize;
use ssz::four_byte_option_impl;
use ssz_derive::{Decode, Encode};
use state_processing::{

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::bls_setting::BlsSetting;
use crate::case_result::{check_state_diff, compare_beacon_state_results_without_caches};
use crate::decode::{ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
use serde_derive::Deserialize;
use serde::Deserialize;
use state_processing::{
per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy,
ConsensusContext, StateProcessingStrategy, VerifyBlockRoot,

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::bls_setting::BlsSetting;
use crate::case_result::{check_state_diff, compare_beacon_state_results_without_caches};
use crate::decode::{ssz_decode_state, yaml_decode_file};
use serde_derive::Deserialize;
use serde::Deserialize;
use state_processing::per_slot_processing;
use types::{BeaconState, EthSpec, ForkName};

View File

@@ -1,7 +1,7 @@
use super::*;
use crate::case_result::compare_result;
use crate::decode::yaml_decode_file;
use serde_derive::Deserialize;
use serde::Deserialize;
use std::marker::PhantomData;
use swap_or_not_shuffle::{compute_shuffled_index, shuffle_list};
use types::ForkName;

View File

@@ -4,8 +4,7 @@ use super::*;
use crate::cases::common::{SszStaticType, TestU128, TestU256};
use crate::cases::ssz_static::{check_serialization, check_tree_hash};
use crate::decode::{log_file_access, snappy_decode_file, yaml_decode_file};
use serde::{de::Error as SerdeError, Deserializer};
use serde_derive::Deserialize;
use serde::{de::Error as SerdeError, Deserialize, Deserializer};
use ssz_derive::{Decode, Encode};
use std::path::{Path, PathBuf};
use tree_hash_derive::TreeHash;

View File

@@ -2,7 +2,7 @@ use super::*;
use crate::case_result::compare_result;
use crate::cases::common::SszStaticType;
use crate::decode::{snappy_decode_file, yaml_decode_file};
use serde_derive::Deserialize;
use serde::Deserialize;
use ssz::Decode;
use tree_hash::TreeHash;
use types::{BeaconBlock, BeaconState, ForkName, Hash256, SignedBeaconBlock};

View File

@@ -1,7 +1,7 @@
use super::*;
use crate::case_result::compare_beacon_state_results_without_caches;
use crate::decode::{ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
use serde_derive::Deserialize;
use serde::Deserialize;
use state_processing::{
per_block_processing, state_advance::complete_state_advance, BlockSignatureStrategy,
ConsensusContext, StateProcessingStrategy, VerifyBlockRoot,

View File

@@ -66,6 +66,7 @@ pub fn get_latest_release(repo_dir: &Path, branch_name: &str) -> Result<String,
Command::new("git")
.arg("fetch")
.arg("--tags")
.arg("--force")
.current_dir(repo_dir)
.output()
.map_err(|e| format!("Failed to fetch tags for {repo_dir:?}: Err: {e}"))?,

View File

@@ -18,11 +18,10 @@ pub fn geth_genesis_json() -> Value {
"muirGlacierBlock":0,
"berlinBlock":0,
"londonBlock":0,
"clique": {
"period": 5,
"epoch": 30000
},
"terminalTotalDifficulty":0
"mergeNetsplitBlock": 0,
"shanghaiTime": 0,
"terminalTotalDifficulty": 0,
"terminalTotalDifficultyPassed": true
},
"nonce":"0x42",
"timestamp":"0x0",
@@ -72,8 +71,10 @@ pub fn nethermind_genesis_json() -> Value {
"accountStartNonce": "0x0",
"maximumExtraDataSize": "0x20",
"minGasLimit": "0x1388",
"networkID": "0x1469ca",
"MergeForkIdTransition": "0x3e8",
"networkID": "0x00146A2E",
"MergeForkIdTransition": "0x0",
"maxCodeSize": "0x6000",
"maxCodeSizeTransition": "0x0",
"eip150Transition": "0x0",
"eip158Transition": "0x0",
"eip160Transition": "0x0",
@@ -101,7 +102,15 @@ pub fn nethermind_genesis_json() -> Value {
"eip1559Transition": "0x0",
"eip3198Transition": "0x0",
"eip3529Transition": "0x0",
"eip3541Transition": "0x0"
"eip3541Transition": "0x0",
"eip3540TransitionTimestamp": "0x0",
"eip3651TransitionTimestamp": "0x0",
"eip3670TransitionTimestamp": "0x0",
"eip3675TransitionTimestamp": "0x0",
"eip3855TransitionTimestamp": "0x0",
"eip3860TransitionTimestamp": "0x0",
"eip4895TransitionTimestamp": "0x0",
"terminalTotalDifficulty": "0x0"
},
"genesis": {
"seal": {
@@ -112,10 +121,10 @@ pub fn nethermind_genesis_json() -> Value {
},
"difficulty": "0x01",
"author": "0x0000000000000000000000000000000000000000",
"timestamp": "0x0",
"timestamp": "0x63585F88",
"parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"extraData": "",
"gasLimit": "0x1C9C380"
"gasLimit": "0x400000"
},
"accounts": {
"0x7b8C3a386C0eea54693fFB0DA17373ffC9228139": {
@@ -123,9 +132,9 @@ pub fn nethermind_genesis_json() -> Value {
},
"0xdA2DD7560DB7e212B945fC72cEB54B7D8C886D77": {
"balance": "10000000000000000000000000"
},
}
},
"nodes": []
}
}
)
}

View File

@@ -3,7 +3,7 @@ use crate::execution_engine::GenericExecutionEngine;
use crate::genesis_json::geth_genesis_json;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output};
use std::{env, fs::File};
use std::{env, fs};
use tempfile::TempDir;
use unused_port::unused_tcp4_port;
@@ -36,6 +36,13 @@ pub fn build(execution_clients_dir: &Path) {
});
}
pub fn clean(execution_clients_dir: &Path) {
let repo_dir = execution_clients_dir.join("go-ethereum");
if let Err(e) = fs::remove_dir_all(repo_dir) {
eprintln!("Error while deleting folder: {}", e);
}
}
/*
* Geth-specific Implementation for GenericExecutionEngine
*/
@@ -60,7 +67,7 @@ impl GenericExecutionEngine for GethEngine {
let datadir = TempDir::new().unwrap();
let genesis_json_path = datadir.path().join("genesis.json");
let mut file = File::create(&genesis_json_path).unwrap();
let mut file = fs::File::create(&genesis_json_path).unwrap();
let json = geth_genesis_json();
serde_json::to_writer(&mut file, &json).unwrap();

View File

@@ -1,3 +1,5 @@
#![recursion_limit = "256"] // for inline json
/// This binary runs integration tests between Lighthouse and execution engines.
///
/// It will first attempt to build any supported integration clients, then it will run tests.
@@ -31,6 +33,7 @@ fn test_geth() {
let test_dir = build_utils::prepare_dir();
geth::build(&test_dir);
TestRig::new(GethEngine).perform_tests_blocking();
geth::clean(&test_dir);
}
fn test_nethermind() {

View File

@@ -2,7 +2,7 @@ use crate::build_utils;
use crate::execution_engine::GenericExecutionEngine;
use crate::genesis_json::nethermind_genesis_json;
use std::env;
use std::fs::File;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output};
use tempfile::TempDir;
@@ -11,7 +11,7 @@ use unused_port::unused_tcp4_port;
/// We've pinned the Nethermind version since our method of using the `master` branch to
/// find the latest tag isn't working. It appears Nethermind don't always tag on `master`.
/// We should fix this so we always pull the latest version of Nethermind.
const NETHERMIND_BRANCH: &str = "release/1.18.2";
const NETHERMIND_BRANCH: &str = "release/1.21.0";
const NETHERMIND_REPO_URL: &str = "https://github.com/NethermindEth/nethermind";
fn build_result(repo_dir: &Path) -> Output {
@@ -47,6 +47,12 @@ pub fn build(execution_clients_dir: &Path) {
build_utils::check_command_output(build_result(&repo_dir), || {
format!("nethermind build failed using release {last_release}")
});
// Cleanup some disk space by removing nethermind's tests
let tests_dir = execution_clients_dir.join("nethermind/src/tests");
if let Err(e) = fs::remove_dir_all(tests_dir) {
eprintln!("Error while deleting folder: {}", e);
}
}
/*
@@ -68,7 +74,8 @@ impl NethermindEngine {
.join("bin")
.join("Release")
.join("net7.0")
.join("Nethermind.Runner")
.join("linux-x64")
.join("nethermind")
}
}
@@ -76,7 +83,7 @@ impl GenericExecutionEngine for NethermindEngine {
fn init_datadir() -> TempDir {
let datadir = TempDir::new().unwrap();
let genesis_json_path = datadir.path().join("genesis.json");
let mut file = File::create(genesis_json_path).unwrap();
let mut file = fs::File::create(genesis_json_path).unwrap();
let json = nethermind_genesis_json();
serde_json::to_writer(&mut file, &json).unwrap();
datadir

View File

@@ -18,7 +18,9 @@ use types::{
Address, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader,
ForkName, FullPayload, Hash256, MainnetEthSpec, PublicKeyBytes, Slot, Uint256,
};
const EXECUTION_ENGINE_START_TIMEOUT: Duration = Duration::from_secs(30);
const EXECUTION_ENGINE_START_TIMEOUT: Duration = Duration::from_secs(60);
const TEST_FORK: ForkName = ForkName::Capella;
struct ExecutionPair<E, T: EthSpec> {
/// The Lighthouse `ExecutionLayer` struct, connected to the `execution_engine` via HTTP.
@@ -110,7 +112,7 @@ impl<E: GenericExecutionEngine> TestRig<E> {
let (runtime_shutdown, exit) = exit_future::signal();
let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
let executor = TaskExecutor::new(Arc::downgrade(&runtime), exit, log.clone(), shutdown_tx);
let mut spec = MainnetEthSpec::default_spec();
let mut spec = TEST_FORK.make_genesis_spec(MainnetEthSpec::default_spec());
spec.terminal_total_difficulty = Uint256::zero();
let fee_recipient = None;
@@ -269,12 +271,11 @@ impl<E: GenericExecutionEngine> TestRig<E> {
Slot::new(1), // Insert proposer for the next slot
head_root,
proposer_index,
// TODO: think about how to test different forks
PayloadAttributes::new(
timestamp,
prev_randao,
Address::repeat_byte(42),
None,
Some(vec![]),
None,
),
)
@@ -314,8 +315,13 @@ impl<E: GenericExecutionEngine> TestRig<E> {
.execution_layer
.get_suggested_fee_recipient(proposer_index)
.await;
let payload_attributes =
PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None, None);
let payload_attributes = PayloadAttributes::new(
timestamp,
prev_randao,
suggested_fee_recipient,
Some(vec![]),
None,
);
let valid_payload = self
.ee_a
.execution_layer
@@ -324,8 +330,7 @@ impl<E: GenericExecutionEngine> TestRig<E> {
&payload_attributes,
forkchoice_update_params,
builder_params,
// FIXME: think about how to test other forks
ForkName::Merge,
TEST_FORK,
&self.spec,
)
.await
@@ -456,8 +461,13 @@ impl<E: GenericExecutionEngine> TestRig<E> {
.execution_layer
.get_suggested_fee_recipient(proposer_index)
.await;
let payload_attributes =
PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None, None);
let payload_attributes = PayloadAttributes::new(
timestamp,
prev_randao,
suggested_fee_recipient,
Some(vec![]),
None,
);
let second_payload = self
.ee_a
.execution_layer
@@ -466,8 +476,7 @@ impl<E: GenericExecutionEngine> TestRig<E> {
&payload_attributes,
forkchoice_update_params,
builder_params,
// FIXME: think about how to test other forks
ForkName::Merge,
TEST_FORK,
&self.spec,
)
.await
@@ -498,11 +507,15 @@ impl<E: GenericExecutionEngine> TestRig<E> {
*/
let head_block_hash = valid_payload.block_hash();
let finalized_block_hash = ExecutionBlockHash::zero();
// TODO: think about how to handle different forks
// To save sending proposer preparation data, just set the fee recipient
// to the fee recipient configured for EE A.
let payload_attributes =
PayloadAttributes::new(timestamp, prev_randao, Address::repeat_byte(42), None, None);
let payload_attributes = PayloadAttributes::new(
timestamp,
prev_randao,
Address::repeat_byte(42),
Some(vec![]),
None,
);
let slot = Slot::new(42);
let head_block_root = Hash256::repeat_byte(100);
let validator_index = 0;
@@ -536,11 +549,7 @@ impl<E: GenericExecutionEngine> TestRig<E> {
.notify_new_payload(second_payload.clone().try_into().unwrap())
.await
.unwrap();
// TODO: we should remove the `Accepted` status here once Geth fixes it
assert!(matches!(
status,
PayloadStatus::Syncing | PayloadStatus::Accepted
));
assert!(matches!(status, PayloadStatus::Syncing));
/*
* Execution Engine B:
@@ -641,11 +650,13 @@ async fn check_payload_reconstruction<E: GenericExecutionEngine>(
.get_engine_capabilities(None)
.await
.unwrap();
assert!(
// if the engine doesn't have these capabilities, we need to update the client in our tests
capabilities.get_payload_bodies_by_hash_v1 && capabilities.get_payload_bodies_by_range_v1,
"Testing engine does not support payload bodies methods"
);
let mut bodies = ee
.execution_layer
.get_payload_bodies_by_hash(vec![payload.block_hash()])