Merge branch 'unstable' into deneb-free-blobs

# Conflicts:
#	.github/workflows/docker.yml
#	.github/workflows/local-testnet.yml
#	.github/workflows/test-suite.yml
#	Cargo.lock
#	Cargo.toml
#	beacon_node/beacon_chain/src/beacon_chain.rs
#	beacon_node/beacon_chain/src/builder.rs
#	beacon_node/beacon_chain/src/test_utils.rs
#	beacon_node/execution_layer/src/engine_api/json_structures.rs
#	beacon_node/network/src/beacon_processor/mod.rs
#	beacon_node/network/src/beacon_processor/worker/gossip_methods.rs
#	beacon_node/network/src/sync/backfill_sync/mod.rs
#	beacon_node/store/src/config.rs
#	beacon_node/store/src/hot_cold_store.rs
#	common/eth2_network_config/Cargo.toml
#	consensus/ssz/src/decode/impls.rs
#	consensus/ssz_derive/src/lib.rs
#	consensus/ssz_derive/tests/tests.rs
#	consensus/ssz_types/src/serde_utils/mod.rs
#	consensus/tree_hash/src/impls.rs
#	consensus/tree_hash/src/lib.rs
#	consensus/types/Cargo.toml
#	consensus/types/src/beacon_state.rs
#	consensus/types/src/chain_spec.rs
#	consensus/types/src/eth_spec.rs
#	consensus/types/src/fork_name.rs
#	lcli/Cargo.toml
#	lcli/src/main.rs
#	lcli/src/new_testnet.rs
#	scripts/local_testnet/el_bootnode.sh
#	scripts/local_testnet/genesis.json
#	scripts/local_testnet/geth.sh
#	scripts/local_testnet/setup.sh
#	scripts/local_testnet/start_local_testnet.sh
#	scripts/local_testnet/vars.env
#	scripts/tests/doppelganger_protection.sh
#	scripts/tests/genesis.json
#	scripts/tests/vars.env
#	testing/ef_tests/Cargo.toml
#	validator_client/src/block_service.rs
This commit is contained in:
Jimmy Chen
2023-05-30 11:26:33 +10:00
333 changed files with 5930 additions and 13386 deletions

View File

@@ -2,19 +2,18 @@ use clap::ArgMatches;
use environment::Environment;
use types::EthSpec;
use web3::{transports::Http, Web3};
use eth1_test_rig::{Http, Provider};
pub fn run<T: EthSpec>(env: Environment<T>, matches: &ArgMatches<'_>) -> Result<(), String> {
let eth1_http: String = clap_utils::parse_required(matches, "eth1-http")?;
let confirmations: usize = clap_utils::parse_required(matches, "confirmations")?;
let validator_count: Option<usize> = clap_utils::parse_optional(matches, "validator-count")?;
let transport =
Http::new(&eth1_http).map_err(|e| format!("Unable to connect to eth1 HTTP: {:?}", e))?;
let web3 = Web3::new(transport);
let client = Provider::<Http>::try_from(&eth1_http)
.map_err(|e| format!("Unable to connect to eth1 HTTP: {:?}", e))?;
env.runtime().block_on(async {
let contract = eth1_test_rig::DepositContract::deploy(web3, confirmations, None)
let contract = eth1_test_rig::DepositContract::deploy(client, confirmations, None)
.await
.map_err(|e| format!("Failed to deploy deposit contract: {:?}", e))?;

View File

@@ -10,6 +10,7 @@ mod generate_bootnode_enr;
mod indexed_attestations;
mod insecure_validators;
mod interop_genesis;
mod mnemonic_validators;
mod new_testnet;
mod parse_ssz;
mod replace_state_pubkeys;
@@ -449,6 +450,22 @@ fn main() {
"If present, a interop-style genesis.ssz file will be generated.",
),
)
.arg(
Arg::with_name("derived-genesis-state")
.long("derived-genesis-state")
.takes_value(false)
.help(
"If present, a genesis.ssz file will be generated with keys generated from a given mnemonic.",
),
)
.arg(
Arg::with_name("mnemonic-phrase")
.long("mnemonic-phrase")
.value_name("MNEMONIC_PHRASE")
.takes_value(true)
.requires("derived-genesis-state")
.help("The mnemonic with which we generate the validator keys for a derived genesis state"),
)
.arg(
Arg::with_name("min-genesis-time")
.long("min-genesis-time")
@@ -722,6 +739,7 @@ fn main() {
.long("count")
.value_name("COUNT")
.takes_value(true)
.required(true)
.help("Produces validators in the range of 0..count."),
)
.arg(
@@ -729,6 +747,7 @@ fn main() {
.long("base-dir")
.value_name("BASE_DIR")
.takes_value(true)
.required(true)
.help("The base directory where validator keypairs and secrets are stored"),
)
.arg(
@@ -739,6 +758,43 @@ fn main() {
.help("The number of nodes to divide the validator keys to"),
)
)
.subcommand(
SubCommand::with_name("mnemonic-validators")
.about("Produces validator directories by deriving the keys from \
a mnemonic. For testing purposes only, DO NOT USE IN \
PRODUCTION!")
.arg(
Arg::with_name("count")
.long("count")
.value_name("COUNT")
.takes_value(true)
.required(true)
.help("Produces validators in the range of 0..count."),
)
.arg(
Arg::with_name("base-dir")
.long("base-dir")
.value_name("BASE_DIR")
.takes_value(true)
.required(true)
.help("The base directory where validator keypairs and secrets are stored"),
)
.arg(
Arg::with_name("node-count")
.long("node-count")
.value_name("NODE_COUNT")
.takes_value(true)
.help("The number of nodes to divide the validator keys to"),
)
.arg(
Arg::with_name("mnemonic-phrase")
.long("mnemonic-phrase")
.value_name("MNEMONIC_PHRASE")
.takes_value(true)
.required(true)
.help("The mnemonic with which we generate the validator keys"),
)
)
.subcommand(
SubCommand::with_name("indexed-attestations")
.about("Convert attestations to indexed form, using the committees from a state.")
@@ -834,6 +890,7 @@ fn run<T: EthSpec>(
max_log_number: 0,
compression: false,
is_restricted: true,
sse_logging: false, // No SSE Logging in LCLI
})
.map_err(|e| format!("should start logger: {:?}", e))?
.build()
@@ -880,6 +937,8 @@ fn run<T: EthSpec>(
.map_err(|e| format!("Failed to run generate-bootnode-enr command: {}", e)),
("insecure-validators", Some(matches)) => insecure_validators::run(matches)
.map_err(|e| format!("Failed to run insecure-validators command: {}", e)),
("mnemonic-validators", Some(matches)) => mnemonic_validators::run(matches)
.map_err(|e| format!("Failed to run mnemonic-validators command: {}", e)),
("indexed-attestations", Some(matches)) => indexed_attestations::run::<T>(matches)
.map_err(|e| format!("Failed to run indexed-attestations command: {}", e)),
("block-root", Some(matches)) => block_root::run::<T>(env, matches)

View File

@@ -0,0 +1,104 @@
use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
use account_utils::random_password;
use clap::ArgMatches;
use eth2_wallet::bip39::Seed;
use eth2_wallet::bip39::{Language, Mnemonic};
use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType};
use rayon::prelude::*;
use std::fs;
use std::path::PathBuf;
use validator_dir::Builder as ValidatorBuilder;
/// Generates validator directories with keys derived from the given mnemonic.
pub fn generate_validator_dirs(
indices: &[usize],
mnemonic_phrase: &str,
validators_dir: PathBuf,
secrets_dir: PathBuf,
) -> Result<(), String> {
if !validators_dir.exists() {
fs::create_dir_all(&validators_dir)
.map_err(|e| format!("Unable to create validators dir: {:?}", e))?;
}
if !secrets_dir.exists() {
fs::create_dir_all(&secrets_dir)
.map_err(|e| format!("Unable to create secrets dir: {:?}", e))?;
}
let mnemonic = Mnemonic::from_phrase(mnemonic_phrase, Language::English).map_err(|e| {
format!(
"Unable to derive mnemonic from string {:?}: {:?}",
mnemonic_phrase, e
)
})?;
let seed = Seed::new(&mnemonic, "");
let _: Vec<_> = indices
.par_iter()
.map(|index| {
let voting_password = random_password();
let derive = |key_type: KeyType, password: &[u8]| -> Result<Keystore, String> {
let (secret, path) = recover_validator_secret_from_mnemonic(
seed.as_bytes(),
*index as u32,
key_type,
)
.map_err(|e| format!("Unable to recover validator keys: {:?}", e))?;
let keypair = keypair_from_secret(secret.as_bytes())
.map_err(|e| format!("Unable build keystore: {:?}", e))?;
KeystoreBuilder::new(&keypair, password, format!("{}", path))
.map_err(|e| format!("Unable build keystore: {:?}", e))?
.build()
.map_err(|e| format!("Unable build keystore: {:?}", e))
};
let voting_keystore = derive(KeyType::Voting, voting_password.as_bytes()).unwrap();
println!("Validator {}", index + 1);
ValidatorBuilder::new(validators_dir.clone())
.password_dir(secrets_dir.clone())
.store_withdrawal_keystore(false)
.voting_keystore(voting_keystore, voting_password.as_bytes())
.build()
.map_err(|e| format!("Unable to build validator: {:?}", e))
.unwrap()
})
.collect();
Ok(())
}
pub fn run(matches: &ArgMatches) -> Result<(), String> {
let validator_count: usize = clap_utils::parse_required(matches, "count")?;
let base_dir: PathBuf = clap_utils::parse_required(matches, "base-dir")?;
let node_count: Option<usize> = clap_utils::parse_optional(matches, "node-count")?;
let mnemonic_phrase: String = clap_utils::parse_required(matches, "mnemonic-phrase")?;
if let Some(node_count) = node_count {
let validators_per_node = validator_count / node_count;
let validator_range = (0..validator_count).collect::<Vec<_>>();
let indices_range = validator_range
.chunks(validators_per_node)
.collect::<Vec<_>>();
for (i, indices) in indices_range.iter().enumerate() {
let validators_dir = base_dir.join(format!("node_{}", i + 1)).join("validators");
let secrets_dir = base_dir.join(format!("node_{}", i + 1)).join("secrets");
generate_validator_dirs(indices, &mnemonic_phrase, validators_dir, secrets_dir)?;
}
} else {
let validators_dir = base_dir.join("validators");
let secrets_dir = base_dir.join("secrets");
generate_validator_dirs(
(0..validator_count).collect::<Vec<_>>().as_slice(),
&mnemonic_phrase,
validators_dir,
secrets_dir,
)?;
}
Ok(())
}

View File

@@ -1,7 +1,11 @@
use account_utils::eth2_keystore::keypair_from_secret;
use clap::ArgMatches;
use clap_utils::{parse_optional, parse_required, parse_ssz_optional};
use eth2_hashing::hash;
use eth2_network_config::{Eth2NetworkConfig, TRUSTED_SETUP};
use eth2_wallet::bip39::Seed;
use eth2_wallet::bip39::{Language, Mnemonic};
use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType};
use ethereum_hashing::hash;
use kzg::TrustedSetup;
use ssz::Decode;
use ssz::Encode;
@@ -16,8 +20,8 @@ use types::ExecutionBlockHash;
use types::{
test_utils::generate_deterministic_keypairs, Address, BeaconState, ChainSpec, Config, Epoch,
Eth1Data, EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella,
ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderMerge, ForkName, Hash256, Keypair,
PublicKey, Validator,
ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRefMut,
ForkName, Hash256, Keypair, PublicKey, Validator,
};
pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Result<(), String> {
@@ -90,61 +94,59 @@ pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul
spec.terminal_total_difficulty = ttd;
}
let genesis_state_bytes = if matches.is_present("interop-genesis-state") {
let execution_payload_header: Option<ExecutionPayloadHeader<T>> =
parse_optional(matches, "execution-payload-header")?
.map(|filename: String| {
let mut bytes = vec![];
let mut file = File::open(filename.as_str())
.map_err(|e| format!("Unable to open {}: {}", filename, e))?;
file.read_to_end(&mut bytes)
.map_err(|e| format!("Unable to read {}: {}", filename, e))?;
let fork_name = spec.fork_name_at_epoch(Epoch::new(0));
match fork_name {
ForkName::Base | ForkName::Altair => Err(ssz::DecodeError::BytesInvalid(
"genesis fork must be post-merge".to_string(),
)),
ForkName::Merge => {
ExecutionPayloadHeaderMerge::<T>::from_ssz_bytes(bytes.as_slice())
.map(ExecutionPayloadHeader::Merge)
}
ForkName::Capella => {
ExecutionPayloadHeaderCapella::<T>::from_ssz_bytes(bytes.as_slice())
.map(ExecutionPayloadHeader::Capella)
}
ForkName::Deneb => {
ExecutionPayloadHeaderDeneb::<T>::from_ssz_bytes(bytes.as_slice())
.map(ExecutionPayloadHeader::Deneb)
}
let validator_count = parse_required(matches, "validator-count")?;
let execution_payload_header: Option<ExecutionPayloadHeader<T>> =
parse_optional(matches, "execution-payload-header")?
.map(|filename: String| {
let mut bytes = vec![];
let mut file = File::open(filename.as_str())
.map_err(|e| format!("Unable to open {}: {}", filename, e))?;
file.read_to_end(&mut bytes)
.map_err(|e| format!("Unable to read {}: {}", filename, e))?;
let fork_name = spec.fork_name_at_epoch(Epoch::new(0));
match fork_name {
ForkName::Base | ForkName::Altair => Err(ssz::DecodeError::BytesInvalid(
"genesis fork must be post-merge".to_string(),
)),
ForkName::Merge => {
ExecutionPayloadHeaderMerge::<T>::from_ssz_bytes(bytes.as_slice())
.map(ExecutionPayloadHeader::Merge)
}
.map_err(|e| format!("SSZ decode failed: {:?}", e))
})
.transpose()?;
ForkName::Capella => {
ExecutionPayloadHeaderCapella::<T>::from_ssz_bytes(bytes.as_slice())
.map(ExecutionPayloadHeader::Capella)
}
ForkName::Deneb => {
ExecutionPayloadHeaderDeneb::<T>::from_ssz_bytes(bytes.as_slice())
.map(ExecutionPayloadHeader::Deneb)
}
}
.map_err(|e| format!("SSZ decode failed: {:?}", e))
})
.transpose()?;
let (eth1_block_hash, genesis_time) = if let Some(payload) =
execution_payload_header.as_ref()
{
let eth1_block_hash =
parse_optional(matches, "eth1-block-hash")?.unwrap_or_else(|| payload.block_hash());
let genesis_time =
parse_optional(matches, "genesis-time")?.unwrap_or_else(|| payload.timestamp());
(eth1_block_hash, genesis_time)
} else {
let eth1_block_hash = parse_required(matches, "eth1-block-hash").map_err(|_| {
"One of `--execution-payload-header` or `--eth1-block-hash` must be set".to_string()
})?;
let genesis_time = parse_optional(matches, "genesis-time")?.unwrap_or(
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|e| format!("Unable to get time: {:?}", e))?
.as_secs(),
);
(eth1_block_hash, genesis_time)
};
let validator_count = parse_required(matches, "validator-count")?;
let (eth1_block_hash, genesis_time) = if let Some(payload) = execution_payload_header.as_ref() {
let eth1_block_hash =
parse_optional(matches, "eth1-block-hash")?.unwrap_or_else(|| payload.block_hash());
let genesis_time =
parse_optional(matches, "genesis-time")?.unwrap_or_else(|| payload.timestamp());
(eth1_block_hash, genesis_time)
} else {
let eth1_block_hash = parse_required(matches, "eth1-block-hash").map_err(|_| {
"One of `--execution-payload-header` or `--eth1-block-hash` must be set".to_string()
})?;
let genesis_time = parse_optional(matches, "genesis-time")?.unwrap_or(
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|e| format!("Unable to get time: {:?}", e))?
.as_secs(),
);
(eth1_block_hash, genesis_time)
};
let genesis_state_bytes = if matches.is_present("interop-genesis-state") {
let keypairs = generate_deterministic_keypairs(validator_count);
let keypairs: Vec<_> = keypairs.into_iter().map(|kp| (kp.clone(), kp)).collect();
let genesis_state = initialize_state_with_validators::<T>(
&keypairs,
@@ -154,6 +156,41 @@ pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul
&spec,
)?;
Some(genesis_state.as_ssz_bytes())
} else if matches.is_present("derived-genesis-state") {
let mnemonic_phrase: String = clap_utils::parse_required(matches, "mnemonic-phrase")?;
let mnemonic = Mnemonic::from_phrase(&mnemonic_phrase, Language::English).map_err(|e| {
format!(
"Unable to derive mnemonic from string {:?}: {:?}",
mnemonic_phrase, e
)
})?;
let seed = Seed::new(&mnemonic, "");
let keypairs = (0..validator_count as u32)
.map(|index| {
let (secret, _) =
recover_validator_secret_from_mnemonic(seed.as_bytes(), index, KeyType::Voting)
.unwrap();
let voting_keypair = keypair_from_secret(secret.as_bytes()).unwrap();
let (secret, _) = recover_validator_secret_from_mnemonic(
seed.as_bytes(),
index,
KeyType::Withdrawal,
)
.unwrap();
let withdrawal_keypair = keypair_from_secret(secret.as_bytes()).unwrap();
(voting_keypair, withdrawal_keypair)
})
.collect::<Vec<_>>();
let genesis_state = initialize_state_with_validators::<T>(
&keypairs,
genesis_time,
eth1_block_hash.into_root(),
execution_payload_header,
&spec,
)?;
Some(genesis_state.as_ssz_bytes())
} else {
None
@@ -182,25 +219,30 @@ pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul
testnet.write_to_file(testnet_dir_path, overwrite_files)
}
/// Returns a `BeaconState` with the given validator keypairs embedded into the
/// genesis state. This allows us to start testnets without having to deposit validators
/// manually.
///
/// The optional `execution_payload_header` allows us to start a network from the bellatrix
/// fork without the need to transition to altair and bellatrix.
///
/// We need to ensure that `eth1_block_hash` is equal to the genesis block hash that is
/// generated from the execution side `genesis.json`.
fn initialize_state_with_validators<T: EthSpec>(
keypairs: &[Keypair],
keypairs: &[(Keypair, Keypair)], // Voting and Withdrawal keypairs
genesis_time: u64,
eth1_block_hash: Hash256,
execution_payload_header: Option<ExecutionPayloadHeader<T>>,
spec: &ChainSpec,
) -> Result<BeaconState<T>, String> {
let default_header = ExecutionPayloadHeaderMerge {
gas_limit: 10,
base_fee_per_gas: 10.into(),
timestamp: genesis_time,
block_hash: ExecutionBlockHash(eth1_block_hash),
prev_randao: Hash256::random(),
parent_hash: ExecutionBlockHash::zero(),
transactions_root: Hash256::random(),
..ExecutionPayloadHeaderMerge::default()
};
let execution_payload_header =
execution_payload_header.or(Some(ExecutionPayloadHeader::Merge(default_header)));
// If no header is provided, then start from a Bellatrix state by default
let default_header: ExecutionPayloadHeader<T> =
ExecutionPayloadHeader::Merge(ExecutionPayloadHeaderMerge {
block_hash: ExecutionBlockHash::from_root(eth1_block_hash),
parent_hash: ExecutionBlockHash::zero(),
..ExecutionPayloadHeaderMerge::default()
});
let execution_payload_header = execution_payload_header.unwrap_or(default_header);
// Empty eth1 data
let eth1_data = Eth1Data {
block_hash: eth1_block_hash,
@@ -224,8 +266,8 @@ fn initialize_state_with_validators<T: EthSpec>(
let amount = spec.max_effective_balance;
// Create a new validator.
let validator = Validator {
pubkey: keypair.pk.clone().into(),
withdrawal_credentials: withdrawal_credentials(&keypair.pk),
pubkey: keypair.0.pk.clone().into(),
withdrawal_credentials: withdrawal_credentials(&keypair.1.pk),
activation_eligibility_epoch: spec.far_future_epoch,
activation_epoch: spec.far_future_epoch,
exit_epoch: spec.far_future_epoch,
@@ -264,12 +306,24 @@ fn initialize_state_with_validators<T: EthSpec>(
// Override latest execution payload header.
// See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/merge/beacon-chain.md#testing
if let Some(ExecutionPayloadHeader::Merge(ref header)) = execution_payload_header {
*state
.latest_execution_payload_header_merge_mut()
.map_err(|_| {
"State must contain bellatrix execution payload header".to_string()
})? = header.clone();
// Currently, we only support starting from a bellatrix state
match state
.latest_execution_payload_header_mut()
.map_err(|e| format!("Failed to get execution payload header: {:?}", e))?
{
ExecutionPayloadHeaderRefMut::Merge(header_mut) => {
if let ExecutionPayloadHeader::Merge(eph) = execution_payload_header {
*header_mut = eph;
} else {
return Err("Execution payload header must be a bellatrix header".to_string());
}
}
ExecutionPayloadHeaderRefMut::Capella(_) => {
return Err("Cannot start genesis from a capella state".to_string())
}
ExecutionPayloadHeaderRefMut::Deneb(_) => {
return Err("Cannot start genesis from a deneb state".to_string())
}
}
}

View File

@@ -74,7 +74,7 @@ use eth2::{
use ssz::Encode;
use state_processing::{
block_signature_verifier::BlockSignatureVerifier, per_block_processing, per_slot_processing,
BlockSignatureStrategy, ConsensusContext, VerifyBlockRoot,
BlockSignatureStrategy, ConsensusContext, StateProcessingStrategy, VerifyBlockRoot,
};
use std::borrow::Cow;
use std::fs::File;
@@ -381,6 +381,7 @@ fn do_transition<T: EthSpec>(
&mut pre_state,
&block,
BlockSignatureStrategy::NoVerification,
StateProcessingStrategy::Accurate,
VerifyBlockRoot::True,
&mut ctxt,
spec,