gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-30 03:03:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for YamlConfig in Eth2TestnetDir

Browse Source
This commit is contained in:
Paul Hauner
2019-11-28 14:49:15 +11:00
parent defe28b183
commit 2577136ba7
11 changed files with 118 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
beacon_node/Cargo.toml
View File

@@ -36,3 +36,4 @@ environment = { path = "../lighthouse/environment" }
genesis = { path = "genesis" }
eth2_testnet = { path = "../eth2/utils/eth2_testnet" }
eth2-libp2p = { path = "./eth2-libp2p" }
eth2_ssz = { path = "../eth2/utils/ssz" }

1
beacon_node/client/Cargo.toml
View File

@@ -38,3 +38,4 @@ eth1 = { path = "../eth1" }
genesis = { path = "../genesis" }
environment = { path = "../../lighthouse/environment" }
lighthouse_bootstrap = { path = "../../eth2/utils/lighthouse_bootstrap" }
eth2_ssz = { path = "../../eth2/utils/ssz" }

21
beacon_node/client/src/builder.rs
View File

@@ -23,13 +23,14 @@ use lighthouse_bootstrap::Bootstrapper;
use lmd_ghost::LmdGhost;
use network::{NetworkConfig, NetworkMessage, Service as NetworkService};
use slog::{debug, error, info, warn};
use ssz::Decode;
use std::net::SocketAddr;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::mpsc::UnboundedSender;
use tokio::timer::Interval;
use types::{ChainSpec, EthSpec};
use types::{BeaconState, ChainSpec, EthSpec};
use websocket_server::{Config as WebSocketConfig, WebSocketSender};
/// The interval between notifier events.
@@ -187,6 +188,24 @@ where
Box::new(future)
}
ClientGenesis::SszBytes {
genesis_state_bytes,
} => {
info!(
context.log,
"Using trusted genesis state";
);
let result = BeaconState::from_ssz_bytes(&genesis_state_bytes)
.map_err(|e| format!("Unable to parse genesis state SSZ: {:?}", e));
let future = result
.and_then(move |genesis_state| builder.genesis_state(genesis_state))
.into_future()
.map(|v| (v, None));
Box::new(future)
}
ClientGenesis::DepositContract => {
info!(
context.log,

5
beacon_node/client/src/config.rs
View File

@@ -24,6 +24,11 @@ pub enum ClientGenesis {
DepositContract,
/// Loads the genesis state from a SSZ-encoded `BeaconState` file.
SszFile { path: PathBuf },
/// Loads the genesis state from SSZ-encoded `BeaconState` bytes.
///
/// We include the bytes instead of the `BeaconState<E>` because the `EthSpec` type
/// parameter would be very annoying.
SszBytes { genesis_state_bytes: Vec<u8> },
/// Connects to another Lighthouse instance and reads the genesis state and other data via the
/// HTTP API.
RemoteNode { server: String, port: Option<u16> },

69
beacon_node/src/config.rs
View File

@@ -6,6 +6,7 @@ use eth2_testnet::Eth2TestnetDir;
use genesis::recent_genesis_time;
use rand::{distributions::Alphanumeric, Rng};
use slog::{crit, info, Logger};
use ssz::Encode;
use std::fs;
use std::net::Ipv4Addr;
use std::path::PathBuf;
@@ -17,8 +18,6 @@ pub const ETH2_TESTNET_DIR: &str = "testnet";
pub const BEACON_NODE_DIR: &str = "beacon";
pub const NETWORK_DIR: &str = "network";
pub const SECONDS_PER_ETH1_BLOCK: u64 = 15;
type Result<T> = std::result::Result<T, String>;
type Config = (ClientConfig, Eth2Config, Logger);
@@ -207,12 +206,6 @@ pub fn get_configs<E: EthSpec>(
}
// No sub-command assumes a resume operation.
_ => {
info!(
log,
"Resuming from existing datadir";
"path" => format!("{:?}", client_config.data_dir)
);
// If no primary subcommand was given, start the beacon chain from an existing
// database.
client_config.genesis = ClientGenesis::Resume;
@@ -220,8 +213,18 @@ pub fn get_configs<E: EthSpec>(
// Whilst there is no large testnet or mainnet force the user to specify how they want
// to start a new chain (e.g., from a genesis YAML file, another node, etc).
if !client_config.data_dir.exists() {
info!(
log,
"Starting from an empty database";
"data_dir" => format!("{:?}", client_config.data_dir)
);
init_new_client::<E>(&mut client_config, &mut eth2_config)?
} else {
info!(
log,
"Resuming from existing datadir";
"data_dir" => format!("{:?}", client_config.data_dir)
);
// If the `testnet` command was not provided, attempt to load an existing datadir and
// continue with an existing chain.
load_from_datadir(&mut client_config, &mut eth2_config)?
@@ -291,6 +294,8 @@ fn load_from_datadir(client_config: &mut ClientConfig, eth2_config: &mut Eth2Con
.map_err(|e| format!("Unable to parse {:?} file: {:?}", path, e))?
.ok_or_else(|| format!("{:?} file does not exist", path))?;
client_config.genesis = ClientGenesis::Resume;
Ok(())
}
@@ -299,44 +304,46 @@ fn init_new_client<E: EthSpec>(
client_config: &mut ClientConfig,
eth2_config: &mut Eth2Config,
) -> Result<()> {
let spec = &mut eth2_config.spec;
spec.min_deposit_amount = 100;
spec.max_effective_balance = 3_200_000_000;
spec.ejection_balance = 1_600_000_000;
spec.effective_balance_increment = 100_000_000;
let testnet_dir = client_config.testnet_dir.clone();
let eth2_testnet_dir: Eth2TestnetDir<E> = Eth2TestnetDir::load(testnet_dir.clone())
.map_err(|e| format!("Unable to open testnet dir at {:?}: {}", testnet_dir, e))?;
eth2_config.spec = eth2_testnet_dir
.yaml_config
.as_ref()
.ok_or_else(|| "The testnet directory must contain a spec config".to_string())?
.apply_to_chain_spec::<E>(&eth2_config.spec)
.ok_or_else(|| {
format!(
"The loaded config is not compatible with the {} spec",
&eth2_config.spec_constants
)
})?;
let spec = &mut eth2_config.spec;
client_config.eth1.deposit_contract_address =
format!("{:?}", eth2_testnet_dir.deposit_contract_address()?);
client_config.eth1.deposit_contract_deploy_block =
eth2_testnet_dir.deposit_contract_deploy_block;
spec.min_genesis_time = eth2_testnet_dir.min_genesis_time;
client_config.eth1.follow_distance = 16;
client_config.eth1.follow_distance = spec.eth1_follow_distance / 2;
client_config.dummy_eth1_backend = false;
client_config.sync_eth1_chain = true;
client_config.eth1.lowest_cached_block_number = client_config
.eth1
.deposit_contract_deploy_block
.saturating_sub(client_config.eth1.follow_distance * 2);
// Note: these constants _should_ only be used during genesis to determine the genesis
// time. This allows the testnet to start shortly after the time + validator count
// conditions are satisfied, not 1-2 days.
//
// This value must be at least 2x the `ETH1_FOLLOW_DISTANCE` otherwise `all_eth1_data`
// can become a subset of `new_eth1_data`.
//
// See:
// https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/validator/0_beacon-chain-validator.md#eth1-data
// TODO: set from 10 back to 2
spec.seconds_per_day = SECONDS_PER_ETH1_BLOCK * spec.eth1_follow_distance * 2;
client_config.genesis = ClientGenesis::DepositContract;
if let Some(genesis_state) = eth2_testnet_dir.genesis_state {
// Note: re-serializing the genesis state is not so efficient, however it avoids adding
// trait bounds to the `ClientGenesis` enum. This would have significant flow-on
// effects.
client_config.genesis = ClientGenesis::SszBytes {
genesis_state_bytes: genesis_state.as_ssz_bytes(),
};
} else {
client_config.genesis = ClientGenesis::DepositContract;
}
create_new_datadir(&client_config, &eth2_config)?;

2
eth2/types/src/chain_spec.rs
View File

@@ -319,7 +319,7 @@ mod tests {
}
// Yaml Config is declared here in order to access domain fields of ChainSpec which are private fields.
#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
#[serde(rename_all = "UPPERCASE")]
#[serde(default)]
#[serde(deny_unknown_fields)]

2
eth2/types/src/lib.rs
View File

@@ -48,7 +48,7 @@ pub use crate::beacon_block_body::BeaconBlockBody;
pub use crate::beacon_block_header::BeaconBlockHeader;
pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee};
pub use crate::beacon_state::{Error as BeaconStateError, *};
pub use crate::chain_spec::{ChainSpec, Domain};
pub use crate::chain_spec::{ChainSpec, Domain, YamlConfig};
pub use crate::checkpoint::Checkpoint;
pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH};
pub use crate::deposit_data::DepositData;

2
eth2/utils/eth2_testnet/Cargo.toml
View File

@@ -11,6 +11,6 @@ tempdir = "0.3"
[dependencies]
serde = "1.0"
serde_json = "^1.0"
serde_yaml = "0.8"
types = { path = "../../types"}
eth2-libp2p = { path = "../../../beacon_node/eth2-libp2p"}

29
eth2/utils/eth2_testnet/src/lib.rs
View File

@@ -10,21 +10,21 @@
use eth2_libp2p::Enr;
use std::fs::{create_dir_all, File};
use std::path::PathBuf;
use types::{Address, BeaconState, EthSpec};
use types::{Address, BeaconState, EthSpec, YamlConfig};
pub const ADDRESS_FILE: &str = "deposit_contract.txt";
pub const DEPLOY_BLOCK_FILE: &str = "deploy_block.txt";
pub const MIN_GENESIS_TIME_FILE: &str = "min_genesis_time.txt";
pub const BOOT_NODES_FILE: &str = "boot_enr.json";
pub const BOOT_NODES_FILE: &str = "boot_enr.yaml";
pub const GENESIS_STATE_FILE: &str = "genesis.ssz";
pub const YAML_CONFIG_FILE: &str = "config.yaml";
#[derive(Clone, PartialEq, Debug)]
pub struct Eth2TestnetDir<E: EthSpec> {
pub deposit_contract_address: String,
pub deposit_contract_deploy_block: u64,
pub min_genesis_time: u64,
pub boot_enr: Option<Vec<Enr>>,
pub genesis_state: Option<BeaconState<E>>,
pub yaml_config: Option<YamlConfig>,
}
impl<E: EthSpec> Eth2TestnetDir<E> {
@@ -47,7 +47,7 @@ impl<E: EthSpec> Eth2TestnetDir<E> {
File::create(base_dir.join($file))
.map_err(|e| format!("Unable to create {}: {:?}", $file, e))
.and_then(|file| {
serde_json::to_writer(file, &$variable)
serde_yaml::to_writer(file, &$variable)
.map_err(|e| format!("Unable to write {}: {:?}", $file, e))
})?;
};
@@ -55,7 +55,6 @@ impl<E: EthSpec> Eth2TestnetDir<E> {
write_to_file!(ADDRESS_FILE, self.deposit_contract_address);
write_to_file!(DEPLOY_BLOCK_FILE, self.deposit_contract_deploy_block);
write_to_file!(MIN_GENESIS_TIME_FILE, self.min_genesis_time);
if let Some(boot_enr) = &self.boot_enr {
write_to_file!(BOOT_NODES_FILE, boot_enr);
@@ -65,6 +64,10 @@ impl<E: EthSpec> Eth2TestnetDir<E> {
write_to_file!(GENESIS_STATE_FILE, genesis_state);
}
if let Some(yaml_config) = &self.yaml_config {
write_to_file!(YAML_CONFIG_FILE, yaml_config);
}
Ok(())
}
@@ -74,7 +77,7 @@ impl<E: EthSpec> Eth2TestnetDir<E> {
File::open(base_dir.join($file))
.map_err(|e| format!("Unable to open {}: {:?}", $file, e))
.and_then(|file| {
serde_json::from_reader(file)
serde_yaml::from_reader(file)
.map_err(|e| format!("Unable to parse {}: {:?}", $file, e))
})?;
};
@@ -92,16 +95,16 @@ impl<E: EthSpec> Eth2TestnetDir<E> {
let deposit_contract_address = load_from_file!(ADDRESS_FILE);
let deposit_contract_deploy_block = load_from_file!(DEPLOY_BLOCK_FILE);
let min_genesis_time = load_from_file!(MIN_GENESIS_TIME_FILE);
let boot_enr = optional_load_from_file!(BOOT_NODES_FILE);
let genesis_state = optional_load_from_file!(GENESIS_STATE_FILE);
let yaml_config = optional_load_from_file!(YAML_CONFIG_FILE);
Ok(Self {
deposit_contract_address,
deposit_contract_deploy_block,
min_genesis_time,
boot_enr,
genesis_state,
yaml_config,
})
}
@@ -130,16 +133,16 @@ mod tests {
let base_dir = PathBuf::from(temp_dir.path().join("my_testnet"));
let deposit_contract_address = "0xBB9bc244D798123fDe783fCc1C72d3Bb8C189413".to_string();
let deposit_contract_deploy_block = 42;
let min_genesis_time = 1337;
let testnet: Eth2TestnetDir<E> = Eth2TestnetDir {
deposit_contract_address: deposit_contract_address.clone(),
deposit_contract_deploy_block: deposit_contract_deploy_block,
min_genesis_time,
// TODO: add some Enr for testing.
boot_enr: None,
// TODO: add a genesis state for testing.
genesis_state: None,
// TODO: add a yaml config for testing.
yaml_config: None,
};
testnet
@@ -156,10 +159,6 @@ mod tests {
decoded.deposit_contract_deploy_block, deposit_contract_deploy_block,
"deposit_contract_deploy_block"
);
assert_eq!(
decoded.min_genesis_time, min_genesis_time,
"min_genesis_time"
);
assert_eq!(testnet, decoded, "should decode as encoded");
}

22
lcli/src/deploy_deposit_contract.rs
View File

@@ -5,9 +5,11 @@ use eth2_testnet::Eth2TestnetDir;
use std::fs::File;
use std::io::Read;
use std::path::PathBuf;
use types::EthSpec;
use types::{EthSpec, YamlConfig};
use web3::{transports::Http, Web3};
pub const SECONDS_PER_ETH1_BLOCK: u64 = 15;
pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
let min_genesis_time = matches
.value_of("min-genesis-time")
@@ -86,12 +88,28 @@ pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches) -> Result<
info!("Writing config to {:?}", output_dir);
let mut spec = env.core_context().eth2_config.spec.clone();
spec.min_deposit_amount = 100;
spec.max_effective_balance = 3_200_000_000;
spec.ejection_balance = 1_600_000_000;
spec.effective_balance_increment = 100_000_000;
spec.min_genesis_time = min_genesis_time;
// This value must be at least 2x the `ETH1_FOLLOW_DISTANCE` otherwise `all_eth1_data`
// can become a subset of `new_eth1_data`.
//
// See:
// https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/validator/0_beacon-chain-validator.md#eth1-data
// TODO: set from 10 back to 2
spec.seconds_per_day = SECONDS_PER_ETH1_BLOCK * spec.eth1_follow_distance * 2;
let testnet_dir: Eth2TestnetDir<T> = Eth2TestnetDir {
deposit_contract_address: format!("{}", deposit_contract.address()),
deposit_contract_deploy_block: deploy_block.as_u64(),
min_genesis_time,
boot_enr: None,
genesis_state: None,
yaml_config: Some(YamlConfig::from_spec::<T>(&spec)),
};
testnet_dir.write_to_file(output_dir)?;

36
lcli/src/eth1_genesis.rs
View File

@@ -5,11 +5,10 @@ use futures::Future;
use genesis::{Eth1Config, Eth1GenesisService};
use std::path::PathBuf;
use std::time::Duration;
use types::{EthSpec, Fork};
use types::EthSpec;
/// Interval between polling the eth1 node for genesis information.
pub const ETH1_GENESIS_UPDATE_INTERVAL_MILLIS: u64 = 7_000;
pub const SECONDS_PER_ETH1_BLOCK: u64 = 15;
pub const ETH1_GENESIS_UPDATE_INTERVAL: Duration = Duration::from_millis(7_000);
pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
let endpoint = matches
@@ -28,32 +27,29 @@ pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches) -> Result<
let mut eth2_testnet_dir: Eth2TestnetDir<T> = Eth2TestnetDir::load(testnet_dir.clone())?;
let spec = eth2_testnet_dir
.yaml_config
.as_ref()
.ok_or_else(|| "The testnet directory must contain a spec config".to_string())?
.apply_to_chain_spec::<T>(&env.core_context().eth2_config.spec)
.ok_or_else(|| {
format!(
"The loaded config is not compatible with the {} spec",
&env.core_context().eth2_config.spec_constants
)
})?;
let mut config = Eth1Config::default();
config.endpoint = endpoint.to_string();
config.deposit_contract_address = eth2_testnet_dir.deposit_contract_address.clone();
config.deposit_contract_deploy_block = eth2_testnet_dir.deposit_contract_deploy_block;
config.lowest_cached_block_number = eth2_testnet_dir.deposit_contract_deploy_block;
config.follow_distance = spec.eth1_follow_distance / 2;
let genesis_service = Eth1GenesisService::new(config, env.core_context().log.clone());
let mut spec = env.core_context().eth2_config.spec.clone();
spec.min_genesis_time = eth2_testnet_dir.min_genesis_time;
spec.min_deposit_amount = 100;
spec.max_effective_balance = 3_200_000_000;
spec.ejection_balance = 1_600_000_000;
spec.effective_balance_increment = 100_000_000;
// Note: these are hard-coded hacky values. This should be fixed when we can load a testnet
// dir from the `Eth2TestnetDir`.
spec.eth1_follow_distance = 16;
spec.seconds_per_day = SECONDS_PER_ETH1_BLOCK * spec.eth1_follow_distance * 2;
let future = genesis_service
.wait_for_genesis_state(
Duration::from_millis(ETH1_GENESIS_UPDATE_INTERVAL_MILLIS),
spec,
)
.wait_for_genesis_state(ETH1_GENESIS_UPDATE_INTERVAL, spec)
.map(move |genesis_state| {
eth2_testnet_dir.genesis_state = Some(genesis_state);
eth2_testnet_dir.force_write_to_file(testnet_dir)