mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-09 19:51:47 +00:00
Eth1 Integration (#542)
* Refactor to cache Eth1Data * Fix merge conflicts and minor refactorings * Rename Eth1Cache to Eth1DataCache * Refactor events subscription * Add deposits module to interface with BeaconChain deposits * Remove utils * Rename to types.rs and add trait constraints to Eth1DataFetcher * Confirm to trait constraints. Make Web3DataFetcher cloneable * Make fetcher object member of deposit and eth1_data cache and other fixes * Fix update_cache function * Move fetch_eth1_data to impl block * Fix deposit tests * Create Eth1 object for interfacing with Beacon chain * Add `run` function for running update_cache and subscribe_deposit_logs tasks * Add logging * Run `cargo fmt` and make tests pass * Convert sync functions to async * Add timeouts to web3 functions * Return futures from cache functions * Add failed chaining of futures * Working cache updation * Clean up tests and `update_cache` function * Refactor `get_eth1_data` functions to work with future returning functions * Refactor eth1 `run` function to work with modified `update_cache` api * Minor changes * Add distance parameter to `update_cache` * Fix tests and other minor fixes * Working integration with cache and deposits * Add merkle_tree construction, proof generation and verification code * Add function to construct and fetch Deposits for BeaconNode * Add error handling * Import ssz * Add error handling to eth1 cache and fix minor errors * Run rustfmt * Fix minor bug * Rename Eth1Error and change to Result<T> * Change deposit fetching mechanism from notification based to poll based * Add deposits from eth1 chain in a given range every `x` blocks * Modify `run` function to accommodate changes * Minor fixes * Fix formatting * Initial commit. web3 api working. * Tidied up lib. Add function for fetching logs. * Refactor with `Eth1DataFetcher` trait * Add parsing for deposit contract logs and get_eth1_data function * Add `get_eth1_votes` function * Refactor to cache Eth1Data * Fix merge conflicts and minor refactorings * Rename Eth1Cache to Eth1DataCache * Refactor events subscription * Add deposits module to interface with BeaconChain deposits * Remove utils * Rename to types.rs and add trait constraints to Eth1DataFetcher * Confirm to trait constraints. Make Web3DataFetcher cloneable * Make fetcher object member of deposit and eth1_data cache and other fixes * Fix update_cache function * Move fetch_eth1_data to impl block * Fix deposit tests * Create Eth1 object for interfacing with Beacon chain * Add `run` function for running update_cache and subscribe_deposit_logs tasks * Add logging * Run `cargo fmt` and make tests pass * Convert sync functions to async * Add timeouts to web3 functions * Return futures from cache functions * Add failed chaining of futures * Working cache updation * Clean up tests and `update_cache` function * Refactor `get_eth1_data` functions to work with future returning functions * Refactor eth1 `run` function to work with modified `update_cache` api * Minor changes * Add distance parameter to `update_cache` * Fix tests and other minor fixes * Working integration with cache and deposits * Add merkle_tree construction, proof generation and verification code * Add function to construct and fetch Deposits for BeaconNode * Add error handling * Import ssz * Add error handling to eth1 cache and fix minor errors * Run rustfmt * Fix minor bug * Rename Eth1Error and change to Result<T> * Change deposit fetching mechanism from notification based to poll based * Add deposits from eth1 chain in a given range every `x` blocks * Modify `run` function to accommodate changes * Minor fixes * Fix formatting * Fix merge issue * Refactor with `Config` struct. Remote `ContractConfig` * Rename eth1_chain crate to eth1 * Rename files and read abi file using `fs::read` * Move eth1 to lib * Remove unnecessary mutability constraint * Add `Web3Backend` for returning actual eth1 data * Refactor `get_eth1_votes` to return a Result * Delete `eth1_chain` crate * Return `Result` from `get_deposits` * Fix range of deposits to return to beacon chain * Add `get_block_height_by_hash` trait function * Add naive method for getting `previous_eth1_distance` * Add eth1 config params to main config * Add instructions for setting up eth1 testing environment * Add build script to fetch deposit contract abi * Contract ABI is part of compiled binary * Fix minor bugs * Move docs to lib * Add timeout to config * Remove print statements * Change warn to error * Fix typos * Removed prints in test and get timeout value from config * Fixed error types * Added logging to web3_fetcher * Refactor for modified web3 api * Fix minor stuff * Add build script * Tidy, hide eth1 integration tests behind flag * Add http crate * Add first stages of eth1_test_rig * Fix deposits on test rig * Fix bug with deposit count method * Add block hash getter to http eth1 * Clean eth1 http crate and tests * Add script to start ganache * Adds deposit tree to eth1-http * Extend deposit tree tests * Tidy tests in eth1-http * Add more detail to get block request * Add block cache to eth1-http * Rename deposit tree to deposit cache * Add inital updating to eth1-http * Tidy updater * Fix compile bugs in tests * Adds an Eth1DataCache builder * Reorg eth1-http files * Add (failing) tests for eth1 updater * Rename files, fix bug in eth1-http * Ensure that ganache timestamps are increasing * Fix bugs with getting eth1data ancestors * Improve eth1 testing, fix bugs * Add truncate method to block cache * Add pruning to block cache update process * Add tests for block pruning * Allow for dropping an expired cache. * Add more comments * Add first compiling version of deposit updater * Add common fn for getting range of required blocks * Add passing deposit update test * Improve tests * Fix block pruning bug * Add tests for running two updates at once * Add updater services to eth1 * Add deposit collection to beacon chain * Add incomplete builder experiments * Add first working version of beacon chain builder * Update test harness to new beacon chain type * Rename builder file, tidy * Add first working client builder * Progress further on client builder * Update becaon node binary to use client builder * Ensure release tests compile * Remove old eth1 crate * Add first pass of new lighthouse binary * Fix websocket server startup * Remove old binary code from beacon_node crate * Add first working beacon node tests * Add genesis crate, new eth1 cache_2 * Add Serivce to Eth1Cache * Refactor with general eth1 improvements * Add passing genesis test * Tidy, add comments * Add more comments to eth1 service * Add further eth1 progress * Fix some bugs with genesis * Fix eth1 bugs, make eth1 linking more efficient * Shift logic in genesis service * Add more comments to genesis service * Add gzip, max request values, timeouts to http * Update testnet parameters to suit goerli testnet * Add ability to vary Fork, fix custom spec * Be more explicit about deposit fork version * Start adding beacon chain eth1 option * Add more flexibility to prod client * Further runtime refactoring * Allow for starting from store * Add bootstrapping to client config * Add remote_beacon_node crate * Update eth1 service for more configurability * Update eth1 tests to use less runtimes * Patch issues with tests using too many files * Move dummy eth1 backend flag * Ensure all tests pass * Add ganache-cli to Dockerfile * Use a special docker hub image for testing * Appease clippy * Move validator client into lighthouse binary * Allow starting with dummy eth1 backend * Improve logging * Fix dummy eth1 backend from cli * Add extra testnet command * Ensure consistent spec in beacon node * Update eth1 rig to work on goerli * Tidy lcli, start adding support for yaml config * Add incomplete YamlConfig struct * Remove efforts at YamlConfig * Add incomplete eth1 voting. Blocked on spec issues * Add (untested) first pass at eth1 vote algo * Add tests for winning vote * Add more tests for eth1 chain * Add more eth1 voting tests * Added more eth1 voting testing * Change test name * Add more tests to eth1 chain * Tidy eth1 generics, add more tests * Improve comments * Tidy beacon_node tests * Tidy, rename JsonRpc.. to Caching.. * Tidy voting logic * Tidy builder docs * Add comments, tidy eth1 * Add more comments to eth1 * Fix bug with winning_vote * Add doc comments to the `ClientBuilder` * Remove commented-out code * Improve `ClientBuilder` docs * Add comments to client config * Add decoding test for `ClientConfig` * Remove unused `DepositSet` struct * Tidy `block_cache` * Remove commented out lines * Remove unused code in `eth1` crate * Remove old validator binary `main.rs` * Tidy, fix tests compile error * Add initial tests for get_deposits * Remove dead code in eth1_test_rig * Update TestingDepositBuilder * Add testing for getting eth1 deposits * Fix duplicate rand dep * Remove dead code * Remove accidentally-added files * Fix comment in eth1_genesis_service * Add .gitignore for eth1_test_rig * Fix bug in eth1_genesis_service * Remove dead code from eth2_config * Fix tabs/spaces in root Cargo.toml * Tidy eth1 crate * Allow for re-use of eth1 service after genesis * Update docs for new CLI * Change README gif * Tidy eth1 http module * Tidy eth1 service * Tidy environment crate * Remove unused file * Tidy, add comments * Remove commented-out code * Address majority of Michael's comments * Address other PR comments * Add link to issue alongside TODO
This commit is contained in:
Paul Hauner
@@ -2,327 +2,58 @@ extern crate slog;
|
||||
|
||||
mod config;
|
||||
|
||||
pub mod builder;
|
||||
pub mod error;
|
||||
pub mod notifier;
|
||||
|
||||
use beacon_chain::{
|
||||
lmd_ghost::ThreadSafeReducedTree, slot_clock::SystemTimeSlotClock, store::Store,
|
||||
test_utils::generate_deterministic_keypairs, BeaconChain, BeaconChainBuilder,
|
||||
};
|
||||
use beacon_chain::BeaconChain;
|
||||
use exit_future::Signal;
|
||||
use futures::{future::Future, Stream};
|
||||
use network::Service as NetworkService;
|
||||
use rest_api::NetworkInfo;
|
||||
use slog::{crit, debug, error, info, o};
|
||||
use slot_clock::SlotClock;
|
||||
use std::marker::PhantomData;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::runtime::TaskExecutor;
|
||||
use tokio::timer::Interval;
|
||||
use types::EthSpec;
|
||||
use websocket_server::WebSocketSender;
|
||||
|
||||
pub use beacon_chain::{BeaconChainTypes, Eth1ChainBackend, InteropEth1ChainBackend};
|
||||
pub use config::{BeaconChainStartMethod, Config as ClientConfig, Eth1BackendMethod};
|
||||
pub use beacon_chain::{BeaconChainTypes, Eth1ChainBackend};
|
||||
pub use builder::ClientBuilder;
|
||||
pub use config::{ClientGenesis, Config as ClientConfig};
|
||||
pub use eth2_config::Eth2Config;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RuntimeBeaconChainTypes<S: Store, E: EthSpec> {
|
||||
_phantom_s: PhantomData<S>,
|
||||
_phantom_e: PhantomData<E>,
|
||||
/// The core "beacon node" client.
|
||||
///
|
||||
/// Holds references to running services, cleanly shutting them down when dropped.
|
||||
pub struct Client<T: BeaconChainTypes> {
|
||||
beacon_chain: Option<Arc<BeaconChain<T>>>,
|
||||
libp2p_network: Option<Arc<NetworkService<T>>>,
|
||||
http_listen_addr: Option<SocketAddr>,
|
||||
websocket_listen_addr: Option<SocketAddr>,
|
||||
/// Exit signals will "fire" when dropped, causing each service to exit gracefully.
|
||||
_exit_signals: Vec<Signal>,
|
||||
}
|
||||
|
||||
impl<S, E> BeaconChainTypes for RuntimeBeaconChainTypes<S, E>
|
||||
where
|
||||
S: Store + 'static,
|
||||
E: EthSpec,
|
||||
{
|
||||
type Store = S;
|
||||
type SlotClock = SystemTimeSlotClock;
|
||||
type LmdGhost = ThreadSafeReducedTree<S, E>;
|
||||
type Eth1Chain = InteropEth1ChainBackend<E>;
|
||||
type EthSpec = E;
|
||||
type EventHandler = WebSocketSender<E>;
|
||||
}
|
||||
impl<T: BeaconChainTypes> Client<T> {
|
||||
/// Returns an `Arc` reference to the client's `BeaconChain`, if it was started.
|
||||
pub fn beacon_chain(&self) -> Option<Arc<BeaconChain<T>>> {
|
||||
self.beacon_chain.clone()
|
||||
}
|
||||
|
||||
/// Main beacon node client service. This provides the connection and initialisation of the clients
|
||||
/// sub-services in multiple threads.
|
||||
pub struct Client<S, E>
|
||||
where
|
||||
S: Store + Clone + 'static,
|
||||
E: EthSpec,
|
||||
{
|
||||
/// Configuration for the lighthouse client.
|
||||
_client_config: ClientConfig,
|
||||
/// The beacon chain for the running client.
|
||||
beacon_chain: Arc<BeaconChain<RuntimeBeaconChainTypes<S, E>>>,
|
||||
/// Reference to the network service.
|
||||
pub network: Arc<NetworkService<RuntimeBeaconChainTypes<S, E>>>,
|
||||
/// Signal to terminate the RPC server.
|
||||
pub rpc_exit_signal: Option<Signal>,
|
||||
/// Signal to terminate the slot timer.
|
||||
pub slot_timer_exit_signal: Option<Signal>,
|
||||
/// Signal to terminate the API
|
||||
pub api_exit_signal: Option<Signal>,
|
||||
/// Signal to terminate the websocket server
|
||||
pub websocket_exit_signal: Option<Signal>,
|
||||
/// The clients logger.
|
||||
log: slog::Logger,
|
||||
}
|
||||
/// Returns the address of the client's HTTP API server, if it was started.
|
||||
pub fn http_listen_addr(&self) -> Option<SocketAddr> {
|
||||
self.http_listen_addr
|
||||
}
|
||||
|
||||
impl<S, E> Client<S, E>
|
||||
where
|
||||
S: Store + Clone + 'static,
|
||||
E: EthSpec,
|
||||
{
|
||||
/// Generate an instance of the client. Spawn and link all internal sub-processes.
|
||||
pub fn new(
|
||||
client_config: ClientConfig,
|
||||
eth2_config: Eth2Config,
|
||||
store: S,
|
||||
log: slog::Logger,
|
||||
executor: &TaskExecutor,
|
||||
) -> error::Result<Self> {
|
||||
let store = Arc::new(store);
|
||||
let milliseconds_per_slot = eth2_config.spec.milliseconds_per_slot;
|
||||
/// Returns the address of the client's WebSocket API server, if it was started.
|
||||
pub fn websocket_listen_addr(&self) -> Option<SocketAddr> {
|
||||
self.websocket_listen_addr
|
||||
}
|
||||
|
||||
let spec = ð2_config.spec.clone();
|
||||
|
||||
let beacon_chain_builder = match &client_config.beacon_chain_start_method {
|
||||
BeaconChainStartMethod::Resume => {
|
||||
info!(
|
||||
log,
|
||||
"Starting beacon chain";
|
||||
"method" => "resume"
|
||||
);
|
||||
BeaconChainBuilder::from_store(spec.clone(), log.clone())
|
||||
}
|
||||
BeaconChainStartMethod::Mainnet => {
|
||||
crit!(log, "No mainnet beacon chain startup specification.");
|
||||
return Err("Mainnet launch is not yet announced.".into());
|
||||
}
|
||||
BeaconChainStartMethod::RecentGenesis {
|
||||
validator_count,
|
||||
minutes,
|
||||
} => {
|
||||
info!(
|
||||
log,
|
||||
"Starting beacon chain";
|
||||
"validator_count" => validator_count,
|
||||
"minutes" => minutes,
|
||||
"method" => "recent"
|
||||
);
|
||||
BeaconChainBuilder::recent_genesis(
|
||||
&generate_deterministic_keypairs(*validator_count),
|
||||
*minutes,
|
||||
spec.clone(),
|
||||
log.clone(),
|
||||
)?
|
||||
}
|
||||
BeaconChainStartMethod::Generated {
|
||||
validator_count,
|
||||
genesis_time,
|
||||
} => {
|
||||
info!(
|
||||
log,
|
||||
"Starting beacon chain";
|
||||
"validator_count" => validator_count,
|
||||
"genesis_time" => genesis_time,
|
||||
"method" => "quick"
|
||||
);
|
||||
BeaconChainBuilder::quick_start(
|
||||
*genesis_time,
|
||||
&generate_deterministic_keypairs(*validator_count),
|
||||
spec.clone(),
|
||||
log.clone(),
|
||||
)?
|
||||
}
|
||||
BeaconChainStartMethod::Yaml { file } => {
|
||||
info!(
|
||||
log,
|
||||
"Starting beacon chain";
|
||||
"file" => format!("{:?}", file),
|
||||
"method" => "yaml"
|
||||
);
|
||||
BeaconChainBuilder::yaml_state(file, spec.clone(), log.clone())?
|
||||
}
|
||||
BeaconChainStartMethod::Ssz { file } => {
|
||||
info!(
|
||||
log,
|
||||
"Starting beacon chain";
|
||||
"file" => format!("{:?}", file),
|
||||
"method" => "ssz"
|
||||
);
|
||||
BeaconChainBuilder::ssz_state(file, spec.clone(), log.clone())?
|
||||
}
|
||||
BeaconChainStartMethod::Json { file } => {
|
||||
info!(
|
||||
log,
|
||||
"Starting beacon chain";
|
||||
"file" => format!("{:?}", file),
|
||||
"method" => "json"
|
||||
);
|
||||
BeaconChainBuilder::json_state(file, spec.clone(), log.clone())?
|
||||
}
|
||||
BeaconChainStartMethod::HttpBootstrap { server, port } => {
|
||||
info!(
|
||||
log,
|
||||
"Starting beacon chain";
|
||||
"port" => port,
|
||||
"server" => server,
|
||||
"method" => "bootstrap"
|
||||
);
|
||||
BeaconChainBuilder::http_bootstrap(server, spec.clone(), log.clone())?
|
||||
}
|
||||
};
|
||||
|
||||
let eth1_backend =
|
||||
InteropEth1ChainBackend::new(String::new()).map_err(|e| format!("{:?}", e))?;
|
||||
|
||||
// Start the websocket server.
|
||||
let (websocket_sender, websocket_exit_signal): (WebSocketSender<E>, Option<_>) =
|
||||
if client_config.websocket_server.enabled {
|
||||
let (sender, exit) = websocket_server::start_server(
|
||||
&client_config.websocket_server,
|
||||
executor,
|
||||
&log,
|
||||
)?;
|
||||
(sender, Some(exit))
|
||||
} else {
|
||||
(WebSocketSender::dummy(), None)
|
||||
};
|
||||
|
||||
let beacon_chain: Arc<BeaconChain<RuntimeBeaconChainTypes<S, E>>> = Arc::new(
|
||||
beacon_chain_builder
|
||||
.build(store, eth1_backend, websocket_sender)
|
||||
.map_err(error::Error::from)?,
|
||||
);
|
||||
|
||||
let since_epoch = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_err(|e| format!("Unable to read system time: {}", e))?;
|
||||
let since_genesis = Duration::from_secs(beacon_chain.head().beacon_state.genesis_time);
|
||||
|
||||
if since_genesis > since_epoch {
|
||||
info!(
|
||||
log,
|
||||
"Starting node prior to genesis";
|
||||
"now" => since_epoch.as_secs(),
|
||||
"genesis_seconds" => since_genesis.as_secs(),
|
||||
);
|
||||
}
|
||||
|
||||
let network_config = &client_config.network;
|
||||
let (network, network_send) =
|
||||
NetworkService::new(beacon_chain.clone(), network_config, executor, log.clone())?;
|
||||
|
||||
// spawn the RPC server
|
||||
let rpc_exit_signal = if client_config.rpc.enabled {
|
||||
Some(rpc::start_server(
|
||||
&client_config.rpc,
|
||||
executor,
|
||||
network_send.clone(),
|
||||
beacon_chain.clone(),
|
||||
&log,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Start the `rest_api` service
|
||||
let api_exit_signal = if client_config.rest_api.enabled {
|
||||
let network_info = NetworkInfo {
|
||||
network_service: network.clone(),
|
||||
network_chan: network_send.clone(),
|
||||
};
|
||||
match rest_api::start_server(
|
||||
&client_config.rest_api,
|
||||
executor,
|
||||
beacon_chain.clone(),
|
||||
network_info,
|
||||
client_config.db_path().expect("unable to read datadir"),
|
||||
eth2_config.clone(),
|
||||
&log,
|
||||
) {
|
||||
Ok(s) => Some(s),
|
||||
Err(e) => {
|
||||
error!(log, "API service failed to start."; "error" => format!("{:?}",e));
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let (slot_timer_exit_signal, exit) = exit_future::signal();
|
||||
if let Some(duration_to_next_slot) = beacon_chain.slot_clock.duration_to_next_slot() {
|
||||
// set up the validator work interval - start at next slot and proceed every slot
|
||||
let interval = {
|
||||
// Set the interval to start at the next slot, and every slot after
|
||||
let slot_duration = Duration::from_millis(milliseconds_per_slot);
|
||||
//TODO: Handle checked add correctly
|
||||
Interval::new(Instant::now() + duration_to_next_slot, slot_duration)
|
||||
};
|
||||
|
||||
let chain = beacon_chain.clone();
|
||||
let log = log.new(o!("Service" => "SlotTimer"));
|
||||
executor.spawn(
|
||||
exit.until(
|
||||
interval
|
||||
.for_each(move |_| {
|
||||
log_new_slot(&chain, &log);
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.map_err(|_| ()),
|
||||
)
|
||||
.map(|_| ()),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(Client {
|
||||
_client_config: client_config,
|
||||
beacon_chain,
|
||||
rpc_exit_signal,
|
||||
slot_timer_exit_signal: Some(slot_timer_exit_signal),
|
||||
api_exit_signal,
|
||||
websocket_exit_signal,
|
||||
log,
|
||||
network,
|
||||
})
|
||||
/// Returns the port of the client's libp2p stack, if it was started.
|
||||
pub fn libp2p_listen_port(&self) -> Option<u16> {
|
||||
self.libp2p_network.as_ref().map(|n| n.listen_port())
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: Store + Clone, E: EthSpec> Drop for Client<S, E> {
|
||||
impl<T: BeaconChainTypes> Drop for Client<T> {
|
||||
fn drop(&mut self) {
|
||||
// Save the beacon chain to it's store before dropping.
|
||||
let _result = self.beacon_chain.persist();
|
||||
if let Some(beacon_chain) = &self.beacon_chain {
|
||||
let _result = beacon_chain.persist();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn log_new_slot<T: BeaconChainTypes>(chain: &Arc<BeaconChain<T>>, log: &slog::Logger) {
|
||||
let best_slot = chain.head().beacon_block.slot;
|
||||
let latest_block_root = chain.head().beacon_block_root;
|
||||
|
||||
if let Ok(current_slot) = chain.slot() {
|
||||
info!(
|
||||
log,
|
||||
"Slot start";
|
||||
"best_slot" => best_slot,
|
||||
"slot" => current_slot,
|
||||
);
|
||||
debug!(
|
||||
log,
|
||||
"Slot info";
|
||||
"skip_slots" => current_slot.saturating_sub(best_slot),
|
||||
"best_block_root" => format!("{}", latest_block_root),
|
||||
"slot" => current_slot,
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
log,
|
||||
"Beacon chain running whilst slot clock is unavailable."
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user