Merge branch 'v0.3.0-staging' into v3-master

This commit is contained in:
Paul Hauner
2020-10-09 15:27:08 +11:00
316 changed files with 15960 additions and 11523 deletions

View File

@@ -5,50 +5,51 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
edition = "2018"
[dependencies]
hex = "0.4.2"
discv5 = { version = "0.1.0-alpha.13", features = ["libp2p"] }
types = { path = "../../consensus/types" }
hashset_delay = { path = "../../common/hashset_delay" }
eth2_ssz_types = { path = "../../consensus/ssz_types" }
serde = { version = "1.0.110", features = ["derive"] }
serde_derive = "1.0.110"
serde = { version = "1.0.116", features = ["derive"] }
serde_derive = "1.0.116"
eth2_ssz = "0.1.2"
eth2_ssz_derive = "0.1.0"
slog = { version = "2.5.2", features = ["max_level_trace"] }
lighthouse_version = { path = "../../common/lighthouse_version" }
tokio = { version = "0.2.22", features = ["time", "macros"] }
futures = "0.3.5"
error-chain = "0.12.2"
dirs = "2.0.2"
error-chain = "0.12.4"
dirs = "3.0.1"
fnv = "1.0.7"
unsigned-varint = { git = "https://github.com/sigp/unsigned-varint", branch = "latest-codecs", features = ["codec"] }
lazy_static = "1.4.0"
lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
smallvec = "1.4.1"
lru = "0.5.1"
smallvec = "1.4.2"
lru = "0.6.0"
parking_lot = "0.11.0"
sha2 = "0.9.1"
base64 = "0.12.1"
snap = "1.0.0"
base64 = "0.13.0"
snap = "1.0.1"
void = "1.0.2"
hex = "0.4.2"
tokio-io-timeout = "0.4.0"
tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
discv5 = { version = "0.1.0-alpha.12", features = ["libp2p"] }
tiny-keccak = "2.0.2"
environment = { path = "../../lighthouse/environment" }
task_executor = { path = "../../common/task_executor" }
rand = "0.7.3"
directory = { path = "../../common/directory" }
regex = "1.3.9"
[dependencies.libp2p]
#version = "0.23.0"
git = "https://github.com/sigp/rust-libp2p"
rev = "03f998022ce2f566a6c6e6c4206bc0ce4d45109f"
rev = "5a9f0819af3990cfefad528e957297af596399b4"
default-features = false
features = ["websocket", "identify", "mplex", "noise", "gossipsub", "dns", "tcp-tokio"]
[dev-dependencies]
tokio = { version = "0.2.22", features = ["full"] }
slog-stdlog = "4.0.0"
slog-term = "2.5.0"
slog-term = "2.6.0"
slog-async = "2.5.0"
tempdir = "0.3.7"
exit-future = "0.2.0"

View File

@@ -54,8 +54,6 @@ impl<TSpec: EthSpec> DelegatingHandler<TSpec> {
}
}
// TODO: this can all be created with macros
/// Wrapper around the `ProtocolsHandler::InEvent` types of the handlers.
/// Simply delegated to the corresponding behaviour's handler.
#[derive(Debug, Clone)]
@@ -115,7 +113,6 @@ pub type DelegateOutProto<TSpec> = EitherUpgrade<
>,
>;
// TODO: prob make this an enum
pub type DelegateOutInfo<TSpec> = EitherOutput<
<GossipHandler as ProtocolsHandler>::OutboundOpenInfo,
EitherOutput<
@@ -216,7 +213,6 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
<Self::OutboundProtocol as OutboundUpgrade<NegotiatedSubstream>>::Error,
>,
) {
// TODO: find how to clean up
match info {
// Gossipsub
EitherOutput::First(info) => match error {

View File

@@ -41,15 +41,9 @@ pub enum BehaviourHandlerIn<TSpec: EthSpec> {
Shutdown(Option<(RequestId, RPCRequest<TSpec>)>),
}
pub enum BehaviourHandlerOut<TSpec: EthSpec> {
Delegate(Box<DelegateOut<TSpec>>),
// TODO: replace custom with events to send
Custom,
}
impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
type InEvent = BehaviourHandlerIn<TSpec>;
type OutEvent = BehaviourHandlerOut<TSpec>;
type OutEvent = DelegateOut<TSpec>;
type Error = DelegateError<TSpec>;
type InboundProtocol = DelegateInProto<TSpec>;
type OutboundProtocol = DelegateOutProto<TSpec>;
@@ -122,9 +116,7 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
match self.delegate.poll(cx) {
Poll::Ready(ProtocolsHandlerEvent::Custom(event)) => {
return Poll::Ready(ProtocolsHandlerEvent::Custom(
BehaviourHandlerOut::Delegate(Box::new(event)),
))
return Poll::Ready(ProtocolsHandlerEvent::Custom(event))
}
Poll::Ready(ProtocolsHandlerEvent::Close(err)) => {
return Poll::Ready(ProtocolsHandlerEvent::Close(err))

View File

@@ -5,7 +5,7 @@ use crate::types::{GossipEncoding, GossipKind, GossipTopic, SubnetDiscovery};
use crate::Eth2Enr;
use crate::{error, metrics, Enr, NetworkConfig, NetworkGlobals, PubsubMessage, TopicHash};
use futures::prelude::*;
use handler::{BehaviourHandler, BehaviourHandlerIn, BehaviourHandlerOut, DelegateIn, DelegateOut};
use handler::{BehaviourHandler, BehaviourHandlerIn, DelegateIn, DelegateOut};
use libp2p::{
core::{
connection::{ConnectedPoint, ConnectionId, ListenerId},
@@ -102,7 +102,7 @@ pub struct Behaviour<TSpec: EthSpec> {
/// The Eth2 RPC specified in the wire-0 protocol.
eth2_rpc: RPC<TSpec>,
/// Keep regular connection to peers and disconnect if absent.
// TODO: Using id for initial interop. This will be removed by mainnet.
// NOTE: The id protocol is used for initial interop. This will be removed by mainnet.
/// Provides IP addresses and peer information.
identify: Identify,
/// The peer manager that keeps track of peer's reputation and status.
@@ -203,9 +203,6 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
self.enr_fork_id.fork_digest,
);
// TODO: Implement scoring
// let topic: Topic = gossip_topic.into();
// self.gossipsub.set_topic_params(t.hash(), TopicScoreParams::default());
self.subscribe(gossip_topic)
}
@@ -227,12 +224,6 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
GossipEncoding::default(),
self.enr_fork_id.fork_digest,
);
// TODO: Implement scoring
/*
let t: Topic = topic.clone().into();
self.gossipsub
.set_topic_params(t.hash(), TopicScoreParams::default());
*/
self.subscribe(topic)
}
@@ -591,7 +582,6 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
} => {
if matches!(error, RPCError::HandlerRejected) {
// this peer's request got canceled
// TODO: cancel processing for this request
}
// Inform the peer manager of the error.
// An inbound error here means we sent an error to the peer, or the stream
@@ -621,11 +611,8 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
RPCRequest::MetaData(_) => {
// send the requested meta-data
self.send_meta_data_response((handler_id, id), peer_id);
// TODO: inform the peer manager?
}
RPCRequest::Goodbye(reason) => {
// let the peer manager know this peer is in the process of disconnecting
self.peer_manager._disconnecting_peer(&peer_id);
// queue for disconnection without a goodbye message
debug!(
self.log, "Peer sent Goodbye";
@@ -975,17 +962,11 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
return;
}
// Events comming from the handler, redirected to each behaviour
match event {
// Events comming from the handler, redirected to each behaviour
BehaviourHandlerOut::Delegate(delegate) => match *delegate {
DelegateOut::Gossipsub(ev) => self.gossipsub.inject_event(peer_id, conn_id, ev),
DelegateOut::RPC(ev) => self.eth2_rpc.inject_event(peer_id, conn_id, ev),
DelegateOut::Identify(ev) => self.identify.inject_event(peer_id, conn_id, *ev),
},
/* Custom events sent BY the handler */
BehaviourHandlerOut::Custom => {
// TODO: implement
}
DelegateOut::Gossipsub(ev) => self.gossipsub.inject_event(peer_id, conn_id, ev),
DelegateOut::RPC(ev) => self.eth2_rpc.inject_event(peer_id, conn_id, ev),
DelegateOut::Identify(ev) => self.identify.inject_event(peer_id, conn_id, *ev),
}
}
@@ -1003,7 +984,6 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
self.waker = Some(cx.waker().clone());
}
// TODO: move where it's less distracting
macro_rules! poll_behaviour {
/* $behaviour: The sub-behaviour being polled.
* $on_event_fn: Function to call if we get an event from the sub-behaviour.

View File

@@ -1,5 +1,8 @@
use crate::types::GossipKind;
use crate::{Enr, PeerIdSerialized};
use directory::{
DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_TESTNET, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR,
};
use discv5::{Discv5Config, Discv5ConfigBuilder};
use libp2p::gossipsub::{
GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode,
@@ -77,9 +80,14 @@ pub struct Config {
impl Default for Config {
/// Generate a default network configuration.
fn default() -> Self {
let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
network_dir.push(".lighthouse");
network_dir.push("network");
// WARNING: this directory default should be always overrided with parameters
// from cli for specific networks.
let network_dir = dirs::home_dir()
.unwrap_or_else(|| PathBuf::from("."))
.join(DEFAULT_ROOT_DIR)
.join(DEFAULT_HARDCODED_TESTNET)
.join(DEFAULT_BEACON_NODE_DIR)
.join(DEFAULT_NETWORK_DIR);
// The function used to generate a gossipsub message id
// We use the first 8 bytes of SHA256(data) for content addressing
@@ -115,6 +123,7 @@ impl Default for Config {
.request_retries(1)
.enr_peer_update_min(10)
.query_parallelism(5)
.disable_report_discovered_peers()
.query_timeout(Duration::from_secs(30))
.query_peer_timeout(Duration::from_secs(2))
.ip_limit() // limits /24 IP's in buckets.

View File

@@ -129,7 +129,6 @@ pub fn create_enr_builder_from_config<T: EnrKey>(config: &NetworkConfig) -> EnrB
builder.udp(udp_port);
}
// we always give it our listening tcp port
// TODO: Add uPnP support to map udp and tcp ports
let tcp_port = config.enr_tcp_port.unwrap_or_else(|| config.libp2p_port);
builder.tcp(tcp_port).tcp(config.libp2p_port);
builder
@@ -144,12 +143,12 @@ pub fn build_enr<T: EthSpec>(
let mut builder = create_enr_builder_from_config(config);
// set the `eth2` field on our ENR
builder.add_value(ETH2_ENR_KEY.into(), enr_fork_id.as_ssz_bytes());
builder.add_value(ETH2_ENR_KEY, &enr_fork_id.as_ssz_bytes());
// set the "attnets" field on our ENR
let bitfield = BitVector::<T::SubnetBitfieldLength>::new();
builder.add_value(BITFIELD_ENR_KEY.into(), bitfield.as_ssz_bytes());
builder.add_value(BITFIELD_ENR_KEY, &bitfield.as_ssz_bytes());
builder
.build(enr_key)

View File

@@ -365,7 +365,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
/// If the external address needs to be modified, use `update_enr_udp_socket.
pub fn update_enr_tcp_port(&mut self, port: u16) -> Result<(), String> {
self.discv5
.enr_insert("tcp", port.to_be_bytes().into())
.enr_insert("tcp", &port.to_be_bytes())
.map_err(|e| format!("{:?}", e))?;
// replace the global version
@@ -383,18 +383,18 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
match socket_addr {
SocketAddr::V4(socket) => {
self.discv5
.enr_insert("ip", socket.ip().octets().into())
.enr_insert("ip", &socket.ip().octets())
.map_err(|e| format!("{:?}", e))?;
self.discv5
.enr_insert("udp", socket.port().to_be_bytes().into())
.enr_insert("udp", &socket.port().to_be_bytes())
.map_err(|e| format!("{:?}", e))?;
}
SocketAddr::V6(socket) => {
self.discv5
.enr_insert("ip6", socket.ip().octets().into())
.enr_insert("ip6", &socket.ip().octets())
.map_err(|e| format!("{:?}", e))?;
self.discv5
.enr_insert("udp6", socket.port().to_be_bytes().into())
.enr_insert("udp6", &socket.port().to_be_bytes())
.map_err(|e| format!("{:?}", e))?;
}
}
@@ -439,7 +439,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
// insert the bitfield into the ENR record
self.discv5
.enr_insert(BITFIELD_ENR_KEY, current_bitfield.as_ssz_bytes())
.enr_insert(BITFIELD_ENR_KEY, &current_bitfield.as_ssz_bytes())
.map_err(|e| format!("{:?}", e))?;
// replace the global version
@@ -468,7 +468,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
let _ = self
.discv5
.enr_insert(ETH2_ENR_KEY, enr_fork_id.as_ssz_bytes())
.enr_insert(ETH2_ENR_KEY, &enr_fork_id.as_ssz_bytes())
.map_err(|e| {
warn!(
self.log,
@@ -858,7 +858,10 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
// Still awaiting the event stream, poll it
if let Poll::Ready(event_stream) = fut.poll_unpin(cx) {
match event_stream {
Ok(stream) => self.event_stream = EventStream::Present(stream),
Ok(stream) => {
debug!(self.log, "Discv5 event stream ready");
self.event_stream = EventStream::Present(stream);
}
Err(e) => {
slog::crit!(self.log, "Discv5 event stream failed"; "error" => e.to_string());
self.event_stream = EventStream::InActive;

View File

@@ -147,8 +147,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
///
/// If the peer doesn't exist, log a warning and insert defaults.
pub fn report_peer(&mut self, peer_id: &PeerId, action: PeerAction) {
// TODO: Remove duplicate code - This is duplicated in the update_peer_scores()
// function.
// NOTE: This is duplicated in the update_peer_scores() and could be improved.
// Variables to update the PeerDb if required.
let mut ban_peer = None;
@@ -179,7 +178,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
GoodbyeReason::BadScore,
));
}
// TODO: Update the peer manager to inform that the peer is disconnecting.
}
ScoreState::Healthy => {
debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
@@ -322,15 +320,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
self.connect_peer(peer_id, ConnectingType::OutgoingConnected { multiaddr })
}
/// Updates the database informing that a peer is being disconnected.
pub fn _disconnecting_peer(&mut self, _peer_id: &PeerId) -> bool {
// TODO: implement
// This informs the database that we are in the process of disconnecting the
// peer. Currently this state only exists for a short period of time before we force the
// disconnection.
true
}
/// Reports if a peer is banned or not.
///
/// This is used to determine if we should accept incoming connections.
@@ -408,10 +397,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
// Not supporting a protocol shouldn't be considered a malicious action, but
// it is an action that in some cases will make the peer unfit to continue
// communicating.
// TODO: To avoid punishing a peer repeatedly for not supporting a protocol, this
// information could be stored and used to prevent sending requests for the given
// protocol to this peer. Similarly, to avoid blacklisting a peer for a protocol
// forever, if stored this information should expire.
match protocol {
Protocol::Ping => PeerAction::Fatal,
Protocol::BlocksByRange => return,
@@ -445,7 +431,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
/// A ping request has been received.
// NOTE: The behaviour responds with a PONG automatically
// TODO: Update last seen
pub fn ping_request(&mut self, peer_id: &PeerId, seq: u64) {
if let Some(peer_info) = self.network_globals.peers.read().peer_info(peer_id) {
// received a ping
@@ -475,7 +460,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
}
/// A PONG has been returned from a peer.
// TODO: Update last seen
pub fn pong_response(&mut self, peer_id: &PeerId, seq: u64) {
if let Some(peer_info) = self.network_globals.peers.read().peer_info(peer_id) {
// received a pong
@@ -501,7 +485,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
}
/// Received a metadata response from a peer.
// TODO: Update last seen
pub fn meta_data_response(&mut self, peer_id: &PeerId, meta_data: MetaData<TSpec>) {
if let Some(peer_info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
if let Some(known_meta_data) = &peer_info.meta_data {
@@ -597,7 +580,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
let connected_or_dialing = self.network_globals.connected_or_dialing_peers();
for (peer_id, min_ttl) in results {
// we attempt a connection if this peer is a subnet peer or if the max peer count
// is not yet filled (including dialling peers)
// is not yet filled (including dialing peers)
if (min_ttl.is_some() || connected_or_dialing + to_dial_peers.len() < self.max_peers)
&& !self
.network_globals
@@ -610,7 +593,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
.read()
.is_banned_or_disconnected(&peer_id)
{
// TODO: Update output
// This should be updated with the peer dialing. In fact created once the peer is
// dialed
if let Some(min_ttl) = min_ttl {
@@ -699,58 +681,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
// Update scores
info.score_update();
/* TODO: Implement logic about connection lifetimes
match info.connection_status {
Connected { .. } => {
// Connected peers gain reputation by sending useful messages
}
Disconnected { since } | Banned { since } => {
// For disconnected peers, lower their reputation by 1 for every hour they
// stay disconnected. This helps us slowly forget disconnected peers.
// In the same way, slowly allow banned peers back again.
let dc_hours = now
.checked_duration_since(since)
.unwrap_or_else(|| Duration::from_secs(0))
.as_secs()
/ 3600;
let last_dc_hours = self
._last_updated
.checked_duration_since(since)
.unwrap_or_else(|| Duration::from_secs(0))
.as_secs()
/ 3600;
if dc_hours > last_dc_hours {
// this should be 1 most of the time
let rep_dif = (dc_hours - last_dc_hours)
.try_into()
.unwrap_or(Rep::max_value());
info.reputation = if info.connection_status.is_banned() {
info.reputation.saturating_add(rep_dif)
} else {
info.reputation.saturating_sub(rep_dif)
};
}
}
Dialing { since } => {
// A peer shouldn't be dialing for more than 2 minutes
if since.elapsed().as_secs() > 120 {
warn!(self.log,"Peer has been dialing for too long"; "peer_id" => id.to_string());
// TODO: decide how to handle this
}
}
Unknown => {} //TODO: Handle this case
}
// Check if the peer gets banned or unbanned and if it should be disconnected
if info.reputation < _MIN_REP_BEFORE_BAN && !info.connection_status.is_banned() {
// This peer gets banned. Check if we should request disconnection
ban_queue.push(id.clone());
} else if info.reputation >= _MIN_REP_BEFORE_BAN && info.connection_status.is_banned() {
// This peer gets unbanned
unban_queue.push(id.clone());
}
*/
// handle score transitions
if previous_state != info.score_state() {
match info.score_state() {
@@ -774,7 +704,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
GoodbyeReason::BadScore,
));
}
// TODO: Update peer manager to report that it's disconnecting.
}
ScoreState::Healthy => {
debug!(self.log, "Peer transitioned to healthy state"; "peer_id" => peer_id.to_string(), "score" => info.score().to_string(), "past_state" => previous_state.to_string());
@@ -838,9 +767,6 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
///
/// NOTE: Discovery will only add a new query if one isn't already queued.
fn heartbeat(&mut self) {
// TODO: Provide a back-off time for discovery queries. I.e Queue many initially, then only
// perform discoveries over a larger fixed interval. Perhaps one every 6 heartbeats. This
// is achievable with a leaky bucket
let peer_count = self.network_globals.connected_or_dialing_peers();
if peer_count < self.target_peers {
// If we need more peers, queue a discovery lookup.

View File

@@ -130,7 +130,6 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
}
/// Returns a mutable reference to a peer's info if known.
/// TODO: make pub(super) to ensure that peer management is unified
pub fn peer_info_mut(&mut self, peer_id: &PeerId) -> Option<&mut PeerInfo<TSpec>> {
self.peers.get_mut(peer_id)
}

View File

@@ -25,8 +25,6 @@ use std::{
use tokio::time::{delay_queue, delay_until, Delay, DelayQueue, Instant as TInstant};
use types::EthSpec;
//TODO: Implement check_timeout() on the substream types
/// The time (in seconds) before a substream that is awaiting a response from the user times out.
pub const RESPONSE_TIMEOUT: u64 = 10;
@@ -163,8 +161,6 @@ struct OutboundInfo<TSpec: EthSpec> {
/// Info over the protocol this substream is handling.
proto: Protocol,
/// Number of chunks to be seen from the peer's response.
// TODO: removing the option could allow clossing the streams after the number of
// expected responses is met for all protocols.
remaining_chunks: Option<u64>,
/// `RequestId` as given by the application that sent the request.
req_id: RequestId,

View File

@@ -59,7 +59,7 @@ pub struct Service<TSpec: EthSpec> {
impl<TSpec: EthSpec> Service<TSpec> {
pub async fn new(
executor: environment::TaskExecutor,
executor: task_executor::TaskExecutor,
config: &NetworkConfig,
enr_fork_id: EnrForkId,
log: &slog::Logger,
@@ -109,7 +109,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
Behaviour::new(&local_keypair, config, network_globals.clone(), &log).await?;
// use the executor for libp2p
struct Executor(environment::TaskExecutor);
struct Executor(task_executor::TaskExecutor);
impl libp2p::core::Executor for Executor {
fn exec(&self, f: Pin<Box<dyn Future<Output = ()> + Send>>) {
self.0.spawn(f, "libp2p");

View File

@@ -99,7 +99,7 @@ pub async fn build_libp2p_instance(boot_nodes: Vec<Enr>, log: slog::Logger) -> L
let (signal, exit) = exit_future::signal();
let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
let executor = environment::TaskExecutor::new(
let executor = task_executor::TaskExecutor::new(
tokio::runtime::Handle::current(),
exit,
log.clone(),