gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-29 20:27:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update libp2p (#2101)

Browse Source 
This is a little bit of a tip-of-the-iceberg PR. It houses a lot of code changes in the libp2p dependency. 

This needs a bit of thorough testing before merging. 

The primary code changes are:
- General libp2p dependency update
- Gossipsub refactor to shift compression into gossipsub providing performance improvements and improved API for handling compression



Co-authored-by: Paul Hauner <paul@paulhauner.com>
This commit is contained in:
Age Manning
2020-12-23 07:53:36 +00:00
parent b5e81eb6b2
commit 2931b05582
22 changed files with 396 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
beacon_node/eth2_libp2p/Cargo.toml
View File

@@ -42,7 +42,7 @@ regex = "1.3.9"
[dependencies.libp2p]
#version = "0.23.0"
git = "https://github.com/sigp/rust-libp2p"
rev = "830e6fabb7ee51281a98f5e092f056668adbef25"
rev = "97000533e4710183124abde017c6c3d68287c1ae"
default-features = false
features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "tcp-tokio"]

8
beacon_node/eth2_libp2p/src/behaviour/gossipsub_scoring_parameters.rs
View File

@@ -1,8 +1,7 @@
use crate::types::{GossipEncoding, GossipKind, GossipTopic};
use crate::{error, TopicHash};
use libp2p::gossipsub::{
GenericGossipsubConfig, IdentTopic as Topic, PeerScoreParams, PeerScoreThresholds,
TopicScoreParams,
GossipsubConfig, IdentTopic as Topic, PeerScoreParams, PeerScoreThresholds, TopicScoreParams,
};
use std::cmp::max;
use std::collections::HashMap;
@@ -37,10 +36,7 @@ pub struct PeerScoreSettings<TSpec: EthSpec> {
}
impl<TSpec: EthSpec> PeerScoreSettings<TSpec> {
pub fn new<T>(
chain_spec: &ChainSpec,
gs_config: &GenericGossipsubConfig<T>,
) -> PeerScoreSettings<TSpec> {
pub fn new(chain_spec: &ChainSpec, gs_config: &GossipsubConfig) -> PeerScoreSettings<TSpec> {
let slot = Duration::from_millis(chain_spec.milliseconds_per_slot);
let beacon_attestation_subnet_weight = 1.0 / chain_spec.attestation_subnet_count as f64;
let max_positive_score = (MAX_IN_MESH_SCORE + MAX_FIRST_MESSAGE_DELIVERIES_SCORE)

88
beacon_node/eth2_libp2p/src/behaviour/mod.rs
View File

@@ -6,17 +6,13 @@ use crate::peer_manager::{
use crate::rpc::*;
use crate::service::METADATA_FILENAME;
use crate::types::{
subnet_id_from_topic_hash, GossipEncoding, GossipKind, GossipTopic, MessageData,
subnet_id_from_topic_hash, GossipEncoding, GossipKind, GossipTopic, SnappyTransform,
SubnetDiscovery,
};
use crate::Eth2Enr;
use crate::{error, metrics, Enr, NetworkConfig, NetworkGlobals, PubsubMessage, TopicHash};
use futures::prelude::*;
use handler::{BehaviourHandler, BehaviourHandlerIn, DelegateIn, DelegateOut};
use libp2p::gossipsub::subscription_filter::{
MaxCountSubscriptionFilter, WhitelistSubscriptionFilter,
};
use libp2p::gossipsub::PeerScoreThresholds;
use libp2p::{
core::{
connection::{ConnectedPoint, ConnectionId, ListenerId},
@@ -24,13 +20,14 @@ use libp2p::{
Multiaddr,
},
gossipsub::{
GenericGossipsub, GenericGossipsubEvent, IdentTopic as Topic, MessageAcceptance,
MessageAuthenticity, MessageId,
subscription_filter::{MaxCountSubscriptionFilter, WhitelistSubscriptionFilter},
Gossipsub as BaseGossipsub, GossipsubEvent, IdentTopic as Topic, MessageAcceptance,
MessageAuthenticity, MessageId, PeerScoreThresholds,
},
identify::{Identify, IdentifyEvent},
swarm::{
NetworkBehaviour, NetworkBehaviourAction as NBAction, NotifyHandler, PollParameters,
ProtocolsHandler,
AddressScore, NetworkBehaviour, NetworkBehaviourAction as NBAction, NotifyHandler,
PollParameters, ProtocolsHandler,
},
PeerId,
};
@@ -58,8 +55,7 @@ pub const GOSSIPSUB_GREYLIST_THRESHOLD: f64 = -16000.0;
pub type PeerRequestId = (ConnectionId, SubstreamId);
pub type SubscriptionFilter = MaxCountSubscriptionFilter<WhitelistSubscriptionFilter>;
pub type Gossipsub = GenericGossipsub<MessageData, SubscriptionFilter>;
pub type GossipsubEvent = GenericGossipsubEvent<MessageData>;
pub type Gossipsub = BaseGossipsub<SnappyTransform, SubscriptionFilter>;
/// The types of events than can be obtained from polling the behaviour.
#[derive(Debug)]
@@ -181,10 +177,14 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
max_subscriptions_per_request: 100, //this is according to the current go implementation
};
let mut gossipsub = Gossipsub::new_with_subscription_filter(
// Initialize the compression transform.
let snappy_transform = SnappyTransform::new(net_conf.gs_config.max_transmit_size());
let mut gossipsub = Gossipsub::new_with_subscription_filter_and_transform(
MessageAuthenticity::Anonymous,
net_conf.gs_config.clone(),
filter,
snappy_transform,
)
.map_err(|e| format!("Could not construct gossipsub: {:?}", e))?;
@@ -390,34 +390,30 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
pub fn publish(&mut self, messages: Vec<PubsubMessage<TSpec>>) {
for message in messages {
for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) {
match message.encode(GossipEncoding::default()) {
Ok(message_data) => {
if let Err(e) = self.gossipsub.publish(topic.clone().into(), message_data) {
slog::warn!(self.log, "Could not publish message";
let message_data = message.encode(GossipEncoding::default());
if let Err(e) = self.gossipsub.publish(topic.clone().into(), message_data) {
slog::warn!(self.log, "Could not publish message";
"error" => ?e);
// add to metrics
match topic.kind() {
GossipKind::Attestation(subnet_id) => {
if let Some(v) = metrics::get_int_gauge(
&metrics::FAILED_ATTESTATION_PUBLISHES_PER_SUBNET,
&[&subnet_id.to_string()],
) {
v.inc()
};
}
kind => {
if let Some(v) = metrics::get_int_gauge(
&metrics::FAILED_PUBLISHES_PER_MAIN_TOPIC,
&[&format!("{:?}", kind)],
) {
v.inc()
};
}
}
// add to metrics
match topic.kind() {
GossipKind::Attestation(subnet_id) => {
if let Some(v) = metrics::get_int_gauge(
&metrics::FAILED_ATTESTATION_PUBLISHES_PER_SUBNET,
&[&subnet_id.to_string()],
) {
v.inc()
};
}
kind => {
if let Some(v) = metrics::get_int_gauge(
&metrics::FAILED_PUBLISHES_PER_MAIN_TOPIC,
&[&format!("{:?}", kind)],
) {
v.inc()
};
}
}
Err(e) => crit!(self.log, "Could not publish message"; "error" => e),
}
}
}
@@ -642,7 +638,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
} => {
// Note: We are keeping track here of the peer that sent us the message, not the
// peer that originally published the message.
match PubsubMessage::decode(&gs_msg.topic, gs_msg.data()) {
match PubsubMessage::decode(&gs_msg.topic, &gs_msg.data) {
Err(e) => {
debug!(self.log, "Could not decode gossipsub message"; "error" => e);
//reject the message
@@ -854,7 +850,10 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
});
}
PeerManagerEvent::SocketUpdated(address) => {
return Poll::Ready(NBAction::ReportObservedAddr { address });
return Poll::Ready(NBAction::ReportObservedAddr {
address,
score: AddressScore::Finite(1),
});
}
PeerManagerEvent::Status(peer_id) => {
// it's time to status. We don't keep a beacon chain reference here, so we inform
@@ -1028,8 +1027,7 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
trace!(self.log, "Disconnecting newly connected peer"; "peer_id" => %peer_id, "reason" => %goodbye_reason)
}
}
self.peers_to_dc
.push_back((peer_id.clone(), Some(goodbye_reason)));
self.peers_to_dc.push_back((*peer_id, Some(goodbye_reason)));
// NOTE: We don't inform the peer manager that this peer is disconnecting. It is simply
// rejected with a goodbye.
return;
@@ -1041,13 +1039,13 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
ConnectedPoint::Listener { send_back_addr, .. } => {
self.peer_manager
.connect_ingoing(&peer_id, send_back_addr.clone());
self.add_event(BehaviourEvent::PeerConnected(peer_id.clone()));
self.add_event(BehaviourEvent::PeerConnected(*peer_id));
debug!(self.log, "Connection established"; "peer_id" => %peer_id, "connection" => "Incoming");
}
ConnectedPoint::Dialer { address } => {
self.peer_manager
.connect_outgoing(&peer_id, address.clone());
self.add_event(BehaviourEvent::PeerDialed(peer_id.clone()));
self.add_event(BehaviourEvent::PeerDialed(*peer_id));
debug!(self.log, "Connection established"; "peer_id" => %peer_id, "connection" => "Dialed");
}
}
@@ -1122,7 +1120,7 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
// Both these cases, the peer has been previously registered in the sub protocols and
// potentially the application layer.
// Inform the application.
self.add_event(BehaviourEvent::PeerDisconnected(peer_id.clone()));
self.add_event(BehaviourEvent::PeerDisconnected(*peer_id));
// Inform the behaviour.
delegate_to_behaviours!(self, inject_disconnected, peer_id);
@@ -1260,8 +1258,8 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
),
});
}
NBAction::ReportObservedAddr { address } => {
return Poll::Ready(NBAction::ReportObservedAddr { address })
NBAction::ReportObservedAddr { address, score } => {
return Poll::Ready(NBAction::ReportObservedAddr { address, score })
}
},
Poll::Pending => break,

43
beacon_node/eth2_libp2p/src/config.rs
View File

@@ -1,12 +1,12 @@
use crate::types::{GossipKind, MessageData};
use crate::types::GossipKind;
use crate::{Enr, PeerIdSerialized};
use directory::{
DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_NETWORK, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR,
};
use discv5::{Discv5Config, Discv5ConfigBuilder};
use libp2p::gossipsub::{
FastMessageId, GenericGossipsubConfig, GenericGossipsubConfigBuilder, GenericGossipsubMessage,
MessageId, RawGossipsubMessage, ValidationMode,
FastMessageId, GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId,
RawGossipsubMessage, ValidationMode,
};
use libp2p::Multiaddr;
use serde_derive::{Deserialize, Serialize};
@@ -15,14 +15,13 @@ use std::path::PathBuf;
use std::time::Duration;
pub const GOSSIP_MAX_SIZE: usize = 1_048_576;
const MESSAGE_DOMAIN_INVALID_SNAPPY: [u8; 4] = [0, 0, 0, 0];
// We treat uncompressed messages as invalid and never use the INVALID_SNAPPY_DOMAIN as in the
// specification. We leave it here for posterity.
// const MESSAGE_DOMAIN_INVALID_SNAPPY: [u8; 4] = [0, 0, 0, 0];
const MESSAGE_DOMAIN_VALID_SNAPPY: [u8; 4] = [1, 0, 0, 0];
pub const MESH_N_LOW: usize = 6;
pub type GossipsubConfig = GenericGossipsubConfig<MessageData>;
pub type GossipsubConfigBuilder = GenericGossipsubConfigBuilder<MessageData>;
pub type GossipsubMessage = GenericGossipsubMessage<MessageData>;
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
/// Network configuration for lighthouse.
@@ -110,35 +109,28 @@ impl Default for Config {
// The function used to generate a gossipsub message id
// We use the first 8 bytes of SHA256(data) for content addressing
let fast_gossip_message_id =
|message: &RawGossipsubMessage| FastMessageId::from(&Sha256::digest(&message.data)[..]);
let fast_gossip_message_id = |message: &RawGossipsubMessage| {
FastMessageId::from(&Sha256::digest(&message.data)[..8])
};
fn prefix(prefix: [u8; 4], data: &[u8]) -> Vec<u8> {
prefix
.to_vec()
.into_iter()
.chain(data.iter().cloned())
.collect()
let mut vec = Vec::with_capacity(prefix.len() + data.len());
vec.extend_from_slice(&prefix);
vec.extend_from_slice(data);
vec
}
let gossip_message_id = |message: &GossipsubMessage| {
MessageId::from(
&Sha256::digest(
{
match &message.data.decompressed {
Ok(decompressed) => prefix(MESSAGE_DOMAIN_VALID_SNAPPY, decompressed),
_ => prefix(MESSAGE_DOMAIN_INVALID_SNAPPY, &message.data.raw),
}
}
.as_slice(),
)[..20],
&Sha256::digest(prefix(MESSAGE_DOMAIN_VALID_SNAPPY, &message.data).as_slice())
[..20],
)
};
// gossipsub configuration
// Note: The topics by default are sent as plain strings. Hashes are an optional
// parameter.
let gs_config = GossipsubConfigBuilder::new()
let gs_config = GossipsubConfigBuilder::default()
.max_transmit_size(GOSSIP_MAX_SIZE)
.heartbeat_interval(Duration::from_millis(700))
.mesh_n(8)
@@ -147,6 +139,7 @@ impl Default for Config {
.gossip_lazy(6)
.fanout_ttl(Duration::from_secs(60))
.history_length(6)
.max_messages_per_rpc(Some(10))
.history_gossip(3)
.validate_messages() // require validation before propagation
.validation_mode(ValidationMode::Anonymous)

2
beacon_node/eth2_libp2p/src/discovery/enr_ext.rs
View File

@@ -241,7 +241,7 @@ impl CombinedKeyExt for CombinedKey {
pub fn peer_id_to_node_id(peer_id: &PeerId) -> Result<discv5::enr::NodeId, String> {
// A libp2p peer id byte representation should be 2 length bytes + 4 protobuf bytes + compressed pk bytes
// if generated from a PublicKey with Identity multihash.
let pk_bytes = &peer_id.as_bytes()[2..];
let pk_bytes = &peer_id.to_bytes()[2..];
match PublicKey::from_protobuf_encoding(pk_bytes).map_err(|e| {
format!(

1
beacon_node/eth2_libp2p/src/lib.rs
View File

@@ -63,7 +63,6 @@ impl<'de> Deserialize<'de> for PeerIdSerialized {
pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage, SubnetDiscovery};
pub use behaviour::{BehaviourEvent, Gossipsub, PeerRequestId, Request, Response};
pub use config::Config as NetworkConfig;
pub use config::{GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage};
pub use discovery::{CombinedKeyExt, EnrExt, Eth2Enr};
pub use discv5;
pub use libp2p::bandwidth::BandwidthSinks;

48
beacon_node/eth2_libp2p/src/peer_manager/mod.rs
View File

@@ -136,7 +136,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
///
/// Returns true if the peer was accepted into the database.
pub fn dial_peer(&mut self, peer_id: &PeerId) -> bool {
self.events.push(PeerManagerEvent::Dial(peer_id.clone()));
self.events.push(PeerManagerEvent::Dial(*peer_id));
self.connect_peer(peer_id, ConnectingType::Dialing)
}
@@ -174,7 +174,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
{
// update the state of the peer.
self.events
.push(PeerManagerEvent::DisconnectPeer(peer_id.clone(), reason));
.push(PeerManagerEvent::DisconnectPeer(*peer_id, reason));
}
}
@@ -282,7 +282,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
/// A STATUS message has been received from a peer. This resets the status timer.
pub fn peer_statusd(&mut self, peer_id: &PeerId) {
self.status_peers.insert(peer_id.clone());
self.status_peers.insert(*peer_id);
}
/// Adds a gossipsub subscription to a peer in the peerdb.
@@ -495,10 +495,10 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
debug!(self.log, "Received a ping request"; "peer_id" => %peer_id, "seq_no" => seq);
match peer_info.connection_direction {
Some(ConnectionDirection::Incoming) => {
self.inbound_ping_peers.insert(peer_id.clone());
self.inbound_ping_peers.insert(*peer_id);
}
Some(ConnectionDirection::Outgoing) => {
self.outbound_ping_peers.insert(peer_id.clone());
self.outbound_ping_peers.insert(*peer_id);
}
None => {
warn!(self.log, "Received a ping from a peer with an unknown connection direction"; "peer_id" => %peer_id);
@@ -510,15 +510,13 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
if meta_data.seq_number < seq {
debug!(self.log, "Requesting new metadata from peer";
"peer_id" => %peer_id, "known_seq_no" => meta_data.seq_number, "ping_seq_no" => seq);
self.events
.push(PeerManagerEvent::MetaData(peer_id.clone()));
self.events.push(PeerManagerEvent::MetaData(*peer_id));
}
} else {
// if we don't know the meta-data, request it
debug!(self.log, "Requesting first metadata from peer";
"peer_id" => %peer_id);
self.events
.push(PeerManagerEvent::MetaData(peer_id.clone()));
self.events.push(PeerManagerEvent::MetaData(*peer_id));
}
} else {
crit!(self.log, "Received a PING from an unknown peer";
@@ -536,15 +534,13 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
if meta_data.seq_number < seq {
debug!(self.log, "Requesting new metadata from peer";
"peer_id" => %peer_id, "known_seq_no" => meta_data.seq_number, "pong_seq_no" => seq);
self.events
.push(PeerManagerEvent::MetaData(peer_id.clone()));
self.events.push(PeerManagerEvent::MetaData(*peer_id));
}
} else {
// if we don't know the meta-data, request it
debug!(self.log, "Requesting first metadata from peer";
"peer_id" => %peer_id);
self.events
.push(PeerManagerEvent::MetaData(peer_id.clone()));
self.events.push(PeerManagerEvent::MetaData(*peer_id));
}
} else {
crit!(self.log, "Received a PONG from an unknown peer"; "peer_id" => %peer_id);
@@ -677,7 +673,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
&& !peers.is_connected_or_dialing(peer_id)
&& !peers.is_banned(peer_id)
{
Some(peer_id.clone())
Some(*peer_id)
} else {
None
}
@@ -755,18 +751,18 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
ConnectingType::IngoingConnected { multiaddr } => {
peerdb.connect_ingoing(peer_id, multiaddr, enr);
// start a timer to ping inbound peers.
self.inbound_ping_peers.insert(peer_id.clone());
self.inbound_ping_peers.insert(*peer_id);
}
ConnectingType::OutgoingConnected { multiaddr } => {
peerdb.connect_outgoing(peer_id, multiaddr, enr);
// start a timer for to ping outbound peers.
self.outbound_ping_peers.insert(peer_id.clone());
self.outbound_ping_peers.insert(*peer_id);
}
}
}
// start a ping and status timer for the peer
self.status_peers.insert(peer_id.clone());
self.status_peers.insert(*peer_id);
// increment prometheus metrics
metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT);
@@ -806,7 +802,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
match info.score_state() {
ScoreState::Banned => {
debug!(log, "Peer has been banned"; "peer_id" => %peer_id, "score" => %info.score());
to_ban_peers.push(peer_id.clone());
to_ban_peers.push(*peer_id);
}
ScoreState::Disconnected => {
debug!(log, "Peer transitioned to disconnect state"; "peer_id" => %peer_id, "score" => %info.score(), "past_state" => %previous_state);
@@ -815,18 +811,18 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
// Change the state to inform that we are disconnecting the peer.
info.disconnecting(false);
events.push(PeerManagerEvent::DisconnectPeer(
peer_id.clone(),
*peer_id,
GoodbyeReason::BadScore,
));
} else if info.is_banned() {
to_unban_peers.push(peer_id.clone());
to_unban_peers.push(*peer_id);
}
}
ScoreState::Healthy => {
debug!(log, "Peer transitioned to healthy state"; "peer_id" => %peer_id, "score" => %info.score(), "past_state" => %previous_state);
// unban the peer if it was previously banned.
if info.is_banned() {
to_unban_peers.push(peer_id.clone());
to_unban_peers.push(*peer_id);
}
}
}
@@ -885,7 +881,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
if peer_db.disconnect_and_ban(peer_id) {
// The peer was currently connected, so we start a disconnection.
self.events.push(PeerManagerEvent::DisconnectPeer(
peer_id.clone(),
*peer_id,
GoodbyeReason::BadScore,
));
}
@@ -960,7 +956,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
//disconnected in update_peer_scores
.filter(|(_, info)| info.score_state() == ScoreState::Healthy)
{
disconnecting_peers.push((*peer_id).clone());
disconnecting_peers.push(**peer_id);
}
}
@@ -996,7 +992,7 @@ impl<TSpec: EthSpec> Stream for PeerManager<TSpec> {
loop {
match self.inbound_ping_peers.poll_next_unpin(cx) {
Poll::Ready(Some(Ok(peer_id))) => {
self.inbound_ping_peers.insert(peer_id.clone());
self.inbound_ping_peers.insert(peer_id);
self.events.push(PeerManagerEvent::Ping(peer_id));
}
Poll::Ready(Some(Err(e))) => {
@@ -1009,7 +1005,7 @@ impl<TSpec: EthSpec> Stream for PeerManager<TSpec> {
loop {
match self.outbound_ping_peers.poll_next_unpin(cx) {
Poll::Ready(Some(Ok(peer_id))) => {
self.outbound_ping_peers.insert(peer_id.clone());
self.outbound_ping_peers.insert(peer_id);
self.events.push(PeerManagerEvent::Ping(peer_id));
}
Poll::Ready(Some(Err(e))) => {
@@ -1024,7 +1020,7 @@ impl<TSpec: EthSpec> Stream for PeerManager<TSpec> {
loop {
match self.status_peers.poll_next_unpin(cx) {
Poll::Ready(Some(Ok(peer_id))) => {
self.status_peers.insert(peer_id.clone());
self.status_peers.insert(peer_id);
self.events.push(PeerManagerEvent::Status(peer_id))
}
Poll::Ready(Some(Err(e))) => {

14
beacon_node/eth2_libp2p/src/peer_manager/peerdb.rs
View File

@@ -322,7 +322,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
/// A peer is being dialed.
pub fn dialing_peer(&mut self, peer_id: &PeerId, enr: Option<Enr>) {
let info = self.peers.entry(peer_id.clone()).or_default();
let info = self.peers.entry(*peer_id).or_default();
info.enr = enr;
if info.is_disconnected() {
@@ -341,7 +341,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
/// Update min ttl of a peer.
pub fn update_min_ttl(&mut self, peer_id: &PeerId, min_ttl: Instant) {
let info = self.peers.entry(peer_id.clone()).or_default();
let info = self.peers.entry(*peer_id).or_default();
// only update if the ttl is longer
if info.min_ttl.is_none() || Some(min_ttl) > info.min_ttl {
@@ -382,7 +382,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
enr: Option<Enr>,
direction: ConnectionDirection,
) {
let info = self.peers.entry(peer_id.clone()).or_default();
let info = self.peers.entry(*peer_id).or_default();
info.enr = enr;
if info.is_disconnected() {
@@ -459,7 +459,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
// peer's score to be a banned state.
pub fn disconnect_and_ban(&mut self, peer_id: &PeerId) -> bool {
let log_ref = &self.log;
let info = self.peers.entry(peer_id.clone()).or_insert_with(|| {
let info = self.peers.entry(*peer_id).or_insert_with(|| {
warn!(log_ref, "Banning unknown peer";
"peer_id" => %peer_id);
PeerInfo::default()
@@ -517,7 +517,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
/// If this is called for a banned peer, it will error.
pub fn unban(&mut self, peer_id: &PeerId) -> Result<(), &'static str> {
let log_ref = &self.log;
let info = self.peers.entry(peer_id.clone()).or_insert_with(|| {
let info = self.peers.entry(*peer_id).or_insert_with(|| {
warn!(log_ref, "UnBanning unknown peer";
"peer_id" => %peer_id);
PeerInfo::default()
@@ -557,7 +557,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
{
self.banned_peers_count
.remove_banned_peer(info.seen_addresses());
Some(id.clone())
Some(*id)
} else {
// If there is no minimum, this is a coding error.
crit!(
@@ -584,7 +584,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
_ => None,
})
.min_by_key(|(_, since)| *since)
.map(|(id, _)| id.clone())
.map(|(id, _)| *id)
{
debug!(self.log, "Removing old disconnected peer"; "peer_id" => %to_drop);
self.peers.remove(&to_drop);

2
beacon_node/eth2_libp2p/src/rpc/mod.rs
View File

@@ -190,7 +190,7 @@ where
debug!(self.log, "Requesting new peer's metadata"; "peer_id" => %peer_id);
let rpc_event = RPCSend::Request(RequestId::Behaviour, RPCRequest::MetaData(PhantomData));
self.events.push(NetworkBehaviourAction::NotifyHandler {
peer_id: peer_id.clone(),
peer_id: *peer_id,
handler: NotifyHandler::Any,
event: rpc_event,
});

24
beacon_node/eth2_libp2p/src/service.rs
View File

@@ -9,7 +9,8 @@ use crate::EnrExt;
use crate::{NetworkConfig, NetworkGlobals, PeerAction, ReportSource};
use futures::prelude::*;
use libp2p::core::{
identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::Boxed,
connection::ConnectionLimits, identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox,
transport::Boxed,
};
use libp2p::{
bandwidth::{BandwidthLogging, BandwidthSinks},
@@ -28,7 +29,7 @@ use types::{ChainSpec, EnrForkId, EthSpec};
pub const NETWORK_KEY_FILENAME: &str = "key";
/// The maximum simultaneous libp2p connections per peer.
const MAX_CONNECTIONS_PER_PEER: usize = 1;
const MAX_CONNECTIONS_PER_PEER: u32 = 1;
/// The filename to store our local metadata.
pub const METADATA_FILENAME: &str = "metadata";
@@ -123,13 +124,20 @@ impl<TSpec: EthSpec> Service<TSpec> {
self.0.spawn(f, "libp2p");
}
}
// sets up the libp2p connection limits
let limits = ConnectionLimits::default()
.with_max_pending_incoming(Some(5))
.with_max_pending_outgoing(Some(16))
.with_max_established_incoming(Some((config.target_peers as f64 * 1.2) as u32))
.with_max_established_outgoing(Some((config.target_peers as f64 * 1.2) as u32))
.with_max_established_per_peer(Some(MAX_CONNECTIONS_PER_PEER));
(
SwarmBuilder::new(transport, behaviour, local_peer_id.clone())
.notify_handler_buffer_size(std::num::NonZeroUsize::new(32).expect("Not zero"))
SwarmBuilder::new(transport, behaviour, local_peer_id)
.notify_handler_buffer_size(std::num::NonZeroUsize::new(7).expect("Not zero"))
.connection_event_buffer_size(64)
.incoming_connection_limit(10)
.outgoing_connection_limit(config.target_peers * 2)
.peer_connection_limit(MAX_CONNECTIONS_PER_PEER)
.connection_limits(limits)
.executor(Box::new(Executor(executor)))
.build(),
bandwidth,
@@ -146,7 +154,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
match Swarm::listen_on(&mut swarm, listen_multiaddr.clone()) {
Ok(_) => {
let mut log_address = listen_multiaddr;
log_address.push(Protocol::P2p(local_peer_id.clone().into()));
log_address.push(Protocol::P2p(local_peer_id.into()));
info!(log, "Listening established"; "address" => %log_address);
}
Err(err) => {

2
beacon_node/eth2_libp2p/src/types/globals.rs
View File

@@ -61,7 +61,7 @@ impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
/// Returns the local libp2p PeerID.
pub fn local_peer_id(&self) -> PeerId {
self.peer_id.read().clone()
*self.peer_id.read()
}
/// Returns the list of `Multiaddr` that the underlying libp2p instance is listening on.

2
beacon_node/eth2_libp2p/src/types/mod.rs
View File

@@ -13,7 +13,7 @@ pub type EnrBitfield<T: EthSpec> = BitVector<T::SubnetBitfieldLength>;
pub type Enr = discv5::enr::Enr<discv5::enr::CombinedKey>;
pub use globals::NetworkGlobals;
pub use pubsub::{MessageData, PubsubMessage};
pub use pubsub::{PubsubMessage, SnappyTransform};
pub use subnet::SubnetDiscovery;
pub use sync_state::SyncState;
pub use topics::{subnet_id_from_topic_hash, GossipEncoding, GossipKind, GossipTopic, CORE_TOPICS};

162
beacon_node/eth2_libp2p/src/types/pubsub.rs
View File

@@ -1,44 +1,18 @@
//! Handles the encoding and decoding of pubsub messages.
use crate::config::GOSSIP_MAX_SIZE;
use crate::types::{GossipEncoding, GossipKind, GossipTopic};
use crate::TopicHash;
use libp2p::gossipsub::{DataTransform, GossipsubMessage, RawGossipsubMessage};
use snap::raw::{decompress_len, Decoder, Encoder};
use ssz::{Decode, Encode};
use std::boxed::Box;
use std::io::{Error, ErrorKind};
use types::SubnetId;
use types::{
Attestation, AttesterSlashing, EthSpec, ProposerSlashing, SignedAggregateAndProof,
SignedBeaconBlock, SignedVoluntaryExit,
};
#[derive(Clone)]
pub struct MessageData {
pub raw: Vec<u8>,
pub decompressed: Result<Vec<u8>, String>,
}
impl AsRef<[u8]> for MessageData {
fn as_ref(&self) -> &[u8] {
self.raw.as_ref()
}
}
impl Into<Vec<u8>> for MessageData {
fn into(self) -> Vec<u8> {
self.raw
}
}
impl From<Vec<u8>> for MessageData {
fn from(raw: Vec<u8>) -> Self {
Self {
decompressed: decompress_snappy(raw.as_ref()),
raw,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum PubsubMessage<T: EthSpec> {
/// Gossipsub message providing notification of a new block.
@@ -55,21 +29,63 @@ pub enum PubsubMessage<T: EthSpec> {
AttesterSlashing(Box<AttesterSlashing<T>>),
}
fn decompress_snappy(data: &[u8]) -> Result<Vec<u8>, String> {
// Exit early if uncompressed data is > GOSSIP_MAX_SIZE
match decompress_len(data) {
Ok(n) if n > GOSSIP_MAX_SIZE => {
return Err("ssz_snappy decoded data > GOSSIP_MAX_SIZE".into());
// Implements the `DataTransform` trait of gossipsub to employ snappy compression
pub struct SnappyTransform {
/// Sets the maximum size we allow gossipsub messages to decompress to.
max_size_per_message: usize,
}
impl SnappyTransform {
pub fn new(max_size_per_message: usize) -> Self {
SnappyTransform {
max_size_per_message,
}
Ok(_) => {}
Err(e) => {
return Err(format!("{}", e));
}
}
impl DataTransform for SnappyTransform {
// Provides the snappy decompression from RawGossipsubMessages
fn inbound_transform(
&self,
raw_message: RawGossipsubMessage,
) -> Result<GossipsubMessage, std::io::Error> {
// check the length of the raw bytes
let len = decompress_len(&raw_message.data)?;
if len > self.max_size_per_message {
return Err(Error::new(
ErrorKind::InvalidData,
"ssz_snappy decoded data > GOSSIP_MAX_SIZE",
));
}
};
let mut decoder = Decoder::new();
match decoder.decompress_vec(data) {
Ok(decompressed_data) => Ok(decompressed_data),
Err(e) => Err(format!("{}", e)),
let mut decoder = Decoder::new();
let decompressed_data = decoder.decompress_vec(&raw_message.data)?;
// Build the GossipsubMessage struct
Ok(GossipsubMessage {
source: raw_message.source,
data: decompressed_data,
sequence_number: raw_message.sequence_number,
topic: raw_message.topic,
})
}
/// Provides the snappy compression logic to gossipsub.
fn outbound_transform(
&self,
_topic: &TopicHash,
data: Vec<u8>,
) -> Result<Vec<u8>, std::io::Error> {
// Currently we are not employing topic-based compression. Everything is expected to be
// snappy compressed.
if data.len() > self.max_size_per_message {
return Err(Error::new(
ErrorKind::InvalidData,
"ssz_snappy Encoded data > GOSSIP_MAX_SIZE",
));
}
let mut encoder = Encoder::new();
encoder.compress_vec(&data).map_err(Into::into)
}
}
@@ -98,48 +114,49 @@ impl<T: EthSpec> PubsubMessage<T> {
/* Note: This is assuming we are not hashing topics. If we choose to hash topics, these will
* need to be modified.
*/
pub fn decode(topic: &TopicHash, data: &MessageData) -> Result<Self, String> {
pub fn decode(topic: &TopicHash, data: &[u8]) -> Result<Self, String> {
match GossipTopic::decode(topic.as_str()) {
Err(_) => Err(format!("Unknown gossipsub topic: {:?}", topic)),
Ok(gossip_topic) => {
let decompressed_data = match gossip_topic.encoding() {
GossipEncoding::SSZSnappy => data.decompressed.as_ref()?.as_slice(),
};
// All topics are currently expected to be compressed and decompressed with snappy.
// This is done in the `SnappyTransform` struct.
// Therefore compression has already been handled for us by the time we are
// decoding the objects here.
// the ssz decoders
match gossip_topic.kind() {
GossipKind::BeaconAggregateAndProof => {
let agg_and_proof =
SignedAggregateAndProof::from_ssz_bytes(decompressed_data)
.map_err(|e| format!("{:?}", e))?;
let agg_and_proof = SignedAggregateAndProof::from_ssz_bytes(data)
.map_err(|e| format!("{:?}", e))?;
Ok(PubsubMessage::AggregateAndProofAttestation(Box::new(
agg_and_proof,
)))
}
GossipKind::Attestation(subnet_id) => {
let attestation = Attestation::from_ssz_bytes(decompressed_data)
.map_err(|e| format!("{:?}", e))?;
let attestation =
Attestation::from_ssz_bytes(data).map_err(|e| format!("{:?}", e))?;
Ok(PubsubMessage::Attestation(Box::new((
*subnet_id,
attestation,
))))
}
GossipKind::BeaconBlock => {
let beacon_block = SignedBeaconBlock::from_ssz_bytes(decompressed_data)
let beacon_block = SignedBeaconBlock::from_ssz_bytes(data)
.map_err(|e| format!("{:?}", e))?;
Ok(PubsubMessage::BeaconBlock(Box::new(beacon_block)))
}
GossipKind::VoluntaryExit => {
let voluntary_exit = SignedVoluntaryExit::from_ssz_bytes(decompressed_data)
let voluntary_exit = SignedVoluntaryExit::from_ssz_bytes(data)
.map_err(|e| format!("{:?}", e))?;
Ok(PubsubMessage::VoluntaryExit(Box::new(voluntary_exit)))
}
GossipKind::ProposerSlashing => {
let proposer_slashing = ProposerSlashing::from_ssz_bytes(decompressed_data)
let proposer_slashing = ProposerSlashing::from_ssz_bytes(data)
.map_err(|e| format!("{:?}", e))?;
Ok(PubsubMessage::ProposerSlashing(Box::new(proposer_slashing)))
}
GossipKind::AttesterSlashing => {
let attester_slashing = AttesterSlashing::from_ssz_bytes(decompressed_data)
let attester_slashing = AttesterSlashing::from_ssz_bytes(data)
.map_err(|e| format!("{:?}", e))?;
Ok(PubsubMessage::AttesterSlashing(Box::new(attester_slashing)))
}
@@ -150,26 +167,18 @@ impl<T: EthSpec> PubsubMessage<T> {
/// Encodes a `PubsubMessage` based on the topic encodings. The first known encoding is used. If
/// no encoding is known, and error is returned.
pub fn encode(&self, encoding: GossipEncoding) -> Result<Vec<u8>, String> {
let data = match &self {
pub fn encode(&self, _encoding: GossipEncoding) -> Vec<u8> {
// Currently do not employ encoding strategies based on the topic. All messages are ssz
// encoded.
// Also note, that the compression is handled by the `SnappyTransform` struct. Gossipsub will compress the
// messages for us.
match &self {
PubsubMessage::BeaconBlock(data) => data.as_ssz_bytes(),
PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(),
PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(),
PubsubMessage::ProposerSlashing(data) => data.as_ssz_bytes(),
PubsubMessage::AttesterSlashing(data) => data.as_ssz_bytes(),
PubsubMessage::Attestation(data) => data.1.as_ssz_bytes(),
};
match encoding {
GossipEncoding::SSZSnappy => {
let mut encoder = Encoder::new();
match encoder.compress_vec(&data) {
Ok(compressed) if compressed.len() > GOSSIP_MAX_SIZE => {
Err("ssz_snappy Encoded data > GOSSIP_MAX_SIZE".into())
}
Ok(compressed) => Ok(compressed),
Err(e) => Err(format!("{}", e)),
}
}
}
}
}
@@ -200,20 +209,3 @@ impl<T: EthSpec> std::fmt::Display for PubsubMessage<T> {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_gossip_max_size() {
// Cannot decode more than gossip max size
let mut encoder = Encoder::new();
let payload = encoder.compress_vec(&[0; GOSSIP_MAX_SIZE + 1]).unwrap();
let message_data: MessageData = payload.into();
assert_eq!(
message_data.decompressed.unwrap_err(),
"ssz_snappy decoded data > GOSSIP_MAX_SIZE".to_string()
);
}
}

3
beacon_node/eth2_libp2p/tests/common/mod.rs
View File

@@ -3,7 +3,8 @@ use eth2_libp2p::Enr;
use eth2_libp2p::EnrExt;
use eth2_libp2p::Multiaddr;
use eth2_libp2p::Service as LibP2PService;
use eth2_libp2p::{GossipsubConfigBuilder, Libp2pEvent, NetworkConfig};
use eth2_libp2p::{Libp2pEvent, NetworkConfig};
use libp2p::gossipsub::GossipsubConfigBuilder;
use slog::{debug, error, o, Drain};
use std::net::{TcpListener, UdpSocket};
use std::sync::Weak;