gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-11 18:04:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Filter gossipsub message duplication (#736)

Browse Source 
* Add duplication prevention to gossipsub

* Clean up topic logs

* Add content addressed messages for gossip
This commit is contained in:
Age Manning
2019-12-20 16:26:30 +11:00
committed by GitHub
parent 74b327b50d
commit 45271abc16
18 changed files with 362 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
beacon_node/eth2-libp2p/src/behaviour.rs
View File

@@ -1,20 +1,20 @@
use crate::config::*;
use crate::discovery::Discovery;
use crate::rpc::{RPCEvent, RPCMessage, RPC};
use crate::GossipTopic;
use crate::{error, NetworkConfig};
use crate::{Topic, TopicHash};
use crate::{BEACON_ATTESTATION_TOPIC, BEACON_BLOCK_TOPIC};
use futures::prelude::*;
use libp2p::{
core::identity::Keypair,
discv5::Discv5Event,
gossipsub::{Gossipsub, GossipsubEvent},
gossipsub::{Gossipsub, GossipsubEvent, MessageId},
identify::{Identify, IdentifyEvent},
ping::{Ping, PingConfig, PingEvent},
swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess},
tokio_io::{AsyncRead, AsyncWrite},
NetworkBehaviour, PeerId,
};
use lru::LruCache;
use slog::{debug, o};
use std::num::NonZeroU32;
use std::time::Duration;
@@ -39,9 +39,13 @@ pub struct Behaviour<TSubstream: AsyncRead + AsyncWrite> {
identify: Identify<TSubstream>,
/// Discovery behaviour.
discovery: Discovery<TSubstream>,
#[behaviour(ignore)]
/// The events generated by this behaviour to be consumed in the swarm poll.
#[behaviour(ignore)]
events: Vec<BehaviourEvent>,
/// A cache of recently seen gossip messages. This is used to filter out any possible
/// duplicates that may still be seen over gossipsub.
#[behaviour(ignore)]
seen_gossip_messages: LruCache<PubsubMessage, ()>,
/// Logger for behaviour actions.
#[behaviour(ignore)]
log: slog::Logger,
@@ -74,6 +78,7 @@ impl<TSubstream: AsyncRead + AsyncWrite> Behaviour<TSubstream> {
discovery: Discovery::new(local_key, net_conf, log)?,
ping: Ping::new(ping_config),
identify,
seen_gossip_messages: LruCache::new(256),
events: Vec::new(),
log: behaviour_log,
})
@@ -94,18 +99,22 @@ impl<TSubstream: AsyncRead + AsyncWrite> NetworkBehaviourEventProcess<GossipsubE
{
fn inject_event(&mut self, event: GossipsubEvent) {
match event {
GossipsubEvent::Message(propagation_source, gs_msg) => {
let id = gs_msg.id();
GossipsubEvent::Message(propagation_source, id, gs_msg) => {
let msg = PubsubMessage::from_topics(&gs_msg.topics, gs_msg.data);
// Note: We are keeping track here of the peer that sent us the message, not the
// peer that originally published the message.
self.events.push(BehaviourEvent::GossipMessage {
id,
source: propagation_source,
topics: gs_msg.topics,
message: msg,
});
if self.seen_gossip_messages.put(msg.clone(), ()).is_none() {
// if this message isn't a duplicate, notify the network
self.events.push(BehaviourEvent::GossipMessage {
id,
source: propagation_source,
topics: gs_msg.topics,
message: msg,
});
} else {
debug!(self.log, "A duplicate message was received"; "message" => format!("{:?}", msg));
}
}
GossipsubEvent::Subscribed { peer_id, topic } => {
self.events
@@ -218,7 +227,7 @@ impl<TSubstream: AsyncRead + AsyncWrite> Behaviour<TSubstream> {
/// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated
/// once validated by the beacon chain.
pub fn propagate_message(&mut self, propagation_source: &PeerId, message_id: String) {
pub fn propagate_message(&mut self, propagation_source: &PeerId, message_id: MessageId) {
self.gossipsub
.propagate_message(&message_id, propagation_source);
}
@@ -263,7 +272,7 @@ pub enum BehaviourEvent {
/// A gossipsub message has been received.
GossipMessage {
/// The gossipsub message id. Used when propagating blocks after validation.
id: String,
id: MessageId,
/// The peer from which we received this message, not the peer that published it.
source: PeerId,
/// The topics that this message was sent on.
@@ -277,7 +286,7 @@ pub enum BehaviourEvent {
/// Messages that are passed to and from the pubsub (Gossipsub) behaviour. These are encoded and
/// decoded upstream.
#[derive(Debug, Clone, PartialEq)]
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum PubsubMessage {
/// Gossipsub message providing notification of a new block.
Block(Vec<u8>),
@@ -302,20 +311,14 @@ impl PubsubMessage {
*/
fn from_topics(topics: &[TopicHash], data: Vec<u8>) -> Self {
for topic in topics {
// compare the prefix and postfix, then match on the topic
let topic_parts: Vec<&str> = topic.as_str().split('/').collect();
if topic_parts.len() == 4
&& topic_parts[1] == TOPIC_PREFIX
&& topic_parts[3] == TOPIC_ENCODING_POSTFIX
{
match topic_parts[2] {
BEACON_BLOCK_TOPIC => return PubsubMessage::Block(data),
BEACON_ATTESTATION_TOPIC => return PubsubMessage::Attestation(data),
VOLUNTARY_EXIT_TOPIC => return PubsubMessage::VoluntaryExit(data),
PROPOSER_SLASHING_TOPIC => return PubsubMessage::ProposerSlashing(data),
ATTESTER_SLASHING_TOPIC => return PubsubMessage::AttesterSlashing(data),
_ => {}
}
match GossipTopic::from(topic.as_str()) {
GossipTopic::BeaconBlock => return PubsubMessage::Block(data),
GossipTopic::BeaconAttestation => return PubsubMessage::Attestation(data),
GossipTopic::VoluntaryExit => return PubsubMessage::VoluntaryExit(data),
GossipTopic::ProposerSlashing => return PubsubMessage::ProposerSlashing(data),
GossipTopic::AttesterSlashing => return PubsubMessage::AttesterSlashing(data),
GossipTopic::Shard => return PubsubMessage::Unknown(data),
GossipTopic::Unknown(_) => continue,
}
}
PubsubMessage::Unknown(data)

40
beacon_node/eth2-libp2p/src/config.rs
View File

@@ -1,22 +1,12 @@
use crate::topics::GossipTopic;
use enr::Enr;
use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder};
use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId};
use libp2p::Multiaddr;
use serde_derive::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::path::PathBuf;
use std::time::Duration;
/// The gossipsub topic names.
// These constants form a topic name of the form /TOPIC_PREFIX/TOPIC/ENCODING_POSTFIX
// For example /eth2/beacon_block/ssz
pub const TOPIC_PREFIX: &str = "eth2";
pub const TOPIC_ENCODING_POSTFIX: &str = "ssz";
pub const BEACON_BLOCK_TOPIC: &str = "beacon_block";
pub const BEACON_ATTESTATION_TOPIC: &str = "beacon_attestation";
pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit";
pub const PROPOSER_SLASHING_TOPIC: &str = "proposer_slashing";
pub const ATTESTER_SLASHING_TOPIC: &str = "attester_slashing";
pub const SHARD_TOPIC_PREFIX: &str = "shard";
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
/// Network configuration for lighthouse.
@@ -59,7 +49,7 @@ pub struct Config {
pub client_version: String,
/// List of extra topics to initially subscribe to as strings.
pub topics: Vec<String>,
pub topics: Vec<GossipTopic>,
/// Introduces randomization in network propagation of messages. This should only be set for
/// testing purposes and will likely be removed in future versions.
@@ -73,6 +63,25 @@ impl Default for Config {
let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
network_dir.push(".lighthouse");
network_dir.push("network");
// The default topics that we will initially subscribe to
let topics = vec![
GossipTopic::BeaconBlock,
GossipTopic::BeaconAttestation,
GossipTopic::VoluntaryExit,
GossipTopic::ProposerSlashing,
GossipTopic::AttesterSlashing,
];
// The function used to generate a gossipsub message id
// We use base64(SHA256(data)) for content addressing
let gossip_message_id = |message: &GossipsubMessage| {
MessageId(base64::encode_config(
&Sha256::digest(&message.data),
base64::URL_SAFE,
))
};
Config {
network_dir,
listen_address: "127.0.0.1".parse().expect("valid ip address"),
@@ -87,11 +96,12 @@ impl Default for Config {
.max_transmit_size(1_048_576)
.heartbeat_interval(Duration::from_secs(20)) // TODO: Reduce for mainnet
.manual_propagation(true) // require validation before propagation
.message_id_fn(gossip_message_id)
.build(),
boot_nodes: vec![],
libp2p_nodes: vec![],
client_version: version::version(),
topics: Vec::new(),
topics,
propagation_percentage: None,
}
}

9
beacon_node/eth2-libp2p/src/lib.rs
View File

@@ -12,14 +12,12 @@ pub mod error;
mod metrics;
pub mod rpc;
mod service;
mod topics;
pub use behaviour::PubsubMessage;
pub use config::{
Config as NetworkConfig, BEACON_ATTESTATION_TOPIC, BEACON_BLOCK_TOPIC, SHARD_TOPIC_PREFIX,
TOPIC_ENCODING_POSTFIX, TOPIC_PREFIX,
};
pub use config::Config as NetworkConfig;
pub use libp2p::enr::Enr;
pub use libp2p::gossipsub::{Topic, TopicHash};
pub use libp2p::gossipsub::{MessageId, Topic, TopicHash};
pub use libp2p::multiaddr;
pub use libp2p::Multiaddr;
pub use libp2p::{
@@ -29,3 +27,4 @@ pub use libp2p::{
pub use rpc::RPCEvent;
pub use service::Libp2pEvent;
pub use service::Service;
pub use topics::GossipTopic;

42
beacon_node/eth2-libp2p/src/service.rs
View File

@@ -1,5 +1,4 @@
use crate::behaviour::{Behaviour, BehaviourEvent, PubsubMessage};
use crate::config::*;
use crate::error;
use crate::multiaddr::Protocol;
use crate::rpc::RPCEvent;
@@ -11,6 +10,7 @@ use libp2p::core::{
identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox, nodes::Substream,
transport::boxed::Boxed, ConnectedPoint,
};
use libp2p::gossipsub::MessageId;
use libp2p::{core, secio, swarm::NetworkBehaviour, PeerId, Swarm, Transport};
use slog::{crit, debug, error, info, trace, warn};
use std::fs::File;
@@ -123,38 +123,18 @@ impl Service {
}
}
// subscribe to default gossipsub topics
let mut topics = vec![];
/* Here we subscribe to all the required gossipsub topics required for interop.
* The topic builder adds the required prefix and postfix to the hardcoded topics that we
* must subscribe to.
*/
let topic_builder = |topic| {
Topic::new(format!(
"/{}/{}/{}",
TOPIC_PREFIX, topic, TOPIC_ENCODING_POSTFIX,
))
};
topics.push(topic_builder(BEACON_BLOCK_TOPIC));
topics.push(topic_builder(BEACON_ATTESTATION_TOPIC));
topics.push(topic_builder(VOLUNTARY_EXIT_TOPIC));
topics.push(topic_builder(PROPOSER_SLASHING_TOPIC));
topics.push(topic_builder(ATTESTER_SLASHING_TOPIC));
// Add any topics specified by the user
topics.append(&mut config.topics.iter().cloned().map(Topic::new).collect());
let mut subscribed_topics = vec![];
for topic in topics {
if swarm.subscribe(topic.clone()) {
trace!(log, "Subscribed to topic"; "topic" => format!("{}", topic));
subscribed_topics.push(topic);
let mut subscribed_topics: Vec<String> = vec![];
for topic in config.topics {
let raw_topic: Topic = topic.into();
let topic_string = raw_topic.no_hash();
if swarm.subscribe(raw_topic.clone()) {
trace!(log, "Subscribed to topic"; "topic" => format!("{}", topic_string));
subscribed_topics.push(topic_string.as_str().into());
} else {
warn!(log, "Could not subscribe to topic"; "topic" => format!("{}", topic));
warn!(log, "Could not subscribe to topic"; "topic" => format!("{}",topic_string));
}
}
info!(log, "Subscribed to topics"; "topics" => format!("{:?}", subscribed_topics.iter().map(|t| format!("{}", t)).collect::<Vec<String>>()));
info!(log, "Subscribed to topics"; "topics" => format!("{:?}", subscribed_topics));
Ok(Service {
local_peer_id,
@@ -337,7 +317,7 @@ pub enum Libp2pEvent {
PeerDisconnected(PeerId),
/// Received pubsub message.
PubsubMessage {
id: String,
id: MessageId,
source: PeerId,
topics: Vec<TopicHash>,
message: PubsubMessage,

71
beacon_node/eth2-libp2p/src/topics.rs Normal file
View File

@@ -0,0 +1,71 @@
use libp2p::gossipsub::Topic;
use serde_derive::{Deserialize, Serialize};
/// The gossipsub topic names.
// These constants form a topic name of the form /TOPIC_PREFIX/TOPIC/ENCODING_POSTFIX
// For example /eth2/beacon_block/ssz
pub const TOPIC_PREFIX: &str = "eth2";
pub const TOPIC_ENCODING_POSTFIX: &str = "ssz";
pub const BEACON_BLOCK_TOPIC: &str = "beacon_block";
pub const BEACON_ATTESTATION_TOPIC: &str = "beacon_attestation";
pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit";
pub const PROPOSER_SLASHING_TOPIC: &str = "proposer_slashing";
pub const ATTESTER_SLASHING_TOPIC: &str = "attester_slashing";
pub const SHARD_TOPIC_PREFIX: &str = "shard";
/// Enum that brings these topics into the rust type system.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum GossipTopic {
BeaconBlock,
BeaconAttestation,
VoluntaryExit,
ProposerSlashing,
AttesterSlashing,
Shard,
Unknown(String),
}
impl From<&str> for GossipTopic {
fn from(topic: &str) -> GossipTopic {
let topic_parts: Vec<&str> = topic.split('/').collect();
if topic_parts.len() == 4
&& topic_parts[1] == TOPIC_PREFIX
&& topic_parts[3] == TOPIC_ENCODING_POSTFIX
{
match topic_parts[2] {
BEACON_BLOCK_TOPIC => GossipTopic::BeaconBlock,
BEACON_ATTESTATION_TOPIC => GossipTopic::BeaconAttestation,
VOLUNTARY_EXIT_TOPIC => GossipTopic::VoluntaryExit,
PROPOSER_SLASHING_TOPIC => GossipTopic::ProposerSlashing,
ATTESTER_SLASHING_TOPIC => GossipTopic::AttesterSlashing,
unknown_topic => GossipTopic::Unknown(unknown_topic.into()),
}
} else {
GossipTopic::Unknown(topic.into())
}
}
}
impl Into<Topic> for GossipTopic {
fn into(self) -> Topic {
Topic::new(self.into())
}
}
impl Into<String> for GossipTopic {
fn into(self) -> String {
match self {
GossipTopic::BeaconBlock => topic_builder(BEACON_BLOCK_TOPIC),
GossipTopic::BeaconAttestation => topic_builder(BEACON_ATTESTATION_TOPIC),
GossipTopic::VoluntaryExit => topic_builder(VOLUNTARY_EXIT_TOPIC),
GossipTopic::ProposerSlashing => topic_builder(PROPOSER_SLASHING_TOPIC),
GossipTopic::AttesterSlashing => topic_builder(ATTESTER_SLASHING_TOPIC),
GossipTopic::Shard => topic_builder(SHARD_TOPIC_PREFIX),
GossipTopic::Unknown(topic) => topic,
}
}
}
fn topic_builder(topic: &'static str) -> String {
format!("/{}/{}/{}", TOPIC_PREFIX, topic, TOPIC_ENCODING_POSTFIX,)
}