gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-15 10:52:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

upgrade to libp2p 0.52 (#4431)

Browse Source 
## Issue Addressed

Upgrade libp2p to v0.52

## Proposed Changes
- **Workflows**: remove installation of `protoc`
- **Book**: remove installation of `protoc`
- **`Dockerfile`s and `cross`**: remove custom base `Dockerfile` for cross since it's no longer needed. Remove `protoc` from remaining `Dockerfiles`s
- **Upgrade `discv5` to `v0.3.1`:** we have some cool stuff in there: no longer needs `protoc` and faster ip updates on cold start
- **Upgrade `prometheus` to `0.21.0`**, now it no longer needs encoding checks
- **things that look like refactors:** bunch of api types were renamed and need to be accessed in a different (clearer) way
- **Lighthouse network**
	- connection limits is now a behaviour
	- banned peers no longer exist on the swarm level, but at the behaviour level
	- `connection_event_buffer_size` now is handled per connection with a buffer size of 4
	- `mplex` is deprecated and was removed
	- rpc handler now logs the peer to which it belongs

## Additional Info

Tried to keep as much behaviour unchanged as possible. However, there is a great deal of improvements we can do _after_ this upgrade:
- Smart connection limits: Connection limits have been checked only based on numbers, we can now use information about the incoming peer to decide if we want it
- More powerful peer management: Dial attempts from other behaviours can be rejected early
- Incoming connections can be rejected early
- Banning can be returned exclusively to the peer management: We should not get connections to banned peers anymore making use of this
- TCP Nat updates: We might be able to take advantage of confirmed external addresses to check out tcp ports/ips


Co-authored-by: Age Manning <Age@AgeManning.com>
Co-authored-by: Akihito Nakano <sora.akatsuki@gmail.com>
This commit is contained in:
Divma
2023-08-02 00:59:34 +00:00
parent 73764d0dd2
commit ff9b09d964
35 changed files with 1594 additions and 2408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
beacon_node/lighthouse_network/src/discovery/enr.rs
View File

@@ -7,7 +7,7 @@ use super::ENR_FILENAME;
use crate::types::{Enr, EnrAttestationBitfield, EnrSyncCommitteeBitfield};
use crate::NetworkConfig;
use discv5::enr::EnrKey;
use libp2p::core::identity::Keypair;
use libp2p::identity::Keypair;
use slog::{debug, warn};
use ssz::{Decode, Encode};
use ssz_types::BitVector;
@@ -133,7 +133,7 @@ pub fn build_or_load_enr<T: EthSpec>(
// Build the local ENR.
// Note: Discovery should update the ENR record's IP to the external IP as seen by the
// majority of our peers, if the CLI doesn't expressly forbid it.
let enr_key = CombinedKey::from_libp2p(&local_key)?;
let enr_key = CombinedKey::from_libp2p(local_key)?;
let mut local_enr = build_enr::<T>(&enr_key, config, enr_fork_id)?;
use_or_load_enr(&enr_key, &mut local_enr, config, log)?;

100
beacon_node/lighthouse_network/src/discovery/enr_ext.rs
View File

@@ -1,10 +1,9 @@
//! ENR extension trait to support libp2p integration.
use crate::{Enr, Multiaddr, PeerId};
use discv5::enr::{CombinedKey, CombinedPublicKey};
use libp2p::{
core::{identity::Keypair, identity::PublicKey, multiaddr::Protocol},
identity::secp256k1,
};
use libp2p::core::multiaddr::Protocol;
use libp2p::identity::{ed25519, secp256k1, KeyType, Keypair, PublicKey};
use tiny_keccak::{Hasher, Keccak};
/// Extend ENR for libp2p types.
@@ -38,7 +37,8 @@ pub trait CombinedKeyPublicExt {
/// Extend ENR CombinedKey for conversion to libp2p keys.
pub trait CombinedKeyExt {
/// Converts a libp2p key into an ENR combined key.
fn from_libp2p(key: &libp2p::core::identity::Keypair) -> Result<CombinedKey, &'static str>;
fn from_libp2p(key: Keypair) -> Result<CombinedKey, &'static str>;
/// Converts a [`secp256k1::Keypair`] into and Enr [`CombinedKey`].
fn from_secp256k1(key: &secp256k1::Keypair) -> CombinedKey;
}
@@ -93,14 +93,14 @@ impl EnrExt for Enr {
if let Some(udp) = self.udp4() {
let mut multiaddr: Multiaddr = ip.into();
multiaddr.push(Protocol::Udp(udp));
multiaddr.push(Protocol::P2p(peer_id.into()));
multiaddr.push(Protocol::P2p(peer_id));
multiaddrs.push(multiaddr);
}
if let Some(tcp) = self.tcp4() {
let mut multiaddr: Multiaddr = ip.into();
multiaddr.push(Protocol::Tcp(tcp));
multiaddr.push(Protocol::P2p(peer_id.into()));
multiaddr.push(Protocol::P2p(peer_id));
multiaddrs.push(multiaddr);
}
}
@@ -108,14 +108,14 @@ impl EnrExt for Enr {
if let Some(udp6) = self.udp6() {
let mut multiaddr: Multiaddr = ip6.into();
multiaddr.push(Protocol::Udp(udp6));
multiaddr.push(Protocol::P2p(peer_id.into()));
multiaddr.push(Protocol::P2p(peer_id));
multiaddrs.push(multiaddr);
}
if let Some(tcp6) = self.tcp6() {
let mut multiaddr: Multiaddr = ip6.into();
multiaddr.push(Protocol::Tcp(tcp6));
multiaddr.push(Protocol::P2p(peer_id.into()));
multiaddr.push(Protocol::P2p(peer_id));
multiaddrs.push(multiaddr);
}
}
@@ -133,7 +133,7 @@ impl EnrExt for Enr {
if let Some(tcp) = self.tcp4() {
let mut multiaddr: Multiaddr = ip.into();
multiaddr.push(Protocol::Tcp(tcp));
multiaddr.push(Protocol::P2p(peer_id.into()));
multiaddr.push(Protocol::P2p(peer_id));
multiaddrs.push(multiaddr);
}
}
@@ -141,7 +141,7 @@ impl EnrExt for Enr {
if let Some(tcp6) = self.tcp6() {
let mut multiaddr: Multiaddr = ip6.into();
multiaddr.push(Protocol::Tcp(tcp6));
multiaddr.push(Protocol::P2p(peer_id.into()));
multiaddr.push(Protocol::P2p(peer_id));
multiaddrs.push(multiaddr);
}
}
@@ -159,7 +159,7 @@ impl EnrExt for Enr {
if let Some(udp) = self.udp4() {
let mut multiaddr: Multiaddr = ip.into();
multiaddr.push(Protocol::Udp(udp));
multiaddr.push(Protocol::P2p(peer_id.into()));
multiaddr.push(Protocol::P2p(peer_id));
multiaddrs.push(multiaddr);
}
}
@@ -167,7 +167,7 @@ impl EnrExt for Enr {
if let Some(udp6) = self.udp6() {
let mut multiaddr: Multiaddr = ip6.into();
multiaddr.push(Protocol::Udp(udp6));
multiaddr.push(Protocol::P2p(peer_id.into()));
multiaddr.push(Protocol::P2p(peer_id));
multiaddrs.push(multiaddr);
}
}
@@ -204,18 +204,16 @@ impl CombinedKeyPublicExt for CombinedPublicKey {
match self {
Self::Secp256k1(pk) => {
let pk_bytes = pk.to_sec1_bytes();
let libp2p_pk = libp2p::core::PublicKey::Secp256k1(
libp2p::core::identity::secp256k1::PublicKey::decode(&pk_bytes)
.expect("valid public key"),
);
let libp2p_pk: PublicKey = secp256k1::PublicKey::try_from_bytes(&pk_bytes)
.expect("valid public key")
.into();
PeerId::from_public_key(&libp2p_pk)
}
Self::Ed25519(pk) => {
let pk_bytes = pk.to_bytes();
let libp2p_pk = libp2p::core::PublicKey::Ed25519(
libp2p::core::identity::ed25519::PublicKey::decode(&pk_bytes)
.expect("valid public key"),
);
let libp2p_pk: PublicKey = ed25519::PublicKey::try_from_bytes(&pk_bytes)
.expect("valid public key")
.into();
PeerId::from_public_key(&libp2p_pk)
}
}
@@ -223,18 +221,25 @@ impl CombinedKeyPublicExt for CombinedPublicKey {
}
impl CombinedKeyExt for CombinedKey {
fn from_libp2p(key: &libp2p::core::identity::Keypair) -> Result<CombinedKey, &'static str> {
match key {
Keypair::Secp256k1(key) => Ok(CombinedKey::from_secp256k1(key)),
Keypair::Ed25519(key) => {
fn from_libp2p(key: Keypair) -> Result<CombinedKey, &'static str> {
match key.key_type() {
KeyType::Secp256k1 => {
let key = key.try_into_secp256k1().expect("right key type");
let secret =
discv5::enr::k256::ecdsa::SigningKey::from_slice(&key.secret().to_bytes())
.expect("libp2p key must be valid");
Ok(CombinedKey::Secp256k1(secret))
}
KeyType::Ed25519 => {
let key = key.try_into_ed25519().expect("right key type");
let ed_keypair = discv5::enr::ed25519_dalek::SigningKey::from_bytes(
&(key.encode()[..32])
&(key.to_bytes()[..32])
.try_into()
.expect("libp2p key must be valid"),
);
Ok(CombinedKey::from(ed_keypair))
}
Keypair::Ecdsa(_) => Err("Ecdsa keypairs not supported"),
_ => Err("Unsupported keypair kind"),
}
}
fn from_secp256k1(key: &secp256k1::Keypair) -> Self {
@@ -251,37 +256,46 @@ pub fn peer_id_to_node_id(peer_id: &PeerId) -> Result<discv5::enr::NodeId, Strin
// if generated from a PublicKey with Identity multihash.
let pk_bytes = &peer_id.to_bytes()[2..];
match PublicKey::from_protobuf_encoding(pk_bytes).map_err(|e| {
let public_key = PublicKey::try_decode_protobuf(pk_bytes).map_err(|e| {
format!(
" Cannot parse libp2p public key public key from peer id: {}",
e
)
})? {
PublicKey::Secp256k1(pk) => {
let uncompressed_key_bytes = &pk.encode_uncompressed()[1..];
})?;
match public_key.key_type() {
KeyType::Secp256k1 => {
let pk = public_key
.clone()
.try_into_secp256k1()
.expect("right key type");
let uncompressed_key_bytes = &pk.to_bytes_uncompressed()[1..];
let mut output = [0_u8; 32];
let mut hasher = Keccak::v256();
hasher.update(uncompressed_key_bytes);
hasher.finalize(&mut output);
Ok(discv5::enr::NodeId::parse(&output).expect("Must be correct length"))
}
PublicKey::Ed25519(pk) => {
let uncompressed_key_bytes = pk.encode();
KeyType::Ed25519 => {
let pk = public_key
.clone()
.try_into_ed25519()
.expect("right key type");
let uncompressed_key_bytes = pk.to_bytes();
let mut output = [0_u8; 32];
let mut hasher = Keccak::v256();
hasher.update(&uncompressed_key_bytes);
hasher.finalize(&mut output);
Ok(discv5::enr::NodeId::parse(&output).expect("Must be correct length"))
}
PublicKey::Ecdsa(_) => Err(format!(
"Unsupported public key (Ecdsa) from peer {}",
peer_id
)),
_ => Err(format!("Unsupported public key from peer {}", peer_id)),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
@@ -290,9 +304,9 @@ mod tests {
let sk_bytes = hex::decode(sk_hex).unwrap();
let secret_key = discv5::enr::k256::ecdsa::SigningKey::from_slice(&sk_bytes).unwrap();
let libp2p_sk = libp2p::identity::secp256k1::SecretKey::from_bytes(sk_bytes).unwrap();
let secp256k1_kp: libp2p::identity::secp256k1::Keypair = libp2p_sk.into();
let libp2p_kp = Keypair::Secp256k1(secp256k1_kp);
let libp2p_sk = secp256k1::SecretKey::try_from_bytes(sk_bytes).unwrap();
let secp256k1_kp: secp256k1::Keypair = libp2p_sk.into();
let libp2p_kp: Keypair = secp256k1_kp.into();
let peer_id = libp2p_kp.public().to_peer_id();
let enr = discv5::enr::EnrBuilder::new("v4")
@@ -311,9 +325,9 @@ mod tests {
&sk_bytes.clone().try_into().unwrap(),
);
let libp2p_sk = libp2p::identity::ed25519::SecretKey::from_bytes(sk_bytes).unwrap();
let secp256k1_kp: libp2p::identity::ed25519::Keypair = libp2p_sk.into();
let libp2p_kp = Keypair::Ed25519(secp256k1_kp);
let libp2p_sk = ed25519::SecretKey::try_from_bytes(sk_bytes).unwrap();
let secp256k1_kp: ed25519::Keypair = libp2p_sk.into();
let libp2p_kp: Keypair = secp256k1_kp.into();
let peer_id = libp2p_kp.public().to_peer_id();
let enr = discv5::enr::EnrBuilder::new("v4")

88
beacon_node/lighthouse_network/src/discovery/mod.rs
View File

@@ -16,19 +16,20 @@ pub use enr::{
Eth2Enr,
};
pub use enr_ext::{peer_id_to_node_id, CombinedKeyExt, EnrExt};
pub use libp2p::core::identity::{Keypair, PublicKey};
pub use libp2p::identity::{Keypair, PublicKey};
use enr::{ATTESTATION_BITFIELD_ENR_KEY, ETH2_ENR_KEY, SYNC_COMMITTEE_BITFIELD_ENR_KEY};
use futures::prelude::*;
use futures::stream::FuturesUnordered;
use libp2p::multiaddr::Protocol;
use libp2p::swarm::behaviour::{DialFailure, FromSwarm};
use libp2p::swarm::AddressScore;
use libp2p::swarm::THandlerInEvent;
pub use libp2p::{
core::{connection::ConnectionId, ConnectedPoint, Multiaddr, PeerId},
core::{ConnectedPoint, Multiaddr},
identity::PeerId,
swarm::{
dummy::ConnectionHandler, DialError, NetworkBehaviour, NetworkBehaviourAction as NBAction,
NotifyHandler, PollParameters, SubstreamProtocol,
dummy::ConnectionHandler, ConnectionId, DialError, NetworkBehaviour, NotifyHandler,
PollParameters, SubstreamProtocol, ToSwarm,
},
};
use lru::LruCache;
@@ -191,7 +192,7 @@ pub struct Discovery<TSpec: EthSpec> {
impl<TSpec: EthSpec> Discovery<TSpec> {
/// NOTE: Creating discovery requires running within a tokio execution environment.
pub async fn new(
local_key: &Keypair,
local_key: Keypair,
config: &NetworkConfig,
network_globals: Arc<NetworkGlobals<TSpec>>,
log: &slog::Logger,
@@ -925,22 +926,51 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
impl<TSpec: EthSpec> NetworkBehaviour for Discovery<TSpec> {
// Discovery is not a real NetworkBehaviour...
type ConnectionHandler = ConnectionHandler;
type OutEvent = DiscoveredPeers;
type ToSwarm = DiscoveredPeers;
fn new_handler(&mut self) -> Self::ConnectionHandler {
ConnectionHandler
fn handle_established_inbound_connection(
&mut self,
_connection_id: ConnectionId,
_peer: PeerId,
_local_addr: &Multiaddr,
_remote_addr: &Multiaddr,
) -> Result<libp2p::swarm::THandler<Self>, libp2p::swarm::ConnectionDenied> {
// TODO: we might want to check discovery's banned ips here in the future.
Ok(ConnectionHandler)
}
// Handles the libp2p request to obtain multiaddrs for peer_id's in order to dial them.
fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec<Multiaddr> {
if let Some(enr) = self.enr_of_peer(peer_id) {
fn handle_established_outbound_connection(
&mut self,
_connection_id: ConnectionId,
_peer: PeerId,
_addr: &Multiaddr,
_role_override: libp2p::core::Endpoint,
) -> Result<libp2p::swarm::THandler<Self>, libp2p::swarm::ConnectionDenied> {
Ok(ConnectionHandler)
}
fn on_connection_handler_event(
&mut self,
_peer_id: PeerId,
_connection_id: ConnectionId,
_event: void::Void,
) {
}
fn handle_pending_outbound_connection(
&mut self,
_connection_id: ConnectionId,
maybe_peer: Option<PeerId>,
_addresses: &[Multiaddr],
_effective_role: libp2p::core::Endpoint,
) -> Result<Vec<Multiaddr>, libp2p::swarm::ConnectionDenied> {
if let Some(enr) = maybe_peer.and_then(|peer_id| self.enr_of_peer(&peer_id)) {
// ENR's may have multiple Multiaddrs. The multi-addr associated with the UDP
// port is removed, which is assumed to be associated with the discv5 protocol (and
// therefore irrelevant for other libp2p components).
enr.multiaddr_tcp()
Ok(enr.multiaddr_tcp())
} else {
// PeerId is not known
Vec::new()
Ok(vec![])
}
}
@@ -949,7 +979,7 @@ impl<TSpec: EthSpec> NetworkBehaviour for Discovery<TSpec> {
&mut self,
cx: &mut Context,
_: &mut impl PollParameters,
) -> Poll<NBAction<Self::OutEvent, Self::ConnectionHandler>> {
) -> Poll<ToSwarm<Self::ToSwarm, THandlerInEvent<Self>>> {
if !self.started {
return Poll::Pending;
}
@@ -960,7 +990,7 @@ impl<TSpec: EthSpec> NetworkBehaviour for Discovery<TSpec> {
// Drive the queries and return any results from completed queries
if let Some(peers) = self.poll_queries(cx) {
// return the result to the peer manager
return Poll::Ready(NBAction::GenerateEvent(DiscoveredPeers { peers }));
return Poll::Ready(ToSwarm::GenerateEvent(DiscoveredPeers { peers }));
}
// Process the server event stream
@@ -1034,10 +1064,7 @@ impl<TSpec: EthSpec> NetworkBehaviour for Discovery<TSpec> {
if let Some(address) = addr {
// NOTE: This doesn't actually track the external TCP port. More sophisticated NAT handling
// should handle this.
return Poll::Ready(NBAction::ReportObservedAddr {
address,
score: AddressScore::Finite(1),
});
return Poll::Ready(ToSwarm::NewExternalAddrCandidate(address));
}
}
Discv5Event::EnrAdded { .. }
@@ -1065,8 +1092,9 @@ impl<TSpec: EthSpec> NetworkBehaviour for Discovery<TSpec> {
| FromSwarm::ExpiredListenAddr(_)
| FromSwarm::ListenerError(_)
| FromSwarm::ListenerClosed(_)
| FromSwarm::NewExternalAddr(_)
| FromSwarm::ExpiredExternalAddr(_) => {
| FromSwarm::NewExternalAddrCandidate(_)
| FromSwarm::ExternalAddrExpired(_)
| FromSwarm::ExternalAddrConfirmed(_) => {
// Ignore events not relevant to discovery
}
}
@@ -1077,10 +1105,8 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
fn on_dial_failure(&mut self, peer_id: Option<PeerId>, error: &DialError) {
if let Some(peer_id) = peer_id {
match error {
DialError::Banned
| DialError::LocalPeerId
| DialError::InvalidPeerId(_)
| DialError::ConnectionIo(_)
DialError::LocalPeerId { .. }
| DialError::Denied { .. }
| DialError::NoAddresses
| DialError::Transport(_)
| DialError::WrongPeerId { .. } => {
@@ -1088,9 +1114,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
debug!(self.log, "Marking peer disconnected in DHT"; "peer_id" => %peer_id);
self.disconnect_peer(&peer_id);
}
DialError::ConnectionLimit(_)
| DialError::DialPeerConditionFalse(_)
| DialError::Aborted => {}
DialError::DialPeerConditionFalse(_) | DialError::Aborted => {}
}
}
}
@@ -1139,8 +1163,8 @@ mod tests {
false,
&log,
);
let keypair = Keypair::Secp256k1(keypair);
Discovery::new(&keypair, &config, Arc::new(globals), &log)
let keypair = keypair.into();
Discovery::new(keypair, &config, Arc::new(globals), &log)
.await
.unwrap()
}