gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-03 00:31:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Create network_utils crate (#7761)

Browse Source 
Anchor currently depends on `lighthouse_network` for a few types and utilities that live within. As we use our own libp2p behaviours, we actually do not use the core logic in that crate. This makes us transitively depend on a bunch of unneeded crates (even a whole separate libp2p if the versions mismatch!)


  Move things we require into it's own lightweight crate.


Co-Authored-By: Daniel Knopik <daniel@dknopik.de>
This commit is contained in:
Daniel Knopik
2025-09-10 14:59:24 +02:00
committed by GitHub
parent caa1df6fc3
commit ee1b6bc81b
42 changed files with 198 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
Cargo.lock generated
View File

@@ -936,6 +936,7 @@ dependencies = [
"hyper 1.6.0",
"lighthouse_network",
"monitoring_api",
"network_utils",
"node_test_rig",
"sensitive_url",
"serde_json",
@@ -945,7 +946,6 @@ dependencies = [
"task_executor",
"tracing",
"types",
"unused_port",
]
[[package]]
@@ -1205,6 +1205,7 @@ dependencies = [
"lighthouse_network",
"log",
"logging",
"network_utils",
"serde",
"tokio",
"tracing",
@@ -3314,6 +3315,7 @@ dependencies = [
"futures",
"hex",
"logging",
"network_utils",
"reqwest 0.11.27",
"sensitive_url",
"serde_json",
@@ -3321,7 +3323,6 @@ dependencies = [
"tempfile",
"tokio",
"types",
"unused_port",
]
[[package]]
@@ -4300,6 +4301,7 @@ dependencies = [
"lru",
"metrics",
"network",
"network_utils",
"operation_pool",
"parking_lot 0.12.3",
"proto_array",
@@ -4334,6 +4336,7 @@ dependencies = [
"logging",
"malloc_utils",
"metrics",
"network_utils",
"reqwest 0.11.27",
"serde",
"slot_clock",
@@ -5068,6 +5071,7 @@ dependencies = [
"lighthouse_version",
"log",
"malloc_utils",
"network_utils",
"rayon",
"serde",
"serde_json",
@@ -5583,6 +5587,7 @@ dependencies = [
"logging",
"malloc_utils",
"metrics",
"network_utils",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry_sdk",
@@ -5599,7 +5604,6 @@ dependencies = [
"tracing-opentelemetry",
"tracing-subscriber",
"types",
"unused_port",
"validator_client",
"validator_dir",
"validator_manager",
@@ -5635,6 +5639,7 @@ dependencies = [
"lru",
"lru_cache",
"metrics",
"network_utils",
"parking_lot 0.12.3",
"prometheus-client",
"quickcheck",
@@ -5650,14 +5655,12 @@ dependencies = [
"superstruct",
"task_executor",
"tempfile",
"tiny-keccak",
"tokio",
"tokio-util",
"tracing",
"tracing-subscriber",
"types",
"unsigned-varint 0.8.0",
"unused_port",
]
[[package]]
@@ -6355,6 +6358,21 @@ dependencies = [
"types",
]
[[package]]
name = "network_utils"
version = "0.1.0"
dependencies = [
"discv5",
"hex",
"libp2p-identity",
"lru_cache",
"metrics",
"multiaddr",
"parking_lot 0.12.3",
"serde",
"tiny-keccak",
]
[[package]]
name = "nix"
version = "0.24.3"
@@ -9179,6 +9197,8 @@ name = "system_health"
version = "0.1.0"
dependencies = [
"lighthouse_network",
"metrics",
"network_utils",
"parking_lot 0.12.3",
"serde",
"sysinfo",
@@ -10022,14 +10042,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "unused_port"
version = "0.1.0"
dependencies = [
"lru_cache",
"parking_lot 0.12.3",
]
[[package]]
name = "url"
version = "2.5.4"

4
Cargo.toml
View File

@@ -37,6 +37,7 @@ members = [
"common/malloc_utils",
"common/metrics",
"common/monitoring_api",
"common/network_utils",
"common/oneshot_broadcast",
"common/pretty_reqwest_error",
"common/sensitive_url",
@@ -45,7 +46,6 @@ members = [
"common/target_check",
"common/task_executor",
"common/test_random_derive",
"common/unused_port",
"common/validator_dir",
"common/warp_utils",
"common/workspace_members",
@@ -194,6 +194,7 @@ mockall_double = "0.3"
mockito = "1.5.0"
monitoring_api = { path = "common/monitoring_api" }
network = { path = "beacon_node/network" }
network_utils = { path = "common/network_utils" }
node_test_rig = { path = "testing/node_test_rig" }
num_cpus = "1"
once_cell = "1.17.1"
@@ -265,7 +266,6 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
tree_hash = "0.10.0"
tree_hash_derive = "0.10.0"
types = { path = "consensus/types" }
unused_port = { path = "common/unused_port" }
url = "2"
uuid = { version = "0.8", features = ["serde", "v4"] }
validator_client = { path = "validator_client" }

2
beacon_node/Cargo.toml
View File

@@ -33,6 +33,7 @@ http_api = { workspace = true }
hyper = { workspace = true }
lighthouse_network = { workspace = true }
monitoring_api = { workspace = true }
network_utils = { workspace = true }
sensitive_url = { workspace = true }
serde_json = { workspace = true }
slasher = { workspace = true }
@@ -41,7 +42,6 @@ strum = { workspace = true }
task_executor = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
unused_port = { workspace = true }
[dev-dependencies]
node_test_rig = { path = "../testing/node_test_rig" }

1
beacon_node/http_api/Cargo.toml
View File

@@ -26,6 +26,7 @@ logging = { workspace = true }
lru = { workspace = true }
metrics = { workspace = true }
network = { workspace = true }
network_utils = { workspace = true }
operation_pool = { workspace = true }
parking_lot = { workspace = true }
proto_array = { workspace = true }

3
beacon_node/http_api/src/lib.rs
View File

@@ -54,10 +54,11 @@ use eth2::types::{
use eth2::{CONSENSUS_VERSION_HEADER, CONTENT_TYPE_HEADER, SSZ_CONTENT_TYPE_HEADER};
use health_metrics::observe::Observe;
use lighthouse_network::rpc::methods::MetaData;
use lighthouse_network::{Enr, EnrExt, NetworkGlobals, PeerId, PubsubMessage, types::SyncState};
use lighthouse_network::{Enr, NetworkGlobals, PeerId, PubsubMessage, types::SyncState};
use lighthouse_version::version_with_platform;
use logging::{SSELoggingComponents, crit};
use network::{NetworkMessage, NetworkSenders, ValidatorSubscriptionMessage};
use network_utils::enr_ext::EnrExt;
use operation_pool::ReceivedPreCapella;
use parking_lot::RwLock;
pub use publish_blocks::{

3
beacon_node/http_api/tests/tests.rs
View File

@@ -26,8 +26,9 @@ use http_api::{
BlockId, StateId,
test_utils::{ApiServer, create_api_server},
};
use lighthouse_network::{Enr, EnrExt, PeerId, types::SyncState};
use lighthouse_network::{Enr, PeerId, types::SyncState};
use network::NetworkReceivers;
use network_utils::enr_ext::EnrExt;
use operation_pool::attestation_storage::CheckpointKey;
use proto_array::ExecutionStatus;
use sensitive_url::SensitiveUrl;

1
beacon_node/http_metrics/Cargo.toml
View File

@@ -13,6 +13,7 @@ lighthouse_version = { workspace = true }
logging = { workspace = true }
malloc_utils = { workspace = true }
metrics = { workspace = true }
network_utils = { workspace = true }
serde = { workspace = true }
slot_clock = { workspace = true }
store = { workspace = true }

2
beacon_node/http_metrics/src/metrics.rs
View File

@@ -37,7 +37,7 @@ pub fn gather_prometheus_metrics<T: BeaconChainTypes>(
store::scrape_for_metrics(db_path, freezer_db_path);
}
lighthouse_network::scrape_discovery_metrics();
network_utils::discovery_metrics::scrape_discovery_metrics();
health_metrics::metrics::scrape_health_metrics();

3
beacon_node/lighthouse_network/Cargo.toml
View File

@@ -31,6 +31,7 @@ logging = { workspace = true }
lru = { workspace = true }
lru_cache = { workspace = true }
metrics = { workspace = true }
network_utils = { workspace = true }
parking_lot = { workspace = true }
prometheus-client = "0.23.0"
rand = { workspace = true }
@@ -43,14 +44,12 @@ ssz_types = { workspace = true }
strum = { workspace = true }
superstruct = { workspace = true }
task_executor = { workspace = true }
tiny-keccak = "2"
tokio = { workspace = true }
tokio-util = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
types = { workspace = true }
unsigned-varint = { version = "0.8", features = ["codec"] }
unused_port = { workspace = true }
[dependencies.libp2p]
version = "0.56"

2
beacon_node/lighthouse_network/src/config.rs
View File

@@ -1,4 +1,3 @@
use crate::listen_addr::{ListenAddr, ListenAddress};
use crate::peer_manager::config::DEFAULT_TARGET_PEERS;
use crate::rpc::config::{InboundRateLimiterConfig, OutboundRateLimiterConfig};
use crate::types::GossipKind;
@@ -8,6 +7,7 @@ use directory::{
};
use libp2p::Multiaddr;
use local_ip_address::local_ipv6;
use network_utils::listen_addr::{ListenAddr, ListenAddress};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::net::{Ipv4Addr, Ipv6Addr};

4
beacon_node/lighthouse_network/src/discovery/enr.rs
View File

@@ -3,13 +3,13 @@
pub use discv5::enr::CombinedKey;
use super::ENR_FILENAME;
use super::enr_ext::CombinedKeyExt;
use super::enr_ext::{EnrExt, QUIC_ENR_KEY, QUIC6_ENR_KEY};
use crate::NetworkConfig;
use crate::types::{Enr, EnrAttestationBitfield, EnrSyncCommitteeBitfield};
use alloy_rlp::bytes::Bytes;
use libp2p::identity::Keypair;
use lighthouse_version::{client_name, version};
use network_utils::enr_ext::CombinedKeyExt;
use network_utils::enr_ext::{EnrExt, QUIC_ENR_KEY, QUIC6_ENR_KEY};
use ssz::{Decode, Encode};
use ssz_types::BitVector;
use std::fs::File;

16
beacon_node/lighthouse_network/src/discovery/mod.rs
View File

@@ -4,7 +4,6 @@
//! queries and manages access to the discovery routing table.
pub(crate) mod enr;
pub mod enr_ext;
// Allow external use of the lighthouse ENR builder
use crate::service::TARGET_SUBNET_PEERS;
@@ -12,8 +11,8 @@ use crate::{ClearDialError, metrics};
use crate::{Enr, NetworkConfig, NetworkGlobals, Subnet, SubnetDiscovery};
use discv5::{Discv5, enr::NodeId};
pub use enr::{CombinedKey, Eth2Enr, build_enr, load_enr_from_disk, use_or_load_enr};
pub use enr_ext::{CombinedKeyExt, EnrExt, peer_id_to_node_id};
pub use libp2p::identity::{Keypair, PublicKey};
use network_utils::enr_ext::{CombinedKeyExt, EnrExt, peer_id_to_node_id};
use alloy_rlp::bytes::Bytes;
use enr::{ATTESTATION_BITFIELD_ENR_KEY, ETH2_ENR_KEY, SYNC_COMMITTEE_BITFIELD_ENR_KEY};
@@ -33,6 +32,7 @@ pub use libp2p::{
};
use logging::crit;
use lru::LruCache;
use network_utils::discovery_metrics;
use ssz::Encode;
use std::num::NonZeroUsize;
use std::{
@@ -687,7 +687,10 @@ impl<E: EthSpec> Discovery<E> {
min_ttl,
retries,
});
metrics::set_gauge(&metrics::DISCOVERY_QUEUE, self.queued_queries.len() as i64);
metrics::set_gauge(
&discovery_metrics::DISCOVERY_QUEUE,
self.queued_queries.len() as i64,
);
}
}
@@ -722,7 +725,10 @@ impl<E: EthSpec> Discovery<E> {
}
}
// Update the queue metric
metrics::set_gauge(&metrics::DISCOVERY_QUEUE, self.queued_queries.len() as i64);
metrics::set_gauge(
&discovery_metrics::DISCOVERY_QUEUE,
self.queued_queries.len() as i64,
);
processed
}
@@ -1233,7 +1239,7 @@ mod tests {
let spec = Arc::new(ChainSpec::default());
let keypair = secp256k1::Keypair::generate();
let mut config = NetworkConfig::default();
config.set_listening_addr(crate::ListenAddress::unused_v4_ports());
config.set_listening_addr(network_utils::listen_addr::ListenAddress::unused_v4_ports());
let config = Arc::new(config);
let enr_key: CombinedKey = CombinedKey::from_secp256k1(&keypair);
let next_fork_digest = [0; 4];

5
beacon_node/lighthouse_network/src/lib.rs
View File

@@ -6,14 +6,12 @@ mod config;
pub mod service;
pub mod discovery;
pub mod listen_addr;
pub mod metrics;
pub mod peer_manager;
pub mod rpc;
pub mod types;
use libp2p::swarm::DialError;
pub use listen_addr::*;
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
use std::str::FromStr;
@@ -107,13 +105,12 @@ pub use crate::types::{
pub use prometheus_client;
pub use config::Config as NetworkConfig;
pub use discovery::{CombinedKeyExt, EnrExt, Eth2Enr};
pub use discovery::Eth2Enr;
pub use discv5;
pub use gossipsub::{IdentTopic, MessageAcceptance, MessageId, Topic, TopicHash};
pub use libp2p;
pub use libp2p::{Multiaddr, multiaddr};
pub use libp2p::{PeerId, Swarm, core::ConnectedPoint};
pub use metrics::scrape_discovery_metrics;
pub use peer_manager::{
ConnectionDirection, PeerConnectionStatus, PeerInfo, PeerManager, SyncInfo, SyncStatus,
peerdb::PeerDB,

44
beacon_node/lighthouse_network/src/metrics.rs
View File

@@ -1,14 +1,6 @@
pub use metrics::*;
use std::sync::LazyLock;
pub static NAT_OPEN: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
try_create_int_gauge_vec(
"nat_open",
"An estimate indicating if the local node is reachable from external nodes",
&["protocol"],
)
});
pub static ADDRESS_UPDATE_COUNT: LazyLock<Result<IntCounter>> = LazyLock::new(|| {
try_create_int_counter(
"libp2p_address_update_total",
@@ -53,31 +45,6 @@ pub static PEER_DISCONNECT_EVENT_COUNT: LazyLock<Result<IntCounter>> = LazyLock:
"Count of libp2p peer disconnect events",
)
});
pub static DISCOVERY_BYTES: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
try_create_int_gauge_vec(
"discovery_bytes",
"The number of bytes sent and received in discovery",
&["direction"],
)
});
pub static DISCOVERY_QUEUE: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge(
"discovery_queue_size",
"The number of discovery queries awaiting execution",
)
});
pub static DISCOVERY_REQS: LazyLock<Result<Gauge>> = LazyLock::new(|| {
try_create_float_gauge(
"discovery_requests",
"The number of unsolicited discovery requests per second",
)
});
pub static DISCOVERY_SESSIONS: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge(
"discovery_sessions",
"The number of active discovery sessions with peers",
)
});
pub static DISCOVERY_NO_USEFUL_ENRS: LazyLock<Result<IntCounter>> = LazyLock::new(|| {
try_create_int_counter(
"discovery_no_useful_enrs_found",
@@ -219,14 +186,3 @@ pub static RESPONSE_IDLING: LazyLock<Result<Histogram>> = LazyLock::new(|| {
"The time our response remained idle in the response limiter",
)
});
pub fn scrape_discovery_metrics() {
let metrics =
discv5::metrics::Metrics::from(discv5::Discv5::<discv5::DefaultProtocolId>::raw_metrics());
set_float_gauge(&DISCOVERY_REQS, metrics.unsolicited_requests_per_second);
set_gauge(&DISCOVERY_SESSIONS, metrics.active_sessions as i64);
set_gauge_vec(&DISCOVERY_BYTES, &["inbound"], metrics.bytes_recv as i64);
set_gauge_vec(&DISCOVERY_BYTES, &["outbound"], metrics.bytes_sent as i64);
set_gauge_vec(&NAT_OPEN, &["discv5_ipv4"], metrics.ipv4_contactable as i64);
set_gauge_vec(&NAT_OPEN, &["discv5_ipv6"], metrics.ipv6_contactable as i64);
}

12
beacon_node/lighthouse_network/src/peer_manager/mod.rs
View File

@@ -1,7 +1,5 @@
//! Implementation of Lighthouse's peer management system.
use crate::discovery::enr_ext::EnrExt;
use crate::discovery::peer_id_to_node_id;
use crate::rpc::{GoodbyeReason, MetaData, Protocol, RPCError, RpcErrorResponse};
use crate::service::TARGET_SUBNET_PEERS;
use crate::{Gossipsub, NetworkGlobals, PeerId, Subnet, SubnetDiscovery, metrics};
@@ -26,6 +24,8 @@ pub mod peerdb;
use crate::peer_manager::peerdb::client::ClientKind;
use libp2p::multiaddr;
use network_utils::discovery_metrics;
use network_utils::enr_ext::{EnrExt, peer_id_to_node_id};
pub use peerdb::peer_info::{ConnectionDirection, PeerConnectionStatus, PeerInfo};
use peerdb::score::{PeerAction, ReportSource};
pub use peerdb::sync_status::{SyncInfo, SyncStatus};
@@ -1586,16 +1586,16 @@ impl<E: EthSpec> PeerManager<E> {
// Set ipv4 nat_open metric flag if threshold of peercount is met, unset if below threshold
if inbound_ipv4_peers_connected >= LIBP2P_NAT_OPEN_THRESHOLD {
metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p_ipv4"], 1);
metrics::set_gauge_vec(&discovery_metrics::NAT_OPEN, &["libp2p_ipv4"], 1);
} else {
metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p_ipv4"], 0);
metrics::set_gauge_vec(&discovery_metrics::NAT_OPEN, &["libp2p_ipv4"], 0);
}
// Set ipv6 nat_open metric flag if threshold of peercount is met, unset if below threshold
if inbound_ipv6_peers_connected >= LIBP2P_NAT_OPEN_THRESHOLD {
metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p_ipv6"], 1);
metrics::set_gauge_vec(&discovery_metrics::NAT_OPEN, &["libp2p_ipv6"], 1);
} else {
metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p_ipv6"], 0);
metrics::set_gauge_vec(&discovery_metrics::NAT_OPEN, &["libp2p_ipv6"], 0);
}
// PEERS_CONNECTED

5
beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs
View File

@@ -12,11 +12,12 @@ use libp2p::swarm::behaviour::{ConnectionClosed, ConnectionEstablished, DialFail
use libp2p::swarm::dial_opts::{DialOpts, PeerCondition};
use libp2p::swarm::dummy::ConnectionHandler;
use libp2p::swarm::{ConnectionDenied, ConnectionId, NetworkBehaviour, ToSwarm};
pub use metrics::{NAT_OPEN, set_gauge_vec};
use metrics::set_gauge_vec;
use network_utils::discovery_metrics::NAT_OPEN;
use network_utils::enr_ext::EnrExt;
use tracing::{debug, error, trace};
use types::EthSpec;
use crate::discovery::enr_ext::EnrExt;
use crate::types::SyncState;
use crate::{ClearDialError, metrics};

7
beacon_node/lighthouse_network/src/peer_manager/peerdb.rs
View File

@@ -1,10 +1,9 @@
use crate::discovery::CombinedKey;
use crate::discovery::enr::PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY;
use crate::discovery::{CombinedKey, peer_id_to_node_id};
use crate::{
Enr, EnrExt, Gossipsub, PeerId, SyncInfo, metrics, multiaddr::Multiaddr, types::Subnet,
};
use crate::{Enr, Gossipsub, PeerId, SyncInfo, metrics, multiaddr::Multiaddr, types::Subnet};
use itertools::Itertools;
use logging::crit;
use network_utils::enr_ext::{EnrExt, peer_id_to_node_id};
use peer_info::{ConnectionDirection, PeerConnectionStatus, PeerInfo};
use score::{PeerAction, ReportSource, Score, ScoreState};
use std::net::IpAddr;

2
beacon_node/lighthouse_network/src/service/mod.rs
View File

@@ -1,5 +1,4 @@
use self::gossip_cache::GossipCache;
use crate::EnrExt;
use crate::Eth2Enr;
use crate::config::{GossipsubConfigParams, NetworkLoad, gossipsub_config};
use crate::discovery::{
@@ -33,6 +32,7 @@ use libp2p::swarm::{NetworkBehaviour, Swarm, SwarmEvent};
use libp2p::upnp::tokio::Behaviour as Upnp;
use libp2p::{PeerId, SwarmBuilder, identify};
use logging::crit;
use network_utils::enr_ext::EnrExt;
use std::num::{NonZeroU8, NonZeroUsize};
use std::path::PathBuf;
use std::pin::Pin;

5
beacon_node/lighthouse_network/src/types/globals.rs
View File

@@ -3,7 +3,8 @@ use super::TopicConfig;
use crate::peer_manager::peerdb::PeerDB;
use crate::rpc::{MetaData, MetaDataV3};
use crate::types::{BackFillState, SyncState};
use crate::{Client, Enr, EnrExt, GossipTopic, Multiaddr, NetworkConfig, PeerId};
use crate::{Client, Enr, GossipTopic, Multiaddr, NetworkConfig, PeerId};
use network_utils::enr_ext::EnrExt;
use parking_lot::RwLock;
use std::collections::HashSet;
use std::sync::Arc;
@@ -250,7 +251,7 @@ impl<E: EthSpec> NetworkGlobals<E> {
config: Arc<NetworkConfig>,
spec: Arc<ChainSpec>,
) -> NetworkGlobals<E> {
use crate::CombinedKeyExt;
use network_utils::enr_ext::CombinedKeyExt;
let keypair = libp2p::identity::secp256k1::Keypair::generate();
let enr_key: discv5::enr::CombinedKey = discv5::enr::CombinedKey::from_secp256k1(&keypair);
let enr = discv5::enr::Enr::builder().build(&enr_key).unwrap();

2
beacon_node/lighthouse_network/tests/common.rs
View File

@@ -1,9 +1,9 @@
#![cfg(test)]
use lighthouse_network::Enr;
use lighthouse_network::EnrExt;
use lighthouse_network::Multiaddr;
use lighthouse_network::service::Network as LibP2PService;
use lighthouse_network::{NetworkConfig, NetworkEvent};
use network_utils::enr_ext::EnrExt;
use std::sync::Arc;
use std::sync::Weak;
use tokio::runtime::Runtime;

34
beacon_node/src/config.rs
View File

@@ -13,8 +13,8 @@ use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR};
use environment::RuntimeContext;
use execution_layer::DEFAULT_JWT_FILE;
use http_api::TlsConfig;
use lighthouse_network::ListenAddress;
use lighthouse_network::{Enr, Multiaddr, NetworkConfig, PeerIdSerialized, multiaddr::Protocol};
use network_utils::listen_addr::ListenAddress;
use sensitive_url::SensitiveUrl;
use std::collections::HashSet;
use std::fmt::Debug;
@@ -1011,7 +1011,7 @@ pub fn parse_listening_addresses(cli_args: &ArgMatches) -> Result<ListenAddress,
// use zero ports if required. If not, use the given port.
let tcp_port = use_zero_ports
.then(unused_port::unused_tcp6_port)
.then(network_utils::unused_port::unused_tcp6_port)
.transpose()?
.unwrap_or(port);
@@ -1030,18 +1030,18 @@ pub fn parse_listening_addresses(cli_args: &ArgMatches) -> Result<ListenAddress,
// use zero ports if required. If not, use the specific udp port. If none given, use
// the tcp port.
let disc_port = use_zero_ports
.then(unused_port::unused_udp6_port)
.then(network_utils::unused_port::unused_udp6_port)
.transpose()?
.or(maybe_disc_port)
.unwrap_or(tcp_port);
let quic_port = use_zero_ports
.then(unused_port::unused_udp6_port)
.then(network_utils::unused_port::unused_udp6_port)
.transpose()?
.or(maybe_quic_port)
.unwrap_or(if tcp_port == 0 { 0 } else { tcp_port + 1 });
ListenAddress::V6(lighthouse_network::ListenAddr {
ListenAddress::V6(network_utils::listen_addr::ListenAddr {
addr: ipv6,
quic_port,
disc_port,
@@ -1053,25 +1053,25 @@ pub fn parse_listening_addresses(cli_args: &ArgMatches) -> Result<ListenAddress,
// use zero ports if required. If not, use the given port.
let tcp_port = use_zero_ports
.then(unused_port::unused_tcp4_port)
.then(network_utils::unused_port::unused_tcp4_port)
.transpose()?
.unwrap_or(port);
// use zero ports if required. If not, use the specific discovery port. If none given, use
// the tcp port.
let disc_port = use_zero_ports
.then(unused_port::unused_udp4_port)
.then(network_utils::unused_port::unused_udp4_port)
.transpose()?
.or(maybe_disc_port)
.unwrap_or(tcp_port);
// use zero ports if required. If not, use the specific quic port. If none given, use
// the tcp port + 1.
let quic_port = use_zero_ports
.then(unused_port::unused_udp4_port)
.then(network_utils::unused_port::unused_udp4_port)
.transpose()?
.or(maybe_quic_port)
.unwrap_or(if tcp_port == 0 { 0 } else { tcp_port + 1 });
ListenAddress::V4(lighthouse_network::ListenAddr {
ListenAddress::V4(network_utils::listen_addr::ListenAddr {
addr: ipv4,
disc_port,
quic_port,
@@ -1083,16 +1083,16 @@ pub fn parse_listening_addresses(cli_args: &ArgMatches) -> Result<ListenAddress,
let port6 = maybe_port6.unwrap_or(port);
let ipv4_tcp_port = use_zero_ports
.then(unused_port::unused_tcp4_port)
.then(network_utils::unused_port::unused_tcp4_port)
.transpose()?
.unwrap_or(port);
let ipv4_disc_port = use_zero_ports
.then(unused_port::unused_udp4_port)
.then(network_utils::unused_port::unused_udp4_port)
.transpose()?
.or(maybe_disc_port)
.unwrap_or(ipv4_tcp_port);
let ipv4_quic_port = use_zero_ports
.then(unused_port::unused_udp4_port)
.then(network_utils::unused_port::unused_udp4_port)
.transpose()?
.or(maybe_quic_port)
.unwrap_or(if ipv4_tcp_port == 0 {
@@ -1103,16 +1103,16 @@ pub fn parse_listening_addresses(cli_args: &ArgMatches) -> Result<ListenAddress,
// Defaults to 9000 when required
let ipv6_tcp_port = use_zero_ports
.then(unused_port::unused_tcp6_port)
.then(network_utils::unused_port::unused_tcp6_port)
.transpose()?
.unwrap_or(port6);
let ipv6_disc_port = use_zero_ports
.then(unused_port::unused_udp6_port)
.then(network_utils::unused_port::unused_udp6_port)
.transpose()?
.or(maybe_disc6_port)
.unwrap_or(ipv6_tcp_port);
let ipv6_quic_port = use_zero_ports
.then(unused_port::unused_udp6_port)
.then(network_utils::unused_port::unused_udp6_port)
.transpose()?
.or(maybe_quic6_port)
.unwrap_or(if ipv6_tcp_port == 0 {
@@ -1122,13 +1122,13 @@ pub fn parse_listening_addresses(cli_args: &ArgMatches) -> Result<ListenAddress,
});
ListenAddress::DualStack(
lighthouse_network::ListenAddr {
network_utils::listen_addr::ListenAddr {
addr: ipv4,
disc_port: ipv4_disc_port,
quic_port: ipv4_quic_port,
tcp_port: ipv4_tcp_port,
},
lighthouse_network::ListenAddr {
network_utils::listen_addr::ListenAddr {
addr: ipv6,
disc_port: ipv6_disc_port,
quic_port: ipv6_quic_port,

1
boot_node/Cargo.toml
View File

@@ -15,6 +15,7 @@ hex = { workspace = true }
lighthouse_network = { workspace = true }
log = { workspace = true }
logging = { workspace = true }
network_utils = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }

3
boot_node/src/config.rs
View File

@@ -4,10 +4,11 @@ use clap::ArgMatches;
use eth2_network_config::Eth2NetworkConfig;
use lighthouse_network::discv5::{self, Enr, enr::CombinedKey};
use lighthouse_network::{
CombinedKeyExt, NetworkConfig,
NetworkConfig,
discovery::{load_enr_from_disk, use_or_load_enr},
load_private_key,
};
use network_utils::enr_ext::CombinedKeyExt;
use serde::{Deserialize, Serialize};
use ssz::Encode;
use std::net::{SocketAddrV4, SocketAddrV6};

3
boot_node/src/server.rs
View File

@@ -5,9 +5,10 @@ use crate::config::BootNodeConfigSerialization;
use clap::ArgMatches;
use eth2_network_config::Eth2NetworkConfig;
use lighthouse_network::{
EnrExt, Eth2Enr,
Eth2Enr,
discv5::{self, Discv5, enr::NodeId},
};
use network_utils::enr_ext::EnrExt;
use tracing::{info, warn};
use types::EthSpec;

17
common/network_utils/Cargo.toml Normal file
View File

@@ -0,0 +1,17 @@
[package]
name = "network_utils"
version = "0.1.0"
edition = { workspace = true }
[dependencies]
discv5 = { workspace = true }
libp2p-identity = "0.2"
lru_cache = { workspace = true }
metrics = { workspace = true }
multiaddr = "0.18.2"
parking_lot = { workspace = true }
serde = { workspace = true }
tiny-keccak = { version = "2", features = ["keccak"] }
[dev-dependencies]
hex = { workspace = true }

46
common/network_utils/src/discovery_metrics.rs Normal file
View File

@@ -0,0 +1,46 @@
use metrics::*;
use std::sync::LazyLock;
pub static NAT_OPEN: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
try_create_int_gauge_vec(
"nat_open",
"An estimate indicating if the local node is reachable from external nodes",
&["protocol"],
)
});
pub static DISCOVERY_BYTES: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
try_create_int_gauge_vec(
"discovery_bytes",
"The number of bytes sent and received in discovery",
&["direction"],
)
});
pub static DISCOVERY_QUEUE: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge(
"discovery_queue_size",
"The number of discovery queries awaiting execution",
)
});
pub static DISCOVERY_REQS: LazyLock<Result<Gauge>> = LazyLock::new(|| {
try_create_float_gauge(
"discovery_requests",
"The number of unsolicited discovery requests per second",
)
});
pub static DISCOVERY_SESSIONS: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
try_create_int_gauge(
"discovery_sessions",
"The number of active discovery sessions with peers",
)
});
pub fn scrape_discovery_metrics() {
let metrics =
discv5::metrics::Metrics::from(discv5::Discv5::<discv5::DefaultProtocolId>::raw_metrics());
set_float_gauge(&DISCOVERY_REQS, metrics.unsolicited_requests_per_second);
set_gauge(&DISCOVERY_SESSIONS, metrics.active_sessions as i64);
set_gauge_vec(&DISCOVERY_BYTES, &["inbound"], metrics.bytes_recv as i64);
set_gauge_vec(&DISCOVERY_BYTES, &["outbound"], metrics.bytes_sent as i64);
set_gauge_vec(&NAT_OPEN, &["discv5_ipv4"], metrics.ipv4_contactable as i64);
set_gauge_vec(&NAT_OPEN, &["discv5_ipv6"], metrics.ipv6_contactable as i64);
}

7
beacon_node/lighthouse_network/src/discovery/enr_ext.rs → common/network_utils/src/enr_ext.rs
View File

@@ -1,11 +1,12 @@
//! ENR extension trait to support libp2p integration.
use crate::{Enr, Multiaddr, PeerId};
use discv5::enr::{CombinedKey, CombinedPublicKey};
use libp2p::core::multiaddr::Protocol;
use libp2p::identity::{KeyType, Keypair, PublicKey, ed25519, secp256k1};
use libp2p_identity::{KeyType, Keypair, PublicKey, ed25519, secp256k1};
use multiaddr::{Multiaddr, PeerId, Protocol};
use tiny_keccak::{Hasher, Keccak};
type Enr = discv5::enr::Enr<CombinedKey>;
pub const QUIC_ENR_KEY: &str = "quic";
pub const QUIC6_ENR_KEY: &str = "quic6";

4
common/network_utils/src/lib.rs Normal file
View File

@@ -0,0 +1,4 @@
pub mod discovery_metrics;
pub mod enr_ext;
pub mod listen_addr;
pub mod unused_port;

16
beacon_node/lighthouse_network/src/listen_addr.rs → common/network_utils/src/listen_addr.rs
View File

@@ -1,6 +1,6 @@
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use libp2p::{Multiaddr, multiaddr::Protocol};
use multiaddr::{Multiaddr, Protocol};
use serde::{Deserialize, Serialize};
/// A listening address composed by an Ip, an UDP port and a TCP port.
@@ -84,23 +84,21 @@ impl ListenAddress {
.chain(v6_tcp_multiaddr)
}
#[cfg(test)]
pub fn unused_v4_ports() -> Self {
ListenAddress::V4(ListenAddr {
addr: Ipv4Addr::UNSPECIFIED,
disc_port: unused_port::unused_udp4_port().unwrap(),
quic_port: unused_port::unused_udp4_port().unwrap(),
tcp_port: unused_port::unused_tcp4_port().unwrap(),
disc_port: crate::unused_port::unused_udp4_port().unwrap(),
quic_port: crate::unused_port::unused_udp4_port().unwrap(),
tcp_port: crate::unused_port::unused_tcp4_port().unwrap(),
})
}
#[cfg(test)]
pub fn unused_v6_ports() -> Self {
ListenAddress::V6(ListenAddr {
addr: Ipv6Addr::UNSPECIFIED,
disc_port: unused_port::unused_udp6_port().unwrap(),
quic_port: unused_port::unused_udp6_port().unwrap(),
tcp_port: unused_port::unused_tcp6_port().unwrap(),
disc_port: crate::unused_port::unused_udp6_port().unwrap(),
quic_port: crate::unused_port::unused_udp6_port().unwrap(),
tcp_port: crate::unused_port::unused_tcp6_port().unwrap(),
})
}
}

0
common/unused_port/src/lib.rs → common/network_utils/src/unused_port.rs
View File

2
common/system_health/Cargo.toml
View File

@@ -5,6 +5,8 @@ edition = { workspace = true }
[dependencies]
lighthouse_network = { workspace = true }
metrics = { workspace = true }
network_utils = { workspace = true }
parking_lot = { workspace = true }
serde = { workspace = true }
sysinfo = { workspace = true }

21
common/system_health/src/lib.rs
View File

@@ -1,4 +1,5 @@
use lighthouse_network::{NetworkGlobals, types::SyncState};
use network_utils::discovery_metrics;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
@@ -219,31 +220,19 @@ impl NatState {
/// Observes if NAT traversal is possible.
pub fn observe_nat() -> NatState {
let discv5_ipv4 = lighthouse_network::metrics::get_int_gauge(
&lighthouse_network::metrics::NAT_OPEN,
&["discv5_ipv4"],
)
let discv5_ipv4 = metrics::get_int_gauge(&discovery_metrics::NAT_OPEN, &["discv5_ipv4"])
.map(|g| g.get() == 1)
.unwrap_or_default();
let discv5_ipv6 = lighthouse_network::metrics::get_int_gauge(
&lighthouse_network::metrics::NAT_OPEN,
&["discv5_ipv6"],
)
let discv5_ipv6 = metrics::get_int_gauge(&discovery_metrics::NAT_OPEN, &["discv5_ipv6"])
.map(|g| g.get() == 1)
.unwrap_or_default();
let libp2p_ipv4 = lighthouse_network::metrics::get_int_gauge(
&lighthouse_network::metrics::NAT_OPEN,
&["libp2p_ipv4"],
)
let libp2p_ipv4 = metrics::get_int_gauge(&discovery_metrics::NAT_OPEN, &["libp2p_ipv4"])
.map(|g| g.get() == 1)
.unwrap_or_default();
let libp2p_ipv6 = lighthouse_network::metrics::get_int_gauge(
&lighthouse_network::metrics::NAT_OPEN,
&["libp2p_ipv6"],
)
let libp2p_ipv6 = metrics::get_int_gauge(&discovery_metrics::NAT_OPEN, &["libp2p_ipv6"])
.map(|g| g.get() == 1)
.unwrap_or_default();

9
common/unused_port/Cargo.toml
View File

@@ -1,9 +0,0 @@
[package]
name = "unused_port"
version = "0.1.0"
edition = { workspace = true }
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
lru_cache = { workspace = true }
parking_lot = { workspace = true }

1
lcli/Cargo.toml
View File

@@ -30,6 +30,7 @@ hex = { workspace = true }
lighthouse_network = { workspace = true }
lighthouse_version = { workspace = true }
log = { workspace = true }
network_utils = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }

3
lcli/src/generate_bootnode_enr.rs
View File

@@ -1,9 +1,10 @@
use clap::ArgMatches;
use lighthouse_network::{
NETWORK_KEY_FILENAME, NetworkConfig,
discovery::{CombinedKey, CombinedKeyExt, ENR_FILENAME, build_enr},
discovery::{CombinedKey, ENR_FILENAME, build_enr},
libp2p::identity::secp256k1,
};
use network_utils::enr_ext::CombinedKeyExt;
use std::io::Write;
use std::path::PathBuf;
use std::{fs, net::Ipv4Addr};

2
lighthouse/Cargo.toml
View File

@@ -57,6 +57,7 @@ lighthouse_tracing = { workspace = true }
lighthouse_version = { workspace = true }
logging = { workspace = true }
metrics = { workspace = true }
network_utils = { workspace = true }
opentelemetry = { workspace = true }
opentelemetry-otlp = { workspace = true }
opentelemetry_sdk = { workspace = true }
@@ -70,7 +71,6 @@ tracing = { workspace = true }
tracing-opentelemetry = { workspace = true }
tracing-subscriber = { workspace = true }
types = { workspace = true }
unused_port = { workspace = true }
validator_client = { workspace = true }
validator_manager = { path = "../validator_manager" }

4
lighthouse/tests/beacon_node.rs
View File

@@ -10,6 +10,9 @@ use beacon_node::{
};
use beacon_processor::BeaconProcessorConfig;
use lighthouse_network::PeerId;
use network_utils::unused_port::{
unused_tcp4_port, unused_tcp6_port, unused_udp4_port, unused_udp6_port,
};
use std::fs::File;
use std::io::{Read, Write};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
@@ -22,7 +25,6 @@ use std::time::Duration;
use tempfile::TempDir;
use types::non_zero_usize::new_non_zero_usize;
use types::{Address, Checkpoint, Epoch, Hash256, MainnetEthSpec};
use unused_port::{unused_tcp4_port, unused_tcp6_port, unused_udp4_port, unused_udp6_port};
const DEFAULT_EXECUTION_ENDPOINT: &str = "http://localhost:8551/";
const DEFAULT_EXECUTION_JWT_SECRET_KEY: &str =

5
lighthouse/tests/boot_node.rs
View File

@@ -3,8 +3,8 @@ use boot_node::config::BootNodeConfigSerialization;
use crate::exec::{CommandLineTestExec, CompletedTest};
use clap::ArgMatches;
use clap_utils::get_eth2_network_config;
use lighthouse_network::Enr;
use lighthouse_network::discovery::ENR_FILENAME;
use lighthouse_network::{Enr, discovery::ENR_FILENAME};
use network_utils::unused_port::unused_udp4_port;
use std::fs::File;
use std::io::Write;
use std::net::Ipv4Addr;
@@ -12,7 +12,6 @@ use std::path::{Path, PathBuf};
use std::process::Command;
use std::str::FromStr;
use tempfile::TempDir;
use unused_port::unused_udp4_port;
const IP_ADDRESS: &str = "192.168.2.108";

2
testing/execution_engine_integration/Cargo.toml
View File

@@ -18,6 +18,7 @@ fork_choice = { workspace = true }
futures = { workspace = true }
hex = { workspace = true }
logging = { workspace = true }
network_utils = { workspace = true }
reqwest = { workspace = true }
sensitive_url = { workspace = true }
serde_json = { workspace = true }
@@ -25,4 +26,3 @@ task_executor = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true }
types = { workspace = true }
unused_port = { workspace = true }

2
testing/execution_engine_integration/src/execution_engine.rs
View File

@@ -1,10 +1,10 @@
use ethers_providers::{Http, Provider};
use execution_layer::DEFAULT_JWT_FILE;
use network_utils::unused_port::unused_tcp4_port;
use sensitive_url::SensitiveUrl;
use std::path::PathBuf;
use std::process::Child;
use tempfile::TempDir;
use unused_port::unused_tcp4_port;
pub const KEYSTORE_PASSWORD: &str = "testpwd";
pub const ACCOUNT1: &str = "7b8C3a386C0eea54693fFB0DA17373ffC9228139";

2
testing/execution_engine_integration/src/geth.rs
View File

@@ -1,11 +1,11 @@
use crate::build_utils;
use crate::execution_engine::GenericExecutionEngine;
use crate::genesis_json::geth_genesis_json;
use network_utils::unused_port::unused_tcp4_port;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output};
use std::{env, fs};
use tempfile::TempDir;
use unused_port::unused_tcp4_port;
const GETH_BRANCH: &str = "master";
const GETH_REPO_URL: &str = "https://github.com/ethereum/go-ethereum";

2
testing/execution_engine_integration/src/nethermind.rs
View File

@@ -1,12 +1,12 @@
use crate::build_utils;
use crate::execution_engine::GenericExecutionEngine;
use crate::genesis_json::nethermind_genesis_json;
use network_utils::unused_port::unused_tcp4_port;
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output};
use tempfile::TempDir;
use unused_port::unused_tcp4_port;
/// We've pinned the Nethermind version since our method of using the `master` branch to
/// find the latest tag isn't working. It appears Nethermind don't always tag on `master`.