Merge branch 'unstable' into deneb-free-blobs

# Conflicts:
#	.github/workflows/docker.yml
#	.github/workflows/local-testnet.yml
#	.github/workflows/test-suite.yml
#	Cargo.lock
#	Cargo.toml
#	beacon_node/beacon_chain/src/beacon_chain.rs
#	beacon_node/beacon_chain/src/builder.rs
#	beacon_node/beacon_chain/src/test_utils.rs
#	beacon_node/execution_layer/src/engine_api/json_structures.rs
#	beacon_node/network/src/beacon_processor/mod.rs
#	beacon_node/network/src/beacon_processor/worker/gossip_methods.rs
#	beacon_node/network/src/sync/backfill_sync/mod.rs
#	beacon_node/store/src/config.rs
#	beacon_node/store/src/hot_cold_store.rs
#	common/eth2_network_config/Cargo.toml
#	consensus/ssz/src/decode/impls.rs
#	consensus/ssz_derive/src/lib.rs
#	consensus/ssz_derive/tests/tests.rs
#	consensus/ssz_types/src/serde_utils/mod.rs
#	consensus/tree_hash/src/impls.rs
#	consensus/tree_hash/src/lib.rs
#	consensus/types/Cargo.toml
#	consensus/types/src/beacon_state.rs
#	consensus/types/src/chain_spec.rs
#	consensus/types/src/eth_spec.rs
#	consensus/types/src/fork_name.rs
#	lcli/Cargo.toml
#	lcli/src/main.rs
#	lcli/src/new_testnet.rs
#	scripts/local_testnet/el_bootnode.sh
#	scripts/local_testnet/genesis.json
#	scripts/local_testnet/geth.sh
#	scripts/local_testnet/setup.sh
#	scripts/local_testnet/start_local_testnet.sh
#	scripts/local_testnet/vars.env
#	scripts/tests/doppelganger_protection.sh
#	scripts/tests/genesis.json
#	scripts/tests/vars.env
#	testing/ef_tests/Cargo.toml
#	validator_client/src/block_service.rs
This commit is contained in:
Jimmy Chen
2023-05-30 11:26:33 +10:00
333 changed files with 5930 additions and 13386 deletions

View File

@@ -8,13 +8,13 @@ edition = "2021"
discv5 = { version = "0.2.2", features = ["libp2p"] }
unsigned-varint = { version = "0.6.0", features = ["codec"] }
types = { path = "../../consensus/types" }
eth2_ssz_types = "0.2.2"
ssz_types = "0.5.0"
serde = { version = "1.0.116", features = ["derive"] }
serde_derive = "1.0.116"
eth2_ssz = "0.4.1"
eth2_ssz_derive = "0.3.0"
tree_hash = "0.4.1"
tree_hash_derive = "0.4.0"
ethereum_ssz = "0.5.0"
ethereum_ssz_derive = "0.5.0"
tree_hash = "0.5.0"
tree_hash_derive = "0.5.0"
slog = { version = "2.5.2", features = ["max_level_trace"] }
lighthouse_version = { path = "../../common/lighthouse_version" }
tokio = { version = "1.14.0", features = ["time", "macros"] }

View File

@@ -134,6 +134,9 @@ pub struct Config {
/// List of extra topics to initially subscribe to as strings.
pub topics: Vec<GossipKind>,
/// Whether we are running a block proposer only node.
pub proposer_only: bool,
/// Whether metrics are enabled.
pub metrics_enabled: bool,
@@ -142,6 +145,9 @@ pub struct Config {
/// Configuration for the outbound rate limiter (requests made by this node).
pub outbound_rate_limiter_config: Option<OutboundRateLimiterConfig>,
/// Configures if/where invalid blocks should be stored.
pub invalid_block_storage: Option<PathBuf>,
}
impl Config {
@@ -322,9 +328,11 @@ impl Default for Config {
import_all_attestations: false,
shutdown_after_sync: false,
topics: Vec::new(),
proposer_only: false,
metrics_enabled: false,
enable_light_client_server: false,
outbound_rate_limiter_config: None,
invalid_block_storage: None,
}
}
}

View File

@@ -952,6 +952,10 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
/// MIN_SYNC_COMMITTEE_PEERS
/// number should be set low as an absolute lower bound to maintain peers on the sync
/// committees.
/// - Do not prune trusted peers. NOTE: This means if a user has more trusted peers than the
/// excess peer limit, all of the following logic is subverted as we will not prune any peers.
/// Also, the more trusted peers a user has, the less room Lighthouse has to efficiently manage
/// its peers across the subnets.
///
/// Prune peers in the following order:
/// 1. Remove worst scoring peers
@@ -982,7 +986,9 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
.read()
.worst_connected_peers()
.iter()
.filter(|(_, info)| !info.has_future_duty() && $filter(*info))
.filter(|(_, info)| {
!info.has_future_duty() && !info.is_trusted() && $filter(*info)
})
{
if peers_to_prune.len()
>= connected_peer_count.saturating_sub(self.target_peers)
@@ -1032,8 +1038,8 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
> = HashMap::new();
for (peer_id, info) in self.network_globals.peers.read().connected_peers() {
// Ignore peers we are already pruning
if peers_to_prune.contains(peer_id) {
// Ignore peers we trust or that we are already pruning
if info.is_trusted() || peers_to_prune.contains(peer_id) {
continue;
}
@@ -1330,25 +1336,47 @@ mod tests {
..Default::default()
};
let log = build_log(slog::Level::Debug, false);
let globals = NetworkGlobals::new_test_globals(&log);
let globals = NetworkGlobals::new_test_globals(vec![], &log);
PeerManager::new(config, Arc::new(globals), &log).unwrap()
}
async fn build_peer_manager_with_trusted_peers(
trusted_peers: Vec<PeerId>,
target_peer_count: usize,
) -> PeerManager<E> {
let config = config::Config {
target_peer_count,
discovery_enabled: false,
..Default::default()
};
let log = build_log(slog::Level::Debug, false);
let globals = NetworkGlobals::new_test_globals(trusted_peers, &log);
PeerManager::new(config, Arc::new(globals), &log).unwrap()
}
#[tokio::test]
async fn test_peer_manager_disconnects_correctly_during_heartbeat() {
let mut peer_manager = build_peer_manager(3).await;
// Create 5 peers to connect to.
// Create 6 peers to connect to with a target of 3.
// 2 will be outbound-only, and have the lowest score.
// 1 will be a trusted peer.
// The other 3 will be ingoing peers.
// We expect this test to disconnect from 3 peers. 1 from the outbound peer (the other must
// remain due to the outbound peer limit) and 2 from the ingoing peers (the trusted peer
// should remain connected).
let peer0 = PeerId::random();
let peer1 = PeerId::random();
let peer2 = PeerId::random();
let outbound_only_peer1 = PeerId::random();
let outbound_only_peer2 = PeerId::random();
let trusted_peer = PeerId::random();
let mut peer_manager = build_peer_manager_with_trusted_peers(vec![trusted_peer], 3).await;
peer_manager.inject_connect_ingoing(&peer0, "/ip4/0.0.0.0".parse().unwrap(), None);
peer_manager.inject_connect_ingoing(&peer1, "/ip4/0.0.0.0".parse().unwrap(), None);
peer_manager.inject_connect_ingoing(&peer2, "/ip4/0.0.0.0".parse().unwrap(), None);
peer_manager.inject_connect_ingoing(&trusted_peer, "/ip4/0.0.0.0".parse().unwrap(), None);
peer_manager.inject_connect_outgoing(
&outbound_only_peer1,
"/ip4/0.0.0.0".parse().unwrap(),
@@ -1378,7 +1406,7 @@ mod tests {
.add_to_score(-2.0);
// Check initial connected peers.
assert_eq!(peer_manager.network_globals.connected_or_dialing_peers(), 5);
assert_eq!(peer_manager.network_globals.connected_or_dialing_peers(), 6);
peer_manager.heartbeat();
@@ -1397,8 +1425,22 @@ mod tests {
.read()
.is_connected(&outbound_only_peer2));
// The trusted peer remains connected
assert!(peer_manager
.network_globals
.peers
.read()
.is_connected(&trusted_peer));
peer_manager.heartbeat();
// The trusted peer remains connected, even after subsequent heartbeats.
assert!(peer_manager
.network_globals
.peers
.read()
.is_connected(&trusted_peer));
// Check that if we are at target number of peers, we do not disconnect any.
assert_eq!(peer_manager.network_globals.connected_or_dialing_peers(), 3);
}
@@ -2143,7 +2185,7 @@ mod tests {
#[cfg(test)]
mod property_based_tests {
use crate::peer_manager::config::DEFAULT_TARGET_PEERS;
use crate::peer_manager::tests::build_peer_manager;
use crate::peer_manager::tests::build_peer_manager_with_trusted_peers;
use crate::rpc::MetaData;
use libp2p::PeerId;
use quickcheck::{Arbitrary, Gen, TestResult};
@@ -2154,10 +2196,12 @@ mod tests {
#[derive(Clone, Debug)]
struct PeerCondition {
peer_id: PeerId,
outgoing: bool,
attestation_net_bitfield: Vec<bool>,
sync_committee_net_bitfield: Vec<bool>,
score: f64,
trusted: bool,
gossipsub_score: f64,
}
@@ -2182,10 +2226,12 @@ mod tests {
};
PeerCondition {
peer_id: PeerId::random(),
outgoing: bool::arbitrary(g),
attestation_net_bitfield,
sync_committee_net_bitfield,
score: f64::arbitrary(g),
trusted: bool::arbitrary(g),
gossipsub_score: f64::arbitrary(g),
}
}
@@ -2197,26 +2243,36 @@ mod tests {
if peer_conditions.len() < target_peer_count {
return TestResult::discard();
}
let trusted_peers: Vec<_> = peer_conditions
.iter()
.filter_map(|p| if p.trusted { Some(p.peer_id) } else { None })
.collect();
// If we have a high percentage of trusted peers, it is very difficult to reason about
// the expected results of the pruning.
if trusted_peers.len() > peer_conditions.len() / 3_usize {
return TestResult::discard();
}
let rt = Runtime::new().unwrap();
rt.block_on(async move {
let mut peer_manager = build_peer_manager(target_peer_count).await;
// Collect all the trusted peers
let mut peer_manager =
build_peer_manager_with_trusted_peers(trusted_peers, target_peer_count).await;
// Create peers based on the randomly generated conditions.
for condition in &peer_conditions {
let peer = PeerId::random();
let mut attnets = crate::types::EnrAttestationBitfield::<E>::new();
let mut syncnets = crate::types::EnrSyncCommitteeBitfield::<E>::new();
if condition.outgoing {
peer_manager.inject_connect_outgoing(
&peer,
&condition.peer_id,
"/ip4/0.0.0.0".parse().unwrap(),
None,
);
} else {
peer_manager.inject_connect_ingoing(
&peer,
&condition.peer_id,
"/ip4/0.0.0.0".parse().unwrap(),
None,
);
@@ -2237,22 +2293,51 @@ mod tests {
};
let mut peer_db = peer_manager.network_globals.peers.write();
let peer_info = peer_db.peer_info_mut(&peer).unwrap();
let peer_info = peer_db.peer_info_mut(&condition.peer_id).unwrap();
peer_info.set_meta_data(MetaData::V2(metadata));
peer_info.set_gossipsub_score(condition.gossipsub_score);
peer_info.add_to_score(condition.score);
for subnet in peer_info.long_lived_subnets() {
peer_db.add_subscription(&peer, subnet);
peer_db.add_subscription(&condition.peer_id, subnet);
}
}
// Perform the heartbeat.
peer_manager.heartbeat();
TestResult::from_bool(
// The minimum number of connected peers cannot be less than the target peer count
// or submitted peers.
let expected_peer_count = target_peer_count.min(peer_conditions.len());
// Trusted peers could make this larger however.
let no_of_trusted_peers = peer_conditions
.iter()
.filter(|condition| condition.trusted)
.count();
let expected_peer_count = expected_peer_count.max(no_of_trusted_peers);
let target_peer_condition =
peer_manager.network_globals.connected_or_dialing_peers()
== target_peer_count.min(peer_conditions.len()),
== expected_peer_count;
// It could be that we reach our target outbound limit and are unable to prune any
// extra, which violates the target_peer_condition.
let outbound_peers = peer_manager.network_globals.connected_outbound_only_peers();
let hit_outbound_limit = outbound_peers == peer_manager.target_outbound_peers();
// No trusted peers should be disconnected
let trusted_peer_disconnected = peer_conditions.iter().any(|condition| {
condition.trusted
&& !peer_manager
.network_globals
.peers
.read()
.is_connected(&condition.peer_id)
});
TestResult::from_bool(
(target_peer_condition || hit_outbound_limit) && !trusted_peer_disconnected,
)
})
}

View File

@@ -1062,7 +1062,7 @@ impl<TSpec: EthSpec> PeerDB<TSpec> {
if let Some(to_drop) = self
.peers
.iter()
.filter(|(_, info)| info.is_disconnected())
.filter(|(_, info)| info.is_disconnected() && !info.is_trusted())
.filter_map(|(id, info)| match info.connection_status() {
PeerConnectionStatus::Disconnected { since } => Some((id, since)),
_ => None,

View File

@@ -129,7 +129,10 @@ impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
}
/// TESTING ONLY. Build a dummy NetworkGlobals instance.
pub fn new_test_globals(log: &slog::Logger) -> NetworkGlobals<TSpec> {
pub fn new_test_globals(
trusted_peers: Vec<PeerId>,
log: &slog::Logger,
) -> NetworkGlobals<TSpec> {
use crate::CombinedKeyExt;
let keypair = libp2p::identity::Keypair::generate_secp256k1();
let enr_key: discv5::enr::CombinedKey =
@@ -144,7 +147,7 @@ impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
attnets: Default::default(),
syncnets: Default::default(),
}),
vec![],
trusted_peers,
false,
log,
)

View File

@@ -13,7 +13,7 @@ pub enum SyncState {
/// The node is undertaking a backfill sync. This occurs when a user has specified a trusted
/// state. The node first syncs "forward" by downloading blocks up to the current head as
/// specified by its peers. Once completed, the node enters this sync state and attempts to
/// download all required historical blocks to complete its chain.
/// download all required historical blocks.
BackFillSyncing { completed: usize, remaining: usize },
/// The node has completed syncing a finalized chain and is in the process of re-evaluating
/// which sync state to progress to.

View File

@@ -126,36 +126,6 @@ pub fn get_enr(node: &LibP2PService<ReqId, E>) -> Enr {
node.local_enr()
}
// Returns `n` libp2p peers in fully connected topology.
#[allow(dead_code)]
/*
pub async fn build_full_mesh(
rt: Weak<Runtime>,
log: slog::Logger,
n: usize,
fork_name: ForkName,
) -> Vec<Libp2pInstance> {
let mut nodes = Vec::with_capacity(n);
for _ in 0..n {
nodes.push(build_libp2p_instance(rt.clone(), vec![], log.clone(), fork_name).await);
}
let multiaddrs: Vec<Multiaddr> = nodes
.iter()
.map(|x| get_enr(x).multiaddr()[1].clone())
.collect();
for (i, node) in nodes.iter_mut().enumerate().take(n) {
for (j, multiaddr) in multiaddrs.iter().enumerate().skip(i) {
if i != j {
match libp2p::Swarm::dial(&mut node.swarm, multiaddr.clone()) {
Ok(()) => debug!(log, "Connected"),
Err(_) => error!(log, "Failed to connect"),
};
}
}
}
nodes
}*/
// Constructs a pair of nodes with separate loggers. The sender dials the receiver.
// This returns a (sender, receiver) pair.
#[allow(dead_code)]

View File

@@ -1,171 +0,0 @@
/* These are temporarily disabled due to their non-deterministic behaviour and impending update to
* gossipsub 1.1. We leave these here as a template for future test upgrades
#![cfg(test)]
use crate::types::GossipEncoding;
use ::types::{BeaconBlock, EthSpec, MinimalEthSpec, Signature, SignedBeaconBlock};
use lighthouse_network::*;
use slog::{debug, Level};
type E = MinimalEthSpec;
mod common;
/* Gossipsub tests */
// Note: The aim of these tests is not to test the robustness of the gossip network
// but to check if the gossipsub implementation is behaving according to the specifications.
// Test if gossipsub message are forwarded by nodes with a simple linear topology.
//
// Topology used in test
//
// node1 <-> node2 <-> node3 ..... <-> node(n-1) <-> node(n)
#[tokio::test]
async fn test_gossipsub_forward() {
// set up the logging. The level and enabled or not
let log = common::build_log(Level::Info, false);
let num_nodes = 20;
let mut nodes = common::build_linear(log.clone(), num_nodes);
let mut received_count = 0;
let spec = E::default_spec();
let empty_block = BeaconBlock::empty(&spec);
let signed_block = SignedBeaconBlock {
message: empty_block,
signature: Signature::empty_signature(),
};
let pubsub_message = PubsubMessage::BeaconBlock(Box::new(signed_block));
let publishing_topic: String = pubsub_message
.topics(GossipEncoding::default(), [0, 0, 0, 0])
.first()
.unwrap()
.clone()
.into();
let mut subscribed_count = 0;
let fut = async move {
for node in nodes.iter_mut() {
loop {
match node.next_event().await {
Libp2pEvent::Behaviour(b) => match b {
BehaviourEvent::PubsubMessage {
topics,
message,
source,
id,
} => {
assert_eq!(topics.len(), 1);
// Assert topic is the published topic
assert_eq!(
topics.first().unwrap(),
&TopicHash::from_raw(publishing_topic.clone())
);
// Assert message received is the correct one
assert_eq!(message, pubsub_message.clone());
received_count += 1;
// Since `propagate_message` is false, need to propagate manually
node.swarm.propagate_message(&source, id);
// Test should succeed if all nodes except the publisher receive the message
if received_count == num_nodes - 1 {
debug!(log.clone(), "Received message at {} nodes", num_nodes - 1);
return;
}
}
BehaviourEvent::PeerSubscribed(_, topic) => {
// Publish on beacon block topic
if topic == TopicHash::from_raw(publishing_topic.clone()) {
subscribed_count += 1;
// Every node except the corner nodes are connected to 2 nodes.
if subscribed_count == (num_nodes * 2) - 2 {
node.swarm.publish(vec![pubsub_message.clone()]);
}
}
}
_ => break,
},
_ => break,
}
}
}
};
tokio::select! {
_ = fut => {}
_ = tokio::time::delay_for(tokio::time::Duration::from_millis(800)) => {
panic!("Future timed out");
}
}
}
// Test publishing of a message with a full mesh for the topic
// Not very useful but this is the bare minimum functionality.
#[tokio::test]
async fn test_gossipsub_full_mesh_publish() {
// set up the logging. The level and enabled or not
let log = common::build_log(Level::Debug, false);
// Note: This test does not propagate gossipsub messages.
// Having `num_nodes` > `mesh_n_high` may give inconsistent results
// as nodes may get pruned out of the mesh before the gossipsub message
// is published to them.
let num_nodes = 12;
let mut nodes = common::build_full_mesh(log, num_nodes);
let mut publishing_node = nodes.pop().unwrap();
let spec = E::default_spec();
let empty_block = BeaconBlock::empty(&spec);
let signed_block = SignedBeaconBlock {
message: empty_block,
signature: Signature::empty_signature(),
};
let pubsub_message = PubsubMessage::BeaconBlock(Box::new(signed_block));
let publishing_topic: String = pubsub_message
.topics(GossipEncoding::default(), [0, 0, 0, 0])
.first()
.unwrap()
.clone()
.into();
let mut subscribed_count = 0;
let mut received_count = 0;
let fut = async move {
for node in nodes.iter_mut() {
while let Libp2pEvent::Behaviour(BehaviourEvent::PubsubMessage {
topics,
message,
..
}) = node.next_event().await
{
assert_eq!(topics.len(), 1);
// Assert topic is the published topic
assert_eq!(
topics.first().unwrap(),
&TopicHash::from_raw(publishing_topic.clone())
);
// Assert message received is the correct one
assert_eq!(message, pubsub_message.clone());
received_count += 1;
if received_count == num_nodes - 1 {
return;
}
}
}
while let Libp2pEvent::Behaviour(BehaviourEvent::PeerSubscribed(_, topic)) =
publishing_node.next_event().await
{
// Publish on beacon block topic
if topic == TopicHash::from_raw(publishing_topic.clone()) {
subscribed_count += 1;
if subscribed_count == num_nodes - 1 {
publishing_node.swarm.publish(vec![pubsub_message.clone()]);
}
}
}
};
tokio::select! {
_ = fut => {}
_ = tokio::time::delay_for(tokio::time::Duration::from_millis(800)) => {
panic!("Future timed out");
}
}
}
*/