gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-01 11:43:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge unstable 20230925 into deneb-free-blobs.

Browse Source
This commit is contained in:
Jimmy Chen
2023-09-26 10:32:18 +10:00
parent 9244f7f7bc 441fc1691b
commit c0b6b92f27
164 changed files with 3844 additions and 3057 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
beacon_node/http_api/Cargo.toml
View File

@@ -2,54 +2,53 @@
name = "http_api"
version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2021"
edition = { workspace = true }
autotests = false # using a single test binary compiles faster
[dependencies]
warp = { version = "0.3.2", features = ["tls"] }
serde = { version = "1.0.116", features = ["derive"] }
tokio = { version = "1.14.0", features = ["macros", "sync"] }
tokio-stream = { version = "0.1.3", features = ["sync"] }
types = { path = "../../consensus/types" }
hex = "0.4.2"
beacon_chain = { path = "../beacon_chain" }
eth2 = { path = "../../common/eth2", features = ["lighthouse"] }
slog = "2.5.2"
network = { path = "../network" }
lighthouse_network = { path = "../lighthouse_network" }
eth1 = { path = "../eth1" }
state_processing = { path = "../../consensus/state_processing" }
lighthouse_version = { path = "../../common/lighthouse_version" }
lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
lazy_static = "1.4.0"
warp_utils = { path = "../../common/warp_utils" }
slot_clock = { path = "../../common/slot_clock" }
ethereum_ssz = "0.5.0"
warp = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
tokio-stream = { workspace = true }
types = { workspace = true }
hex = { workspace = true }
beacon_chain = { workspace = true }
eth2 = { workspace = true }
slog = { workspace = true }
network = { workspace = true }
lighthouse_network = { workspace = true }
eth1 = { workspace = true }
state_processing = { workspace = true }
lighthouse_version = { workspace = true }
lighthouse_metrics = { workspace = true }
lazy_static = { workspace = true }
warp_utils = { workspace = true }
slot_clock = { workspace = true }
ethereum_ssz = { workspace = true }
bs58 = "0.4.0"
futures = "0.3.8"
execution_layer = { path = "../execution_layer" }
parking_lot = "0.12.0"
safe_arith = { path = "../../consensus/safe_arith" }
task_executor = { path = "../../common/task_executor" }
lru = "0.7.7"
tree_hash = "0.5.2"
sysinfo = "0.26.5"
futures = { workspace = true }
execution_layer = { workspace = true }
parking_lot = { workspace = true }
safe_arith = { workspace = true }
task_executor = { workspace = true }
lru = { workspace = true }
tree_hash = { workspace = true }
sysinfo = { workspace = true }
system_health = { path = "../../common/system_health" }
directory = { path = "../../common/directory" }
logging = { path = "../../common/logging" }
ethereum_serde_utils = "0.5.0"
operation_pool = { path = "../operation_pool" }
sensitive_url = { path = "../../common/sensitive_url" }
unused_port = { path = "../../common/unused_port" }
store = { path = "../store" }
bytes = "1.1.0"
beacon_processor = { path = "../beacon_processor" }
directory = { workspace = true }
logging = { workspace = true }
ethereum_serde_utils = { workspace = true }
operation_pool = { workspace = true }
sensitive_url = { workspace = true }
store = { workspace = true }
bytes = { workspace = true }
beacon_processor = { workspace = true }
[dev-dependencies]
environment = { path = "../../lighthouse/environment" }
serde_json = "1.0.58"
proto_array = { path = "../../consensus/proto_array" }
genesis = { path = "../genesis" }
environment = { workspace = true }
serde_json = { workspace = true }
proto_array = { workspace = true }
genesis = { workspace = true }
[[test]]
name = "bn_http_api_tests"

73
beacon_node/http_api/src/lib.rs
View File

@@ -2869,12 +2869,8 @@ pub fn serve<T: BeaconChainTypes>(
})?;
if let Some(peer_info) = network_globals.peers.read().peer_info(&peer_id) {
let address = if let Some(socket_addr) = peer_info.seen_addresses().next() {
let mut addr = lighthouse_network::Multiaddr::from(socket_addr.ip());
addr.push(lighthouse_network::multiaddr::Protocol::Tcp(
socket_addr.port(),
));
addr.to_string()
let address = if let Some(multiaddr) = peer_info.seen_multiaddrs().next() {
multiaddr.to_string()
} else if let Some(addr) = peer_info.listening_addresses().first() {
addr.to_string()
} else {
@@ -2922,13 +2918,8 @@ pub fn serve<T: BeaconChainTypes>(
.peers()
.for_each(|(peer_id, peer_info)| {
let address =
if let Some(socket_addr) = peer_info.seen_addresses().next() {
let mut addr =
lighthouse_network::Multiaddr::from(socket_addr.ip());
addr.push(lighthouse_network::multiaddr::Protocol::Tcp(
socket_addr.port(),
));
addr.to_string()
if let Some(multiaddr) = peer_info.seen_multiaddrs().next() {
multiaddr.to_string()
} else if let Some(addr) = peer_info.listening_addresses().first() {
addr.to_string()
} else {
@@ -3055,6 +3046,7 @@ pub fn serve<T: BeaconChainTypes>(
.and(warp::path::end())
.and(not_while_syncing_filter.clone())
.and(warp::query::<api_types::ValidatorBlocksQuery>())
.and(warp::header::optional::<api_types::Accept>("accept"))
.and(task_spawner_filter.clone())
.and(chain_filter.clone())
.and(log_filter.clone())
@@ -3062,6 +3054,7 @@ pub fn serve<T: BeaconChainTypes>(
|endpoint_version: EndpointVersion,
slot: Slot,
query: api_types::ValidatorBlocksQuery,
accept_header: Option<api_types::Accept>,
task_spawner: TaskSpawner<T::EthSpec>,
chain: Arc<BeaconChain<T>>,
log: Logger| {
@@ -3109,9 +3102,24 @@ pub fn serve<T: BeaconChainTypes>(
let block_contents =
build_block_contents::build_block_contents(fork_name, block, maybe_blobs)?;
fork_versioned_response(endpoint_version, fork_name, block_contents)
.map(|response| warp::reply::json(&response).into_response())
.map(|res| add_consensus_version_header(res, fork_name))
match accept_header {
Some(api_types::Accept::Ssz) => Response::builder()
.status(200)
.header("Content-Type", "application/octet-stream")
.body(block_contents.as_ssz_bytes().into())
.map(|res: Response<Bytes>| {
add_consensus_version_header(res, fork_name)
})
.map_err(|e| {
warp_utils::reject::custom_server_error(format!(
"failed to create response: {}",
e
))
}),
_ => fork_versioned_response(endpoint_version, fork_name, block_contents)
.map(|response| warp::reply::json(&response).into_response())
.map(|res| add_consensus_version_header(res, fork_name)),
}
})
},
);
@@ -3128,11 +3136,13 @@ pub fn serve<T: BeaconChainTypes>(
.and(warp::path::end())
.and(not_while_syncing_filter.clone())
.and(warp::query::<api_types::ValidatorBlocksQuery>())
.and(warp::header::optional::<api_types::Accept>("accept"))
.and(task_spawner_filter.clone())
.and(chain_filter.clone())
.then(
|slot: Slot,
query: api_types::ValidatorBlocksQuery,
accept_header: Option<api_types::Accept>,
task_spawner: TaskSpawner<T::EthSpec>,
chain: Arc<BeaconChain<T>>| {
task_spawner.spawn_async_with_rejection(Priority::P0, async move {
@@ -3176,10 +3186,25 @@ pub fn serve<T: BeaconChainTypes>(
maybe_blobs,
)?;
// Pose as a V2 endpoint so we return the fork `version`.
fork_versioned_response(V2, fork_name, block_contents)
.map(|response| warp::reply::json(&response).into_response())
.map(|res| add_consensus_version_header(res, fork_name))
match accept_header {
Some(api_types::Accept::Ssz) => Response::builder()
.status(200)
.header("Content-Type", "application/octet-stream")
.body(block_contents.as_ssz_bytes().into())
.map(|res: Response<Bytes>| {
add_consensus_version_header(res, fork_name)
})
.map_err(|e| {
warp_utils::reject::custom_server_error(format!(
"failed to create response: {}",
e
))
}),
// Pose as a V2 endpoint so we return the fork `version`.
_ => fork_versioned_response(V2, fork_name, block_contents)
.map(|response| warp::reply::json(&response).into_response())
.map(|res| add_consensus_version_header(res, fork_name)),
}
})
},
);
@@ -3697,12 +3722,13 @@ pub fn serve<T: BeaconChainTypes>(
// send the response back to our original HTTP request
// task via a channel.
let builder_future = async move {
let builder = chain
let arc_builder = chain
.execution_layer
.as_ref()
.ok_or(BeaconChainError::ExecutionLayerMissing)
.map_err(warp_utils::reject::beacon_chain_error)?
.builder()
.builder();
let builder = arc_builder
.as_ref()
.ok_or(BeaconChainError::BuilderMissing)
.map_err(warp_utils::reject::beacon_chain_error)?;
@@ -4363,7 +4389,8 @@ pub fn serve<T: BeaconChainTypes>(
.then(
|task_spawner: TaskSpawner<T::EthSpec>, chain: Arc<BeaconChain<T>>| {
task_spawner.spawn_async_with_rejection(Priority::P1, async move {
let merge_readiness = chain.check_merge_readiness().await;
let current_slot = chain.slot_clock.now_or_genesis().unwrap_or(Slot::new(0));
let merge_readiness = chain.check_merge_readiness(current_slot).await;
Ok::<_, warp::reject::Rejection>(
warp::reply::json(&api_types::GenericResponse::from(merge_readiness))
.into_response(),

14
beacon_node/http_api/src/test_utils.rs
View File

@@ -126,17 +126,9 @@ pub async fn create_api_server<T: BeaconChainTypes>(
test_runtime: &TestRuntime,
log: Logger,
) -> ApiServer<T::EthSpec, impl Future<Output = ()>> {
// Get a random unused port.
let port = unused_port::unused_tcp4_port().unwrap();
create_api_server_on_port(chain, test_runtime, log, port).await
}
// Use port 0 to allocate a new unused port.
let port = 0;
pub async fn create_api_server_on_port<T: BeaconChainTypes>(
chain: Arc<BeaconChain<T>>,
test_runtime: &TestRuntime,
log: Logger,
port: u16,
) -> ApiServer<T::EthSpec, impl Future<Output = ()>> {
let (network_senders, network_receivers) = NetworkSenders::new();
// Default metadata
@@ -149,8 +141,6 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
let enr = EnrBuilder::new("v4").build(&enr_key).unwrap();
let network_globals = Arc::new(NetworkGlobals::new(
enr.clone(),
Some(TCP_PORT),
None,
meta_data,
vec![],
false,

190
beacon_node/http_api/tests/tests.rs
View File

@@ -10,15 +10,14 @@ use eth2::{
types::{BlockId as CoreBlockId, ForkChoiceNode, StateId as CoreStateId, *},
BeaconNodeHttpClient, Error, StatusCode, Timeouts,
};
use execution_layer::test_utils::TestingBuilder;
use execution_layer::test_utils::DEFAULT_BUILDER_THRESHOLD_WEI;
use execution_layer::test_utils::{
Operation, DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI,
MockBuilder, Operation, DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, DEFAULT_BUILDER_THRESHOLD_WEI,
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI,
};
use futures::stream::{Stream, StreamExt};
use futures::FutureExt;
use http_api::{
test_utils::{create_api_server, create_api_server_on_port, ApiServer},
test_utils::{create_api_server, ApiServer},
BlockId, StateId,
};
use lighthouse_network::{Enr, EnrExt, PeerId};
@@ -73,7 +72,7 @@ struct ApiTester {
network_rx: NetworkReceivers<E>,
local_enr: Enr,
external_peer_id: PeerId,
mock_builder: Option<Arc<TestingBuilder<E>>>,
mock_builder: Option<Arc<MockBuilder<E>>>,
}
struct ApiTesterConfig {
@@ -120,24 +119,28 @@ impl ApiTester {
}
pub async fn new_from_config(config: ApiTesterConfig) -> Self {
// Get a random unused port
let spec = config.spec;
let port = unused_port::unused_tcp4_port().unwrap();
let beacon_url = SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap();
let harness = Arc::new(
BeaconChainHarness::builder(MainnetEthSpec)
.spec(spec.clone())
.chain_config(ChainConfig {
reconstruct_historic_states: config.retain_historic_states,
..ChainConfig::default()
})
.logger(logging::test_logger())
.deterministic_keypairs(VALIDATOR_COUNT)
.fresh_ephemeral_store()
.mock_execution_layer_with_builder(beacon_url.clone(), config.builder_threshold)
.build(),
);
let mut harness = BeaconChainHarness::builder(MainnetEthSpec)
.spec(spec.clone())
.chain_config(ChainConfig {
reconstruct_historic_states: config.retain_historic_states,
..ChainConfig::default()
})
.logger(logging::test_logger())
.deterministic_keypairs(VALIDATOR_COUNT)
.fresh_ephemeral_store()
.mock_execution_layer_with_config()
.build();
harness
.mock_execution_layer
.as_ref()
.unwrap()
.server
.execution_block_generator()
.move_to_terminal_block()
.unwrap();
harness.advance_slot();
@@ -247,29 +250,40 @@ impl ApiTester {
let ApiServer {
server,
listening_socket: _,
listening_socket,
network_rx,
local_enr,
external_peer_id,
} = create_api_server_on_port(chain.clone(), &harness.runtime, log, port).await;
} = create_api_server(chain.clone(), &harness.runtime, log).await;
harness.runtime.task_executor.spawn(server, "api_server");
// Late-initalize the mock builder now that the mock execution node and beacon API ports
// have been allocated.
let beacon_api_port = listening_socket.port();
let beacon_url =
SensitiveUrl::parse(format!("http://127.0.0.1:{beacon_api_port}").as_str()).unwrap();
let mock_builder_server = harness.set_mock_builder(beacon_url.clone());
// Start the mock builder service prior to building the chain out.
harness.runtime.task_executor.spawn(
async move {
if let Err(e) = mock_builder_server.await {
panic!("error in mock builder server: {e:?}");
}
},
"mock_builder_server",
);
let mock_builder = harness.mock_builder.clone();
let client = BeaconNodeHttpClient::new(
beacon_url,
Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)),
);
let builder_ref = harness.mock_builder.as_ref().unwrap().clone();
harness.runtime.task_executor.spawn(
async move { builder_ref.run().await },
"mock_builder_server",
);
let mock_builder = harness.mock_builder.clone();
Self {
harness,
harness: Arc::new(harness),
chain,
client,
next_block,
@@ -383,7 +397,6 @@ impl ApiTester {
.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::Value(Uint256::from(
DEFAULT_BUILDER_THRESHOLD_WEI,
)));
@@ -406,7 +419,6 @@ impl ApiTester {
.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::Value(Uint256::from(
DEFAULT_BUILDER_PAYLOAD_VALUE_WEI,
)));
@@ -2528,6 +2540,70 @@ impl ApiTester {
self
}
pub async fn test_block_production_ssz(self) -> Self {
let fork = self.chain.canonical_head.cached_head().head_fork();
let genesis_validators_root = self.chain.genesis_validators_root;
for _ in 0..E::slots_per_epoch() * 3 {
let slot = self.chain.slot().unwrap();
let epoch = self.chain.epoch().unwrap();
let proposer_pubkey_bytes = self
.client
.get_validator_duties_proposer(epoch)
.await
.unwrap()
.data
.into_iter()
.find(|duty| duty.slot == slot)
.map(|duty| duty.pubkey)
.unwrap();
let proposer_pubkey = (&proposer_pubkey_bytes).try_into().unwrap();
let sk = self
.validator_keypairs()
.iter()
.find(|kp| kp.pk == proposer_pubkey)
.map(|kp| kp.sk.clone())
.unwrap();
let randao_reveal = {
let domain = self.chain.spec.get_domain(
epoch,
Domain::Randao,
&fork,
genesis_validators_root,
);
let message = epoch.signing_root(domain);
sk.sign(message).into()
};
let block_bytes = self
.client
.get_validator_blocks_ssz::<E, FullPayload<E>>(slot, &randao_reveal, None)
.await
.unwrap()
.expect("block bytes");
let block =
BeaconBlock::<E, FullPayload<E>>::from_ssz_bytes(&block_bytes, &self.chain.spec)
.expect("block bytes can be decoded");
let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec);
self.client
.post_beacon_blocks_ssz(&signed_block)
.await
.unwrap();
assert_eq!(self.chain.head_beacon_block().as_ref(), &signed_block);
self.chain.slot_clock.set_slot(slot.as_u64() + 1);
}
self
}
pub async fn test_block_production_no_verify_randao(self) -> Self {
for _ in 0..E::slots_per_epoch() {
let slot = self.chain.slot().unwrap();
@@ -2713,12 +2789,18 @@ impl ApiTester {
sk.sign(message).into()
};
let block_contents = self
let block_contents_bytes = self
.client
.get_validator_blinded_blocks::<E, Payload>(slot, &randao_reveal, None)
.get_validator_blinded_blocks_ssz::<E, Payload>(slot, &randao_reveal, None)
.await
.unwrap()
.data;
.expect("block bytes");
let block_contents = BlockContents::<E, Payload>::from_ssz_bytes(
&block_contents_bytes,
&self.chain.spec,
)
.expect("block bytes can be decoded");
let signed_block_contents =
block_contents.sign(&sk, &fork, genesis_validators_root, &self.chain.spec);
@@ -3309,6 +3391,7 @@ impl ApiTester {
.unwrap()
.get_payload_by_root(&payload.tree_hash_root())
.is_none());
self
}
@@ -3317,7 +3400,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::GasLimit(30_000_000));
let slot = self.chain.slot().unwrap();
@@ -3361,7 +3443,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::FeeRecipient(test_fee_recipient));
let slot = self.chain.slot().unwrap();
@@ -3404,7 +3485,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::ParentHash(invalid_parent_hash));
let slot = self.chain.slot().unwrap();
@@ -3454,7 +3534,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::PrevRandao(invalid_prev_randao));
let slot = self.chain.slot().unwrap();
@@ -3500,7 +3579,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::BlockNumber(invalid_block_number));
let slot = self.chain.slot().unwrap();
@@ -3548,7 +3626,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::Timestamp(invalid_timestamp));
let slot = self.chain.slot().unwrap();
@@ -3589,11 +3666,7 @@ impl ApiTester {
}
pub async fn test_payload_rejects_invalid_signature(self) -> Self {
self.mock_builder
.as_ref()
.unwrap()
.builder
.invalid_signatures();
self.mock_builder.as_ref().unwrap().invalid_signatures();
let slot = self.chain.slot().unwrap();
let epoch = self.chain.epoch().unwrap();
@@ -3878,7 +3951,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::Value(Uint256::from(
DEFAULT_BUILDER_THRESHOLD_WEI - 1,
)));
@@ -3916,7 +3988,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::Value(Uint256::from(
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
)));
@@ -3954,7 +4025,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::Value(Uint256::from(
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI,
)));
@@ -3992,7 +4062,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::Value(Uint256::from(
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI - 1,
)));
@@ -4030,7 +4099,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::Value(Uint256::from(
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
)));
@@ -4104,7 +4172,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::Value(Uint256::from(
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
)));
@@ -4112,7 +4179,6 @@ impl ApiTester {
self.mock_builder
.as_ref()
.unwrap()
.builder
.add_operation(Operation::WithdrawalsRoot(Hash256::repeat_byte(0x42)));
let slot = self.chain.slot().unwrap();
@@ -4976,6 +5042,20 @@ async fn block_production_verify_randao_invalid() {
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn block_production_ssz_full_payload() {
ApiTester::new().await.test_block_production_ssz().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn block_production_ssz_with_skip_slots() {
ApiTester::new()
.await
.skip_slots(E::slots_per_epoch() * 2)
.test_block_production_ssz()
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn blinded_block_production_full_payload_premerge() {
ApiTester::new()