gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-16 19:32:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Shift HTTP server heavy-lifting to blocking executor (#1518)

Browse Source 
## Issue Addressed

NA

## Proposed Changes

Shift practically all HTTP endpoint handlers to the blocking executor (some very light tasks are left on the core executor).

## Additional Info

This PR covers the `rest_api` which will soon be refactored to suit the standard API. As such, I've cut a few corners and left some existing issues open in this patch. What I have done here should leave the API in state that is not necessary *exactly* the same, but good enough for us to run validators with. Specifically, the number of blocking workers that can be spawned is unbounded and I have not implemented a queue; this will need to be fixed when we implement the standard API.
This commit is contained in:
Paul Hauner
2020-08-24 03:06:10 +00:00
parent 2bc9115a94
commit c895dc8971
22 changed files with 828 additions and 906 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
beacon_node/rest_api/src/validator.rs
View File

@@ -1,15 +1,12 @@
use crate::helpers::{
check_content_type_for_json, parse_hex_ssz_bytes, publish_beacon_block_to_network,
};
use crate::response_builder::ResponseBuilder;
use crate::{ApiError, ApiResult, NetworkChannel, UrlQuery};
use crate::helpers::{parse_hex_ssz_bytes, publish_beacon_block_to_network};
use crate::{ApiError, Context, NetworkChannel, UrlQuery};
use beacon_chain::{
attestation_verification::Error as AttnError, BeaconChain, BeaconChainError, BeaconChainTypes,
BlockError, ForkChoiceError, StateSkipConfig,
};
use bls::PublicKeyBytes;
use eth2_libp2p::PubsubMessage;
use hyper::{Body, Request};
use hyper::Request;
use network::NetworkMessage;
use rayon::prelude::*;
use rest_types::{ValidatorDutiesRequest, ValidatorDutyBytes, ValidatorSubscription};
@@ -17,25 +14,20 @@ use slog::{error, info, trace, warn, Logger};
use std::sync::Arc;
use types::beacon_state::EthSpec;
use types::{
Attestation, AttestationData, BeaconState, Epoch, RelativeEpoch, SelectionProof,
Attestation, AttestationData, BeaconBlock, BeaconState, Epoch, RelativeEpoch, SelectionProof,
SignedAggregateAndProof, SignedBeaconBlock, SubnetId,
};
/// HTTP Handler to retrieve the duties for a set of validators during a particular epoch. This
/// method allows for collecting bulk sets of validator duties without risking exceeding the max
/// URL length with query pairs.
pub async fn post_validator_duties<T: BeaconChainTypes>(
req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>,
) -> ApiResult {
let response_builder = ResponseBuilder::new(&req);
pub fn post_validator_duties<T: BeaconChainTypes>(
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
let body = req.into_body();
let chunks = hyper::body::to_bytes(body)
.await
.map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
serde_json::from_slice::<ValidatorDutiesRequest>(&chunks)
serde_json::from_slice::<ValidatorDutiesRequest>(&body)
.map_err(|e| {
ApiError::BadRequest(format!(
"Unable to parse JSON into ValidatorDutiesRequest: {:?}",
@@ -44,29 +36,22 @@ pub async fn post_validator_duties<T: BeaconChainTypes>(
})
.and_then(|bulk_request| {
return_validator_duties(
beacon_chain,
&ctx.beacon_chain.clone(),
bulk_request.epoch,
bulk_request.pubkeys.into_iter().map(Into::into).collect(),
)
})
.and_then(|duties| response_builder?.body_no_ssz(&duties))
}
/// HTTP Handler to retrieve subscriptions for a set of validators. This allows the node to
/// organise peer discovery and topic subscription for known validators.
pub async fn post_validator_subscriptions<T: BeaconChainTypes>(
req: Request<Body>,
network_chan: NetworkChannel<T::EthSpec>,
) -> ApiResult {
try_future!(check_content_type_for_json(&req));
let response_builder = ResponseBuilder::new(&req);
pub fn post_validator_subscriptions<T: BeaconChainTypes>(
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<(), ApiError> {
let body = req.into_body();
let chunks = hyper::body::to_bytes(body)
.await
.map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
serde_json::from_slice(&chunks)
serde_json::from_slice(&body)
.map_err(|e| {
ApiError::BadRequest(format!(
"Unable to parse JSON into ValidatorSubscriptions: {:?}",
@@ -74,7 +59,7 @@ pub async fn post_validator_subscriptions<T: BeaconChainTypes>(
))
})
.and_then(move |subscriptions: Vec<ValidatorSubscription>| {
network_chan
ctx.network_chan
.send(NetworkMessage::Subscribe { subscriptions })
.map_err(|e| {
ApiError::ServerError(format!(
@@ -84,19 +69,18 @@ pub async fn post_validator_subscriptions<T: BeaconChainTypes>(
})?;
Ok(())
})
.and_then(|_| response_builder?.body_no_ssz(&()))
}
/// HTTP Handler to retrieve all validator duties for the given epoch.
pub fn get_all_validator_duties<T: BeaconChainTypes>(
req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>,
) -> ApiResult {
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
let query = UrlQuery::from_request(&req)?;
let epoch = query.epoch()?;
let state = get_state_for_epoch(&beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
let state = get_state_for_epoch(&ctx.beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
let validator_pubkeys = state
.validators
@@ -104,21 +88,19 @@ pub fn get_all_validator_duties<T: BeaconChainTypes>(
.map(|validator| validator.pubkey.clone())
.collect();
let duties = return_validator_duties(beacon_chain, epoch, validator_pubkeys)?;
ResponseBuilder::new(&req)?.body_no_ssz(&duties)
return_validator_duties(&ctx.beacon_chain, epoch, validator_pubkeys)
}
/// HTTP Handler to retrieve all active validator duties for the given epoch.
pub fn get_active_validator_duties<T: BeaconChainTypes>(
req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>,
) -> ApiResult {
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
let query = UrlQuery::from_request(&req)?;
let epoch = query.epoch()?;
let state = get_state_for_epoch(&beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
let state = get_state_for_epoch(&ctx.beacon_chain, epoch, StateSkipConfig::WithoutStateRoots)?;
let validator_pubkeys = state
.validators
@@ -127,9 +109,7 @@ pub fn get_active_validator_duties<T: BeaconChainTypes>(
.map(|validator| validator.pubkey.clone())
.collect();
let duties = return_validator_duties(beacon_chain, epoch, validator_pubkeys)?;
ResponseBuilder::new(&req)?.body_no_ssz(&duties)
return_validator_duties(&ctx.beacon_chain, epoch, validator_pubkeys)
}
/// Helper function to return the state that can be used to determine the duties for some `epoch`.
@@ -165,7 +145,7 @@ pub fn get_state_for_epoch<T: BeaconChainTypes>(
/// Helper function to get the duties for some `validator_pubkeys` in some `epoch`.
fn return_validator_duties<T: BeaconChainTypes>(
beacon_chain: Arc<BeaconChain<T>>,
beacon_chain: &BeaconChain<T>,
epoch: Epoch,
validator_pubkeys: Vec<PublicKeyBytes>,
) -> Result<Vec<ValidatorDutyBytes>, ApiError> {
@@ -281,10 +261,9 @@ fn return_validator_duties<T: BeaconChainTypes>(
/// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator.
pub fn get_new_beacon_block<T: BeaconChainTypes>(
req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>,
log: Logger,
) -> ApiResult {
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<BeaconBlock<T::EthSpec>, ApiError> {
let query = UrlQuery::from_request(&req)?;
let slot = query.slot()?;
@@ -296,11 +275,12 @@ pub fn get_new_beacon_block<T: BeaconChainTypes>(
None
};
let (new_block, _state) = beacon_chain
let (new_block, _state) = ctx
.beacon_chain
.produce_block(randao_reveal, slot, validator_graffiti)
.map_err(|e| {
error!(
log,
ctx.log,
"Error whilst producing block";
"error" => format!("{:?}", e)
);
@@ -311,48 +291,40 @@ pub fn get_new_beacon_block<T: BeaconChainTypes>(
))
})?;
ResponseBuilder::new(&req)?.body(&new_block)
Ok(new_block)
}
/// HTTP Handler to publish a SignedBeaconBlock, which has been signed by a validator.
pub async fn publish_beacon_block<T: BeaconChainTypes>(
req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>,
network_chan: NetworkChannel<T::EthSpec>,
log: Logger,
) -> ApiResult {
try_future!(check_content_type_for_json(&req));
let response_builder = ResponseBuilder::new(&req);
pub fn publish_beacon_block<T: BeaconChainTypes>(
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<(), ApiError> {
let body = req.into_body();
let chunks = hyper::body::to_bytes(body)
.await
.map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
serde_json::from_slice(&chunks).map_err(|e| {
serde_json::from_slice(&body).map_err(|e| {
ApiError::BadRequest(format!("Unable to parse JSON into SignedBeaconBlock: {:?}", e))
})
.and_then(move |block: SignedBeaconBlock<T::EthSpec>| {
let slot = block.slot();
match beacon_chain.process_block(block.clone()) {
match ctx.beacon_chain.process_block(block.clone()) {
Ok(block_root) => {
// Block was processed, publish via gossipsub
info!(
log,
ctx.log,
"Block from local validator";
"block_root" => format!("{}", block_root),
"block_slot" => slot,
);
publish_beacon_block_to_network::<T>(network_chan, block)?;
publish_beacon_block_to_network::<T>(&ctx.network_chan, block)?;
// Run the fork choice algorithm and enshrine a new canonical head, if
// found.
//
// The new head may or may not be the block we just received.
if let Err(e) = beacon_chain.fork_choice() {
if let Err(e) = ctx.beacon_chain.fork_choice() {
error!(
log,
ctx.log,
"Failed to find beacon chain head";
"error" => format!("{:?}", e)
);
@@ -366,9 +338,9 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
// - Excessive time between block produce and publish.
// - A validator is using another beacon node to produce blocks and
// submitting them here.
if beacon_chain.head()?.beacon_block_root != block_root {
if ctx.beacon_chain.head()?.beacon_block_root != block_root {
warn!(
log,
ctx.log,
"Block from validator is not head";
"desc" => "potential re-org",
);
@@ -380,7 +352,7 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
}
Err(BlockError::BeaconChainError(e)) => {
error!(
log,
ctx.log,
"Error whilst processing block";
"error" => format!("{:?}", e)
);
@@ -392,7 +364,7 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
}
Err(other) => {
warn!(
log,
ctx.log,
"Invalid block from local validator";
"outcome" => format!("{:?}", other)
);
@@ -404,41 +376,41 @@ pub async fn publish_beacon_block<T: BeaconChainTypes>(
}
}
})
.and_then(|_| response_builder?.body_no_ssz(&()))
}
/// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator.
pub fn get_new_attestation<T: BeaconChainTypes>(
req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>,
) -> ApiResult {
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<Attestation<T::EthSpec>, ApiError> {
let query = UrlQuery::from_request(&req)?;
let slot = query.slot()?;
let index = query.committee_index()?;
let attestation = beacon_chain
ctx.beacon_chain
.produce_unaggregated_attestation(slot, index)
.map_err(|e| ApiError::BadRequest(format!("Unable to produce attestation: {:?}", e)))?;
ResponseBuilder::new(&req)?.body(&attestation)
.map_err(|e| ApiError::BadRequest(format!("Unable to produce attestation: {:?}", e)))
}
/// HTTP Handler to retrieve the aggregate attestation for a slot
pub fn get_aggregate_attestation<T: BeaconChainTypes>(
req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>,
) -> ApiResult {
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<Attestation<T::EthSpec>, ApiError> {
let query = UrlQuery::from_request(&req)?;
let attestation_data = query.attestation_data()?;
match beacon_chain.get_aggregated_attestation(&attestation_data) {
Ok(Some(attestation)) => ResponseBuilder::new(&req)?.body(&attestation),
match ctx
.beacon_chain
.get_aggregated_attestation(&attestation_data)
{
Ok(Some(attestation)) => Ok(attestation),
Ok(None) => Err(ApiError::NotFound(format!(
"No matching aggregate attestation for slot {:?} is known in slot {:?}",
attestation_data.slot,
beacon_chain.slot()
ctx.beacon_chain.slot()
))),
Err(e) => Err(ApiError::ServerError(format!(
"Unable to obtain attestation: {:?}",
@@ -448,22 +420,13 @@ pub fn get_aggregate_attestation<T: BeaconChainTypes>(
}
/// HTTP Handler to publish a list of Attestations, which have been signed by a number of validators.
pub async fn publish_attestations<T: BeaconChainTypes>(
req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>,
network_chan: NetworkChannel<T::EthSpec>,
log: Logger,
) -> ApiResult {
try_future!(check_content_type_for_json(&req));
let response_builder = ResponseBuilder::new(&req);
pub fn publish_attestations<T: BeaconChainTypes>(
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<(), ApiError> {
let bytes = req.into_body();
let body = req.into_body();
let chunk = hyper::body::to_bytes(body)
.await
.map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
let chunks = chunk.iter().cloned().collect::<Vec<u8>>();
serde_json::from_slice(&chunks.as_slice())
serde_json::from_slice(&bytes)
.map_err(|e| {
ApiError::BadRequest(format!(
"Unable to deserialize JSON into a list of attestations: {:?}",
@@ -478,12 +441,12 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
.enumerate()
.map(|(i, (attestation, subnet_id))| {
process_unaggregated_attestation(
&beacon_chain,
network_chan.clone(),
&ctx.beacon_chain,
ctx.network_chan.clone(),
attestation,
subnet_id,
i,
&log,
&ctx.log,
)
})
.collect::<Vec<Result<_, _>>>()
@@ -493,7 +456,7 @@ pub async fn publish_attestations<T: BeaconChainTypes>(
//
// Note: this will only provide info about the _first_ failure, not all failures.
.and_then(|processing_results| processing_results.into_iter().try_for_each(|result| result))
.and_then(|_| response_builder?.body_no_ssz(&()))
.map(|_| ())
}
/// Processes an unaggregrated attestation that was included in a list of attestations with the
@@ -566,21 +529,13 @@ fn process_unaggregated_attestation<T: BeaconChainTypes>(
}
/// HTTP Handler to publish an Attestation, which has been signed by a validator.
#[allow(clippy::redundant_clone)] // false positives in this function.
pub async fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>,
network_chan: NetworkChannel<T::EthSpec>,
log: Logger,
) -> ApiResult {
try_future!(check_content_type_for_json(&req));
let response_builder = ResponseBuilder::new(&req);
pub fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
req: Request<Vec<u8>>,
ctx: Arc<Context<T>>,
) -> Result<(), ApiError> {
let body = req.into_body();
let chunk = hyper::body::to_bytes(body)
.await
.map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}", e)))?;
let chunks = chunk.iter().cloned().collect::<Vec<u8>>();
serde_json::from_slice(&chunks.as_slice())
serde_json::from_slice(&body)
.map_err(|e| {
ApiError::BadRequest(format!(
"Unable to deserialize JSON into a list of SignedAggregateAndProof: {:?}",
@@ -595,11 +550,11 @@ pub async fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
.enumerate()
.map(|(i, signed_aggregate)| {
process_aggregated_attestation(
&beacon_chain,
network_chan.clone(),
&ctx.beacon_chain,
ctx.network_chan.clone(),
signed_aggregate,
i,
&log,
&ctx.log,
)
})
.collect::<Vec<Result<_, _>>>()
@@ -609,7 +564,6 @@ pub async fn publish_aggregate_and_proofs<T: BeaconChainTypes>(
//
// Note: this will only provide info about the _first_ failure, not all failures.
.and_then(|processing_results| processing_results.into_iter().try_for_each(|result| result))
.and_then(|_| response_builder?.body_no_ssz(&()))
}
/// Processes an aggregrated attestation that was included in a list of attestations with the index