gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-10 12:11:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement standard eth2.0 API (#1569)

Browse Source 
- Resolves #1550
- Resolves #824
- Resolves #825
- Resolves #1131
- Resolves #1411
- Resolves #1256
- Resolve #1177

- Includes the `ShufflingId` struct initially defined in #1492. That PR is now closed and the changes are included here, with significant bug fixes.
- Implement the https://github.com/ethereum/eth2.0-APIs in a new `http_api` crate using `warp`. This replaces the `rest_api` crate.
- Add a new `common/eth2` crate which provides a wrapper around `reqwest`, providing the HTTP client that is used by the validator client and for testing. This replaces the `common/remote_beacon_node` crate.
- Create a `http_metrics` crate which is a dedicated server for Prometheus metrics (they are no longer served on the same port as the REST API). We now have flags for `--metrics`, `--metrics-address`, etc.
- Allow the `subnet_id` to be an optional parameter for `VerifiedUnaggregatedAttestation::verify`. This means it does not need to be provided unnecessarily by the validator client.
- Move `fn map_attestation_committee` in `mod beacon_chain::attestation_verification` to a new `fn with_committee_cache` on the `BeaconChain` so the same cache can be used for obtaining validator duties.
- Add some other helpers to `BeaconChain` to assist with common API duties (e.g., `block_root_at_slot`, `head_beacon_block_root`).
- Change the `NaiveAggregationPool` so it can index attestations by `hash_tree_root(attestation.data)`. This is a requirement of the API.
- Add functions to `BeaconChainHarness` to allow it to create slashings and exits.
- Allow for `eth1::Eth1NetworkId` to go to/from a `String`.
- Add functions to the `OperationPool` to allow getting all objects in the pool.
- Add function to `BeaconState` to check if a committee cache is initialized.
- Fix bug where `seconds_per_eth1_block` was not transferring over from `YamlConfig` to `ChainSpec`.
- Add the `deposit_contract_address` to `YamlConfig` and `ChainSpec`. We needed to be able to return it in an API response.
- Change some uses of serde `serialize_with` and `deserialize_with` to a single use of `with` (code quality).
- Impl `Display` and `FromStr` for several BLS fields.
- Check for clock discrepancy when VC polls BN for sync state (with +/- 1 slot tolerance). This is not intended to be comprehensive, it was just easy to do.

- See #1434 for a per-endpoint overview.
- Seeking clarity here: https://github.com/ethereum/eth2.0-APIs/issues/75

- [x] Add docs for prom port to close #1256
- [x] Follow up on this #1177
- [x] ~~Follow up with #1424~~ Will fix in future PR.
- [x] Follow up with #1411
- [x] ~~Follow up with  #1260~~ Will fix in future PR.
- [x] Add quotes to all integers.
- [x] Remove `rest_types`
- [x] Address missing beacon block error. (#1629)
- [x] ~~Add tests for lighthouse/peers endpoints~~ Wontfix
- [x] ~~Follow up with validator status proposal~~ Tracked in #1434
- [x] Unify graffiti structs
- [x] ~~Start server when waiting for genesis?~~ Will fix in future PR.
- [x] TODO in http_api tests
- [x] Move lighthouse endpoints off /eth/v1
- [x] Update docs to link to standard

- ~~Blocked on #1586~~

Co-authored-by: Michael Sproul <michael@sigmaprime.io>
This commit is contained in:
Paul Hauner
2020-09-29 03:46:54 +00:00
parent 8e20176337
commit cdec3cec18
156 changed files with 8862 additions and 8916 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
beacon_node/beacon_chain/src/attestation_verification.rs
View File

@@ -28,8 +28,7 @@
use crate::{
beacon_chain::{
ATTESTATION_CACHE_LOCK_TIMEOUT, HEAD_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY,
VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT,
HEAD_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY, VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT,
},
metrics,
observed_attestations::ObserveOutcome,
@@ -38,12 +37,10 @@ use crate::{
};
use bls::verify_signature_sets;
use proto_array::Block as ProtoBlock;
use slog::debug;
use slot_clock::SlotClock;
use state_processing::{
common::get_indexed_attestation,
per_block_processing::errors::AttestationValidationError,
per_slot_processing,
signature_sets::{
indexed_attestation_signature_set_from_pubkeys,
signed_aggregate_selection_proof_signature_set, signed_aggregate_signature_set,
@@ -53,7 +50,7 @@ use std::borrow::Cow;
use tree_hash::TreeHash;
use types::{
Attestation, BeaconCommittee, CommitteeIndex, Epoch, EthSpec, Hash256, IndexedAttestation,
RelativeEpoch, SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
};
/// Returned when an attestation was not successfully verified. It might not have been verified for
@@ -267,6 +264,7 @@ pub struct VerifiedAggregatedAttestation<T: BeaconChainTypes> {
pub struct VerifiedUnaggregatedAttestation<T: BeaconChainTypes> {
attestation: Attestation<T::EthSpec>,
indexed_attestation: IndexedAttestation<T::EthSpec>,
subnet_id: SubnetId,
}
/// Custom `Clone` implementation is to avoid the restrictive trait bounds applied by the usual derive
@@ -276,6 +274,7 @@ impl<T: BeaconChainTypes> Clone for VerifiedUnaggregatedAttestation<T> {
Self {
attestation: self.attestation.clone(),
indexed_attestation: self.indexed_attestation.clone(),
subnet_id: self.subnet_id,
}
}
}
@@ -428,6 +427,11 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
pub fn attestation(&self) -> &Attestation<T::EthSpec> {
&self.signed_aggregate.message.aggregate
}
/// Returns the underlying `signed_aggregate`.
pub fn aggregate(&self) -> &SignedAggregateAndProof<T::EthSpec> {
&self.signed_aggregate
}
}
impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
@@ -438,7 +442,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
/// verify that it was received on the correct subnet.
pub fn verify(
attestation: Attestation<T::EthSpec>,
subnet_id: SubnetId,
subnet_id: Option<SubnetId>,
chain: &BeaconChain<T>,
) -> Result<Self, Error> {
let attestation_epoch = attestation.data.slot.epoch(T::EthSpec::slots_per_epoch());
@@ -513,13 +517,15 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
)
.map_err(BeaconChainError::from)?;
// Ensure the attestation is from the correct subnet.
if subnet_id != expected_subnet_id {
return Err(Error::InvalidSubnetId {
received: subnet_id,
expected: expected_subnet_id,
});
}
// If a subnet was specified, ensure that subnet is correct.
if let Some(subnet_id) = subnet_id {
if subnet_id != expected_subnet_id {
return Err(Error::InvalidSubnetId {
received: subnet_id,
expected: expected_subnet_id,
});
}
};
let validator_index = *indexed_attestation
.attesting_indices
@@ -564,6 +570,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
Ok(Self {
attestation,
indexed_attestation,
subnet_id: expected_subnet_id,
})
}
@@ -572,6 +579,11 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
chain.add_to_naive_aggregation_pool(self)
}
/// Returns the correct subnet for the attestation.
pub fn subnet_id(&self) -> SubnetId {
self.subnet_id
}
/// Returns the wrapped `attestation`.
pub fn attestation(&self) -> &Attestation<T::EthSpec> {
&self.attestation
@@ -587,6 +599,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
}
/// Returns `Ok(())` if the `attestation.data.beacon_block_root` is known to this chain.
/// You can use this `shuffling_id` to read from the shuffling cache.
///
/// The block root may not be known for two reasons:
///
@@ -615,6 +628,7 @@ fn verify_head_block_is_known<T: BeaconChainTypes>(
});
}
}
Ok(block)
} else {
Err(Error::UnknownHeadBlock {
@@ -770,7 +784,7 @@ type CommitteesPerSlot = u64;
/// Returns the `indexed_attestation` and committee count per slot for the `attestation` using the
/// public keys cached in the `chain`.
pub fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
chain: &BeaconChain<T>,
attestation: &Attestation<T::EthSpec>,
) -> Result<(IndexedAttestation<T::EthSpec>, CommitteesPerSlot), Error> {
@@ -790,8 +804,8 @@ pub fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
///
/// If the committee for `attestation` isn't found in the `shuffling_cache`, we will read a state
/// from disk and then update the `shuffling_cache`.
pub fn map_attestation_committee<'a, T, F, R>(
chain: &'a BeaconChain<T>,
fn map_attestation_committee<T, F, R>(
chain: &BeaconChain<T>,
attestation: &Attestation<T::EthSpec>,
map_fn: F,
) -> Result<R, Error>
@@ -809,104 +823,23 @@ where
// processing an attestation that does not include our latest finalized block in its chain.
//
// We do not delay consideration for later, we simply drop the attestation.
let target_block = chain
.fork_choice
.read()
.get_block(&target.root)
.ok_or_else(|| Error::UnknownTargetRoot(target.root))?;
// Obtain the shuffling cache, timing how long we wait.
let cache_wait_timer =
metrics::start_timer(&metrics::ATTESTATION_PROCESSING_SHUFFLING_CACHE_WAIT_TIMES);
let mut shuffling_cache = chain
.shuffling_cache
.try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
.ok_or_else(|| BeaconChainError::AttestationCacheLockTimeout)?;
metrics::stop_timer(cache_wait_timer);
if let Some(committee_cache) = shuffling_cache.get(attestation_epoch, target.root) {
let committees_per_slot = committee_cache.committees_per_slot();
committee_cache
.get_beacon_committee(attestation.data.slot, attestation.data.index)
.map(|committee| map_fn((committee, committees_per_slot)))
.unwrap_or_else(|| {
Err(Error::NoCommitteeForSlotAndIndex {
slot: attestation.data.slot,
index: attestation.data.index,
})
})
} else {
// Drop the shuffling cache to avoid holding the lock for any longer than
// required.
drop(shuffling_cache);
debug!(
chain.log,
"Attestation processing cache miss";
"attn_epoch" => attestation_epoch.as_u64(),
"target_block_epoch" => target_block.slot.epoch(T::EthSpec::slots_per_epoch()).as_u64(),
);
let state_read_timer =
metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_READ_TIMES);
let mut state = chain
.store
.get_inconsistent_state_for_attestation_verification_only(
&target_block.state_root,
Some(target_block.slot),
)
.map_err(BeaconChainError::from)?
.ok_or_else(|| BeaconChainError::MissingBeaconState(target_block.state_root))?;
metrics::stop_timer(state_read_timer);
let state_skip_timer =
metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_SKIP_TIMES);
while state.current_epoch() + 1 < attestation_epoch {
// Here we tell `per_slot_processing` to skip hashing the state and just
// use the zero hash instead.
//
// The state roots are not useful for the shuffling, so there's no need to
// compute them.
per_slot_processing(&mut state, Some(Hash256::zero()), &chain.spec)
.map_err(BeaconChainError::from)?;
}
metrics::stop_timer(state_skip_timer);
let committee_building_timer =
metrics::start_timer(&metrics::ATTESTATION_PROCESSING_COMMITTEE_BUILDING_TIMES);
let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), attestation_epoch)
.map_err(BeaconChainError::IncorrectStateForAttestation)?;
state
.build_committee_cache(relative_epoch, &chain.spec)
.map_err(BeaconChainError::from)?;
let committee_cache = state
.committee_cache(relative_epoch)
.map_err(BeaconChainError::from)?;
chain
.shuffling_cache
.try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
.ok_or_else(|| BeaconChainError::AttestationCacheLockTimeout)?
.insert(attestation_epoch, target.root, committee_cache);
metrics::stop_timer(committee_building_timer);
let committees_per_slot = committee_cache.committees_per_slot();
committee_cache
.get_beacon_committee(attestation.data.slot, attestation.data.index)
.map(|committee| map_fn((committee, committees_per_slot)))
.unwrap_or_else(|| {
Err(Error::NoCommitteeForSlotAndIndex {
slot: attestation.data.slot,
index: attestation.data.index,
})
})
if !chain.fork_choice.read().contains_block(&target.root) {
return Err(Error::UnknownTargetRoot(target.root));
}
chain
.with_committee_cache(target.root, attestation_epoch, |committee_cache| {
let committees_per_slot = committee_cache.committees_per_slot();
Ok(committee_cache
.get_beacon_committee(attestation.data.slot, attestation.data.index)
.map(|committee| map_fn((committee, committees_per_slot)))
.unwrap_or_else(|| {
Err(Error::NoCommitteeForSlotAndIndex {
slot: attestation.data.slot,
index: attestation.data.index,
})
}))
})
.map_err(BeaconChainError::from)?
}

282
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@@ -21,7 +21,7 @@ use crate::observed_block_producers::ObservedBlockProducers;
use crate::observed_operations::{ObservationOutcome, ObservedOperations};
use crate::persisted_beacon_chain::PersistedBeaconChain;
use crate::persisted_fork_choice::PersistedForkChoice;
use crate::shuffling_cache::ShufflingCache;
use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache};
use crate::snapshot_cache::SnapshotCache;
use crate::timeout_rw_lock::TimeoutRwLock;
use crate::validator_pubkey_cache::ValidatorPubkeyCache;
@@ -31,7 +31,6 @@ use fork_choice::ForkChoice;
use itertools::process_results;
use operation_pool::{OperationPool, PersistedOperationPool};
use parking_lot::RwLock;
use regex::bytes::Regex;
use slog::{crit, debug, error, info, trace, warn, Logger};
use slot_clock::SlotClock;
use state_processing::{
@@ -201,6 +200,8 @@ pub struct BeaconChain<T: BeaconChainTypes> {
pub(crate) canonical_head: TimeoutRwLock<BeaconSnapshot<T::EthSpec>>,
/// The root of the genesis block.
pub genesis_block_root: Hash256,
/// The root of the genesis state.
pub genesis_state_root: Hash256,
/// The root of the list of genesis validators, used during syncing.
pub genesis_validators_root: Hash256,
@@ -459,6 +460,30 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
}
}
/// Returns the block at the given slot, if any. Only returns blocks in the canonical chain.
///
/// ## Errors
///
/// May return a database error.
pub fn state_root_at_slot(&self, slot: Slot) -> Result<Option<Hash256>, Error> {
process_results(self.rev_iter_state_roots()?, |mut iter| {
iter.find(|(_, this_slot)| *this_slot == slot)
.map(|(root, _)| root)
})
}
/// Returns the block root at the given slot, if any. Only returns roots in the canonical chain.
///
/// ## Errors
///
/// May return a database error.
pub fn block_root_at_slot(&self, slot: Slot) -> Result<Option<Hash256>, Error> {
process_results(self.rev_iter_block_roots()?, |mut iter| {
iter.find(|(_, this_slot)| *this_slot == slot)
.map(|(root, _)| root)
})
}
/// Returns the block at the given root, if any.
///
/// ## Errors
@@ -506,6 +531,30 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
f(&head_lock)
}
/// Returns the beacon block root at the head of the canonical chain.
///
/// See `Self::head` for more information.
pub fn head_beacon_block_root(&self) -> Result<Hash256, Error> {
self.with_head(|s| Ok(s.beacon_block_root))
}
/// Returns the beacon block at the head of the canonical chain.
///
/// See `Self::head` for more information.
pub fn head_beacon_block(&self) -> Result<SignedBeaconBlock<T::EthSpec>, Error> {
self.with_head(|s| Ok(s.beacon_block.clone()))
}
/// Returns the beacon state at the head of the canonical chain.
///
/// See `Self::head` for more information.
pub fn head_beacon_state(&self) -> Result<BeaconState<T::EthSpec>, Error> {
self.with_head(|s| {
Ok(s.beacon_state
.clone_with(CloneConfig::committee_caches_only()))
})
}
/// Returns info representing the head block and state.
///
/// A summarized version of `Self::head` that involves less cloning.
@@ -719,46 +768,20 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.map_err(Into::into)
}
/// Returns the attestation slot and committee index for a given validator index.
/// Returns the attestation duties for a given validator index.
///
/// Information is read from the current state, so only information from the present and prior
/// epoch is available.
pub fn validator_attestation_slot_and_index(
pub fn validator_attestation_duty(
&self,
validator_index: usize,
epoch: Epoch,
) -> Result<Option<(Slot, u64)>, Error> {
let as_epoch = |slot: Slot| slot.epoch(T::EthSpec::slots_per_epoch());
let head_state = &self.head()?.beacon_state;
) -> Result<Option<AttestationDuty>, Error> {
let head_block_root = self.head_beacon_block_root()?;
let mut state = if epoch == as_epoch(head_state.slot) {
self.head()?.beacon_state
} else {
// The block proposer shuffling is not affected by the state roots, so we don't need to
// calculate them.
self.state_at_slot(
epoch.start_slot(T::EthSpec::slots_per_epoch()),
StateSkipConfig::WithoutStateRoots,
)?
};
state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
if as_epoch(state.slot) != epoch {
return Err(Error::InvariantViolated(format!(
"Epochs in consistent in attestation duties lookup: state: {}, requested: {}",
as_epoch(state.slot),
epoch
)));
}
if let Some(attestation_duty) =
state.get_attestation_duties(validator_index, RelativeEpoch::Current)?
{
Ok(Some((attestation_duty.slot, attestation_duty.index)))
} else {
Ok(None)
}
self.with_committee_cache(head_block_root, epoch, |committee_cache| {
Ok(committee_cache.get_attestation_duties(validator_index))
})
}
/// Returns an aggregated `Attestation`, if any, that has a matching `attestation.data`.
@@ -767,11 +790,22 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
pub fn get_aggregated_attestation(
&self,
data: &AttestationData,
) -> Result<Option<Attestation<T::EthSpec>>, Error> {
) -> Option<Attestation<T::EthSpec>> {
self.naive_aggregation_pool.read().get(data)
}
/// Returns an aggregated `Attestation`, if any, that has a matching
/// `attestation.data.tree_hash_root()`.
///
/// The attestation will be obtained from `self.naive_aggregation_pool`.
pub fn get_aggregated_attestation_by_slot_and_root(
&self,
slot: Slot,
attestation_data_root: &Hash256,
) -> Option<Attestation<T::EthSpec>> {
self.naive_aggregation_pool
.read()
.get(data)
.map_err(Into::into)
.get_by_slot_and_root(slot, attestation_data_root)
}
/// Produce an unaggregated `Attestation` that is valid for the given `slot` and `index`.
@@ -898,7 +932,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
pub fn verify_unaggregated_attestation_for_gossip(
&self,
attestation: Attestation<T::EthSpec>,
subnet_id: SubnetId,
subnet_id: Option<SubnetId>,
) -> Result<VerifiedUnaggregatedAttestation<T>, AttestationError> {
metrics::inc_counter(&metrics::UNAGGREGATED_ATTESTATION_PROCESSING_REQUESTS);
let _timer =
@@ -1320,11 +1354,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
block: SignedBeaconBlock<T::EthSpec>,
) -> Result<GossipVerifiedBlock<T>, BlockError<T::EthSpec>> {
let slot = block.message.slot;
#[allow(clippy::invalid_regex)]
let re = Regex::new("\\p{C}").expect("regex is valid");
let graffiti_string =
String::from_utf8_lossy(&re.replace_all(&block.message.body.graffiti[..], &b""[..]))
.to_string();
let graffiti_string = block.message.body.graffiti.as_utf8_lossy();
match GossipVerifiedBlock::new(block, self) {
Ok(verified) => {
@@ -1449,8 +1479,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
) -> Result<Hash256, BlockError<T::EthSpec>> {
let signed_block = fully_verified_block.block;
let block_root = fully_verified_block.block_root;
let state = fully_verified_block.state;
let parent_block = fully_verified_block.parent_block;
let mut state = fully_verified_block.state;
let current_slot = self.slot()?;
let mut ops = fully_verified_block.intermediate_states;
@@ -1482,29 +1511,25 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.ok_or_else(|| Error::ValidatorPubkeyCacheLockTimeout)?
.import_new_pubkeys(&state)?;
// If the imported block is in the previous or current epochs (according to the
// wall-clock), check to see if this is the first block of the epoch. If so, add the
// committee to the shuffling cache.
if state.current_epoch() + 1 >= self.epoch()?
&& parent_block.slot().epoch(T::EthSpec::slots_per_epoch()) != state.current_epoch()
{
let mut shuffling_cache = self
// For the current and next epoch of this state, ensure we have the shuffling from this
// block in our cache.
for relative_epoch in &[RelativeEpoch::Current, RelativeEpoch::Next] {
let shuffling_id = ShufflingId::new(block_root, &state, *relative_epoch)?;
let shuffling_is_cached = self
.shuffling_cache
.try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
.ok_or_else(|| Error::AttestationCacheLockTimeout)?;
.try_read_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
.ok_or_else(|| Error::AttestationCacheLockTimeout)?
.contains(&shuffling_id);
let committee_cache = state.committee_cache(RelativeEpoch::Current)?;
let epoch_start_slot = state
.current_epoch()
.start_slot(T::EthSpec::slots_per_epoch());
let target_root = if state.slot == epoch_start_slot {
block_root
} else {
*state.get_block_root(epoch_start_slot)?
};
shuffling_cache.insert(state.current_epoch(), target_root, committee_cache);
if !shuffling_is_cached {
state.build_committee_cache(*relative_epoch, &self.spec)?;
let committee_cache = state.committee_cache(*relative_epoch)?;
self.shuffling_cache
.try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
.ok_or_else(|| Error::AttestationCacheLockTimeout)?
.insert(shuffling_id, committee_cache);
}
}
let mut fork_choice = self.fork_choice.write();
@@ -1992,6 +2017,129 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(())
}
/// Runs the `map_fn` with the committee cache for `shuffling_epoch` from the chain with head
/// `head_block_root`.
///
/// It's not necessary that `head_block_root` matches our current view of the chain, it can be
/// any block that is:
///
/// - Known to us.
/// - The finalized block or a descendant of the finalized block.
///
/// It would be quite common for attestation verification operations to use a `head_block_root`
/// that differs from our view of the head.
///
/// ## Important
///
/// This function is **not** suitable for determining proposer duties.
///
/// ## Notes
///
/// This function exists in this odd "map" pattern because efficiently obtaining a committee
/// can be complex. It might involve reading straight from the `beacon_chain.shuffling_cache`
/// or it might involve reading it from a state from the DB. Due to the complexities of
/// `RwLock`s on the shuffling cache, a simple `Cow` isn't suitable here.
///
/// If the committee for `(head_block_root, shuffling_epoch)` isn't found in the
/// `shuffling_cache`, we will read a state from disk and then update the `shuffling_cache`.
pub(crate) fn with_committee_cache<F, R>(
&self,
head_block_root: Hash256,
shuffling_epoch: Epoch,
map_fn: F,
) -> Result<R, Error>
where
F: Fn(&CommitteeCache) -> Result<R, Error>,
{
let head_block = self
.fork_choice
.read()
.get_block(&head_block_root)
.ok_or_else(|| Error::MissingBeaconBlock(head_block_root))?;
let shuffling_id = BlockShufflingIds {
current: head_block.current_epoch_shuffling_id.clone(),
next: head_block.next_epoch_shuffling_id.clone(),
block_root: head_block.root,
}
.id_for_epoch(shuffling_epoch)
.ok_or_else(|| Error::InvalidShufflingId {
shuffling_epoch,
head_block_epoch: head_block.slot.epoch(T::EthSpec::slots_per_epoch()),
})?;
// Obtain the shuffling cache, timing how long we wait.
let cache_wait_timer =
metrics::start_timer(&metrics::ATTESTATION_PROCESSING_SHUFFLING_CACHE_WAIT_TIMES);
let mut shuffling_cache = self
.shuffling_cache
.try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
.ok_or_else(|| Error::AttestationCacheLockTimeout)?;
metrics::stop_timer(cache_wait_timer);
if let Some(committee_cache) = shuffling_cache.get(&shuffling_id) {
map_fn(committee_cache)
} else {
// Drop the shuffling cache to avoid holding the lock for any longer than
// required.
drop(shuffling_cache);
debug!(
self.log,
"Committee cache miss";
"shuffling_epoch" => shuffling_epoch.as_u64(),
"head_block_root" => head_block_root.to_string(),
);
let state_read_timer =
metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_READ_TIMES);
let mut state = self
.store
.get_inconsistent_state_for_attestation_verification_only(
&head_block.state_root,
Some(head_block.slot),
)?
.ok_or_else(|| Error::MissingBeaconState(head_block.state_root))?;
metrics::stop_timer(state_read_timer);
let state_skip_timer =
metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_SKIP_TIMES);
while state.current_epoch() + 1 < shuffling_epoch {
// Here we tell `per_slot_processing` to skip hashing the state and just
// use the zero hash instead.
//
// The state roots are not useful for the shuffling, so there's no need to
// compute them.
per_slot_processing(&mut state, Some(Hash256::zero()), &self.spec)
.map_err(Error::from)?;
}
metrics::stop_timer(state_skip_timer);
let committee_building_timer =
metrics::start_timer(&metrics::ATTESTATION_PROCESSING_COMMITTEE_BUILDING_TIMES);
let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), shuffling_epoch)
.map_err(Error::IncorrectStateForAttestation)?;
state.build_committee_cache(relative_epoch, &self.spec)?;
let committee_cache = state.committee_cache(relative_epoch)?;
self.shuffling_cache
.try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
.ok_or_else(|| Error::AttestationCacheLockTimeout)?
.insert(shuffling_id, committee_cache);
metrics::stop_timer(committee_building_timer);
map_fn(&committee_cache)
}
}
/// Returns `true` if the given block root has not been processed.
pub fn is_new_block_root(&self, beacon_block_root: &Hash256) -> Result<bool, Error> {
Ok(!self

10
beacon_node/beacon_chain/src/builder.rs
View File

@@ -374,8 +374,13 @@ where
let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis);
let fork_choice = ForkChoice::from_genesis(fc_store, &genesis.beacon_block.message)
.map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?;
let fork_choice = ForkChoice::from_genesis(
fc_store,
genesis.beacon_block_root,
&genesis.beacon_block.message,
&genesis.beacon_state,
)
.map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?;
self.fork_choice = Some(fork_choice);
self.genesis_time = Some(genesis.beacon_state.genesis_time);
@@ -561,6 +566,7 @@ where
observed_attester_slashings: <_>::default(),
eth1_chain: self.eth1_chain,
genesis_validators_root: canonical_head.beacon_state.genesis_validators_root,
genesis_state_root: canonical_head.beacon_state_root,
canonical_head: TimeoutRwLock::new(canonical_head.clone()),
genesis_block_root,
fork_choice: RwLock::new(fork_choice),

4
beacon_node/beacon_chain/src/errors.rs
View File

@@ -83,6 +83,10 @@ pub enum BeaconChainError {
ObservedBlockProducersError(ObservedBlockProducersError),
PruningError(PruningError),
ArithError(ArithError),
InvalidShufflingId {
shuffling_epoch: Epoch,
head_block_epoch: Epoch,
},
}
easy_from_to!(SlotProcessingError, BeaconChainError);

47
beacon_node/beacon_chain/src/naive_aggregation_pool.rs
View File

@@ -1,7 +1,9 @@
use crate::metrics;
use std::collections::HashMap;
use types::{Attestation, AttestationData, EthSpec, Slot};
use tree_hash::TreeHash;
use types::{Attestation, AttestationData, EthSpec, Hash256, Slot};
type AttestationDataRoot = Hash256;
/// The number of slots that will be stored in the pool.
///
/// For example, if `SLOTS_RETAINED == 3` and the pool is pruned at slot `6`, then all attestations
@@ -53,7 +55,7 @@ pub enum Error {
/// A collection of `Attestation` objects, keyed by their `attestation.data`. Enforces that all
/// `attestation` are from the same slot.
struct AggregatedAttestationMap<E: EthSpec> {
map: HashMap<AttestationData, Attestation<E>>,
map: HashMap<AttestationDataRoot, Attestation<E>>,
}
impl<E: EthSpec> AggregatedAttestationMap<E> {
@@ -87,7 +89,9 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
return Err(Error::MoreThanOneAggregationBitSet(set_bits.len()));
}
if let Some(existing_attestation) = self.map.get_mut(&a.data) {
let attestation_data_root = a.data.tree_hash_root();
if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) {
if existing_attestation
.aggregation_bits
.get(committee_index)
@@ -107,7 +111,7 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
));
}
self.map.insert(a.data.clone(), a.clone());
self.map.insert(attestation_data_root, a.clone());
Ok(InsertOutcome::NewAttestationData { committee_index })
}
}
@@ -115,8 +119,13 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
/// Returns an aggregated `Attestation` with the given `data`, if any.
///
/// The given `a.data.slot` must match the slot that `self` was initialized with.
pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
Ok(self.map.get(data).cloned())
pub fn get(&self, data: &AttestationData) -> Option<Attestation<E>> {
self.map.get(&data.tree_hash_root()).cloned()
}
/// Returns an aggregated `Attestation` with the given `root`, if any.
pub fn get_by_root(&self, root: &AttestationDataRoot) -> Option<&Attestation<E>> {
self.map.get(root)
}
/// Iterate all attestations in `self`.
@@ -220,12 +229,19 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
}
/// Returns an aggregated `Attestation` with the given `data`, if any.
pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
pub fn get(&self, data: &AttestationData) -> Option<Attestation<E>> {
self.maps.get(&data.slot).and_then(|map| map.get(data))
}
/// Returns an aggregated `Attestation` with the given `data`, if any.
pub fn get_by_slot_and_root(
&self,
slot: Slot,
root: &AttestationDataRoot,
) -> Option<Attestation<E>> {
self.maps
.iter()
.find(|(slot, _map)| **slot == data.slot)
.map(|(_slot, map)| map.get(data))
.unwrap_or_else(|| Ok(None))
.get(&slot)
.and_then(|map| map.get_by_root(root).cloned())
}
/// Iterate all attestations in all slots of `self`.
@@ -338,8 +354,7 @@ mod tests {
let retrieved = pool
.get(&a.data)
.expect("should not error while getting attestation")
.expect("should get an attestation");
.expect("should not error while getting attestation");
assert_eq!(
retrieved, a,
"retrieved attestation should equal the one inserted"
@@ -378,8 +393,7 @@ mod tests {
let retrieved = pool
.get(&a_0.data)
.expect("should not error while getting attestation")
.expect("should get an attestation");
.expect("should not error while getting attestation");
let mut a_01 = a_0.clone();
a_01.aggregate(&a_1);
@@ -408,8 +422,7 @@ mod tests {
assert_eq!(
pool.get(&a_0.data)
.expect("should not error while getting attestation")
.expect("should get an attestation"),
.expect("should not error while getting attestation"),
retrieved,
"should not have aggregated different attestation data"
);

38
beacon_node/beacon_chain/src/shuffling_cache.rs
View File

@@ -1,6 +1,6 @@
use crate::metrics;
use lru::LruCache;
use types::{beacon_state::CommitteeCache, Epoch, Hash256};
use types::{beacon_state::CommitteeCache, Epoch, Hash256, ShufflingId};
/// The size of the LRU cache that stores committee caches for quicker verification.
///
@@ -14,7 +14,7 @@ const CACHE_SIZE: usize = 16;
/// It has been named `ShufflingCache` because `CommitteeCacheCache` is a bit weird and looks like
/// a find/replace error.
pub struct ShufflingCache {
cache: LruCache<(Epoch, Hash256), CommitteeCache>,
cache: LruCache<ShufflingId, CommitteeCache>,
}
impl ShufflingCache {
@@ -24,8 +24,8 @@ impl ShufflingCache {
}
}
pub fn get(&mut self, epoch: Epoch, root: Hash256) -> Option<&CommitteeCache> {
let opt = self.cache.get(&(epoch, root));
pub fn get(&mut self, key: &ShufflingId) -> Option<&CommitteeCache> {
let opt = self.cache.get(key);
if opt.is_some() {
metrics::inc_counter(&metrics::SHUFFLING_CACHE_HITS);
@@ -36,11 +36,37 @@ impl ShufflingCache {
opt
}
pub fn insert(&mut self, epoch: Epoch, root: Hash256, committee_cache: &CommitteeCache) {
let key = (epoch, root);
pub fn contains(&self, key: &ShufflingId) -> bool {
self.cache.contains(key)
}
pub fn insert(&mut self, key: ShufflingId, committee_cache: &CommitteeCache) {
if !self.cache.contains(&key) {
self.cache.put(key, committee_cache.clone());
}
}
}
/// Contains the shuffling IDs for a beacon block.
pub struct BlockShufflingIds {
pub current: ShufflingId,
pub next: ShufflingId,
pub block_root: Hash256,
}
impl BlockShufflingIds {
/// Returns the shuffling ID for the given epoch.
///
/// Returns `None` if `epoch` is prior to `self.current.shuffling_epoch`.
pub fn id_for_epoch(&self, epoch: Epoch) -> Option<ShufflingId> {
if epoch == self.current.shuffling_epoch {
Some(self.current.clone())
} else if epoch == self.next.shuffling_epoch {
Some(self.next.clone())
} else if epoch > self.next.shuffling_epoch {
Some(ShufflingId::from_components(epoch, self.block_root))
} else {
None
}
}
}

114
beacon_node/beacon_chain/src/test_utils.rs
View File

@@ -26,9 +26,11 @@ use store::{config::StoreConfig, BlockReplay, HotColdDB, ItemStore, LevelDB, Mem
use tempfile::{tempdir, TempDir};
use tree_hash::TreeHash;
use types::{
AggregateSignature, Attestation, BeaconState, BeaconStateHash, ChainSpec, Domain, Epoch,
EthSpec, Hash256, Keypair, SelectionProof, SignedAggregateAndProof, SignedBeaconBlock,
SignedBeaconBlockHash, SignedRoot, Slot, SubnetId,
AggregateSignature, Attestation, AttestationData, AttesterSlashing, BeaconState,
BeaconStateHash, ChainSpec, Checkpoint, Domain, Epoch, EthSpec, Hash256, IndexedAttestation,
Keypair, ProposerSlashing, SelectionProof, SignedAggregateAndProof, SignedBeaconBlock,
SignedBeaconBlockHash, SignedRoot, SignedVoluntaryExit, Slot, SubnetId, VariableList,
VoluntaryExit,
};
pub use types::test_utils::generate_deterministic_keypairs;
@@ -129,7 +131,7 @@ impl<E: EthSpec> BeaconChainHarness<BlockingMigratorEphemeralHarnessType<E>> {
let decorator = slog_term::PlainDecorator::new(slog_term::TestStdoutWriter);
let drain = slog_term::FullFormat::new(decorator).build();
let debug_level = slog::LevelFilter::new(drain, slog::Level::Debug);
let debug_level = slog::LevelFilter::new(drain, slog::Level::Critical);
let log = slog::Logger::root(std::sync::Mutex::new(debug_level).fuse(), o!());
let config = StoreConfig::default();
@@ -193,7 +195,7 @@ impl<E: EthSpec> BeaconChainHarness<NullMigratorEphemeralHarnessType<E>> {
let decorator = slog_term::PlainDecorator::new(slog_term::TestStdoutWriter);
let drain = slog_term::FullFormat::new(decorator).build();
let debug_level = slog::LevelFilter::new(drain, slog::Level::Debug);
let debug_level = slog::LevelFilter::new(drain, slog::Level::Critical);
let log = slog::Logger::root(std::sync::Mutex::new(debug_level).fuse(), o!());
let store = HotColdDB::open_ephemeral(config, spec.clone(), log.clone()).unwrap();
@@ -238,7 +240,7 @@ impl<E: EthSpec> BeaconChainHarness<BlockingMigratorDiskHarnessType<E>> {
let decorator = slog_term::PlainDecorator::new(slog_term::TestStdoutWriter);
let drain = slog_term::FullFormat::new(decorator).build();
let debug_level = slog::LevelFilter::new(drain, slog::Level::Debug);
let debug_level = slog::LevelFilter::new(drain, slog::Level::Critical);
let log = slog::Logger::root(std::sync::Mutex::new(debug_level).fuse(), o!());
let chain = BeaconChainBuilder::new(eth_spec_instance)
@@ -397,7 +399,7 @@ where
// If we produce two blocks for the same slot, they hash up to the same value and
// BeaconChain errors out with `BlockIsAlreadyKnown`. Vary the graffiti so that we produce
// different blocks each time.
self.chain.set_graffiti(self.rng.gen::<[u8; 32]>());
self.chain.set_graffiti(self.rng.gen::<[u8; 32]>().into());
let randao_reveal = {
let epoch = slot.epoch(E::slots_per_epoch());
@@ -442,8 +444,8 @@ where
let committee_count = state.get_committee_count_at_slot(state.slot).unwrap();
state
.get_beacon_committees_at_slot(state.slot)
.unwrap()
.get_beacon_committees_at_slot(attestation_slot)
.expect("should get committees")
.iter()
.map(|bc| {
bc.committee
@@ -570,7 +572,6 @@ where
let aggregate = self
.chain
.get_aggregated_attestation(&attestation.data)
.unwrap()
.unwrap_or_else(|| {
committee_attestations.iter().skip(1).fold(attestation.clone(), |mut agg, (att, _)| {
agg.aggregate(att);
@@ -601,6 +602,94 @@ where
.collect()
}
pub fn make_attester_slashing(&self, validator_indices: Vec<u64>) -> AttesterSlashing<E> {
let mut attestation_1 = IndexedAttestation {
attesting_indices: VariableList::new(validator_indices).unwrap(),
data: AttestationData {
slot: Slot::new(0),
index: 0,
beacon_block_root: Hash256::zero(),
target: Checkpoint {
root: Hash256::zero(),
epoch: Epoch::new(0),
},
source: Checkpoint {
root: Hash256::zero(),
epoch: Epoch::new(0),
},
},
signature: AggregateSignature::infinity(),
};
let mut attestation_2 = attestation_1.clone();
attestation_2.data.index += 1;
for attestation in &mut [&mut attestation_1, &mut attestation_2] {
for &i in &attestation.attesting_indices {
let sk = &self.validators_keypairs[i as usize].sk;
let fork = self.chain.head_info().unwrap().fork;
let genesis_validators_root = self.chain.genesis_validators_root;
let domain = self.chain.spec.get_domain(
attestation.data.target.epoch,
Domain::BeaconAttester,
&fork,
genesis_validators_root,
);
let message = attestation.data.signing_root(domain);
attestation.signature.add_assign(&sk.sign(message));
}
}
AttesterSlashing {
attestation_1,
attestation_2,
}
}
pub fn make_proposer_slashing(&self, validator_index: u64) -> ProposerSlashing {
let mut block_header_1 = self
.chain
.head_beacon_block()
.unwrap()
.message
.block_header();
block_header_1.proposer_index = validator_index;
let mut block_header_2 = block_header_1.clone();
block_header_2.state_root = Hash256::zero();
let sk = &self.validators_keypairs[validator_index as usize].sk;
let fork = self.chain.head_info().unwrap().fork;
let genesis_validators_root = self.chain.genesis_validators_root;
let mut signed_block_headers = vec![block_header_1, block_header_2]
.into_iter()
.map(|block_header| {
block_header.sign::<E>(&sk, &fork, genesis_validators_root, &self.chain.spec)
})
.collect::<Vec<_>>();
ProposerSlashing {
signed_header_2: signed_block_headers.remove(1),
signed_header_1: signed_block_headers.remove(0),
}
}
pub fn make_voluntary_exit(&self, validator_index: u64, epoch: Epoch) -> SignedVoluntaryExit {
let sk = &self.validators_keypairs[validator_index as usize].sk;
let fork = self.chain.head_info().unwrap().fork;
let genesis_validators_root = self.chain.genesis_validators_root;
VoluntaryExit {
epoch,
validator_index,
}
.sign(sk, &fork, genesis_validators_root, &self.chain.spec)
}
pub fn process_block(&self, slot: Slot, block: SignedBeaconBlock<E>) -> SignedBeaconBlockHash {
assert_eq!(self.chain.slot().unwrap(), slot);
let block_hash: SignedBeaconBlockHash = self.chain.process_block(block).unwrap().into();
@@ -612,7 +701,10 @@ where
for (unaggregated_attestations, maybe_signed_aggregate) in attestations.into_iter() {
for (attestation, subnet_id) in unaggregated_attestations {
self.chain
.verify_unaggregated_attestation_for_gossip(attestation.clone(), subnet_id)
.verify_unaggregated_attestation_for_gossip(
attestation.clone(),
Some(subnet_id),
)
.unwrap()
.add_to_pool(&self.chain)
.unwrap();