mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-22 14:24:44 +00:00
Add attestation gossip pre-verification (#983)
* Add PH & MS slot clock changes * Account for genesis time * Add progress on duties refactor * Add simple is_aggregator bool to val subscription * Start work on attestation_verification.rs * Add progress on ObservedAttestations * Progress with ObservedAttestations * Fix tests * Add observed attestations to the beacon chain * Add attestation observation to processing code * Add progress on attestation verification * Add first draft of ObservedAttesters * Add more tests * Add observed attesters to beacon chain * Add observers to attestation processing * Add more attestation verification * Create ObservedAggregators map * Remove commented-out code * Add observed aggregators into chain * Add progress * Finish adding features to attestation verification * Ensure beacon chain compiles * Link attn verification into chain * Integrate new attn verification in chain * Remove old attestation processing code * Start trying to fix beacon_chain tests * Split adding into pools into two functions * Add aggregation to harness * Get test harness working again * Adjust the number of aggregators for test harness * Fix edge-case in harness * Integrate new attn processing in network * Fix compile bug in validator_client * Update validator API endpoints * Fix aggreagation in test harness * Fix enum thing * Fix attestation observation bug: * Patch failing API tests * Start adding comments to attestation verification * Remove unused attestation field * Unify "is block known" logic * Update comments * Supress fork choice errors for network processing * Add todos * Tidy * Add gossip attn tests * Disallow test harness to produce old attns * Comment out in-progress tests * Partially address pruning tests * Fix failing store test * Add aggregate tests * Add comments about which spec conditions we check * Dont re-aggregate * Split apart test harness attn production * Fix compile error in network * Make progress on commented-out test * Fix skipping attestation test * Add fork choice verification tests * Tidy attn tests, remove dead code * Remove some accidentally added code * Fix clippy lint * Rename test file * Add block tests, add cheap block proposer check * Rename block testing file * Add observed_block_producers * Tidy * Switch around block signature verification * Finish block testing * Remove gossip from signature tests * First pass of self review * Fix deviation in spec * Update test spec tags * Start moving over to hashset * Finish moving observed attesters to hashmap * Move aggregation pool over to hashmap * Make fc attn borrow again * Fix rest_api compile error * Fix missing comments * Fix monster test * Uncomment increasing slots test * Address remaining comments * Remove unsafe, use cfg test * Remove cfg test flag * Fix dodgy comment * Ignore aggregates that are already known. * Unify aggregator modulo logic * Fix typo in logs * Refactor validator subscription logic * Avoid reproducing selection proof * Skip HTTP call if no subscriptions * Rename DutyAndState -> DutyAndProof * Tidy logs * Print root as dbg * Fix compile errors in tests * Fix compile error in test
This commit is contained in:
Paul Hauner
@@ -7,7 +7,7 @@ use crate::{
|
||||
builder::{BeaconChainBuilder, Witness},
|
||||
eth1_chain::CachingEth1Backend,
|
||||
events::NullEventHandler,
|
||||
AttestationProcessingOutcome, AttestationType, BeaconChain, BeaconChainTypes, StateSkipConfig,
|
||||
BeaconChain, BeaconChainTypes, StateSkipConfig,
|
||||
};
|
||||
use genesis::interop_genesis_state;
|
||||
use rayon::prelude::*;
|
||||
@@ -23,8 +23,8 @@ use tempfile::{tempdir, TempDir};
|
||||
use tree_hash::TreeHash;
|
||||
use types::{
|
||||
AggregateSignature, Attestation, BeaconState, BeaconStateHash, ChainSpec, Domain, EthSpec,
|
||||
Hash256, Keypair, SecretKey, Signature, SignedBeaconBlock, SignedBeaconBlockHash, SignedRoot,
|
||||
Slot,
|
||||
Hash256, Keypair, SecretKey, SelectionProof, Signature, SignedAggregateAndProof,
|
||||
SignedBeaconBlock, SignedBeaconBlockHash, SignedRoot, Slot,
|
||||
};
|
||||
|
||||
pub use types::test_utils::generate_deterministic_keypairs;
|
||||
@@ -85,8 +85,24 @@ pub struct BeaconChainHarness<T: BeaconChainTypes> {
|
||||
impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
|
||||
/// Instantiate a new harness with `validator_count` initial validators.
|
||||
pub fn new(eth_spec_instance: E, keypairs: Vec<Keypair>) -> Self {
|
||||
// Setting the target aggregators to really high means that _all_ validators in the
|
||||
// committee are required to produce an aggregate. This is overkill, however with small
|
||||
// validator counts it's the only way to be certain there is _at least one_ aggregator per
|
||||
// committee.
|
||||
Self::new_with_target_aggregators(eth_spec_instance, keypairs, 1 << 32)
|
||||
}
|
||||
|
||||
/// Instantiate a new harness with `validator_count` initial validators and a custom
|
||||
/// `target_aggregators_per_committee` spec value
|
||||
pub fn new_with_target_aggregators(
|
||||
eth_spec_instance: E,
|
||||
keypairs: Vec<Keypair>,
|
||||
target_aggregators_per_committee: u64,
|
||||
) -> Self {
|
||||
let data_dir = tempdir().expect("should create temporary data_dir");
|
||||
let spec = E::default_spec();
|
||||
let mut spec = E::default_spec();
|
||||
|
||||
spec.target_aggregators_per_committee = target_aggregators_per_committee;
|
||||
|
||||
let log = NullLoggerBuilder.build().expect("logger should build");
|
||||
|
||||
@@ -268,9 +284,9 @@ where
|
||||
.expect("should not error during block processing");
|
||||
|
||||
self.chain.fork_choice().expect("should find head");
|
||||
|
||||
head_block_root = Some(block_root);
|
||||
self.add_free_attestations(&attestation_strategy, &new_state, block_root, slot);
|
||||
|
||||
self.add_attestations_for_slot(&attestation_strategy, &new_state, block_root, slot);
|
||||
|
||||
state = new_state;
|
||||
slot += 1;
|
||||
@@ -312,7 +328,7 @@ where
|
||||
self.chain.fork_choice().expect("should find head");
|
||||
|
||||
let attestation_strategy = AttestationStrategy::SomeValidators(validators.to_vec());
|
||||
self.add_free_attestations(&attestation_strategy, &new_state, block_root, slot);
|
||||
self.add_attestations_for_slot(&attestation_strategy, &new_state, block_root, slot);
|
||||
(block_root.into(), new_state)
|
||||
}
|
||||
|
||||
@@ -448,114 +464,183 @@ where
|
||||
(signed_block, state)
|
||||
}
|
||||
|
||||
/// Adds attestations to the `BeaconChain` operations pool and fork choice.
|
||||
/// A list of attestations for each committee for the given slot.
|
||||
///
|
||||
/// The `attestation_strategy` dictates which validators should attest.
|
||||
fn add_free_attestations(
|
||||
/// The first layer of the Vec is organised per committee. For example, if the return value is
|
||||
/// called `all_attestations`, then all attestations in `all_attestations[0]` will be for
|
||||
/// committee 0, whilst all in `all_attestations[1]` will be for committee 1.
|
||||
pub fn get_unaggregated_attestations(
|
||||
&self,
|
||||
attestation_strategy: &AttestationStrategy,
|
||||
state: &BeaconState<E>,
|
||||
head_block_root: Hash256,
|
||||
attestation_slot: Slot,
|
||||
) -> Vec<Vec<Attestation<E>>> {
|
||||
let spec = &self.spec;
|
||||
let fork = &state.fork;
|
||||
|
||||
let attesting_validators = self.get_attesting_validators(attestation_strategy);
|
||||
|
||||
state
|
||||
.get_beacon_committees_at_slot(state.slot)
|
||||
.expect("should get committees")
|
||||
.iter()
|
||||
.map(|bc| {
|
||||
bc.committee
|
||||
.par_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, validator_index)| {
|
||||
if !attesting_validators.contains(validator_index) {
|
||||
return None;
|
||||
}
|
||||
let mut attestation = self
|
||||
.chain
|
||||
.produce_unaggregated_attestation_for_block(
|
||||
attestation_slot,
|
||||
bc.index,
|
||||
head_block_root,
|
||||
Cow::Borrowed(state),
|
||||
)
|
||||
.expect("should produce attestation");
|
||||
|
||||
attestation
|
||||
.aggregation_bits
|
||||
.set(i, true)
|
||||
.expect("should be able to set aggregation bits");
|
||||
|
||||
attestation.signature = {
|
||||
let domain = spec.get_domain(
|
||||
attestation.data.target.epoch,
|
||||
Domain::BeaconAttester,
|
||||
fork,
|
||||
state.genesis_validators_root,
|
||||
);
|
||||
|
||||
let message = attestation.data.signing_root(domain);
|
||||
|
||||
let mut agg_sig = AggregateSignature::new();
|
||||
|
||||
agg_sig.add(&Signature::new(
|
||||
message.as_bytes(),
|
||||
self.get_sk(*validator_index),
|
||||
));
|
||||
|
||||
agg_sig
|
||||
};
|
||||
|
||||
Some(attestation)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn get_attesting_validators(&self, attestation_strategy: &AttestationStrategy) -> Vec<usize> {
|
||||
match attestation_strategy {
|
||||
AttestationStrategy::AllValidators => (0..self.keypairs.len()).collect(),
|
||||
AttestationStrategy::SomeValidators(vec) => vec.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates a `Vec<Attestation>` for some attestation strategy and head_block.
|
||||
pub fn add_attestations_for_slot(
|
||||
&self,
|
||||
attestation_strategy: &AttestationStrategy,
|
||||
state: &BeaconState<E>,
|
||||
head_block_root: Hash256,
|
||||
head_block_slot: Slot,
|
||||
) {
|
||||
self.get_free_attestations(
|
||||
// These attestations will not be accepted by the chain so no need to generate them.
|
||||
if state.slot + E::slots_per_epoch() < self.chain.slot().expect("should get slot") {
|
||||
return;
|
||||
}
|
||||
|
||||
let spec = &self.spec;
|
||||
let fork = &state.fork;
|
||||
|
||||
let attesting_validators = self.get_attesting_validators(attestation_strategy);
|
||||
|
||||
let unaggregated_attestations = self.get_unaggregated_attestations(
|
||||
attestation_strategy,
|
||||
state,
|
||||
head_block_root,
|
||||
head_block_slot,
|
||||
)
|
||||
.into_iter()
|
||||
.for_each(|attestation| {
|
||||
match self
|
||||
.chain
|
||||
.process_attestation(attestation, AttestationType::Aggregated)
|
||||
.expect("should not error during attestation processing")
|
||||
{
|
||||
// PastEpoch can occur if we fork over several epochs
|
||||
AttestationProcessingOutcome::Processed
|
||||
| AttestationProcessingOutcome::PastEpoch { .. } => (),
|
||||
other => panic!("did not successfully process attestation: {:?}", other),
|
||||
}
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
/// Generates a `Vec<Attestation>` for some attestation strategy and head_block.
|
||||
pub fn get_free_attestations(
|
||||
&self,
|
||||
attestation_strategy: &AttestationStrategy,
|
||||
state: &BeaconState<E>,
|
||||
head_block_root: Hash256,
|
||||
head_block_slot: Slot,
|
||||
) -> Vec<Attestation<E>> {
|
||||
let spec = &self.spec;
|
||||
let fork = &state.fork;
|
||||
// Loop through all unaggregated attestations, submit them to the chain and also submit a
|
||||
// single aggregate.
|
||||
unaggregated_attestations
|
||||
.into_iter()
|
||||
.for_each(|committee_attestations| {
|
||||
// Submit each unaggregated attestation to the chain.
|
||||
for attestation in &committee_attestations {
|
||||
self.chain
|
||||
.verify_unaggregated_attestation_for_gossip(attestation.clone())
|
||||
.expect("should not error during attestation processing")
|
||||
.add_to_pool(&self.chain)
|
||||
.expect("should add attestation to naive pool");
|
||||
}
|
||||
|
||||
let attesting_validators: Vec<usize> = match attestation_strategy {
|
||||
AttestationStrategy::AllValidators => (0..self.keypairs.len()).collect(),
|
||||
AttestationStrategy::SomeValidators(vec) => vec.clone(),
|
||||
};
|
||||
// If there are any attestations in this committee, create an aggregate.
|
||||
if let Some(attestation) = committee_attestations.first() {
|
||||
let bc = state.get_beacon_committee(attestation.data.slot, attestation.data.index)
|
||||
.expect("should get committee");
|
||||
|
||||
let mut attestations = vec![];
|
||||
let aggregator_index = bc.committee
|
||||
.iter()
|
||||
.find(|&validator_index| {
|
||||
if !attesting_validators.contains(validator_index) {
|
||||
return false
|
||||
}
|
||||
|
||||
state
|
||||
.get_beacon_committees_at_slot(state.slot)
|
||||
.expect("should get committees")
|
||||
.iter()
|
||||
.for_each(|bc| {
|
||||
let mut local_attestations: Vec<Attestation<E>> = bc
|
||||
.committee
|
||||
.par_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, validator_index)| {
|
||||
// Note: searching this array is worst-case `O(n)`. A hashset could be a better
|
||||
// alternative.
|
||||
if attesting_validators.contains(validator_index) {
|
||||
let mut attestation = self
|
||||
.chain
|
||||
.produce_attestation_for_block(
|
||||
head_block_slot,
|
||||
bc.index,
|
||||
head_block_root,
|
||||
Cow::Borrowed(state),
|
||||
)
|
||||
.expect("should produce attestation");
|
||||
let selection_proof = SelectionProof::new::<E>(
|
||||
state.slot,
|
||||
self.get_sk(*validator_index),
|
||||
fork,
|
||||
state.genesis_validators_root,
|
||||
spec,
|
||||
);
|
||||
|
||||
attestation
|
||||
.aggregation_bits
|
||||
.set(i, true)
|
||||
.expect("should be able to set aggregation bits");
|
||||
selection_proof.is_aggregator(bc.committee.len(), spec).unwrap_or(false)
|
||||
})
|
||||
.copied()
|
||||
.expect(&format!(
|
||||
"Committee {} at slot {} with {} attesting validators does not have any aggregators",
|
||||
bc.index, state.slot, bc.committee.len()
|
||||
));
|
||||
|
||||
attestation.signature = {
|
||||
let domain = spec.get_domain(
|
||||
attestation.data.target.epoch,
|
||||
Domain::BeaconAttester,
|
||||
fork,
|
||||
state.genesis_validators_root,
|
||||
);
|
||||
// If the chain is able to produce an aggregate, use that. Otherwise, build an
|
||||
// aggregate locally.
|
||||
let aggregate = self
|
||||
.chain
|
||||
.get_aggregated_attestation(&attestation.data)
|
||||
.expect("should not error whilst finding aggregate")
|
||||
.unwrap_or_else(|| {
|
||||
committee_attestations.iter().skip(1).fold(attestation.clone(), |mut agg, att| {
|
||||
agg.aggregate(att);
|
||||
agg
|
||||
})
|
||||
});
|
||||
|
||||
let message = attestation.data.signing_root(domain);
|
||||
let signed_aggregate = SignedAggregateAndProof::from_aggregate(
|
||||
aggregator_index as u64,
|
||||
aggregate,
|
||||
None,
|
||||
self.get_sk(aggregator_index),
|
||||
fork,
|
||||
state.genesis_validators_root,
|
||||
spec,
|
||||
);
|
||||
|
||||
let mut agg_sig = AggregateSignature::new();
|
||||
|
||||
agg_sig.add(&Signature::new(
|
||||
message.as_bytes(),
|
||||
self.get_sk(*validator_index),
|
||||
));
|
||||
|
||||
agg_sig
|
||||
};
|
||||
|
||||
Some(attestation)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
attestations.append(&mut local_attestations);
|
||||
self.chain
|
||||
.verify_aggregated_attestation_for_gossip(signed_aggregate)
|
||||
.expect("should not error during attestation processing")
|
||||
.add_to_pool(&self.chain)
|
||||
.expect("should add attestation to naive aggregation pool")
|
||||
.add_to_fork_choice(&self.chain)
|
||||
.expect("should add attestation to fork choice");
|
||||
}
|
||||
});
|
||||
|
||||
attestations
|
||||
}
|
||||
|
||||
/// Creates two forks:
|
||||
|
||||
Reference in New Issue
Block a user