gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-18 13:28:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add attestation gossip pre-verification (#983)

Browse Source 
* Add PH & MS slot clock changes

* Account for genesis time

* Add progress on duties refactor

* Add simple is_aggregator bool to val subscription

* Start work on attestation_verification.rs

* Add progress on ObservedAttestations

* Progress with ObservedAttestations

* Fix tests

* Add observed attestations to the beacon chain

* Add attestation observation to processing code

* Add progress on attestation verification

* Add first draft of ObservedAttesters

* Add more tests

* Add observed attesters to beacon chain

* Add observers to attestation processing

* Add more attestation verification

* Create ObservedAggregators map

* Remove commented-out code

* Add observed aggregators into chain

* Add progress

* Finish adding features to attestation verification

* Ensure beacon chain compiles

* Link attn verification into chain

* Integrate new attn verification in chain

* Remove old attestation processing code

* Start trying to fix beacon_chain tests

* Split adding into pools into two functions

* Add aggregation to harness

* Get test harness working again

* Adjust the number of aggregators for test harness

* Fix edge-case in harness

* Integrate new attn processing in network

* Fix compile bug in validator_client

* Update validator API endpoints

* Fix aggreagation in test harness

* Fix enum thing

* Fix attestation observation bug:

* Patch failing API tests

* Start adding comments to attestation verification

* Remove unused attestation field

* Unify "is block known" logic

* Update comments

* Supress fork choice errors for network processing

* Add todos

* Tidy

* Add gossip attn tests

* Disallow test harness to produce old attns

* Comment out in-progress tests

* Partially address pruning tests

* Fix failing store test

* Add aggregate tests

* Add comments about which spec conditions we check

* Dont re-aggregate

* Split apart test harness attn production

* Fix compile error in network

* Make progress on commented-out test

* Fix skipping attestation test

* Add fork choice verification tests

* Tidy attn tests, remove dead code

* Remove some accidentally added code

* Fix clippy lint

* Rename test file

* Add block tests, add cheap block proposer check

* Rename block testing file

* Add observed_block_producers

* Tidy

* Switch around block signature verification

* Finish block testing

* Remove gossip from signature tests

* First pass of self review

* Fix deviation in spec

* Update test spec tags

* Start moving over to hashset

* Finish moving observed attesters to hashmap

* Move aggregation pool over to hashmap

* Make fc attn borrow again

* Fix rest_api compile error

* Fix missing comments

* Fix monster test

* Uncomment increasing slots test

* Address remaining comments

* Remove unsafe, use cfg test

* Remove cfg test flag

* Fix dodgy comment

* Ignore aggregates that are already known.

* Unify aggregator modulo logic

* Fix typo in logs

* Refactor validator subscription logic

* Avoid reproducing selection proof

* Skip HTTP call if no subscriptions

* Rename DutyAndState -> DutyAndProof

* Tidy logs

* Print root as dbg

* Fix compile errors in tests

* Fix compile error in test
This commit is contained in:
Paul Hauner
2020-05-06 21:42:56 +10:00
committed by GitHub
parent 1552f9997e
commit ad5bd6412a
38 changed files with 4952 additions and 1479 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
beacon_node/beacon_chain/src/naive_aggregation_pool.rs
View File

@@ -46,8 +46,6 @@ pub enum Error {
/// stored. This indicates a fairly serious error somewhere in the code that called this
/// function.
InconsistentBitfieldLengths,
/// The function to obtain a map index failed, this is an internal error.
InvalidMapIndex(usize),
/// The given `attestation` was for the incorrect slot. This is an internal error.
IncorrectSlot { expected: Slot, attestation: Slot },
}
@@ -56,30 +54,20 @@ pub enum Error {
/// `attestation` are from the same slot.
struct AggregatedAttestationMap<E: EthSpec> {
map: HashMap<AttestationData, Attestation<E>>,
slot: Slot,
}
impl<E: EthSpec> AggregatedAttestationMap<E> {
/// Create an empty collection that will only contain attestation for the given `slot`.
pub fn new(slot: Slot) -> Self {
/// Create an empty collection with the given `initial_capacity`.
pub fn new(initial_capacity: usize) -> Self {
Self {
slot,
map: <_>::default(),
map: HashMap::with_capacity(initial_capacity),
}
}
/// Insert an attestation into `self`, aggregating it into the pool.
///
/// The given attestation (`a`) must only have one signature and be from the slot that `self`
/// was initialized with.
/// The given attestation (`a`) must only have one signature.
pub fn insert(&mut self, a: &Attestation<E>) -> Result<InsertOutcome, Error> {
if a.data.slot != self.slot {
return Err(Error::IncorrectSlot {
expected: self.slot,
attestation: a.data.slot,
});
}
let set_bits = a
.aggregation_bits
.iter()
@@ -124,15 +112,12 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
///
/// The given `a.data.slot` must match the slot that `self` was initialized with.
pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
if data.slot != self.slot {
return Err(Error::IncorrectSlot {
expected: self.slot,
attestation: data.slot,
});
}
Ok(self.map.get(data).cloned())
}
pub fn len(&self) -> usize {
self.map.len()
}
}
/// A pool of `Attestation` that is specially designed to store "unaggregated" attestations from
@@ -158,14 +143,14 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
/// receives and it can be triggered manually.
pub struct NaiveAggregationPool<E: EthSpec> {
lowest_permissible_slot: RwLock<Slot>,
maps: RwLock<Vec<AggregatedAttestationMap<E>>>,
maps: RwLock<HashMap<Slot, AggregatedAttestationMap<E>>>,
}
impl<E: EthSpec> Default for NaiveAggregationPool<E> {
fn default() -> Self {
Self {
lowest_permissible_slot: RwLock::new(Slot::new(0)),
maps: RwLock::new(vec![]),
maps: RwLock::new(HashMap::new()),
}
}
}
@@ -179,28 +164,46 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
/// The pool may be pruned if the given `attestation.data` has a slot higher than any
/// previously seen.
pub fn insert(&self, attestation: &Attestation<E>) -> Result<InsertOutcome, Error> {
let slot = attestation.data.slot;
let lowest_permissible_slot = *self.lowest_permissible_slot.read();
// Reject any attestations that are too old.
if attestation.data.slot < lowest_permissible_slot {
if slot < lowest_permissible_slot {
return Err(Error::SlotTooLow {
slot: attestation.data.slot,
slot,
lowest_permissible_slot,
});
}
// Prune the pool if this attestation indicates that the current slot has advanced.
if (lowest_permissible_slot + SLOTS_RETAINED as u64) < attestation.data.slot + 1 {
self.prune(attestation.data.slot)
}
let mut maps = self.maps.write();
let index = self.get_map_index(attestation.data.slot);
let outcome = if let Some(map) = maps.get_mut(&slot) {
map.insert(attestation)
} else {
// To avoid re-allocations, try and determine a rough initial capacity for the new item
// by obtaining the mean size of all items in earlier epoch.
let (count, sum) = maps
.iter()
// Only include epochs that are less than the given slot in the average. This should
// generally avoid including recent epochs that are still "filling up".
.filter(|(map_slot, _item)| **map_slot < slot)
.map(|(_slot, map)| map.len())
.fold((0, 0), |(count, sum), len| (count + 1, sum + len));
self.maps
.write()
.get_mut(index)
.ok_or_else(|| Error::InvalidMapIndex(index))?
.insert(attestation)
// Use the mainnet default committee size if we can't determine an average.
let initial_capacity = sum.checked_div(count).unwrap_or(128);
let mut item = AggregatedAttestationMap::new(initial_capacity);
let outcome = item.insert(attestation);
maps.insert(slot, item);
outcome
};
drop(maps);
self.prune(slot);
outcome
}
/// Returns an aggregated `Attestation` with the given `data`, if any.
@@ -208,8 +211,8 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
self.maps
.read()
.iter()
.find(|map| map.slot == data.slot)
.map(|map| map.get(data))
.find(|(slot, _map)| **slot == data.slot)
.map(|(_slot, map)| map.get(data))
.unwrap_or_else(|| Ok(None))
}
@@ -218,41 +221,26 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
pub fn prune(&self, current_slot: Slot) {
// Taking advantage of saturating subtraction on `Slot`.
let lowest_permissible_slot = current_slot - Slot::from(SLOTS_RETAINED);
self.maps
.write()
.retain(|map| map.slot >= lowest_permissible_slot);
*self.lowest_permissible_slot.write() = lowest_permissible_slot;
}
/// Returns the index of `self.maps` that matches `slot`.
///
/// If there is no existing map for this slot one will be created. If `self.maps.len() >=
/// SLOTS_RETAINED`, the map with the lowest slot will be replaced.
fn get_map_index(&self, slot: Slot) -> usize {
let mut maps = self.maps.write();
if let Some(index) = maps.iter().position(|map| map.slot == slot) {
return index;
// Remove any maps that are definitely expired.
maps.retain(|slot, _map| *slot >= lowest_permissible_slot);
// If we have too many maps, remove the lowest amount to ensure we only have
// `SLOTS_RETAINED` left.
if maps.len() > SLOTS_RETAINED {
let mut slots = maps.iter().map(|(slot, _map)| *slot).collect::<Vec<_>>();
// Sort is generally pretty slow, however `SLOTS_RETAINED` is quite low so it should be
// negligible.
slots.sort_unstable();
slots
.into_iter()
.take(maps.len().saturating_sub(SLOTS_RETAINED))
.for_each(|slot| {
maps.remove(&slot);
})
}
if maps.len() < SLOTS_RETAINED || maps.is_empty() {
let index = maps.len();
maps.push(AggregatedAttestationMap::new(slot));
return index;
}
let index = maps
.iter()
.enumerate()
.min_by_key(|(_i, map)| map.slot)
.map(|(i, _map)| i)
.expect("maps cannot be empty due to previous .is_empty() check");
maps[index] = AggregatedAttestationMap::new(slot);
index
}
}
@@ -432,7 +420,7 @@ mod tests {
.maps
.read()
.iter()
.map(|map| map.slot)
.map(|(slot, _map)| *slot)
.collect::<Vec<_>>();
pool_slots.sort_unstable();