gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-20 14:28:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

De-duplicate attestations in the slasher (#2767)

Browse Source 
## Issue Addressed

Closes https://github.com/sigp/lighthouse/issues/2112
Closes https://github.com/sigp/lighthouse/issues/1861

## Proposed Changes

Collect attestations by validator index in the slasher, and use the magic of reference counting to automatically discard redundant attestations. This results in us storing only 1-2% of the attestations observed when subscribed to all subnets, which carries over to a 50-100x reduction in data stored 🎉 

## Additional Info

There's some nuance to the configuration of the `slot-offset`. It has a profound effect on the effictiveness of de-duplication, see the docs added to the book for an explanation: 5442e695e5/book/src/slasher.md (slot-offset)
This commit is contained in:
Michael Sproul
2021-11-08 00:01:09 +00:00
parent fadb8b2b2b
commit df02639b71
13 changed files with 252 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
slasher/src/slasher.rs
View File

@@ -1,11 +1,12 @@
use crate::batch_stats::{AttestationStats, BatchStats, BlockStats};
use crate::metrics::{
self, SLASHER_NUM_ATTESTATIONS_DEFERRED, SLASHER_NUM_ATTESTATIONS_DROPPED,
SLASHER_NUM_ATTESTATIONS_VALID, SLASHER_NUM_BLOCKS_PROCESSED,
SLASHER_NUM_ATTESTATIONS_STORED_PER_BATCH, SLASHER_NUM_ATTESTATIONS_VALID,
SLASHER_NUM_BLOCKS_PROCESSED,
};
use crate::{
array, AttestationBatch, AttestationQueue, AttesterRecord, BlockQueue, Config, Error,
ProposerSlashingStatus, SlasherDB,
ProposerSlashingStatus, SimpleBatch, SlasherDB,
};
use lmdb::{RwTransaction, Transaction};
use parking_lot::Mutex;
@@ -132,33 +133,56 @@ impl<E: EthSpec> Slasher<E> {
// Filter attestations for relevance.
let (snapshot, deferred, num_dropped) = self.validate(snapshot, current_epoch);
let num_valid = snapshot.len();
let num_deferred = deferred.len();
self.attestation_queue.requeue(deferred);
// Insert attestations into database.
debug!(
self.log,
"Storing attestations in slasher DB";
"num_valid" => snapshot.len(),
"Pre-processing attestations for slasher";
"num_valid" => num_valid,
"num_deferred" => num_deferred,
"num_dropped" => num_dropped,
);
metrics::set_gauge(&SLASHER_NUM_ATTESTATIONS_VALID, snapshot.len() as i64);
metrics::set_gauge(&SLASHER_NUM_ATTESTATIONS_VALID, num_valid as i64);
metrics::set_gauge(&SLASHER_NUM_ATTESTATIONS_DEFERRED, num_deferred as i64);
metrics::set_gauge(&SLASHER_NUM_ATTESTATIONS_DROPPED, num_dropped as i64);
for attestation in snapshot.attestations.iter() {
self.db.store_indexed_attestation(
txn,
attestation.1.indexed_attestation_hash,
&attestation.0,
)?;
// De-duplicate attestations and sort by validator index.
let mut batch = AttestationBatch::default();
for indexed_record in snapshot {
batch.queue(indexed_record);
}
// Group attestations into batches and process them.
let grouped_attestations = snapshot.group_by_validator_index(&self.config);
// Insert relevant attestations into database.
let mut num_stored = 0;
for weak_record in &batch.attestations {
if let Some(indexed_record) = weak_record.upgrade() {
self.db.store_indexed_attestation(
txn,
indexed_record.record.indexed_attestation_hash,
&indexed_record.indexed,
)?;
num_stored += 1;
}
}
debug!(
self.log,
"Stored attestations in slasher DB";
"num_stored" => num_stored,
"num_valid" => num_valid,
);
metrics::set_gauge(
&SLASHER_NUM_ATTESTATIONS_STORED_PER_BATCH,
num_stored as i64,
);
// Group attestations into chunked batches and process them.
let grouped_attestations = batch.group_by_validator_chunk_index(&self.config);
for (subqueue_id, subqueue) in grouped_attestations.subqueues.into_iter().enumerate() {
self.process_batch(txn, subqueue_id, subqueue.attestations, current_epoch)?;
self.process_batch(txn, subqueue_id, subqueue, current_epoch)?;
}
Ok(AttestationStats { num_processed })
}
@@ -168,12 +192,17 @@ impl<E: EthSpec> Slasher<E> {
&self,
txn: &mut RwTransaction<'_>,
subqueue_id: usize,
batch: Vec<Arc<(IndexedAttestation<E>, AttesterRecord)>>,
batch: SimpleBatch<E>,
current_epoch: Epoch,
) -> Result<(), Error> {
// First, check for double votes.
for attestation in &batch {
match self.check_double_votes(txn, subqueue_id, &attestation.0, attestation.1) {
match self.check_double_votes(
txn,
subqueue_id,
&attestation.indexed,
attestation.record,
) {
Ok(slashings) => {
if !slashings.is_empty() {
info!(
@@ -270,15 +299,15 @@ impl<E: EthSpec> Slasher<E> {
/// Returns `(valid, deferred, num_dropped)`.
fn validate(
&self,
batch: AttestationBatch<E>,
batch: SimpleBatch<E>,
current_epoch: Epoch,
) -> (AttestationBatch<E>, AttestationBatch<E>, usize) {
) -> (SimpleBatch<E>, SimpleBatch<E>, usize) {
let mut keep = Vec::with_capacity(batch.len());
let mut defer = vec![];
let mut drop_count = 0;
for tuple in batch.attestations.into_iter() {
let attestation = &tuple.0;
for indexed_record in batch {
let attestation = &indexed_record.indexed;
let target_epoch = attestation.data.target.epoch;
let source_epoch = attestation.data.source.epoch;
@@ -292,20 +321,14 @@ impl<E: EthSpec> Slasher<E> {
// Check that the attestation's target epoch is acceptable, and defer it
// if it's not.
if target_epoch > current_epoch {
defer.push(tuple);
defer.push(indexed_record);
} else {
// Otherwise the attestation is OK to process.
keep.push(tuple);
keep.push(indexed_record);
}
}
(
AttestationBatch { attestations: keep },
AttestationBatch {
attestations: defer,
},
drop_count,
)
(keep, defer, drop_count)
}
/// Prune unnecessary attestations and blocks from the on-disk database.