gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-02 16:21:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Minify slashing protection interchange data (#2380)

Browse Source 
## Issue Addressed

Closes #2354

## Proposed Changes

Add a `minify` method to `slashing_protection::Interchange` that keeps only the maximum-epoch attestation and maximum-slot block for each validator. Specifically, `minify` constructs "synthetic" attestations (with no `signing_root`) containing the maximum source epoch _and_ the maximum target epoch from the input. This is equivalent to the `minify_synth` algorithm that I've formally verified in this repository:

https://github.com/michaelsproul/slashing-proofs

## Additional Info

Includes the JSON loading optimisation from #2347
This commit is contained in:
Michael Sproul
2021-06-21 05:46:36 +00:00
parent b84ff9f793
commit 6583ce325b
11 changed files with 441 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
validator_client/slashing_protection/Cargo.toml
View File

@@ -19,4 +19,5 @@ serde_utils = { path = "../../consensus/serde_utils" }
filesystem = { path = "../../common/filesystem" }
[dev-dependencies]
lazy_static = "1.4.0"
rayon = "1.4.1"

2
validator_client/slashing_protection/Makefile
View File

@@ -1,4 +1,4 @@
TESTS_TAG := f495032df9c26c678536cd2b7854e836ea94c217
TESTS_TAG := v5.1.0
GENERATE_DIR := generated-tests
OUTPUT_DIR := interchange-tests
TARBALL := $(OUTPUT_DIR)-$(TESTS_TAG).tar.gz

71
validator_client/slashing_protection/src/bin/test_generator.rs
View File

@@ -220,13 +220,40 @@ fn main() {
vec![
TestCase::new(interchange(vec![(0, vec![40], vec![])])),
TestCase::new(interchange(vec![(0, vec![20], vec![])]))
.allow_partial_import()
.contains_slashable_data()
.with_blocks(vec![(0, 20, false)]),
],
),
MultiTestCase::new(
"multiple_interchanges_single_validator_fail_iff_imported",
vec![
TestCase::new(interchange(vec![(0, vec![40], vec![])])),
TestCase::new(interchange(vec![(0, vec![20, 50], vec![])]))
.contains_slashable_data()
.with_blocks(vec![(0, 20, false), (0, 50, false)]),
],
),
MultiTestCase::single(
"single_validator_source_greater_than_target",
TestCase::new(interchange(vec![(0, vec![], vec![(8, 7)])])).allow_partial_import(),
TestCase::new(interchange(vec![(0, vec![], vec![(8, 7)])])).contains_slashable_data(),
),
MultiTestCase::single(
"single_validator_source_greater_than_target_surrounding",
TestCase::new(interchange(vec![(0, vec![], vec![(5, 2)])]))
.contains_slashable_data()
.with_attestations(vec![(0, 3, 4, false)]),
),
MultiTestCase::single(
"single_validator_source_greater_than_target_surrounded",
TestCase::new(interchange(vec![(0, vec![], vec![(5, 2)])]))
.contains_slashable_data()
.with_attestations(vec![(0, 6, 1, false)]),
),
MultiTestCase::single(
"single_validator_source_greater_than_target_sensible_iff_minified",
TestCase::new(interchange(vec![(0, vec![], vec![(5, 2), (6, 7)])]))
.contains_slashable_data()
.with_attestations(vec![(0, 5, 8, false), (0, 6, 8, true)]),
),
MultiTestCase::single(
"single_validator_out_of_order_blocks",
@@ -304,11 +331,11 @@ fn main() {
vec![(10, Some(0)), (10, Some(11))],
vec![],
)]))
.allow_partial_import(),
.contains_slashable_data(),
),
MultiTestCase::single(
"single_validator_slashable_blocks_no_root",
TestCase::new(interchange(vec![(0, vec![10, 10], vec![])])).allow_partial_import(),
TestCase::new(interchange(vec![(0, vec![10, 10], vec![])])).contains_slashable_data(),
),
MultiTestCase::single(
"single_validator_slashable_attestations_double_vote",
@@ -317,17 +344,17 @@ fn main() {
vec![],
vec![(2, 3, Some(0)), (2, 3, Some(1))],
)]))
.allow_partial_import(),
.contains_slashable_data(),
),
MultiTestCase::single(
"single_validator_slashable_attestations_surrounds_existing",
TestCase::new(interchange(vec![(0, vec![], vec![(2, 3), (0, 4)])]))
.allow_partial_import(),
.contains_slashable_data(),
),
MultiTestCase::single(
"single_validator_slashable_attestations_surrounded_by_existing",
TestCase::new(interchange(vec![(0, vec![], vec![(0, 4), (2, 3)])]))
.allow_partial_import(),
.contains_slashable_data(),
),
MultiTestCase::single(
"duplicate_pubkey_not_slashable",
@@ -338,6 +365,29 @@ fn main() {
.with_blocks(vec![(0, 10, false), (0, 13, false), (0, 14, true)])
.with_attestations(vec![(0, 0, 2, false), (0, 1, 3, false)]),
),
MultiTestCase::single(
"duplicate_pubkey_slashable_block",
TestCase::new(interchange(vec![
(0, vec![10], vec![(0, 2)]),
(0, vec![10], vec![(1, 3)]),
]))
.contains_slashable_data()
.with_blocks(vec![(0, 10, false), (0, 11, true)]),
),
MultiTestCase::single(
"duplicate_pubkey_slashable_attestation",
TestCase::new(interchange_with_signing_roots(vec![
(0, vec![], vec![(0, 3, Some(3))]),
(0, vec![], vec![(1, 2, None)]),
]))
.contains_slashable_data()
.with_attestations(vec![
(0, 0, 1, false),
(0, 0, 2, false),
(0, 0, 4, false),
(0, 1, 4, true),
]),
),
];
let args = std::env::args().collect::<Vec<_>>();
@@ -345,7 +395,12 @@ fn main() {
fs::create_dir_all(output_dir).unwrap();
for test in tests {
test.run();
// Check that test case passes without minification
test.run(false);
// Check that test case passes with minification
test.run(true);
let f = File::create(output_dir.join(format!("{}.json", test.name))).unwrap();
serde_json::to_writer_pretty(&f, &test).unwrap();
writeln!(&f).unwrap();

84
validator_client/slashing_protection/src/interchange.rs
View File

@@ -1,5 +1,8 @@
use crate::InterchangeError;
use serde_derive::{Deserialize, Serialize};
use std::collections::HashSet;
use std::cmp::max;
use std::collections::{HashMap, HashSet};
use std::io;
use types::{Epoch, Hash256, PublicKeyBytes, Slot};
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
@@ -49,8 +52,12 @@ impl Interchange {
serde_json::from_str(json)
}
pub fn from_json_reader(reader: impl std::io::Read) -> Result<Self, serde_json::Error> {
serde_json::from_reader(reader)
pub fn from_json_reader(mut reader: impl std::io::Read) -> Result<Self, io::Error> {
// We read the entire file into memory first, as this is *a lot* faster than using
// `serde_json::from_reader`. See https://github.com/serde-rs/json/issues/160
let mut json_str = String::new();
reader.read_to_string(&mut json_str)?;
Ok(Interchange::from_json_str(&json_str)?)
}
pub fn write_to(&self, writer: impl std::io::Write) -> Result<(), serde_json::Error> {
@@ -73,4 +80,75 @@ impl Interchange {
pub fn is_empty(&self) -> bool {
self.len() == 0
}
/// Minify an interchange by constructing a synthetic block & attestation for each validator.
pub fn minify(&self) -> Result<Self, InterchangeError> {
// Map from pubkey to optional max block and max attestation.
let mut validator_data =
HashMap::<PublicKeyBytes, (Option<SignedBlock>, Option<SignedAttestation>)>::new();
for data in self.data.iter() {
// Existing maximum attestation and maximum block.
let (max_block, max_attestation) = validator_data
.entry(data.pubkey)
.or_insert_with(|| (None, None));
// Find maximum source and target epochs.
let max_source_epoch = data
.signed_attestations
.iter()
.map(|attestation| attestation.source_epoch)
.max();
let max_target_epoch = data
.signed_attestations
.iter()
.map(|attestation| attestation.target_epoch)
.max();
match (max_source_epoch, max_target_epoch) {
(Some(source_epoch), Some(target_epoch)) => {
if let Some(prev_max) = max_attestation {
prev_max.source_epoch = max(prev_max.source_epoch, source_epoch);
prev_max.target_epoch = max(prev_max.target_epoch, target_epoch);
} else {
*max_attestation = Some(SignedAttestation {
source_epoch,
target_epoch,
signing_root: None,
});
}
}
(None, None) => {}
_ => return Err(InterchangeError::MinAndMaxInconsistent),
};
// Find maximum block slot.
let max_block_slot = data.signed_blocks.iter().map(|block| block.slot).max();
if let Some(max_slot) = max_block_slot {
if let Some(prev_max) = max_block {
prev_max.slot = max(prev_max.slot, max_slot);
} else {
*max_block = Some(SignedBlock {
slot: max_slot,
signing_root: None,
});
}
}
}
let data = validator_data
.into_iter()
.map(|(pubkey, (maybe_block, maybe_att))| InterchangeData {
pubkey,
signed_blocks: maybe_block.into_iter().collect(),
signed_attestations: maybe_att.into_iter().collect(),
})
.collect();
Ok(Self {
metadata: self.metadata.clone(),
data,
})
}
}

143
validator_client/slashing_protection/src/interchange_test.rs
View File

@@ -1,9 +1,10 @@
use crate::{
interchange::Interchange,
interchange::{Interchange, SignedAttestation, SignedBlock},
test_utils::{pubkey, DEFAULT_GENESIS_VALIDATORS_ROOT},
SigningRoot, SlashingDatabase,
};
use serde_derive::{Deserialize, Serialize};
use std::collections::HashSet;
use tempfile::tempdir;
use types::{Epoch, Hash256, PublicKeyBytes, Slot};
@@ -17,7 +18,7 @@ pub struct MultiTestCase {
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct TestCase {
pub should_succeed: bool,
pub allow_partial_import: bool,
pub contains_slashable_data: bool,
pub interchange: Interchange,
pub blocks: Vec<TestBlock>,
pub attestations: Vec<TestAttestation>,
@@ -58,41 +59,53 @@ impl MultiTestCase {
self
}
pub fn run(&self) {
pub fn run(&self, minify: bool) {
let dir = tempdir().unwrap();
let slashing_db_file = dir.path().join("slashing_protection.sqlite");
let slashing_db = SlashingDatabase::create(&slashing_db_file).unwrap();
// If minification is used, false positives are allowed, i.e. there may be some situations
// in which signing is safe but the minified file prevents it.
let allow_false_positives = minify;
for test_case in &self.steps {
match slashing_db.import_interchange_info(
test_case.interchange.clone(),
self.genesis_validators_root,
) {
// If the test case is marked as containing slashable data, then it is permissible
// that we fail to import the file, in which case execution of the whole test should
// be aborted.
let allow_import_failure = test_case.contains_slashable_data;
let interchange = if minify {
let minified = test_case.interchange.minify().unwrap();
check_minification_invariants(&test_case.interchange, &minified);
minified
} else {
test_case.interchange.clone()
};
match slashing_db.import_interchange_info(interchange, self.genesis_validators_root) {
Ok(import_outcomes) => {
let failed_records = import_outcomes
.iter()
.filter(|o| o.failed())
.collect::<Vec<_>>();
let none_failed = import_outcomes.iter().all(|o| !o.failed());
assert!(
none_failed,
"test `{}` failed to import some records: {:#?}",
self.name, import_outcomes
);
if !test_case.should_succeed {
panic!(
"test `{}` succeeded on import when it should have failed",
self.name
);
}
if !failed_records.is_empty() && !test_case.allow_partial_import {
}
Err(e) => {
if test_case.should_succeed && !allow_import_failure {
panic!(
"test `{}` failed to import some records but should have succeeded: {:#?}",
self.name, failed_records,
"test `{}` failed on import when it should have succeeded, error: {:?}",
self.name, e
);
}
break;
}
Err(e) if test_case.should_succeed => {
panic!(
"test `{}` failed on import when it should have succeeded, error: {:?}",
self.name, e
);
}
_ => (),
}
for (i, block) in test_case.blocks.iter().enumerate() {
@@ -107,7 +120,7 @@ impl MultiTestCase {
i, self.name, safe
);
}
Err(e) if block.should_succeed => {
Err(e) if block.should_succeed && !allow_false_positives => {
panic!(
"block {} from `{}` failed when it should have succeeded: {:?}",
i, self.name, e
@@ -130,7 +143,7 @@ impl MultiTestCase {
i, self.name, safe
);
}
Err(e) if att.should_succeed => {
Err(e) if att.should_succeed && !allow_false_positives => {
panic!(
"attestation {} from `{}` failed when it should have succeeded: {:?}",
i, self.name, e
@@ -147,7 +160,7 @@ impl TestCase {
pub fn new(interchange: Interchange) -> Self {
TestCase {
should_succeed: true,
allow_partial_import: false,
contains_slashable_data: false,
interchange,
blocks: vec![],
attestations: vec![],
@@ -159,8 +172,8 @@ impl TestCase {
self
}
pub fn allow_partial_import(mut self) -> Self {
self.allow_partial_import = true;
pub fn contains_slashable_data(mut self) -> Self {
self.contains_slashable_data = true;
self
}
@@ -216,3 +229,81 @@ impl TestCase {
self
}
}
fn check_minification_invariants(interchange: &Interchange, minified: &Interchange) {
// Metadata should be unchanged.
assert_eq!(interchange.metadata, minified.metadata);
// Minified data should contain one entry per *unique* public key.
let uniq_pubkeys = get_uniq_pubkeys(interchange);
assert_eq!(uniq_pubkeys, get_uniq_pubkeys(minified));
assert_eq!(uniq_pubkeys.len(), minified.data.len());
for &pubkey in uniq_pubkeys.iter() {
// Minified data should contain 1 block per validator, unless the validator never signed any
// blocks. All of those blocks should have slots <= the slot of the minified block.
let original_blocks = get_blocks_of_validator(interchange, pubkey);
let minified_blocks = get_blocks_of_validator(minified, pubkey);
if original_blocks.is_empty() {
assert!(minified_blocks.is_empty());
} else {
// Should have exactly 1 block.
assert_eq!(minified_blocks.len(), 1);
// That block should have no signing root (it's synthetic).
let mini_block = minified_blocks.first().unwrap();
assert_eq!(mini_block.signing_root, None);
// All original blocks should have slots <= the mini block.
assert!(original_blocks
.iter()
.all(|block| block.slot <= mini_block.slot));
}
// Minified data should contain 1 attestation per validator, unless the validator never
// signed any attestations. All attestations should have source and target <= the source
// and target of the minified attestation.
let original_attestations = get_attestations_of_validator(interchange, pubkey);
let minified_attestations = get_attestations_of_validator(minified, pubkey);
if original_attestations.is_empty() {
assert!(minified_attestations.is_empty());
} else {
assert_eq!(minified_attestations.len(), 1);
let mini_attestation = minified_attestations.first().unwrap();
assert_eq!(mini_attestation.signing_root, None);
assert!(original_attestations
.iter()
.all(|att| att.source_epoch <= mini_attestation.source_epoch
&& att.target_epoch <= mini_attestation.target_epoch));
}
}
}
fn get_uniq_pubkeys(interchange: &Interchange) -> HashSet<PublicKeyBytes> {
interchange.data.iter().map(|data| data.pubkey).collect()
}
fn get_blocks_of_validator(interchange: &Interchange, pubkey: PublicKeyBytes) -> Vec<&SignedBlock> {
interchange
.data
.iter()
.filter(|data| data.pubkey == pubkey)
.flat_map(|data| data.signed_blocks.iter())
.collect()
}
fn get_attestations_of_validator(
interchange: &Interchange,
pubkey: PublicKeyBytes,
) -> Vec<&SignedAttestation> {
interchange
.data
.iter()
.filter(|data| data.pubkey == pubkey)
.flat_map(|data| data.signed_attestations.iter())
.collect()
}

3
validator_client/slashing_protection/src/lib.rs
View File

@@ -12,7 +12,8 @@ pub mod test_utils;
pub use crate::signed_attestation::{InvalidAttestation, SignedAttestation};
pub use crate::signed_block::{InvalidBlock, SignedBlock};
pub use crate::slashing_database::{
InterchangeImportOutcome, SlashingDatabase, SUPPORTED_INTERCHANGE_FORMAT_VERSION,
InterchangeError, InterchangeImportOutcome, SlashingDatabase,
SUPPORTED_INTERCHANGE_FORMAT_VERSION,
};
use rusqlite::Error as SQLError;
use std::io::{Error as IOError, ErrorKind};

22
validator_client/slashing_protection/src/slashing_database.rs
View File

@@ -562,8 +562,8 @@ impl SlashingDatabase {
/// Import slashing protection from another client in the interchange format.
///
/// Return a vector of public keys and errors for any validators whose data could not be
/// imported.
/// This function will atomically import the entire interchange, failing if *any*
/// record cannot be imported.
pub fn import_interchange_info(
&self,
interchange: Interchange,
@@ -581,25 +581,33 @@ impl SlashingDatabase {
});
}
// Create a single transaction for the entire batch, which will only be committed if
// all records are imported successfully.
let mut conn = self.conn_pool.get()?;
let txn = conn.transaction()?;
let mut import_outcomes = vec![];
let mut commit = true;
for record in interchange.data {
let pubkey = record.pubkey;
let txn = conn.transaction()?;
match self.import_interchange_record(record, &txn) {
Ok(summary) => {
import_outcomes.push(InterchangeImportOutcome::Success { pubkey, summary });
txn.commit()?;
}
Err(error) => {
import_outcomes.push(InterchangeImportOutcome::Failure { pubkey, error });
commit = false;
}
}
}
Ok(import_outcomes)
if commit {
txn.commit()?;
Ok(import_outcomes)
} else {
Err(InterchangeError::AtomicBatchAborted(import_outcomes))
}
}
pub fn import_interchange_record(
@@ -914,12 +922,14 @@ pub enum InterchangeError {
interchange_file: Hash256,
client: Hash256,
},
MinimalAttestationSourceAndTargetInconsistent,
MinAndMaxInconsistent,
SQLError(String),
SQLPoolError(r2d2::Error),
SerdeJsonError(serde_json::Error),
InvalidPubkey(String),
NotSafe(NotSafe),
/// One or more records were found to be slashable, so the whole batch was aborted.
AtomicBatchAborted(Vec<InterchangeImportOutcome>),
}
impl From<NotSafe> for InterchangeError {

23
validator_client/slashing_protection/tests/interop.rs
View File

@@ -1,7 +1,12 @@
use lazy_static::lazy_static;
use slashing_protection::interchange_test::MultiTestCase;
use std::fs::File;
use std::path::PathBuf;
lazy_static! {
pub static ref TEST_ROOT_DIR: PathBuf = test_root_dir();
}
fn download_tests() {
let make_output = std::process::Command::new("make")
.current_dir(std::env::var("CARGO_MANIFEST_DIR").unwrap())
@@ -22,7 +27,7 @@ fn test_root_dir() -> PathBuf {
#[test]
fn generated() {
for entry in test_root_dir()
for entry in TEST_ROOT_DIR
.join("generated")
.read_dir()
.unwrap()
@@ -30,6 +35,20 @@ fn generated() {
{
let file = File::open(entry.path()).unwrap();
let test_case: MultiTestCase = serde_json::from_reader(&file).unwrap();
test_case.run();
test_case.run(false);
}
}
#[test]
fn generated_with_minification() {
for entry in TEST_ROOT_DIR
.join("generated")
.read_dir()
.unwrap()
.map(Result::unwrap)
{
let file = File::open(entry.path()).unwrap();
let test_case: MultiTestCase = serde_json::from_reader(&file).unwrap();
test_case.run(true);
}
}