From b6f4d1c968c53c13df399aef20b0f3e0eaa5037c Mon Sep 17 00:00:00 2001 From: Age Date: Sat, 15 Sep 2018 22:53:45 +1000 Subject: [PATCH 01/74] add initial attestation validation structure --- lighthouse/state/mod.rs | 1 + .../validation/attestation_validation.rs | 66 +++++++++++++++++++ lighthouse/state/validation/mod.rs | 1 + lighthouse/utils/errors.rs | 12 ++++ lighthouse/utils/mod.rs | 1 + 5 files changed, 81 insertions(+) create mode 100644 lighthouse/state/validation/attestation_validation.rs create mode 100644 lighthouse/state/validation/mod.rs create mode 100644 lighthouse/utils/errors.rs diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index d10acf8fda..2c938cf29b 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -15,3 +15,4 @@ pub mod crosslink_record; pub mod shard_and_committee; pub mod transition; pub mod validator_record; +pub mod validation; diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation_validation.rs new file mode 100644 index 0000000000..483fef55ac --- /dev/null +++ b/lighthouse/state/validation/attestation_validation.rs @@ -0,0 +1,66 @@ +use super::super::crystallized_state::CrystallizedState; +use super::super::active_state::ActiveState; +use super::super::attestation_record::AttestationRecord; +use super::super::block::Block; +use super::super::chain_config::ChainConfig; + +use ::utils::errors::AttestationValidationError; + +// implementation of validate_attestation in the v2.1 python reference implementation +// see: https://github.com/ethereum/beacon_chain/blob/a79ab2c6f03cbdabf2b6d9d435c26e2b216e09a5/beacon_chain/state/state_transition.py#L61 +pub fn validate_attestation( + crystallized_state: &CrystallizedState, + active_state: &ActiveState, + attestation: &AttestationRecord, + block: &Block, + chain_config: &ChainConfig) + -> Result { + + if !(attestation.slot < block.slot_number) { + return Err(AttestationValidationError::SlotTooHigh); + } + + if !(attestation.slot > (block.slot_number - chain_config.cycle_length as u64)) { + return Err(AttestationValidationError::SlotTooLow(format!("Attestation slot number too low\n\tFound: {:?}, Needed greater than: {:?}", attestation.slot, block.slot_number - chain_config.cycle_length as u64))); + } + + return Ok(true); + } + + +#[cfg(test)] +mod tests { + + use super::*; + // test helper functions + + fn generate_standard_state() -> ( + CrystallizedState, + ActiveState, + AttestationRecord, + Block, + ChainConfig) { + + let mut crystallized_state = CrystallizedState::zero(); + let mut active_state = ActiveState::zero(); + let mut attestation_record = AttestationRecord::zero(); + let mut block = Block::zero(); + let chain_config = ChainConfig::standard(); + + return (crystallized_state, active_state, attestation_record, block, chain_config); + + } + + + #[test] + fn test_attestation_validation_slot_high() { + // generate standard state + let (mut crystallized_state, mut active_state, mut attestation_record, mut block, mut chain_config) = generate_standard_state(); + // set slot too high + attestation_record.slot = 30; + block.slot_number = 10; + + let result = validate_attestation(&crystallized_state, &active_state, &attestation_record, &block, &chain_config); + assert_eq!(result, Err(AttestationValidationError::SlotTooHigh)); + } +} diff --git a/lighthouse/state/validation/mod.rs b/lighthouse/state/validation/mod.rs new file mode 100644 index 0000000000..98db2cb665 --- /dev/null +++ b/lighthouse/state/validation/mod.rs @@ -0,0 +1 @@ +mod attestation_validation; diff --git a/lighthouse/utils/errors.rs b/lighthouse/utils/errors.rs new file mode 100644 index 0000000000..4f12cfbdd6 --- /dev/null +++ b/lighthouse/utils/errors.rs @@ -0,0 +1,12 @@ +// Collection of custom errors + +#[derive(Debug,PartialEq)] +pub enum AttestationValidationError { + SlotTooHigh, + SlotTooLow(String), + IncorrectBitField, + NonZeroTrailingBits, + AggregateSignatureFail +} + + diff --git a/lighthouse/utils/mod.rs b/lighthouse/utils/mod.rs index d116422be5..4cb6ddf38d 100644 --- a/lighthouse/utils/mod.rs +++ b/lighthouse/utils/mod.rs @@ -7,3 +7,4 @@ pub mod types; pub mod bls; pub mod test_helpers; pub mod logging; +pub mod errors; From b8ca18f87bdcf6bdffe0047d2f0d57517beb0440 Mon Sep 17 00:00:00 2001 From: Age Date: Sat, 15 Sep 2018 23:22:05 +1000 Subject: [PATCH 02/74] correct attestation warnings, add unit test --- .../validation/attestation_validation.rs | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation_validation.rs index 483fef55ac..4e0707dc3b 100644 --- a/lighthouse/state/validation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation_validation.rs @@ -41,21 +41,20 @@ mod tests { Block, ChainConfig) { - let mut crystallized_state = CrystallizedState::zero(); - let mut active_state = ActiveState::zero(); - let mut attestation_record = AttestationRecord::zero(); - let mut block = Block::zero(); + let crystallized_state = CrystallizedState::zero(); + let active_state = ActiveState::zero(); + let attestation_record = AttestationRecord::zero(); + let block = Block::zero(); let chain_config = ChainConfig::standard(); return (crystallized_state, active_state, attestation_record, block, chain_config); - } #[test] fn test_attestation_validation_slot_high() { // generate standard state - let (mut crystallized_state, mut active_state, mut attestation_record, mut block, mut chain_config) = generate_standard_state(); + let (crystallized_state, active_state, mut attestation_record, mut block, chain_config) = generate_standard_state(); // set slot too high attestation_record.slot = 30; block.slot_number = 10; @@ -63,4 +62,16 @@ mod tests { let result = validate_attestation(&crystallized_state, &active_state, &attestation_record, &block, &chain_config); assert_eq!(result, Err(AttestationValidationError::SlotTooHigh)); } + + #[test] + fn test_attestation_validation_slot_low() { + // generate standard state + let (crystallized_state, active_state, mut attestation_record, mut block, chain_config) = generate_standard_state(); + // set slot too high + attestation_record.slot = 2; + block.slot_number = 10; + + let result = validate_attestation(&crystallized_state, &active_state, &attestation_record, &block, &chain_config); + //assert_eq!(result, Err(AttestationValidationError::SlotTooLow)); + } } From 815180d88c338394d2e9af455a9fac9e388f6d85 Mon Sep 17 00:00:00 2001 From: Age Date: Wed, 19 Sep 2018 14:00:35 +1000 Subject: [PATCH 03/74] Adds assert_error macro --- lighthouse/main.rs | 3 ++- lighthouse/utils/macros.rs | 10 ++++++++++ lighthouse/utils/mod.rs | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 lighthouse/utils/macros.rs diff --git a/lighthouse/main.rs b/lighthouse/main.rs index 0a9f35288d..7f5e4f7eaf 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -6,11 +6,12 @@ extern crate clap; extern crate network_libp2p; extern crate futures; +#[macro_use] +pub mod utils; pub mod db; pub mod client; pub mod state; pub mod sync; -pub mod utils; pub mod config; use std::path::PathBuf; diff --git a/lighthouse/utils/macros.rs b/lighthouse/utils/macros.rs new file mode 100644 index 0000000000..e6e3be01ec --- /dev/null +++ b/lighthouse/utils/macros.rs @@ -0,0 +1,10 @@ +#[macro_export] +macro_rules! assert_error { + ($exp: expr, $err: expr) => { + if ( !$exp ) { + return Err($err); + } + } +} + + diff --git a/lighthouse/utils/mod.rs b/lighthouse/utils/mod.rs index 4cb6ddf38d..4d2ae39428 100644 --- a/lighthouse/utils/mod.rs +++ b/lighthouse/utils/mod.rs @@ -3,8 +3,12 @@ extern crate blake2; extern crate crypto_mac; extern crate boolean_bitfield; +#[macro_use] +pub mod macros; pub mod types; pub mod bls; pub mod test_helpers; pub mod logging; pub mod errors; + + From 693d635f8c3d863408bee518480b1b9d01f67d9a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 14:45:03 +1000 Subject: [PATCH 04/74] Add canonical hash function --- lighthouse/utils/hash.rs | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 lighthouse/utils/hash.rs diff --git a/lighthouse/utils/hash.rs b/lighthouse/utils/hash.rs new file mode 100644 index 0000000000..7b52eb9d13 --- /dev/null +++ b/lighthouse/utils/hash.rs @@ -0,0 +1,6 @@ +use super::blake2::blake2b::blake2b; + +pub fn canonical_hash(input: &[u8]) -> Vec { + let result = blake2b(64, &[], input); + result.as_bytes()[0..32].to_vec() +} From 6c8c4d9b1e874bba6c8d56f766316f4fd5b23902 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 14:45:55 +1000 Subject: [PATCH 05/74] Move state/block.rs into own mod, add SszBlock --- lighthouse/state/{ => block}/block.rs | 0 lighthouse/state/block/mod.rs | 11 ++++ lighthouse/state/block/ssz_block.rs | 90 +++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) rename lighthouse/state/{ => block}/block.rs (100%) create mode 100644 lighthouse/state/block/mod.rs create mode 100644 lighthouse/state/block/ssz_block.rs diff --git a/lighthouse/state/block.rs b/lighthouse/state/block/block.rs similarity index 100% rename from lighthouse/state/block.rs rename to lighthouse/state/block/block.rs diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs new file mode 100644 index 0000000000..20467eca7f --- /dev/null +++ b/lighthouse/state/block/mod.rs @@ -0,0 +1,11 @@ +extern crate blake2_rfc; + +use super::ssz; +use super::utils; +use super::attestation_record; + +mod block; +mod ssz_block; + +pub use self::block::Block; +pub use self::ssz_block::SszBlock; diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs new file mode 100644 index 0000000000..d9ad3945f4 --- /dev/null +++ b/lighthouse/state/block/ssz_block.rs @@ -0,0 +1,90 @@ +use super::ssz::decode::{ + decode_length, + Decodable, +}; +use super::utils::hash::canonical_hash; + +pub enum BlockValidatorError { + SszInvalid, + BadPowHash, + SlotTooLow, + SlotTooHigh, +} + +const LENGTH_BYTES: usize = 4; +const MIN_SSZ_BLOCK_LENGTH: usize = { + 32 + // parent_hash + 8 + // slot_number + 32 + // randao_reveal + LENGTH_BYTES + // attestations (assuming zero) + 32 + // pow_chain_ref + 32 + // active_state_root + 32 // crystallized_state_root +}; +const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + 2^24; + + +pub struct SszBlock<'a> { + ssz: &'a [u8], + attestation_len: usize, +} + +impl<'a> SszBlock<'a> { + pub fn from_vec(vec: &'a Vec) + -> Result + { + if vec.len() < MIN_SSZ_BLOCK_LENGTH { + return Err(BlockValidatorError::SszInvalid); + } + if vec.len() > MAX_SSZ_BLOCK_LENGTH { + return Err(BlockValidatorError::SszInvalid); + } + let attestation_len = decode_length(&vec[72..76], LENGTH_BYTES) + .map_err(|_| BlockValidatorError::SszInvalid)?; + // Is the length adequate now we know now many attestation + // records exist? + if vec.len() < (76 + attestation_len + 96) { + return Err(BlockValidatorError::SszInvalid); + } + Ok(Self{ + ssz: &vec[..], + attestation_len + }) + } + + pub fn block_hash(&self) -> Vec { + canonical_hash(self.ssz) + } + + pub fn parent_hash(&self) -> &[u8] { + &self.ssz[0..32] + } + + pub fn slot_number(&self) -> u64 { + u64::ssz_decode(&self.ssz[32..40]).unwrap() + } + + pub fn randao_reveal(&self) -> &[u8] { + &self.ssz[40..72] + } + + pub fn attestations(&self) -> &[u8] { + let start = 72 + LENGTH_BYTES; + &self.ssz[start..(start + self.attestation_len)] + } + + pub fn pow_chain_ref(&self) -> &[u8] { + let len = self.ssz.len(); + &self.ssz[(len - 96)..(len - 64)] + } + + pub fn act_state_root(&self) -> &[u8] { + let len = self.ssz.len(); + &self.ssz[(len - 64)..(len - 32)] + } + + pub fn cry_state_root(&self) -> &[u8] { + let len = self.ssz.len(); + &self.ssz[(len - 32)..(len)] + } +} From 718b9ed7dee87aa9b697e2c7326e2947547779a8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 14:46:58 +1000 Subject: [PATCH 06/74] Move to blake2b, start adding validate_block fn --- lighthouse/state/mod.rs | 3 ++- lighthouse/state/transition/mod.rs | 6 ++++++ lighthouse/state/transition/validate_block.rs | 4 ++++ lighthouse/utils/mod.rs | 3 ++- lighthouse/utils/types.rs | 1 - 5 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 lighthouse/state/transition/validate_block.rs diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index d10acf8fda..737839721d 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -1,10 +1,11 @@ extern crate rlp; extern crate ethereum_types; -extern crate blake2; +extern crate blake2_rfc as blake2; extern crate bytes; extern crate ssz; use super::utils; +use super::db; pub mod active_state; pub mod attestation_record; diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index dd8967e652..8ba28d05f7 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,7 +1,13 @@ use super::super::utils::types::Hash256; +use super::crystallized_state; +use super::super::db; +use super::ssz; +use super::blake2; +use super::utils; mod attestation_parent_hashes; mod shuffling; +mod validate_block; pub use self::attestation_parent_hashes::attestation_parent_hashes; pub use self::shuffling::shuffle; diff --git a/lighthouse/state/transition/validate_block.rs b/lighthouse/state/transition/validate_block.rs new file mode 100644 index 0000000000..bbd784d327 --- /dev/null +++ b/lighthouse/state/transition/validate_block.rs @@ -0,0 +1,4 @@ +enum BlockStatus { + NewBlock, + KnownBlock, +} diff --git a/lighthouse/utils/mod.rs b/lighthouse/utils/mod.rs index d116422be5..cfb302d4b4 100644 --- a/lighthouse/utils/mod.rs +++ b/lighthouse/utils/mod.rs @@ -1,8 +1,9 @@ extern crate ethereum_types; -extern crate blake2; +extern crate blake2_rfc as blake2; extern crate crypto_mac; extern crate boolean_bitfield; +pub mod hash; pub mod types; pub mod bls; pub mod test_helpers; diff --git a/lighthouse/utils/types.rs b/lighthouse/utils/types.rs index 8e9bf2006e..949b7a549d 100644 --- a/lighthouse/utils/types.rs +++ b/lighthouse/utils/types.rs @@ -3,7 +3,6 @@ extern crate boolean_bitfield; use super::ethereum_types::{ H256, H160 }; use self::boolean_bitfield::BooleanBitfield; -pub use super::blake2::Blake2s; pub use super::ethereum_types::U256; pub type Hash256 = H256; From fe2127ea31265ba0e755bb558c889975bd8d5b0b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 14:52:36 +1000 Subject: [PATCH 07/74] Update SszBlock to use new SSZ API --- lighthouse/state/block/ssz_block.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index d9ad3945f4..11af7c820a 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -33,13 +33,14 @@ impl<'a> SszBlock<'a> { pub fn from_vec(vec: &'a Vec) -> Result { + let ssz = &vec[..]; if vec.len() < MIN_SSZ_BLOCK_LENGTH { return Err(BlockValidatorError::SszInvalid); } if vec.len() > MAX_SSZ_BLOCK_LENGTH { return Err(BlockValidatorError::SszInvalid); } - let attestation_len = decode_length(&vec[72..76], LENGTH_BYTES) + let attestation_len = decode_length(ssz, 72, LENGTH_BYTES) .map_err(|_| BlockValidatorError::SszInvalid)?; // Is the length adequate now we know now many attestation // records exist? @@ -47,7 +48,7 @@ impl<'a> SszBlock<'a> { return Err(BlockValidatorError::SszInvalid); } Ok(Self{ - ssz: &vec[..], + ssz, attestation_len }) } @@ -61,7 +62,7 @@ impl<'a> SszBlock<'a> { } pub fn slot_number(&self) -> u64 { - u64::ssz_decode(&self.ssz[32..40]).unwrap() + u64::ssz_decode(&self.ssz, 32).unwrap().0 } pub fn randao_reveal(&self) -> &[u8] { From 4d9b53478f91352bbcb08ac7826833280e2713ef Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 15:24:06 +1000 Subject: [PATCH 08/74] Add SszBlock comment --- lighthouse/state/block/ssz_block.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 11af7c820a..8ee7580a2d 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -23,7 +23,8 @@ const MIN_SSZ_BLOCK_LENGTH: usize = { }; const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + 2^24; - +/// Allows for reading of block values directly from serialized +/// ssz bytes. pub struct SszBlock<'a> { ssz: &'a [u8], attestation_len: usize, From 3dea40abe080719065ab23a1613fba214985d83f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 16:14:31 +1000 Subject: [PATCH 09/74] Add temporary (incomplete) aggsig serialization It just sets them to 64 zeros. This needs to be fixed. --- lighthouse/state/attestation_record.rs | 3 ++- lighthouse/state/block/block.rs | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lighthouse/state/attestation_record.rs b/lighthouse/state/attestation_record.rs index fc949b9e6d..9fa8fc73a6 100644 --- a/lighthouse/state/attestation_record.rs +++ b/lighthouse/state/attestation_record.rs @@ -19,7 +19,8 @@ impl Encodable for AttestationRecord { s.append_vec(&self.oblique_parent_hashes); s.append(&self.shard_block_hash); s.append_vec(&self.attester_bitfield.to_be_vec()); - // TODO: add aggregate signature + // TODO: encode the aggregate sig correctly + s.append_vec(&vec![0_u8; 64]) } } diff --git a/lighthouse/state/block/block.rs b/lighthouse/state/block/block.rs index d9c2c05790..bfc2e1b402 100644 --- a/lighthouse/state/block/block.rs +++ b/lighthouse/state/block/block.rs @@ -54,6 +54,8 @@ impl Encodable for Block { s.append(&self.pow_chain_ref); s.append(&self.active_state_root); s.append(&self.crystallized_state_root); + // TODO: encode the aggregate sig correctly + s.append_vec(&vec![0_u8; 64]) } } From 61fddb26606d81b8b06ec6cf1cba439890c5e1b6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 16:15:05 +1000 Subject: [PATCH 10/74] Implement basics for ssz_block --- lighthouse/state/attestation_record.rs | 10 +++ lighthouse/state/block/ssz_block.rs | 97 +++++++++++++++++++++----- 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/lighthouse/state/attestation_record.rs b/lighthouse/state/attestation_record.rs index 9fa8fc73a6..35d61f5837 100644 --- a/lighthouse/state/attestation_record.rs +++ b/lighthouse/state/attestation_record.rs @@ -2,6 +2,16 @@ use super::utils::types::{ Hash256, Bitfield }; use super::utils::bls::{ AggregateSignature }; use super::ssz::{ Encodable, SszStream }; +pub const MIN_SSZ_ATTESTION_RECORD_LENGTH: usize = { + 8 + // slot + 2 + // shard_id + 4 + // oblique_parent_hashes (empty list) + 32 + // shard_block_hash + 5 + // attester_bitfield (assuming 1 byte of bitfield) + 8 + // justified_slot + 32 + // justified_block_hash + 2 * 32 // aggregate sig (two 256 bit points) +}; pub struct AttestationRecord { pub slot: u64, diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 8ee7580a2d..8643a531ed 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -3,9 +3,13 @@ use super::ssz::decode::{ Decodable, }; use super::utils::hash::canonical_hash; +use super::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH; +#[derive(Debug, PartialEq)] pub enum BlockValidatorError { - SszInvalid, + TooShort, + TooLong, + NoAttestationRecords, BadPowHash, SlotTooLow, SlotTooHigh, @@ -21,36 +25,56 @@ const MIN_SSZ_BLOCK_LENGTH: usize = { 32 + // active_state_root 32 // crystallized_state_root }; -const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + 2^24; +const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + (1 << 24); /// Allows for reading of block values directly from serialized /// ssz bytes. +#[derive(Debug, PartialEq)] pub struct SszBlock<'a> { ssz: &'a [u8], attestation_len: usize, + pub len: usize, } impl<'a> SszBlock<'a> { - pub fn from_vec(vec: &'a Vec) + pub fn from_slice(vec: &'a [u8]) -> Result { let ssz = &vec[..]; - if vec.len() < MIN_SSZ_BLOCK_LENGTH { - return Err(BlockValidatorError::SszInvalid); + let len = vec.len(); + /* + * Ensure the SSZ is long enough to be a block. + */ + if len < MIN_SSZ_BLOCK_LENGTH { + return Err(BlockValidatorError::TooShort); } - if vec.len() > MAX_SSZ_BLOCK_LENGTH { - return Err(BlockValidatorError::SszInvalid); + /* + * Ensure the SSZ slice isn't longer than is possible for a block. + */ + if len > MAX_SSZ_BLOCK_LENGTH { + return Err(BlockValidatorError::TooLong); } + /* + * Determine how many bytes are used to store attestation records + * and ensure that length is enough to store at least one attestation + * record. + */ let attestation_len = decode_length(ssz, 72, LENGTH_BYTES) - .map_err(|_| BlockValidatorError::SszInvalid)?; - // Is the length adequate now we know now many attestation - // records exist? - if vec.len() < (76 + attestation_len + 96) { - return Err(BlockValidatorError::SszInvalid); + .map_err(|_| BlockValidatorError::TooShort)?; + if attestation_len < MIN_SSZ_ATTESTION_RECORD_LENGTH { + return Err(BlockValidatorError::NoAttestationRecords) + } + /* + * Ensure that the SSZ is long enough, now we know the + * length of the attestation records field. + */ + if len < (76 + attestation_len + 96) { + return Err(BlockValidatorError::TooShort); } Ok(Self{ ssz, - attestation_len + attestation_len, + len, }) } @@ -76,17 +100,52 @@ impl<'a> SszBlock<'a> { } pub fn pow_chain_ref(&self) -> &[u8] { - let len = self.ssz.len(); - &self.ssz[(len - 96)..(len - 64)] + &self.ssz[(self.len - 96)..(self.len - 64)] } pub fn act_state_root(&self) -> &[u8] { - let len = self.ssz.len(); - &self.ssz[(len - 64)..(len - 32)] + &self.ssz[(self.len - 64)..(self.len - 32)] } pub fn cry_state_root(&self) -> &[u8] { - let len = self.ssz.len(); - &self.ssz[(len - 32)..(len)] + &self.ssz[(self.len - 32)..(self.len)] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::block::Block; + use super::super::attestation_record::AttestationRecord; + use super::super::ssz::SszStream; + + fn get_block_ssz(b: &Block) -> Vec { + let mut ssz_stream = SszStream::new(); + ssz_stream.append(b); + ssz_stream.drain() + } + + #[test] + fn test_ssz_block_zero_attestation_records() { + let mut b = Block::zero(); + b.attestations = vec![]; + let ssz = get_block_ssz(&b); + + assert_eq!( + SszBlock::from_slice(&ssz[..]), + Err(BlockValidatorError::NoAttestationRecords) + ); + } + + #[test] + fn test_ssz_block_single_attestation_record() { + let mut b = Block::zero(); + b.attestations = vec![AttestationRecord::zero()]; + let ssz = get_block_ssz(&b); + + assert_eq!( + SszBlock::from_slice(&ssz[..]), + Err(BlockValidatorError::NoAttestationRecords) + ); } } From 27581c0ff4e191313af8257a402c7ac65b9a642c Mon Sep 17 00:00:00 2001 From: Age Date: Wed, 19 Sep 2018 16:25:39 +1000 Subject: [PATCH 11/74] Implement generalised error ParameterError --- .../state/transition/attestation_parent_hashes.rs | 14 +++++++------- lighthouse/state/transition/mod.rs | 7 +------ lighthouse/utils/errors.rs | 11 ++++++++--- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lighthouse/state/transition/attestation_parent_hashes.rs b/lighthouse/state/transition/attestation_parent_hashes.rs index 624d8c4437..f635972b2c 100644 --- a/lighthouse/state/transition/attestation_parent_hashes.rs +++ b/lighthouse/state/transition/attestation_parent_hashes.rs @@ -1,5 +1,5 @@ use super::Hash256; -use super::TransitionError; +use super::ParameterError; /// This function is used to select the hashes used in /// the signing of an AttestationRecord. @@ -18,22 +18,22 @@ pub fn attestation_parent_hashes( attestation_slot: &u64, current_hashes: &Vec, oblique_hashes: &Vec) - -> Result, TransitionError> + -> Result, ParameterError> { // This cast places a limit on cycle_length. If you change it, check math // for overflow. let cycle_length: u64 = *cycle_length as u64; if current_hashes.len() as u64 != (cycle_length * 2) { - return Err(TransitionError::InvalidInput(String::from( + return Err(ParameterError::InvalidInput(String::from( "current_hashes.len() must equal cycle_length * 2"))); } if attestation_slot >= block_slot { - return Err(TransitionError::InvalidInput(String::from( + return Err(ParameterError::InvalidInput(String::from( "attestation_slot must be less than block_slot"))); } if oblique_hashes.len() as u64 > cycle_length { - return Err(TransitionError::InvalidInput(String::from( + return Err(ParameterError::InvalidInput(String::from( "oblique_hashes.len() must be <= cycle_length * 2"))); } @@ -44,7 +44,7 @@ pub fn attestation_parent_hashes( let attestation_distance = block_slot - attestation_slot; if attestation_distance > cycle_length { - return Err(TransitionError::InvalidInput(String::from( + return Err(ParameterError::InvalidInput(String::from( "attestation_slot must be withing one cycle of block_slot"))); } @@ -63,7 +63,7 @@ pub fn attestation_parent_hashes( */ let end = start.checked_add(cycle_length) .and_then(|x| x.checked_sub(oblique_hashes.len() as u64)) - .ok_or(TransitionError::IntWrapping)?; + .ok_or(ParameterError::IntWrapping)?; let mut hashes = Vec::new(); diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index dd8967e652..f3da910276 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,4 +1,5 @@ use super::super::utils::types::Hash256; +use super::super::utils::errors::ParameterError; mod attestation_parent_hashes; mod shuffling; @@ -6,12 +7,6 @@ mod shuffling; pub use self::attestation_parent_hashes::attestation_parent_hashes; pub use self::shuffling::shuffle; -#[derive(Debug)] -pub enum TransitionError { - IntWrapping, - OutOfBounds, - InvalidInput(String), -} diff --git a/lighthouse/utils/errors.rs b/lighthouse/utils/errors.rs index 4f12cfbdd6..2a25b29b8e 100644 --- a/lighthouse/utils/errors.rs +++ b/lighthouse/utils/errors.rs @@ -1,7 +1,7 @@ -// Collection of custom errors +// Collection of custom errors #[derive(Debug,PartialEq)] -pub enum AttestationValidationError { +pub enum AttestationValidationError { SlotTooHigh, SlotTooLow(String), IncorrectBitField, @@ -9,4 +9,9 @@ pub enum AttestationValidationError { AggregateSignatureFail } - +#[derive(Debug,PartialEq)] +pub enum ParameterError { + IntWrapping, + OutOfBounds, + InvalidInput(String), +} From 7e1819f69367db601fd988b519ac21fa5bb88d35 Mon Sep 17 00:00:00 2001 From: Age Date: Wed, 19 Sep 2018 16:27:19 +1000 Subject: [PATCH 12/74] add helpers - implement get_block_hash --- lighthouse/state/helpers.rs | 71 +++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 lighthouse/state/helpers.rs diff --git a/lighthouse/state/helpers.rs b/lighthouse/state/helpers.rs new file mode 100644 index 0000000000..388fedafbf --- /dev/null +++ b/lighthouse/state/helpers.rs @@ -0,0 +1,71 @@ +/* + * Collection of helper functions used in the state transition modules + */ +use super::active_state::ActiveState; +use super::block::Block; +use super::chain_config::ChainConfig; +use super::utils::errors::ParameterError; +use super::utils::types::Hash256; + +/* + pub fn get_signed_parent_hashes( + active_state: &ActiveState, + block: &Block, + attestation: &AttestationRecord, + chain_config: &ChainConfig) + -> Vec { + } + */ + +pub fn get_block_hash( + active_state_recent_block_hashes: &Vec, + current_block_slot: &u64, + slot: &u64, + cycle_length: &u64, // convert from standard u8 +) -> Result { + // active_state must have at 2*cycle_length hashes + assert_error!( + active_state_recent_block_hashes.len() as u64 == cycle_length * 2, + ParameterError::InvalidInput(String::from( + "active state has incorrect number of block hashes" + )) + ); + + let state_start_slot = (*current_block_slot) + .checked_sub(cycle_length * 2) + .unwrap_or(0); + + assert_error!( + (state_start_slot <= *slot) && (*slot < *current_block_slot), + ParameterError::InvalidInput(String::from("incorrect slot number")) + ); + + let index = 2 * cycle_length + (*slot) - *current_block_slot; // should always be positive + Ok(active_state_recent_block_hashes[index as usize]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_block_hash() { + let block_slot: u64 = 10; + let slot: u64 = 3; + let cycle_length: u64 = 8; + + let mut block_hashes: Vec = Vec::new(); + for _i in 0..2 * cycle_length { + block_hashes.push(Hash256::random()); + } + + let result = get_block_hash(&block_hashes, &block_slot, &slot, &cycle_length).unwrap(); + + assert_eq!( + result, + block_hashes[(2 * cycle_length + slot - block_slot) as usize] + ); + + println!("{:?}", result); + } +} From 048f8b6f0101f76823f727d08c814f439cd491ae Mon Sep 17 00:00:00 2001 From: Age Date: Wed, 19 Sep 2018 16:29:29 +1000 Subject: [PATCH 13/74] slight corrections to attestation validation --- lighthouse/state/mod.rs | 1 + .../validation/attestation_validation.rs | 48 ++++++++++--------- lighthouse/state/validation/mod.rs | 6 +++ 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index 2c938cf29b..a245056a17 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -16,3 +16,4 @@ pub mod shard_and_committee; pub mod transition; pub mod validator_record; pub mod validation; +pub mod helpers; diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation_validation.rs index 4e0707dc3b..4704320638 100644 --- a/lighthouse/state/validation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation_validation.rs @@ -1,20 +1,20 @@ -use super::super::crystallized_state::CrystallizedState; -use super::super::active_state::ActiveState; -use super::super::attestation_record::AttestationRecord; -use super::super::block::Block; -use super::super::chain_config::ChainConfig; +use super::CrystallizedState; +use super::ActiveState; +use super::AttestationRecord; +use super::Block; +use super::ChainConfig; use ::utils::errors::AttestationValidationError; // implementation of validate_attestation in the v2.1 python reference implementation // see: https://github.com/ethereum/beacon_chain/blob/a79ab2c6f03cbdabf2b6d9d435c26e2b216e09a5/beacon_chain/state/state_transition.py#L61 pub fn validate_attestation( - crystallized_state: &CrystallizedState, + crystallized_state: &CrystallizedState, active_state: &ActiveState, attestation: &AttestationRecord, block: &Block, - chain_config: &ChainConfig) - -> Result { + chain_config: &ChainConfig) + -> Result { if !(attestation.slot < block.slot_number) { return Err(AttestationValidationError::SlotTooHigh); @@ -24,24 +24,24 @@ pub fn validate_attestation( return Err(AttestationValidationError::SlotTooLow(format!("Attestation slot number too low\n\tFound: {:?}, Needed greater than: {:?}", attestation.slot, block.slot_number - chain_config.cycle_length as u64))); } - return Ok(true); + Ok(true) } #[cfg(test)] -mod tests { +mod tests { use super::*; // test helper functions - - fn generate_standard_state() -> ( - CrystallizedState, - ActiveState, - AttestationRecord, - Block, - ChainConfig) { - let crystallized_state = CrystallizedState::zero(); + fn generate_standard_state() -> ( + CrystallizedState, + ActiveState, + AttestationRecord, + Block, + ChainConfig) { + + let crystallized_state = CrystallizedState::zero(); let active_state = ActiveState::zero(); let attestation_record = AttestationRecord::zero(); let block = Block::zero(); @@ -49,10 +49,9 @@ mod tests { return (crystallized_state, active_state, attestation_record, block, chain_config); } - #[test] - fn test_attestation_validation_slot_high() { + fn test_attestation_validation_slot_high() { // generate standard state let (crystallized_state, active_state, mut attestation_record, mut block, chain_config) = generate_standard_state(); // set slot too high @@ -64,14 +63,19 @@ mod tests { } #[test] - fn test_attestation_validation_slot_low() { + fn test_attestation_validation_slot_low() { // generate standard state let (crystallized_state, active_state, mut attestation_record, mut block, chain_config) = generate_standard_state(); // set slot too high attestation_record.slot = 2; block.slot_number = 10; - let result = validate_attestation(&crystallized_state, &active_state, &attestation_record, &block, &chain_config); + let result = validate_attestation( + &crystallized_state, + &active_state, + &attestation_record, + &block, + &chain_config); //assert_eq!(result, Err(AttestationValidationError::SlotTooLow)); } } diff --git a/lighthouse/state/validation/mod.rs b/lighthouse/state/validation/mod.rs index 98db2cb665..c11b652612 100644 --- a/lighthouse/state/validation/mod.rs +++ b/lighthouse/state/validation/mod.rs @@ -1 +1,7 @@ +use super::crystallized_state::CrystallizedState; +use super::active_state::ActiveState; +use super::attestation_record::AttestationRecord; +use super::block::Block; +use super::chain_config::ChainConfig; + mod attestation_validation; From 928bafb3da920b9c9172b1e3b0d6e5f92ba4826c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 16:45:32 +1000 Subject: [PATCH 14/74] Ensure an empty bitfield serializes to 1 byte --- boolean-bitfield/src/lib.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/boolean-bitfield/src/lib.rs b/boolean-bitfield/src/lib.rs index 0a690bf3db..a22d9cde41 100644 --- a/boolean-bitfield/src/lib.rs +++ b/boolean-bitfield/src/lib.rs @@ -19,15 +19,17 @@ impl BooleanBitfield { pub fn new() -> Self { Self { len: 0, - vec: vec![] + vec: vec![0] } } /// Create a new bitfield of a certain capacity pub fn with_capacity(capacity: usize) -> Self { + let mut vec = Vec::with_capacity(capacity / 8 + 1); + vec.push(0); Self { len: 0, - vec: Vec::with_capacity(capacity / 8 + 1) + vec } } @@ -112,6 +114,17 @@ impl PartialEq for BooleanBitfield { mod tests { use super::*; + #[test] + fn test_new_bitfield_len() { + let b = BooleanBitfield::new(); + assert_eq!(b.len(), 0); + assert_eq!(b.to_be_vec(), vec![0]); + + let b = BooleanBitfield::with_capacity(100); + assert_eq!(b.len(), 0); + assert_eq!(b.to_be_vec(), vec![0]); + } + #[test] fn test_bitfield_set() { let mut b = BooleanBitfield::new(); From d0d336c3413fc9c0f9befce08fdd6d95a8b2d319 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 16:55:18 +1000 Subject: [PATCH 15/74] Update AttestationRecord fields --- lighthouse/state/attestation_record.rs | 39 ++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/lighthouse/state/attestation_record.rs b/lighthouse/state/attestation_record.rs index 35d61f5837..25abbd3f3a 100644 --- a/lighthouse/state/attestation_record.rs +++ b/lighthouse/state/attestation_record.rs @@ -3,14 +3,14 @@ use super::utils::bls::{ AggregateSignature }; use super::ssz::{ Encodable, SszStream }; pub const MIN_SSZ_ATTESTION_RECORD_LENGTH: usize = { - 8 + // slot - 2 + // shard_id - 4 + // oblique_parent_hashes (empty list) - 32 + // shard_block_hash - 5 + // attester_bitfield (assuming 1 byte of bitfield) - 8 + // justified_slot - 32 + // justified_block_hash - 2 * 32 // aggregate sig (two 256 bit points) + 8 + // slot + 2 + // shard_id + 4 + // oblique_parent_hashes (empty list) + 4 + 32 + // shard_block_hash + 5 + // attester_bitfield (assuming 1 byte of bitfield) + 8 + // justified_slot + 4 + 32 + // justified_block_hash + 4 + (2 * 32) // aggregate sig (two 256 bit points) }; pub struct AttestationRecord { @@ -19,6 +19,8 @@ pub struct AttestationRecord { pub oblique_parent_hashes: Vec, pub shard_block_hash: Hash256, pub attester_bitfield: Bitfield, + pub justified_slot: u64, + pub justified_block_hash: Hash256, pub aggregate_sig: Option, } @@ -29,6 +31,8 @@ impl Encodable for AttestationRecord { s.append_vec(&self.oblique_parent_hashes); s.append(&self.shard_block_hash); s.append_vec(&self.attester_bitfield.to_be_vec()); + s.append(&self.justified_slot); + s.append(&self.justified_block_hash); // TODO: encode the aggregate sig correctly s.append_vec(&vec![0_u8; 64]) } @@ -42,7 +46,26 @@ impl AttestationRecord { oblique_parent_hashes: vec![], shard_block_hash: Hash256::zero(), attester_bitfield: Bitfield::new(), + justified_slot: 0, + justified_block_hash: Hash256::zero(), aggregate_sig: None, } } } + + +#[cfg(test)] +mod tests { + use super::*; + use super::super::ssz::SszStream; + + #[test] + pub fn test_attestation_record_min_ssz_length() { + let ar = AttestationRecord::zero(); + let mut ssz_stream = SszStream::new(); + ssz_stream.append(&ar); + let ssz = ssz_stream.drain(); + + assert_eq!(ssz.len(), MIN_SSZ_ATTESTION_RECORD_LENGTH); + } +} From ca951d03612b5607505d3f897806fc05eb193b98 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 16:57:21 +1000 Subject: [PATCH 16/74] Move ssz block min length into block.rs, add tests --- lighthouse/state/block/block.rs | 47 ++++++++++++++++------------- lighthouse/state/block/ssz_block.rs | 22 +++----------- 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/lighthouse/state/block/block.rs b/lighthouse/state/block/block.rs index bfc2e1b402..115cbcd0ea 100644 --- a/lighthouse/state/block/block.rs +++ b/lighthouse/state/block/block.rs @@ -1,8 +1,20 @@ use super::utils::types::Hash256; -use super::attestation_record::AttestationRecord; +use super::attestation_record::{ + AttestationRecord, + MIN_SSZ_ATTESTION_RECORD_LENGTH, +}; use super::ssz::{ Encodable, SszStream }; -const SSZ_BLOCK_LENGTH: usize = 192; +pub const MIN_SSZ_BLOCK_LENGTH: usize = { + 4 + 32 + // parent_hash + 8 + // slot_number + 4 + 32 + // randao_reveal + 4 + MIN_SSZ_ATTESTION_RECORD_LENGTH + // attestations (minimum one) + 4 + 32 + // pow_chain_ref + 4 + 32 + // active_state_root + 4 + 32 // crystallized_state_root +}; +pub const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + (1 << 24); pub struct Block { pub parent_hash: Hash256, @@ -26,23 +38,6 @@ impl Block { crystallized_state_root: Hash256::zero(), } } - - /// Return the bytes that should be signed in order to - /// attest for this block. - pub fn encode_for_signing(&self) - -> [u8; SSZ_BLOCK_LENGTH] - { - let mut s = SszStream::new(); - s.append(&self.parent_hash); - s.append(&self.slot_number); - s.append(&self.randao_reveal); - s.append(&self.pow_chain_ref); - s.append(&self.active_state_root); - s.append(&self.crystallized_state_root); - let vec = s.drain(); - let mut encoded = [0; SSZ_BLOCK_LENGTH]; - encoded.copy_from_slice(&vec); encoded - } } impl Encodable for Block { @@ -54,8 +49,6 @@ impl Encodable for Block { s.append(&self.pow_chain_ref); s.append(&self.active_state_root); s.append(&self.crystallized_state_root); - // TODO: encode the aggregate sig correctly - s.append_vec(&vec![0_u8; 64]) } } @@ -75,4 +68,16 @@ mod tests { assert!(b.active_state_root.is_zero()); assert!(b.crystallized_state_root.is_zero()); } + + #[test] + pub fn test_block_min_ssz_length() { + let mut b = Block::zero(); + b.attestations = vec![AttestationRecord::zero()]; + + let mut ssz_stream = SszStream::new(); + ssz_stream.append(&b); + let ssz = ssz_stream.drain(); + + assert_eq!(ssz.len(), MIN_SSZ_BLOCK_LENGTH); + } } diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 8643a531ed..055549f4a8 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -3,7 +3,10 @@ use super::ssz::decode::{ Decodable, }; use super::utils::hash::canonical_hash; -use super::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH; +use super::block::{ + MIN_SSZ_BLOCK_LENGTH, + MAX_SSZ_BLOCK_LENGTH, +}; #[derive(Debug, PartialEq)] pub enum BlockValidatorError { @@ -16,16 +19,6 @@ pub enum BlockValidatorError { } const LENGTH_BYTES: usize = 4; -const MIN_SSZ_BLOCK_LENGTH: usize = { - 32 + // parent_hash - 8 + // slot_number - 32 + // randao_reveal - LENGTH_BYTES + // attestations (assuming zero) - 32 + // pow_chain_ref - 32 + // active_state_root - 32 // crystallized_state_root -}; -const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + (1 << 24); /// Allows for reading of block values directly from serialized /// ssz bytes. @@ -61,13 +54,6 @@ impl<'a> SszBlock<'a> { */ let attestation_len = decode_length(ssz, 72, LENGTH_BYTES) .map_err(|_| BlockValidatorError::TooShort)?; - if attestation_len < MIN_SSZ_ATTESTION_RECORD_LENGTH { - return Err(BlockValidatorError::NoAttestationRecords) - } - /* - * Ensure that the SSZ is long enough, now we know the - * length of the attestation records field. - */ if len < (76 + attestation_len + 96) { return Err(BlockValidatorError::TooShort); } From 1dd1714444a2c422696ed11cd8f69165e8e813d4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 17:15:54 +1000 Subject: [PATCH 17/74] Half update some ssz indexes for ssz_block --- lighthouse/state/block/ssz_block.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 055549f4a8..7f672dcdfb 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -52,7 +52,7 @@ impl<'a> SszBlock<'a> { * and ensure that length is enough to store at least one attestation * record. */ - let attestation_len = decode_length(ssz, 72, LENGTH_BYTES) + let attestation_len = decode_length(ssz, 80, LENGTH_BYTES) .map_err(|_| BlockValidatorError::TooShort)?; if len < (76 + attestation_len + 96) { return Err(BlockValidatorError::TooShort); @@ -69,7 +69,7 @@ impl<'a> SszBlock<'a> { } pub fn parent_hash(&self) -> &[u8] { - &self.ssz[0..32] + &self.ssz[5..37] } pub fn slot_number(&self) -> u64 { From a1b826906acf3a24551cbedb814f078337da3eb0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 19 Sep 2018 17:54:14 +1000 Subject: [PATCH 18/74] Fix SszBlock tests --- lighthouse/state/block/ssz_block.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 7f672dcdfb..ea89c6fa7a 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -12,7 +12,6 @@ use super::block::{ pub enum BlockValidatorError { TooShort, TooLong, - NoAttestationRecords, BadPowHash, SlotTooLow, SlotTooHigh, @@ -119,7 +118,7 @@ mod tests { assert_eq!( SszBlock::from_slice(&ssz[..]), - Err(BlockValidatorError::NoAttestationRecords) + Err(BlockValidatorError::TooShort) ); } @@ -129,9 +128,6 @@ mod tests { b.attestations = vec![AttestationRecord::zero()]; let ssz = get_block_ssz(&b); - assert_eq!( - SszBlock::from_slice(&ssz[..]), - Err(BlockValidatorError::NoAttestationRecords) - ); + assert!(SszBlock::from_slice(&ssz[..]).is_ok()); } } From 7020a08b7d47873e3773153263c4b7be5ccfca9e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 20 Sep 2018 13:13:07 +1000 Subject: [PATCH 19/74] Finish SszBlock, add tests --- lighthouse/state/block/ssz_block.rs | 173 ++++++++++++++++++++++++++-- 1 file changed, 166 insertions(+), 7 deletions(-) diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index ea89c6fa7a..1e4d20bb1d 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -68,32 +68,47 @@ impl<'a> SszBlock<'a> { } pub fn parent_hash(&self) -> &[u8] { - &self.ssz[5..37] + &self.ssz[4..36] } pub fn slot_number(&self) -> u64 { - u64::ssz_decode(&self.ssz, 32).unwrap().0 + /* + * An error should be unreachable from this decode + * because we checked the length of the array at + * the initalization of this struct. + * + * If you can make this function panic, please report + * it to paul@sigmaprime.io + */ + if let Ok((n, _)) = u64::ssz_decode(&self.ssz, 36) { + n + } else { + unreachable!(); + } } pub fn randao_reveal(&self) -> &[u8] { - &self.ssz[40..72] + &self.ssz[48..80] } pub fn attestations(&self) -> &[u8] { - let start = 72 + LENGTH_BYTES; + let start = 80 + LENGTH_BYTES; &self.ssz[start..(start + self.attestation_len)] } pub fn pow_chain_ref(&self) -> &[u8] { - &self.ssz[(self.len - 96)..(self.len - 64)] + let start = self.len - (32 + LENGTH_BYTES + 32 + LENGTH_BYTES + 32); + &self.ssz[start..(start + 32)] } pub fn act_state_root(&self) -> &[u8] { - &self.ssz[(self.len - 64)..(self.len - 32)] + let start = self.len - (32 + LENGTH_BYTES + 32); + &self.ssz[start..(start + 32)] } pub fn cry_state_root(&self) -> &[u8] { - &self.ssz[(self.len - 32)..(self.len)] + let start = self.len - 32; + &self.ssz[start..(start + 32)] } } @@ -103,6 +118,7 @@ mod tests { use super::super::block::Block; use super::super::attestation_record::AttestationRecord; use super::super::ssz::SszStream; + use super::super::utils::types::Hash256; fn get_block_ssz(b: &Block) -> Vec { let mut ssz_stream = SszStream::new(); @@ -110,6 +126,12 @@ mod tests { ssz_stream.drain() } + fn get_attestation_record_ssz(ar: &AttestationRecord) -> Vec { + let mut ssz_stream = SszStream::new(); + ssz_stream.append(ar); + ssz_stream.drain() + } + #[test] fn test_ssz_block_zero_attestation_records() { let mut b = Block::zero(); @@ -122,6 +144,18 @@ mod tests { ); } + #[test] + fn test_ssz_block_single_attestation_record_one_byte_short() { + let mut b = Block::zero(); + b.attestations = vec![AttestationRecord::zero()]; + let ssz = get_block_ssz(&b); + + assert_eq!( + SszBlock::from_slice(&ssz[0..(ssz.len() - 1)]), + Err(BlockValidatorError::TooShort) + ); + } + #[test] fn test_ssz_block_single_attestation_record() { let mut b = Block::zero(); @@ -130,4 +164,129 @@ mod tests { assert!(SszBlock::from_slice(&ssz[..]).is_ok()); } + + #[test] + fn test_ssz_block_block_hash() { + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + let hash = ssz_block.block_hash(); + // Note: this hash was not generated by some external program, + // it was simply printed then copied into the code. This test + // will tell us if the hash changes, not that it matches some + // canonical reference. + let expected_hash = [ + 28, 184, 51, 12, 226, 15, 73, 50, 66, 19, 168, 149, + 229, 122, 141, 111, 42, 236, 137, 157, 230, 90, 149, + 58, 145, 52, 47, 62, 158, 131, 46, 147 + ]; + assert_eq!(hash, expected_hash); + } + + #[test] + fn test_ssz_block_parent_hash() { + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + let reference_hash = Hash256::from([42_u8; 32]); + block.parent_hash = reference_hash.clone(); + + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + + assert_eq!(ssz_block.parent_hash(), &reference_hash.to_vec()[..]); + } + + #[test] + fn test_ssz_block_slot_number() { + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + block.slot_number = 42; + + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + + assert_eq!(ssz_block.slot_number(), 42); + } + + #[test] + fn test_ssz_block_randao_reveal() { + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + let reference_hash = Hash256::from([42_u8; 32]); + block.randao_reveal = reference_hash.clone(); + + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + + assert_eq!(ssz_block.randao_reveal(), &reference_hash.to_vec()[..]); + } + + #[test] + fn test_ssz_block_attestations() { + /* + * Single AttestationRecord + */ + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + let ssz_ar = get_attestation_record_ssz(&AttestationRecord::zero()); + + assert_eq!(ssz_block.attestations(), &ssz_ar[..]); + + /* + * Multiple AttestationRecords + */ + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + block.attestations.push(AttestationRecord::zero()); + + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + let mut ssz_ar = get_attestation_record_ssz(&AttestationRecord::zero()); + ssz_ar.append(&mut get_attestation_record_ssz(&AttestationRecord::zero())); + + assert_eq!(ssz_block.attestations(), &ssz_ar[..]); + } + + #[test] + fn test_ssz_block_pow_chain_ref() { + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + let reference_hash = Hash256::from([42_u8; 32]); + block.pow_chain_ref = reference_hash.clone(); + + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + + assert_eq!(ssz_block.pow_chain_ref(), &reference_hash.to_vec()[..]); + } + + #[test] + fn test_ssz_block_act_state_root() { + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + let reference_hash = Hash256::from([42_u8; 32]); + block.active_state_root = reference_hash.clone(); + + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + + assert_eq!(ssz_block.act_state_root(), &reference_hash.to_vec()[..]); + } + + #[test] + fn test_ssz_block_cry_state_root() { + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + let reference_hash = Hash256::from([42_u8; 32]); + block.crystallized_state_root = reference_hash.clone(); + + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + + assert_eq!(ssz_block.cry_state_root(), &reference_hash.to_vec()[..]); + } } From 55ce4998d82ed208b35a0ec5a14fbfaad4c9e944 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 20 Sep 2018 13:51:58 +1000 Subject: [PATCH 20/74] Fix SszBlock bug relating to excess ssz If you gave SszBlock too many bytes it would hash them too. --- lighthouse/state/block/block.rs | 12 +++----- lighthouse/state/block/ssz_block.rs | 45 +++++++++++++++++++++-------- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/lighthouse/state/block/block.rs b/lighthouse/state/block/block.rs index 115cbcd0ea..3fe9a668ed 100644 --- a/lighthouse/state/block/block.rs +++ b/lighthouse/state/block/block.rs @@ -1,15 +1,12 @@ use super::utils::types::Hash256; -use super::attestation_record::{ - AttestationRecord, - MIN_SSZ_ATTESTION_RECORD_LENGTH, -}; +use super::attestation_record::AttestationRecord; use super::ssz::{ Encodable, SszStream }; pub const MIN_SSZ_BLOCK_LENGTH: usize = { 4 + 32 + // parent_hash - 8 + // slot_number + 8 + // slot_number 4 + 32 + // randao_reveal - 4 + MIN_SSZ_ATTESTION_RECORD_LENGTH + // attestations (minimum one) + 4 + // attestations (assuming zero) 4 + 32 + // pow_chain_ref 4 + 32 + // active_state_root 4 + 32 // crystallized_state_root @@ -71,8 +68,7 @@ mod tests { #[test] pub fn test_block_min_ssz_length() { - let mut b = Block::zero(); - b.attestations = vec![AttestationRecord::zero()]; + let b = Block::zero(); let mut ssz_stream = SszStream::new(); ssz_stream.append(&b); diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 1e4d20bb1d..dc2a60a1a5 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -7,6 +7,7 @@ use super::block::{ MIN_SSZ_BLOCK_LENGTH, MAX_SSZ_BLOCK_LENGTH, }; +use super::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH; #[derive(Debug, PartialEq)] pub enum BlockValidatorError { @@ -32,34 +33,41 @@ impl<'a> SszBlock<'a> { pub fn from_slice(vec: &'a [u8]) -> Result { - let ssz = &vec[..]; - let len = vec.len(); + let untrimmed_ssz = &vec[..]; /* - * Ensure the SSZ is long enough to be a block. + * Ensure the SSZ is long enough to be a block with + * one attestation record (not necessarily a valid + * attestation record). */ - if len < MIN_SSZ_BLOCK_LENGTH { + if vec.len() < MIN_SSZ_BLOCK_LENGTH + MIN_SSZ_ATTESTION_RECORD_LENGTH { return Err(BlockValidatorError::TooShort); } /* * Ensure the SSZ slice isn't longer than is possible for a block. */ - if len > MAX_SSZ_BLOCK_LENGTH { + if vec.len() > MAX_SSZ_BLOCK_LENGTH { return Err(BlockValidatorError::TooLong); } /* - * Determine how many bytes are used to store attestation records - * and ensure that length is enough to store at least one attestation - * record. + * Determine how many bytes are used to store attestation records. */ - let attestation_len = decode_length(ssz, 80, LENGTH_BYTES) + let attestation_len = decode_length(untrimmed_ssz, 80, LENGTH_BYTES) .map_err(|_| BlockValidatorError::TooShort)?; - if len < (76 + attestation_len + 96) { + /* + * The block only has one variable field, `attestations`, therefore + * the size of the block must be the minimum size, plus the length + * of the attestations. + */ + let block_ssz_len = { + MIN_SSZ_BLOCK_LENGTH + attestation_len + }; + if vec.len() < block_ssz_len { return Err(BlockValidatorError::TooShort); } Ok(Self{ - ssz, + ssz: &untrimmed_ssz[0..block_ssz_len], attestation_len, - len, + len: block_ssz_len, }) } @@ -156,6 +164,19 @@ mod tests { ); } + #[test] + fn test_ssz_block_single_attestation_record_one_byte_long() { + let mut b = Block::zero(); + b.attestations = vec![AttestationRecord::zero()]; + let mut ssz = get_block_ssz(&b); + let original_len = ssz.len(); + ssz.push(42); + + let ssz_block = SszBlock::from_slice(&ssz[..]).unwrap(); + + assert_eq!(ssz_block.len, original_len); + } + #[test] fn test_ssz_block_single_attestation_record() { let mut b = Block::zero(); From 2beeef815321360765e3eccff2c37e1fc5a367bc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 20 Sep 2018 13:52:45 +1000 Subject: [PATCH 21/74] Add doc comments to SszBlock --- lighthouse/state/block/ssz_block.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index dc2a60a1a5..79175b3c22 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -22,6 +22,11 @@ const LENGTH_BYTES: usize = 4; /// Allows for reading of block values directly from serialized /// ssz bytes. +/// +/// Use this to perform intial checks before we fully de-serialize +/// a block. It should only really be used to verify blocks that come +/// in from the network, for internal operations we should use a full +/// `Block`. #[derive(Debug, PartialEq)] pub struct SszBlock<'a> { ssz: &'a [u8], @@ -30,6 +35,12 @@ pub struct SszBlock<'a> { } impl<'a> SszBlock<'a> { + /// Create a new instance from a slice reference. + /// + /// This function will validate the length of the ssz + /// string, however it will not validate the contents. + /// + /// The slice will not be copied, instead referenced. pub fn from_slice(vec: &'a [u8]) -> Result { @@ -71,14 +82,17 @@ impl<'a> SszBlock<'a> { }) } + /// Return the canonical hash for this block. pub fn block_hash(&self) -> Vec { canonical_hash(self.ssz) } + /// Return the `parent_hash` field. pub fn parent_hash(&self) -> &[u8] { &self.ssz[4..36] } + /// Return the `slot_number` field. pub fn slot_number(&self) -> u64 { /* * An error should be unreachable from this decode @@ -95,25 +109,30 @@ impl<'a> SszBlock<'a> { } } + /// Return the `randao_reveal` field. pub fn randao_reveal(&self) -> &[u8] { &self.ssz[48..80] } + /// Return the `attestations` field. pub fn attestations(&self) -> &[u8] { let start = 80 + LENGTH_BYTES; &self.ssz[start..(start + self.attestation_len)] } + /// Return the `pow_chain_ref` field. pub fn pow_chain_ref(&self) -> &[u8] { let start = self.len - (32 + LENGTH_BYTES + 32 + LENGTH_BYTES + 32); &self.ssz[start..(start + 32)] } + /// Return the `active_state_root` field. pub fn act_state_root(&self) -> &[u8] { let start = self.len - (32 + LENGTH_BYTES + 32); &self.ssz[start..(start + 32)] } + /// Return the `active_state_root` field. pub fn cry_state_root(&self) -> &[u8] { let start = self.len - 32; &self.ssz[start..(start + 32)] From a4e61541aa156ef8388b96552a98d9505745b997 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 20 Sep 2018 13:56:44 +1000 Subject: [PATCH 22/74] Add SszBlock test to check the canonical hash --- lighthouse/state/block/ssz_block.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 79175b3c22..4d51c3a913 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -222,6 +222,15 @@ mod tests { 58, 145, 52, 47, 62, 158, 131, 46, 147 ]; assert_eq!(hash, expected_hash); + + /* + * Test if you give the SszBlock too many ssz bytes + */ + let mut too_long = serialized.clone(); + too_long.push(42); + let ssz_block = SszBlock::from_slice(&too_long).unwrap(); + let hash = ssz_block.block_hash(); + assert_eq!(hash, expected_hash); } #[test] From 43394c074ce64f4e6a204f05a08002d97c57e91f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 20 Sep 2018 15:17:34 +1000 Subject: [PATCH 23/74] Remove some dangling deps causing warnings --- lighthouse/state/mod.rs | 1 - lighthouse/state/transition/mod.rs | 6 ------ lighthouse/state/transition/validate_block.rs | 4 ---- 3 files changed, 11 deletions(-) delete mode 100644 lighthouse/state/transition/validate_block.rs diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index 737839721d..baa7597d35 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -5,7 +5,6 @@ extern crate bytes; extern crate ssz; use super::utils; -use super::db; pub mod active_state; pub mod attestation_record; diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index 8ba28d05f7..dd8967e652 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,13 +1,7 @@ use super::super::utils::types::Hash256; -use super::crystallized_state; -use super::super::db; -use super::ssz; -use super::blake2; -use super::utils; mod attestation_parent_hashes; mod shuffling; -mod validate_block; pub use self::attestation_parent_hashes::attestation_parent_hashes; pub use self::shuffling::shuffle; diff --git a/lighthouse/state/transition/validate_block.rs b/lighthouse/state/transition/validate_block.rs deleted file mode 100644 index bbd784d327..0000000000 --- a/lighthouse/state/transition/validate_block.rs +++ /dev/null @@ -1,4 +0,0 @@ -enum BlockStatus { - NewBlock, - KnownBlock, -} From f64b8e30a10d8052d5b43c60ecc7a399800aa961 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 20 Sep 2018 15:24:51 +1000 Subject: [PATCH 24/74] Update SszBlock doc comments --- lighthouse/state/block/ssz_block.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 4d51c3a913..e0828705ed 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -20,13 +20,18 @@ pub enum BlockValidatorError { const LENGTH_BYTES: usize = 4; -/// Allows for reading of block values directly from serialized -/// ssz bytes. +/// Allows for reading of block values directly from serialized ssz bytes. /// -/// Use this to perform intial checks before we fully de-serialize -/// a block. It should only really be used to verify blocks that come -/// in from the network, for internal operations we should use a full -/// `Block`. +/// The purpose of this struct is to provide the functionality to read block fields directly from +/// some serialized SSZ slice, effectively allowing us to read the block without fully +/// de-serializing it. +/// +/// This struct should be as "zero-copy" as possible. The `ssz` field is a reference to some slice +/// and each function reads from that slice. +/// +/// Use this to perform intial checks before we fully de-serialize a block. It should only really +/// be used to verify blocks that come in from the network, for internal operations we should use a +/// full `Block`. #[derive(Debug, PartialEq)] pub struct SszBlock<'a> { ssz: &'a [u8], @@ -37,10 +42,12 @@ pub struct SszBlock<'a> { impl<'a> SszBlock<'a> { /// Create a new instance from a slice reference. /// - /// This function will validate the length of the ssz - /// string, however it will not validate the contents. + /// This function will validate the length of the ssz string, however it will not validate the + /// contents. /// - /// The slice will not be copied, instead referenced. + /// The returned `SszBlock` instance will contain a `len` field which can be used to determine + /// how many bytes were read from the slice. In the case of multiple, sequentually serialized + /// blocks `len` can be used to assume the location of the next serialized block. pub fn from_slice(vec: &'a [u8]) -> Result { From f6330ce9679409259834e59c8a109d8337b3a69f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 20 Sep 2018 17:36:23 +1000 Subject: [PATCH 25/74] Add ClientDB `exists()` method --- lighthouse/db/disk_db.rs | 15 +++++++++++++++ lighthouse/db/traits.rs | 3 +++ 2 files changed, 18 insertions(+) diff --git a/lighthouse/db/disk_db.rs b/lighthouse/db/disk_db.rs index e4ebdedec4..d0f6eb9707 100644 --- a/lighthouse/db/disk_db.rs +++ b/lighthouse/db/disk_db.rs @@ -107,6 +107,21 @@ impl ClientDB for DiskDB { Some(handle) => self.db.put_cf(handle, key, val).map_err(|e| e.into()) } } + + /// Return true if some key exists in some column. + fn exists(&self, col: &str, key: &[u8]) + -> Result + { + /* + * I'm not sure if this is the correct way to read if some + * block exists. Naievely I would expect this to unncessarily + * copy some data, but I could be wrong. + */ + match self.db.cf_handle(col) { + None => Err(DBError{ message: "Unknown column".to_string() }), + Some(handle) => Ok(self.db.get_cf(handle, key)?.is_some()) + } + } } diff --git a/lighthouse/db/traits.rs b/lighthouse/db/traits.rs index 97759d3b77..35feda2439 100644 --- a/lighthouse/db/traits.rs +++ b/lighthouse/db/traits.rs @@ -26,5 +26,8 @@ pub trait ClientDB: Sync + Send { fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError>; + + fn exists(&self, col: &str, key: &[u8]) + -> Result; } From 106555421601e5604df4183a93ff438cf56b366a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 20 Sep 2018 17:36:49 +1000 Subject: [PATCH 26/74] Add BlockStore basics --- lighthouse/db/mod.rs | 2 + lighthouse/db/stores/block_store.rs | 103 ++++++++++++++++++++++++++++ lighthouse/db/stores/mod.rs | 10 +++ 3 files changed, 115 insertions(+) create mode 100644 lighthouse/db/stores/block_store.rs create mode 100644 lighthouse/db/stores/mod.rs diff --git a/lighthouse/db/mod.rs b/lighthouse/db/mod.rs index c85eaf18b7..57ba8b4e93 100644 --- a/lighthouse/db/mod.rs +++ b/lighthouse/db/mod.rs @@ -2,6 +2,8 @@ extern crate rocksdb; mod disk_db; mod traits; +mod stores; + pub use self::disk_db::DiskDB; pub use self::traits::{ diff --git a/lighthouse/db/stores/block_store.rs b/lighthouse/db/stores/block_store.rs new file mode 100644 index 0000000000..ab755d0536 --- /dev/null +++ b/lighthouse/db/stores/block_store.rs @@ -0,0 +1,103 @@ +use std::sync::Arc; +use super::{ + ClientDB, + DBError, +}; +use super::BLOCKS_DB_COLUMN as DB_COLUMN; + +pub struct BlockStore + where T: ClientDB +{ + db: Arc, +} + +impl BlockStore { + pub fn new(db: Arc) -> Self { + Self { + db, + } + } + + pub fn put_block(&self, hash: &[u8], ssz: &[u8]) + -> Result<(), DBError> + { + self.db.put(DB_COLUMN, hash, ssz) + } + + pub fn get_block(&self, hash: &[u8]) + -> Result>, DBError> + { + self.db.get(DB_COLUMN, hash) + } + + pub fn block_exists(&self, hash: &[u8]) + -> Result + { + self.db.exists(DB_COLUMN, hash) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::ClientDB; + use super::super::super::DiskDB; + use std::{ env, fs, thread }; + use std::sync::Arc; + + #[test] + fn test_block_store_on_disk_db() { + let pwd = env::current_dir().unwrap(); + let path = pwd.join("block_store_testdb_please_remove"); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + + let column_families = vec![ + DB_COLUMN + ]; + + let mut db = DiskDB::open(&path, None); + + for cf in column_families { + db.create_col(&cf).unwrap(); + } + + let db = Arc::new(db); + + let bs = Arc::new(BlockStore::new(db.clone())); + + let thread_count = 10; + let write_count = 10; + + // We're expecting the product of these numbers to fit in one byte. + assert!(thread_count * write_count <= 255); + + let mut handles = vec![]; + for t in 0..thread_count { + let wc = write_count; + let bs = bs.clone(); + let handle = thread::spawn(move || { + for w in 0..wc { + let key = (t * w) as u8; + let val = 42; + bs.put_block(&vec![key], &vec![val]).unwrap(); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + for t in 0..thread_count { + for w in 0..write_count { + let key = (t * w) as u8; + assert!(bs.block_exists(&vec![key]).unwrap()); + let val = bs.get_block(&vec![key]).unwrap().unwrap(); + assert_eq!(vec![42], val); + } + } + fs::remove_dir_all(&path).unwrap(); + } +} diff --git a/lighthouse/db/stores/mod.rs b/lighthouse/db/stores/mod.rs new file mode 100644 index 0000000000..9c564dd71b --- /dev/null +++ b/lighthouse/db/stores/mod.rs @@ -0,0 +1,10 @@ +use super::{ + ClientDB, + DBError, +}; + +mod block_store; + +pub use self::block_store::BlockStore; + +const BLOCKS_DB_COLUMN: &str = "blocks"; From d4e6f12dedf4a63946b1f67d5d2c0fa7f258ffe8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 21 Sep 2018 11:13:07 +1000 Subject: [PATCH 27/74] Add basic PoW chain db store --- lighthouse/db/stores/mod.rs | 3 +++ lighthouse/db/stores/pow_chain_store.rs | 34 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 lighthouse/db/stores/pow_chain_store.rs diff --git a/lighthouse/db/stores/mod.rs b/lighthouse/db/stores/mod.rs index 9c564dd71b..0db6aba38f 100644 --- a/lighthouse/db/stores/mod.rs +++ b/lighthouse/db/stores/mod.rs @@ -4,7 +4,10 @@ use super::{ }; mod block_store; +mod pow_chain_store; pub use self::block_store::BlockStore; +pub use self::pow_chain_store::PoWChainStore; const BLOCKS_DB_COLUMN: &str = "blocks"; +const POW_CHAIN_DB_COLUMN: &str = "powchain"; diff --git a/lighthouse/db/stores/pow_chain_store.rs b/lighthouse/db/stores/pow_chain_store.rs new file mode 100644 index 0000000000..d428054764 --- /dev/null +++ b/lighthouse/db/stores/pow_chain_store.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; +use super::{ + ClientDB, + DBError, +}; +use super::POW_CHAIN_DB_COLUMN as DB_COLUMN; + +pub struct PoWChainStore + where T: ClientDB +{ + db: Arc, +} + +impl PoWChainStore { + pub fn new(db: Arc) -> Self { + Self { + db, + } + } + + pub fn put_block_hash(&self, hash: &[u8]) + -> Result<(), DBError> + { + self.db.put(DB_COLUMN, hash, &vec![0]) + } + + pub fn block_hash_exists(&self, hash: &[u8]) + -> Result + { + self.db.exists(DB_COLUMN, hash) + } +} + +// TODO: add tests once a memory-db is implemented From 51c842c2363488bab5993ba9e6dfc6cb7b1a4b13 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 21 Sep 2018 11:14:28 +1000 Subject: [PATCH 28/74] Begin basics of block validation --- lighthouse/state/chain_config.rs | 2 + lighthouse/state/mod.rs | 1 + lighthouse/state/transition/mod.rs | 3 +- lighthouse/state/transition/validate_block.rs | 56 +++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 lighthouse/state/transition/validate_block.rs diff --git a/lighthouse/state/chain_config.rs b/lighthouse/state/chain_config.rs index 3d285e7a77..857c2f8626 100644 --- a/lighthouse/state/chain_config.rs +++ b/lighthouse/state/chain_config.rs @@ -2,6 +2,7 @@ pub struct ChainConfig { pub cycle_length: u8, pub shard_count: u16, pub min_committee_size: u64, + pub genesis_time: u64, } impl ChainConfig { @@ -10,6 +11,7 @@ impl ChainConfig { cycle_length: 8, shard_count: 1024, min_committee_size: 128, + genesis_time: 1537488655, // arbitrary } } } diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index baa7597d35..737839721d 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -5,6 +5,7 @@ extern crate bytes; extern crate ssz; use super::utils; +use super::db; pub mod active_state; pub mod attestation_record; diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index dd8967e652..518a86db6c 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,4 +1,5 @@ -use super::super::utils::types::Hash256; +use super::utils::types::Hash256; +use super::db; mod attestation_parent_hashes; mod shuffling; diff --git a/lighthouse/state/transition/validate_block.rs b/lighthouse/state/transition/validate_block.rs new file mode 100644 index 0000000000..6ed3a28242 --- /dev/null +++ b/lighthouse/state/transition/validate_block.rs @@ -0,0 +1,56 @@ +use super::block::SszBlock; +use super::Logger; +use super::db::{ + BlockStore, + PoWChainStore, +}; + +pub enum BlockStatus { + NewBlock, + KnownBlock, + UnknownPoWChainRef, +} + +pub enum SszBlockValidationError { + SszInvalid, + FutureSlot, +} + +macro_rules! valid_if { + ($cond:expr, $val:expr) => { + if ($cond) + return Ok($val); + } + }; +} + +macro_rules! invalid_if { + ($cond:expr, $val:expr) => { + if ($cond) + return Err($val); + } + }; +} + +fn slot_from_time() + + +pub fn validate_ssz_block(b: &SszBlock, + expected_slot: &u64, + block_store: &BlockStore, + pow_store: &PoWChainStore, + log: &Logger) + -> Result +{ + valid_if!(block_store.block_exists(b.block_hash()), + BlockStatus::KnownBlock); + + invalid_if!(b.slot_number() > expected_slot, + SszBlockValidationError::FutureSlot); + + invalid_if!(pow_store.block_hash_exists(b.pow_chain_ref()) == false, + SszBlockValidationError::UnknownPoWChainRef); + + // Do validation here + Ok(BlockStatus::NewBlock) +} From e6a693b89cc1c1f9d77d62f5df6e24319507ef9a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 21 Sep 2018 15:02:00 +1000 Subject: [PATCH 29/74] Add `exists()` function for MemoryDB --- lighthouse/db/memory_db.rs | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/lighthouse/db/memory_db.rs b/lighthouse/db/memory_db.rs index c875b55545..a1292da396 100644 --- a/lighthouse/db/memory_db.rs +++ b/lighthouse/db/memory_db.rs @@ -79,6 +79,23 @@ impl ClientDB for MemoryDB { } } } + + /// Return true if some key exists in some column. + fn exists(&self, col: &str, key: &[u8]) + -> Result + { + // Panic if the DB locks are poisoned. + let db = self.db.read().unwrap(); + let known_columns = self.known_columns.read().unwrap(); + + match known_columns.contains(&col.to_string()) { + false => Err(DBError{ message: "Unknown column".to_string() }), + true => { + let column_key = MemoryDB::get_key_for_col(col, key); + Ok(db.contains_key(&column_key)) + } + } + } } @@ -136,6 +153,31 @@ mod tests { assert!(db.get(col_x, "cats".as_bytes()).is_err()); } + #[test] + fn test_memorydb_exists() { + let col_a: &str = "ColumnA"; + let col_b: &str = "ColumnB"; + + let column_families = vec![ + col_a, + col_b, + ]; + + let db = MemoryDB::open(Some(&column_families)); + + /* + * Testing that if we write to the same key in different columns that + * there is not an overlap. + */ + db.put(col_a, "cats".as_bytes(), "lol".as_bytes()).unwrap(); + + assert_eq!(true, db.exists(col_a, "cats".as_bytes()).unwrap()); + assert_eq!(false, db.exists(col_b, "cats".as_bytes()).unwrap()); + + assert_eq!(false, db.exists(col_a, "dogs".as_bytes()).unwrap()); + assert_eq!(false, db.exists(col_b, "dogs".as_bytes()).unwrap()); + } + #[test] fn test_memorydb_threading() { let col_name: &str = "TestColumn"; From 8cbd4f47a00f8a2346aedb478e24092896fdb293 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 21 Sep 2018 15:02:28 +1000 Subject: [PATCH 30/74] Move BlockStore test across to MemoryDB --- lighthouse/db/stores/block_store.rs | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/lighthouse/db/stores/block_store.rs b/lighthouse/db/stores/block_store.rs index ab755d0536..fde09bec87 100644 --- a/lighthouse/db/stores/block_store.rs +++ b/lighthouse/db/stores/block_store.rs @@ -40,30 +40,15 @@ impl BlockStore { #[cfg(test)] mod tests { use super::*; - use super::super::ClientDB; - use super::super::super::DiskDB; - use std::{ env, fs, thread }; + use super::super::super::MemoryDB; + use std::thread; use std::sync::Arc; #[test] fn test_block_store_on_disk_db() { - let pwd = env::current_dir().unwrap(); - let path = pwd.join("block_store_testdb_please_remove"); - let _ = fs::remove_dir_all(&path); - fs::create_dir_all(&path).unwrap(); - - let column_families = vec![ - DB_COLUMN - ]; - - let mut db = DiskDB::open(&path, None); - - for cf in column_families { - db.create_col(&cf).unwrap(); - } - - let db = Arc::new(db); + let column_families = vec![DB_COLUMN]; + let db = Arc::new(MemoryDB::open(Some(&column_families))); let bs = Arc::new(BlockStore::new(db.clone())); let thread_count = 10; @@ -98,6 +83,5 @@ mod tests { assert_eq!(vec![42], val); } } - fs::remove_dir_all(&path).unwrap(); } } From f0b432d81c45d0f3b5153bbe99172d2c6c6282bf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 21 Sep 2018 18:52:03 +1000 Subject: [PATCH 31/74] Add ValidatorStore --- lighthouse/db/stores/mod.rs | 2 ++ lighthouse/db/stores/validator_store.rs | 34 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 lighthouse/db/stores/validator_store.rs diff --git a/lighthouse/db/stores/mod.rs b/lighthouse/db/stores/mod.rs index 0db6aba38f..e7281ad149 100644 --- a/lighthouse/db/stores/mod.rs +++ b/lighthouse/db/stores/mod.rs @@ -5,9 +5,11 @@ use super::{ mod block_store; mod pow_chain_store; +mod validator_store; pub use self::block_store::BlockStore; pub use self::pow_chain_store::PoWChainStore; const BLOCKS_DB_COLUMN: &str = "blocks"; const POW_CHAIN_DB_COLUMN: &str = "powchain"; +const VALIDATOR_DB_COLUMN: &str = "validator"; diff --git a/lighthouse/db/stores/validator_store.rs b/lighthouse/db/stores/validator_store.rs new file mode 100644 index 0000000000..6f095ca2b6 --- /dev/null +++ b/lighthouse/db/stores/validator_store.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; +use super::{ + ClientDB, + DBError, +}; +use super::VALIDATOR_DB_COLUMN as DB_COLUMN; + +pub struct ValidatorStore + where T: ClientDB +{ + db: Arc, +} + +impl ValidatorStore { + pub fn new(db: Arc) -> Self { + Self { + db, + } + } + + pub fn put_validator_record_by_index(&self, hash: &[u8], val: &[u8]) + -> Result<(), DBError> + { + self.db.put(DB_COLUMN, hash, &vec![0]) + } + + pub fn get_validator_record_by_index(&self, hash: &[u8]) + -> Result + { + self.db.exists(DB_COLUMN, hash) + } +} + +// TODO: add tests From d00d109fa683176ba0cc9fdeac919fa8c6a53274 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 07:37:25 +1000 Subject: [PATCH 32/74] Update as per changes to SSZ 32-byte hashes are no longer prefixed with a length sequence. --- lighthouse/state/attestation_record.rs | 4 ++-- lighthouse/state/block/block.rs | 14 ++++++------ lighthouse/state/block/ssz_block.rs | 31 +++++++++++++++++--------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/lighthouse/state/attestation_record.rs b/lighthouse/state/attestation_record.rs index 25abbd3f3a..a665394a4a 100644 --- a/lighthouse/state/attestation_record.rs +++ b/lighthouse/state/attestation_record.rs @@ -6,10 +6,10 @@ pub const MIN_SSZ_ATTESTION_RECORD_LENGTH: usize = { 8 + // slot 2 + // shard_id 4 + // oblique_parent_hashes (empty list) - 4 + 32 + // shard_block_hash + 32 + // shard_block_hash 5 + // attester_bitfield (assuming 1 byte of bitfield) 8 + // justified_slot - 4 + 32 + // justified_block_hash + 32 + // justified_block_hash 4 + (2 * 32) // aggregate sig (two 256 bit points) }; diff --git a/lighthouse/state/block/block.rs b/lighthouse/state/block/block.rs index 3fe9a668ed..e3a75b1c59 100644 --- a/lighthouse/state/block/block.rs +++ b/lighthouse/state/block/block.rs @@ -3,13 +3,13 @@ use super::attestation_record::AttestationRecord; use super::ssz::{ Encodable, SszStream }; pub const MIN_SSZ_BLOCK_LENGTH: usize = { - 4 + 32 + // parent_hash - 8 + // slot_number - 4 + 32 + // randao_reveal - 4 + // attestations (assuming zero) - 4 + 32 + // pow_chain_ref - 4 + 32 + // active_state_root - 4 + 32 // crystallized_state_root + 32 + // parent_hash + 8 + // slot_number + 32 + // randao_reveal + 4 + // attestations (assuming zero) + 32 + // pow_chain_ref + 32 + // active_state_root + 32 // crystallized_state_root }; pub const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + (1 << 24); diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index e0828705ed..86fd5283f8 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -69,7 +69,7 @@ impl<'a> SszBlock<'a> { /* * Determine how many bytes are used to store attestation records. */ - let attestation_len = decode_length(untrimmed_ssz, 80, LENGTH_BYTES) + let attestation_len = decode_length(untrimmed_ssz, 72, LENGTH_BYTES) .map_err(|_| BlockValidatorError::TooShort)?; /* * The block only has one variable field, `attestations`, therefore @@ -96,7 +96,7 @@ impl<'a> SszBlock<'a> { /// Return the `parent_hash` field. pub fn parent_hash(&self) -> &[u8] { - &self.ssz[4..36] + &self.ssz[0..32] } /// Return the `slot_number` field. @@ -109,7 +109,7 @@ impl<'a> SszBlock<'a> { * If you can make this function panic, please report * it to paul@sigmaprime.io */ - if let Ok((n, _)) = u64::ssz_decode(&self.ssz, 36) { + if let Ok((n, _)) = u64::ssz_decode(&self.ssz, 32) { n } else { unreachable!(); @@ -118,24 +118,24 @@ impl<'a> SszBlock<'a> { /// Return the `randao_reveal` field. pub fn randao_reveal(&self) -> &[u8] { - &self.ssz[48..80] + &self.ssz[40..72] } /// Return the `attestations` field. pub fn attestations(&self) -> &[u8] { - let start = 80 + LENGTH_BYTES; + let start = 72 + LENGTH_BYTES; &self.ssz[start..(start + self.attestation_len)] } /// Return the `pow_chain_ref` field. pub fn pow_chain_ref(&self) -> &[u8] { - let start = self.len - (32 + LENGTH_BYTES + 32 + LENGTH_BYTES + 32); + let start = self.len - (32 * 3); &self.ssz[start..(start + 32)] } /// Return the `active_state_root` field. pub fn act_state_root(&self) -> &[u8] { - let start = self.len - (32 + LENGTH_BYTES + 32); + let start = self.len - (32 * 2); &self.ssz[start..(start + 32)] } @@ -212,6 +212,17 @@ mod tests { assert!(SszBlock::from_slice(&ssz[..]).is_ok()); } + #[test] + fn test_ssz_block_attestation_length() { + let mut block = Block::zero(); + block.attestations.push(AttestationRecord::zero()); + + let serialized = get_block_ssz(&block); + let ssz_block = SszBlock::from_slice(&serialized).unwrap(); + + assert_eq!(ssz_block.attestation_len, MIN_SSZ_ATTESTION_RECORD_LENGTH); + } + #[test] fn test_ssz_block_block_hash() { let mut block = Block::zero(); @@ -224,9 +235,9 @@ mod tests { // will tell us if the hash changes, not that it matches some // canonical reference. let expected_hash = [ - 28, 184, 51, 12, 226, 15, 73, 50, 66, 19, 168, 149, - 229, 122, 141, 111, 42, 236, 137, 157, 230, 90, 149, - 58, 145, 52, 47, 62, 158, 131, 46, 147 + 214, 217, 16, 230, 17, 204, 99, 222, 104, 90, 128, 228, + 12, 249, 56, 255, 110, 10, 229, 29, 110, 107, 105, 195, + 219, 132, 138, 206, 204, 34, 21, 159 ]; assert_eq!(hash, expected_hash); From af2ba7eebf0294ae8db4742209bb267292a088b4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 11:13:10 +1000 Subject: [PATCH 33/74] Fix clippy lint in MemoryDB --- lighthouse/db/memory_db.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lighthouse/db/memory_db.rs b/lighthouse/db/memory_db.rs index 154b85560e..41d1f7e6de 100644 --- a/lighthouse/db/memory_db.rs +++ b/lighthouse/db/memory_db.rs @@ -86,12 +86,12 @@ impl ClientDB for MemoryDB { let db = self.db.read().unwrap(); let known_columns = self.known_columns.read().unwrap(); - match known_columns.contains(&col.to_string()) { - false => Err(DBError{ message: "Unknown column".to_string() }), - true => { - let column_key = MemoryDB::get_key_for_col(col, key); - Ok(db.contains_key(&column_key)) - } + if known_columns.contains(&col.to_string()) { + let column_key = MemoryDB::get_key_for_col(col, key); + Ok(db.contains_key(&column_key)) + } else { + Err(DBError{ message: "Unknown column".to_string() }) + } } } From bbf640c40d87e7ce8d58ca98c9d83e4c6d03fc1e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 11:13:55 +1000 Subject: [PATCH 34/74] Add progress to validate_block --- lighthouse/db/mod.rs | 2 +- lighthouse/db/stores/mod.rs | 1 + lighthouse/main.rs | 2 +- lighthouse/state/mod.rs | 1 + lighthouse/state/transition/mod.rs | 3 ++ lighthouse/state/transition/validate_block.rs | 53 +++++++++---------- 6 files changed, 33 insertions(+), 29 deletions(-) diff --git a/lighthouse/db/mod.rs b/lighthouse/db/mod.rs index ab70ede94c..e030fc009a 100644 --- a/lighthouse/db/mod.rs +++ b/lighthouse/db/mod.rs @@ -4,7 +4,7 @@ extern crate blake2_rfc as blake2; mod disk_db; mod memory_db; mod traits; -mod stores; +pub mod stores; pub use self::disk_db::DiskDB; diff --git a/lighthouse/db/stores/mod.rs b/lighthouse/db/stores/mod.rs index e7281ad149..a7e5c60bb8 100644 --- a/lighthouse/db/stores/mod.rs +++ b/lighthouse/db/stores/mod.rs @@ -9,6 +9,7 @@ mod validator_store; pub use self::block_store::BlockStore; pub use self::pow_chain_store::PoWChainStore; +pub use self::validator_store::ValidatorStore; const BLOCKS_DB_COLUMN: &str = "blocks"; const POW_CHAIN_DB_COLUMN: &str = "powchain"; diff --git a/lighthouse/main.rs b/lighthouse/main.rs index 367b719207..257ef1b6b5 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -15,7 +15,7 @@ pub mod config; use std::path::PathBuf; -use slog::Drain; +use slog::{ Drain, Logger }; use clap::{ Arg, App }; use config::LighthouseConfig; use client::Client; diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index 737839721d..8f8f748d87 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -4,6 +4,7 @@ extern crate blake2_rfc as blake2; extern crate bytes; extern crate ssz; +use super::Logger; use super::utils; use super::db; diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index 518a86db6c..2d4592f303 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,8 +1,11 @@ +use super::block; +use super::Logger; use super::utils::types::Hash256; use super::db; mod attestation_parent_hashes; mod shuffling; +mod validate_block; pub use self::attestation_parent_hashes::attestation_parent_hashes; pub use self::shuffling::shuffle; diff --git a/lighthouse/state/transition/validate_block.rs b/lighthouse/state/transition/validate_block.rs index 6ed3a28242..c7c04e10b7 100644 --- a/lighthouse/state/transition/validate_block.rs +++ b/lighthouse/state/transition/validate_block.rs @@ -1,55 +1,54 @@ use super::block::SszBlock; use super::Logger; use super::db::{ + ClientDB, + DBError, +}; +use super::db::stores::{ BlockStore, PoWChainStore, + ValidatorStore, }; pub enum BlockStatus { NewBlock, KnownBlock, - UnknownPoWChainRef, } pub enum SszBlockValidationError { SszInvalid, FutureSlot, + UnknownPoWChainRef, + DatabaseError(String), } -macro_rules! valid_if { - ($cond:expr, $val:expr) => { - if ($cond) - return Ok($val); - } - }; +impl From for SszBlockValidationError { + fn from(e: DBError) -> SszBlockValidationError { + SszBlockValidationError::DatabaseError(e.message) + } } -macro_rules! invalid_if { - ($cond:expr, $val:expr) => { - if ($cond) - return Err($val); - } - }; -} -fn slot_from_time() - - -pub fn validate_ssz_block(b: &SszBlock, - expected_slot: &u64, - block_store: &BlockStore, - pow_store: &PoWChainStore, +pub fn validate_ssz_block(b: &SszBlock, + expected_slot: u64, + block_store: &BlockStore, + pow_store: &PoWChainStore, + validator_store: &ValidatorStore, log: &Logger) -> Result + where T: Sized + ClientDB { - valid_if!(block_store.block_exists(b.block_hash()), - BlockStatus::KnownBlock); + if block_store.block_exists(&b.block_hash())? { + return Ok(BlockStatus::KnownBlock); + } - invalid_if!(b.slot_number() > expected_slot, - SszBlockValidationError::FutureSlot); + if b.slot_number() > expected_slot { + return Err(SszBlockValidationError::FutureSlot); + } - invalid_if!(pow_store.block_hash_exists(b.pow_chain_ref()) == false, - SszBlockValidationError::UnknownPoWChainRef); + if pow_store.block_hash_exists(b.pow_chain_ref())? == false { + return Err(SszBlockValidationError::UnknownPoWChainRef); + } // Do validation here Ok(BlockStatus::NewBlock) From 67f2850d370b26f321b91467d8d9762fac123069 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 15:52:10 +1000 Subject: [PATCH 35/74] Fix bug on BooleanBitfield decoding --- boolean-bitfield/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/boolean-bitfield/src/lib.rs b/boolean-bitfield/src/lib.rs index 9a6effebb0..717f487760 100644 --- a/boolean-bitfield/src/lib.rs +++ b/boolean-bitfield/src/lib.rs @@ -161,7 +161,8 @@ impl ssz::Decodable for BooleanBitfield { Ok((BooleanBitfield::new(), index + ssz::LENGTH_BYTES)) } else { - let b = BooleanBitfield::from(&bytes[(index + 4)..(len + 4)]); + let b = BooleanBitfield:: + from(&bytes[(index + 4)..(index + len + 4)]); let index = index + ssz::LENGTH_BYTES + len; Ok((b, index)) } From 4d3422d332e771b9bc5f553b7435664ef2e5e303 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 15:53:31 +1000 Subject: [PATCH 36/74] Remove warnings, restructure validation files --- lighthouse/db/disk_db.rs | 4 ++++ lighthouse/db/stores/validator_store.rs | 2 +- lighthouse/state/helpers.rs | 3 --- lighthouse/state/transition/mod.rs | 8 -------- lighthouse/state/validation/mod.rs | 8 +++++++- .../validate_block.rs => validation/ssz_block.rs} | 13 ++++++------- lighthouse/utils/macros.rs | 2 +- 7 files changed, 19 insertions(+), 21 deletions(-) rename lighthouse/state/{transition/validate_block.rs => validation/ssz_block.rs} (77%) diff --git a/lighthouse/db/disk_db.rs b/lighthouse/db/disk_db.rs index 338ec47099..2b101355dc 100644 --- a/lighthouse/db/disk_db.rs +++ b/lighthouse/db/disk_db.rs @@ -36,6 +36,9 @@ impl DiskDB { let mut options = Options::default(); options.create_if_missing(true); + // TODO: ensure that columns are created (and remove + // the dead_code allow) + /* * Initialise the path */ @@ -58,6 +61,7 @@ impl DiskDB { /// Create a RocksDB column family. Corresponds to the /// `create_cf()` function on the RocksDB API. + #[allow(dead_code)] fn create_col(&mut self, col: &str) -> Result<(), DBError> { diff --git a/lighthouse/db/stores/validator_store.rs b/lighthouse/db/stores/validator_store.rs index 6f095ca2b6..badea16e20 100644 --- a/lighthouse/db/stores/validator_store.rs +++ b/lighthouse/db/stores/validator_store.rs @@ -21,7 +21,7 @@ impl ValidatorStore { pub fn put_validator_record_by_index(&self, hash: &[u8], val: &[u8]) -> Result<(), DBError> { - self.db.put(DB_COLUMN, hash, &vec![0]) + self.db.put(DB_COLUMN, hash, val) } pub fn get_validator_record_by_index(&self, hash: &[u8]) diff --git a/lighthouse/state/helpers.rs b/lighthouse/state/helpers.rs index 388fedafbf..5ed03daf7d 100644 --- a/lighthouse/state/helpers.rs +++ b/lighthouse/state/helpers.rs @@ -1,9 +1,6 @@ /* * Collection of helper functions used in the state transition modules */ -use super::active_state::ActiveState; -use super::block::Block; -use super::chain_config::ChainConfig; use super::utils::errors::ParameterError; use super::utils::types::Hash256; diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index de80e4961d..9fe3817572 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,16 +1,8 @@ -use super::block; -use super::Logger; use super::utils::types::Hash256; use super::utils::errors::ParameterError; -use super::db; mod attestation_parent_hashes; mod shuffling; -mod validate_block; pub use self::attestation_parent_hashes::attestation_parent_hashes; pub use self::shuffling::shuffle; - - - - diff --git a/lighthouse/state/validation/mod.rs b/lighthouse/state/validation/mod.rs index c11b652612..5630a96e9c 100644 --- a/lighthouse/state/validation/mod.rs +++ b/lighthouse/state/validation/mod.rs @@ -1,7 +1,13 @@ +/* use super::crystallized_state::CrystallizedState; use super::active_state::ActiveState; use super::attestation_record::AttestationRecord; use super::block::Block; use super::chain_config::ChainConfig; +*/ +use super::block; +use super::Logger; +use super::db; -mod attestation_validation; +// mod attestation_validation; +mod ssz_block; diff --git a/lighthouse/state/transition/validate_block.rs b/lighthouse/state/validation/ssz_block.rs similarity index 77% rename from lighthouse/state/transition/validate_block.rs rename to lighthouse/state/validation/ssz_block.rs index c7c04e10b7..5d3d9b59ab 100644 --- a/lighthouse/state/transition/validate_block.rs +++ b/lighthouse/state/validation/ssz_block.rs @@ -16,7 +16,6 @@ pub enum BlockStatus { } pub enum SszBlockValidationError { - SszInvalid, FutureSlot, UnknownPoWChainRef, DatabaseError(String), @@ -28,13 +27,13 @@ impl From for SszBlockValidationError { } } - +#[allow(dead_code)] pub fn validate_ssz_block(b: &SszBlock, - expected_slot: u64, - block_store: &BlockStore, - pow_store: &PoWChainStore, - validator_store: &ValidatorStore, - log: &Logger) + expected_slot: u64, + block_store: &BlockStore, + pow_store: &PoWChainStore, + _validator_store: &ValidatorStore, + _log: &Logger) -> Result where T: Sized + ClientDB { diff --git a/lighthouse/utils/macros.rs b/lighthouse/utils/macros.rs index e6e3be01ec..60bb129142 100644 --- a/lighthouse/utils/macros.rs +++ b/lighthouse/utils/macros.rs @@ -1,7 +1,7 @@ #[macro_export] macro_rules! assert_error { ($exp: expr, $err: expr) => { - if ( !$exp ) { + if !$exp { return Err($err); } } From 78de4e43e2a2287d0377734121290a44f3ca4fba Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 15:54:04 +1000 Subject: [PATCH 37/74] Add SSZ decoding for AttRec, fix length mistake --- lighthouse/state/attestation_record.rs | 67 ++++++++++++++++++++++++-- lighthouse/state/block/ssz_block.rs | 6 +-- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/lighthouse/state/attestation_record.rs b/lighthouse/state/attestation_record.rs index a665394a4a..73ed5e543e 100644 --- a/lighthouse/state/attestation_record.rs +++ b/lighthouse/state/attestation_record.rs @@ -1,6 +1,12 @@ use super::utils::types::{ Hash256, Bitfield }; use super::utils::bls::{ AggregateSignature }; -use super::ssz::{ Encodable, SszStream }; +use super::ssz::{ + Encodable, + Decodable, + DecodeError, + decode_ssz_list, + SszStream, +}; pub const MIN_SSZ_ATTESTION_RECORD_LENGTH: usize = { 8 + // slot @@ -10,9 +16,10 @@ pub const MIN_SSZ_ATTESTION_RECORD_LENGTH: usize = { 5 + // attester_bitfield (assuming 1 byte of bitfield) 8 + // justified_slot 32 + // justified_block_hash - 4 + (2 * 32) // aggregate sig (two 256 bit points) + 4 + (2 * 8) // aggregate sig (two 256 bit points) }; +#[derive(Debug)] pub struct AttestationRecord { pub slot: u64, pub shard_id: u16, @@ -34,7 +41,34 @@ impl Encodable for AttestationRecord { s.append(&self.justified_slot); s.append(&self.justified_block_hash); // TODO: encode the aggregate sig correctly - s.append_vec(&vec![0_u8; 64]) + s.append_vec(&vec![0_u8; 16]) + } +} + +impl Decodable for AttestationRecord { + fn ssz_decode(bytes: &[u8], i: usize) + -> Result<(Self, usize), DecodeError> + { + let (slot, i) = u64::ssz_decode(bytes, i)?; + let (shard_id, i) = u16::ssz_decode(bytes, i)?; + let (oblique_parent_hashes, i) = decode_ssz_list(bytes, i)?; + let (shard_block_hash, i) = Hash256::ssz_decode(bytes, i)?; + let (attester_bitfield, i) = Bitfield::ssz_decode(bytes, i)?; + let (justified_slot, i) = u64::ssz_decode(bytes, i)?; + let (justified_block_hash, i) = Hash256::ssz_decode(bytes, i)?; + // Do aggregate sig decoding properly. + let aggregate_sig = None; let i = i + 20; + let attestation_record = Self { + slot, + shard_id, + oblique_parent_hashes, + shard_block_hash, + attester_bitfield, + justified_slot, + justified_block_hash, + aggregate_sig, + }; + Ok((attestation_record, i)) } } @@ -68,4 +102,31 @@ mod tests { assert_eq!(ssz.len(), MIN_SSZ_ATTESTION_RECORD_LENGTH); } + + #[test] + pub fn test_attestation_record_min_ssz_encode_decode() { + let original = AttestationRecord { + slot: 7, + shard_id: 9, + oblique_parent_hashes: vec![Hash256::from(&vec![14; 32][..])], + shard_block_hash: Hash256::from(&vec![15; 32][..]), + attester_bitfield: Bitfield::from(&vec![17; 42][..]), + justified_slot: 19, + justified_block_hash: Hash256::from(&vec![15; 32][..]), + aggregate_sig: None, + }; + + let mut ssz_stream = SszStream::new(); + ssz_stream.append(&original); + + let (decoded, _) = AttestationRecord:: + ssz_decode(&ssz_stream.drain(), 0).unwrap(); + assert_eq!(original.slot, decoded.slot); + assert_eq!(original.shard_id, decoded.shard_id); + assert_eq!(original.oblique_parent_hashes, decoded.oblique_parent_hashes); + assert_eq!(original.shard_block_hash, decoded.shard_block_hash); + assert_eq!(original.attester_bitfield, decoded.attester_bitfield); + assert_eq!(original.justified_slot, decoded.justified_slot); + assert_eq!(original.justified_block_hash, decoded.justified_block_hash); + } } diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 86fd5283f8..577857a19e 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -235,9 +235,9 @@ mod tests { // will tell us if the hash changes, not that it matches some // canonical reference. let expected_hash = [ - 214, 217, 16, 230, 17, 204, 99, 222, 104, 90, 128, 228, - 12, 249, 56, 255, 110, 10, 229, 29, 110, 107, 105, 195, - 219, 132, 138, 206, 204, 34, 21, 159 + 195, 180, 208, 144, 113, 20, 129, 108, 14, 128, 166, 170, + 137, 15, 191, 186, 34, 171, 79, 214, 74, 86, 89, 202, 255, + 9, 100, 170, 149, 160, 93, 59 ]; assert_eq!(hash, expected_hash); From 890aaf7335431919edcb5586f6494c23b8171cd6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 21:58:54 +1000 Subject: [PATCH 38/74] Add `num_bytes()` to bitfield --- boolean-bitfield/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/boolean-bitfield/src/lib.rs b/boolean-bitfield/src/lib.rs index 717f487760..8328e63509 100644 --- a/boolean-bitfield/src/lib.rs +++ b/boolean-bitfield/src/lib.rs @@ -86,6 +86,9 @@ impl BooleanBitfield { /// vector. pub fn is_empty(&self) -> bool { self.len == 0 } + /// The number of bytes required to represent the bitfield. + pub fn num_bytes(&self) -> usize { self.vec.len() } + /// Iterate through the underlying vector and count the number of /// true bits. pub fn num_true_bits(&self) -> u64 { From 30bf97539cecf24e861ee9abf7d244899c4b88dd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 21:59:44 +1000 Subject: [PATCH 39/74] Create unique errors for att. parent hashes --- .../transition/attestation_parent_hashes.rs | 29 ++++++++++--------- lighthouse/state/transition/mod.rs | 4 +-- lighthouse/utils/errors.rs | 9 ------ 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/lighthouse/state/transition/attestation_parent_hashes.rs b/lighthouse/state/transition/attestation_parent_hashes.rs index 2ba4712d55..f98414c394 100644 --- a/lighthouse/state/transition/attestation_parent_hashes.rs +++ b/lighthouse/state/transition/attestation_parent_hashes.rs @@ -1,5 +1,12 @@ use super::Hash256; -use super::ParameterError; + +pub enum ParentHashesError { + BadCurrentHashes, + BadObliqueHashes, + SlotTooHigh, + SlotTooLow, + IntWrapping, +} /// This function is used to select the hashes used in /// the signing of an AttestationRecord. @@ -18,23 +25,20 @@ pub fn attestation_parent_hashes( attestation_slot: u64, current_hashes: &[Hash256], oblique_hashes: &[Hash256]) - -> Result, ParameterError> + -> Result, ParentHashesError> { // This cast places a limit on cycle_length. If you change it, check math // for overflow. let cycle_length: u64 = u64::from(cycle_length); if current_hashes.len() as u64 != (cycle_length * 2) { - return Err(ParameterError::InvalidInput(String::from( - "current_hashes.len() must equal cycle_length * 2"))); - } - if attestation_slot >= block_slot { - return Err(ParameterError::InvalidInput(String::from( - "attestation_slot must be less than block_slot"))); + return Err(ParentHashesError::BadCurrentHashes); } if oblique_hashes.len() as u64 > cycle_length { - return Err(ParameterError::InvalidInput(String::from( - "oblique_hashes.len() must be <= cycle_length * 2"))); + return Err(ParentHashesError::BadObliqueHashes); + } + if attestation_slot >= block_slot { + return Err(ParentHashesError::SlotTooHigh); } /* @@ -44,8 +48,7 @@ pub fn attestation_parent_hashes( let attestation_distance = block_slot - attestation_slot; if attestation_distance > cycle_length { - return Err(ParameterError::InvalidInput(String::from( - "attestation_slot must be withing one cycle of block_slot"))); + return Err(ParentHashesError::SlotTooLow); } /* @@ -63,7 +66,7 @@ pub fn attestation_parent_hashes( */ let end = start.checked_add(cycle_length) .and_then(|x| x.checked_sub(oblique_hashes.len() as u64)) - .ok_or(ParameterError::IntWrapping)?; + .ok_or(ParentHashesError::IntWrapping)?; let mut hashes = Vec::new(); diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index 9fe3817572..7510796716 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,8 +1,6 @@ use super::utils::types::Hash256; -use super::utils::errors::ParameterError; -mod attestation_parent_hashes; +pub mod attestation_parent_hashes; mod shuffling; -pub use self::attestation_parent_hashes::attestation_parent_hashes; pub use self::shuffling::shuffle; diff --git a/lighthouse/utils/errors.rs b/lighthouse/utils/errors.rs index 2a25b29b8e..b464f6a485 100644 --- a/lighthouse/utils/errors.rs +++ b/lighthouse/utils/errors.rs @@ -1,14 +1,5 @@ // Collection of custom errors -#[derive(Debug,PartialEq)] -pub enum AttestationValidationError { - SlotTooHigh, - SlotTooLow(String), - IncorrectBitField, - NonZeroTrailingBits, - AggregateSignatureFail -} - #[derive(Debug,PartialEq)] pub enum ParameterError { IntWrapping, From 75031f5e69859062c7b05cceba9097593e823a13 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 22:00:11 +1000 Subject: [PATCH 40/74] Progress on attestation validation --- .../validation/attestation_validation.rs | 181 +++++++++++++++++- lighthouse/state/validation/mod.rs | 6 +- lighthouse/state/validation/ssz_block.rs | 2 +- 3 files changed, 179 insertions(+), 10 deletions(-) diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation_validation.rs index 4704320638..9dfa943fb6 100644 --- a/lighthouse/state/validation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation_validation.rs @@ -1,14 +1,178 @@ -use super::CrystallizedState; -use super::ActiveState; use super::AttestationRecord; -use super::Block; -use super::ChainConfig; +use super::attestation_parent_hashes::{ + attestation_parent_hashes, + ParentHashesError, +}; +use super::db::ClientDB; +use super::db::stores::BlockStore; +use super::ssz::SszStream; +use super::utils::bls::{ + AggregateSignature, + PublicKey, +}; +use super::utils::hash::canonical_hash; +use super::utils::types::{ + Hash256, + Bitfield, +}; +use std::collections::HashMap; +use std::sync::Arc; -use ::utils::errors::AttestationValidationError; +#[derive(Debug,PartialEq)] +pub enum AttestationValidationError { + SlotTooHigh, + SlotTooLow, + JustifiedSlotTooHigh, + TooManyObliqueHashes, + BadCurrentHashes, + BadObliqueHashes, + BadAttesterMap, + IntWrapping, + IncorrectBitField, + NoSignatures, + NonZeroTrailingBits, + AggregateSignatureFail +} -// implementation of validate_attestation in the v2.1 python reference implementation -// see: https://github.com/ethereum/beacon_chain/blob/a79ab2c6f03cbdabf2b6d9d435c26e2b216e09a5/beacon_chain/state/state_transition.py#L61 -pub fn validate_attestation( +type Slot = u64; +type ShardId = u64; +type AttesterMap = HashMap<(Slot, ShardId), Vec>; + +fn bytes_for_bits(bits: usize) -> usize { + (bits.saturating_sub(1) / 8) + 1 +} + +pub fn validate_attestation(a: &AttestationRecord, + block_slot: u64, + cycle_length: u8, + known_last_justified_slot: u64, + known_parent_hashes: Arc>, + block_store: BlockStore, + attester_map: Arc) + -> Result + where T: ClientDB + Sized +{ + /* + * The attesation slot must not be higher than the block that contained it. + */ + if a.slot > block_slot { + return Err(AttestationValidationError::SlotTooHigh); + } + + /* + * The slot of this attestation must not be more than cycle_length + 1 distance + * from the block that contained it. + * + * The below code stays overflow-safe as long as cycle length is a < 64 bit integer. + */ + if a.slot < block_slot.saturating_sub(u64::from(cycle_length) + 1) { + return Err(AttestationValidationError::SlotTooLow); + } + + /* + * The attestation must indicate that its last justified slot is the same as the last justified + * slot known to us. + */ + if a.justified_slot > known_last_justified_slot { + return Err(AttestationValidationError::JustifiedSlotTooHigh); + } + + /* + * There is no need to include more oblique parents hashes than there are blocks + * in a cycle. + */ + if a.oblique_parent_hashes.len() > usize::from(cycle_length) { + return Err(AttestationValidationError::TooManyObliqueHashes); + } + + let attestation_indices = attester_map.get(&(a.slot, a.shard_id.into())) + .ok_or(AttestationValidationError::BadAttesterMap)?; + + if a.attester_bitfield.num_bytes() != + bytes_for_bits(attestation_indices.len()) + { + return Err(AttestationValidationError::IncorrectBitField); + } + + let signed_message = { + let parent_hashes = attestation_parent_hashes( + cycle_length, + block_slot, + a.slot, + &known_parent_hashes, + &a.oblique_parent_hashes)?; + generate_signed_message( + a.slot, + &parent_hashes, + a.shard_id, + &a.shard_block_hash, + a.justified_slot) + }; + + Ok(false) +} + +fn collect_pub_keys(attestation_indices: &Vec, + bitfield: &Bitfield) + -> Option> +{ + // cats + None +} + +/// Generates the message used to validate the signature provided with an AttestationRecord. +/// +/// Ensures that the signer of the message has a view of the chain that is compatible with ours. +fn generate_signed_message(slot: u64, + parent_hashes: &[Hash256], + shard_id: u16, + shard_block_hash: &Hash256, + justified_slot: u64) + -> Vec +{ + /* + * Note: it's a little risky here to use SSZ, because the encoding is not necessarily SSZ + * (for example, SSZ might change whilst this doesn't). + * + * I have suggested switching this to ssz here: + * https://github.com/ethereum/eth2.0-specs/issues/5 + * + * If this doesn't happen, it would be safer to not use SSZ at all. + */ + let mut ssz_stream = SszStream::new(); + ssz_stream.append(&slot); + for h in parent_hashes { + ssz_stream.append_encoded_raw(&h.to_vec()) + } + ssz_stream.append(&shard_id); + ssz_stream.append(shard_block_hash); + ssz_stream.append(&justified_slot); + let bytes = ssz_stream.drain(); + canonical_hash(&bytes) +} + +impl From for AttestationValidationError { + fn from(e: ParentHashesError) -> Self { + match e { + ParentHashesError::BadCurrentHashes => + AttestationValidationError::BadCurrentHashes, + ParentHashesError::BadObliqueHashes => + AttestationValidationError::BadObliqueHashes, + ParentHashesError::SlotTooLow => + AttestationValidationError::SlotTooLow, + ParentHashesError::SlotTooHigh => + AttestationValidationError::SlotTooHigh, + ParentHashesError::IntWrapping => + AttestationValidationError::IntWrapping + } + } +} + +/* +// Implementation of validate_attestation in the v2.1 python reference implementation see: +// +// github.com/ethereum/beacon_chain/beacon_chain/state/state_transition.py +pub fn validate_attestation_2( crystallized_state: &CrystallizedState, active_state: &ActiveState, attestation: &AttestationRecord, @@ -79,3 +243,4 @@ mod tests { //assert_eq!(result, Err(AttestationValidationError::SlotTooLow)); } } +*/ diff --git a/lighthouse/state/validation/mod.rs b/lighthouse/state/validation/mod.rs index 5630a96e9c..39663dba11 100644 --- a/lighthouse/state/validation/mod.rs +++ b/lighthouse/state/validation/mod.rs @@ -8,6 +8,10 @@ use super::chain_config::ChainConfig; use super::block; use super::Logger; use super::db; +use super::attestation_record::AttestationRecord; +use super::ssz; +use super::transition::attestation_parent_hashes; +use super::utils; -// mod attestation_validation; +mod attestation_validation; mod ssz_block; diff --git a/lighthouse/state/validation/ssz_block.rs b/lighthouse/state/validation/ssz_block.rs index 5d3d9b59ab..c94f252949 100644 --- a/lighthouse/state/validation/ssz_block.rs +++ b/lighthouse/state/validation/ssz_block.rs @@ -35,7 +35,7 @@ pub fn validate_ssz_block(b: &SszBlock, _validator_store: &ValidatorStore, _log: &Logger) -> Result - where T: Sized + ClientDB + where T: ClientDB + Sized { if block_store.block_exists(&b.block_hash())? { return Ok(BlockStatus::KnownBlock); From 3968aaa0e0918dfe74e5cbcd36b179e7ef286d7a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 22 Sep 2018 22:07:39 +1000 Subject: [PATCH 41/74] Derive debug for some error enum --- lighthouse/state/transition/attestation_parent_hashes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse/state/transition/attestation_parent_hashes.rs b/lighthouse/state/transition/attestation_parent_hashes.rs index f98414c394..ff54233bc2 100644 --- a/lighthouse/state/transition/attestation_parent_hashes.rs +++ b/lighthouse/state/transition/attestation_parent_hashes.rs @@ -1,5 +1,6 @@ use super::Hash256; +#[derive(Debug)] pub enum ParentHashesError { BadCurrentHashes, BadObliqueHashes, @@ -179,7 +180,6 @@ mod tests { attestation_slot, ¤t_hashes, &oblique_hashes); - assert!(result.is_ok()); let result = result.unwrap(); assert_eq!(result.len(), cycle_length as usize); let expected_result = get_range_of_hashes(7, 15); From 0b661c5b113b516e0e669ac290088145ef18b97e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 23 Sep 2018 17:22:27 +1000 Subject: [PATCH 42/74] Implement ssz::Decodable for u8 --- ssz/src/impl_decode.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ssz/src/impl_decode.rs b/ssz/src/impl_decode.rs index bbcb9d863f..12e3e04c9c 100644 --- a/ssz/src/impl_decode.rs +++ b/ssz/src/impl_decode.rs @@ -36,6 +36,18 @@ impl_decodable_for_uint!(u32, 32); impl_decodable_for_uint!(u64, 64); impl_decodable_for_uint!(usize, 64); +impl Decodable for u8 { + fn ssz_decode(bytes: &[u8], index: usize) + -> Result<(Self, usize), DecodeError> + { + if index >= bytes.len() { + Err(DecodeError::TooShort) + } else { + Ok((bytes[index], index + 1)) + } + } +} + impl Decodable for H256 { fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> From 67b11a394e48408661394134ee7c1ba5a8811a4b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 23 Sep 2018 20:19:30 +1000 Subject: [PATCH 43/74] Add new bls-aggregates crates --- Cargo.toml | 2 +- lighthouse/bls/mod.rs | 9 ++++++++ lighthouse/main.rs | 1 + lighthouse/state/attestation_record.rs | 21 ++++++++++++------- lighthouse/state/block/ssz_block.rs | 6 +++--- lighthouse/state/mod.rs | 3 ++- .../validation/attestation_validation.rs | 2 +- lighthouse/state/validation/mod.rs | 1 + lighthouse/state/validator_record.rs | 7 +++---- lighthouse/utils/bls.rs | 13 ------------ lighthouse/utils/mod.rs | 4 ---- lighthouse/utils/test_helpers.rs | 12 ----------- 12 files changed, 34 insertions(+), 47 deletions(-) create mode 100644 lighthouse/bls/mod.rs delete mode 100644 lighthouse/utils/bls.rs delete mode 100644 lighthouse/utils/test_helpers.rs diff --git a/Cargo.toml b/Cargo.toml index 778df551d0..eaca0fd616 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ authors = ["Paul Hauner "] # TODO: remove "blake2" in favor of "blake2-rfc" blake2 = "^0.7.1" blake2-rfc = "0.2.18" -bls = { git = "https://github.com/sigp/bls" } +bls-aggregates = { git = "https://github.com/sigp/signature-schemes" } boolean-bitfield = { path = "boolean-bitfield" } bytes = "" crypto-mac = "^0.6.2" diff --git a/lighthouse/bls/mod.rs b/lighthouse/bls/mod.rs new file mode 100644 index 0000000000..fff21940cb --- /dev/null +++ b/lighthouse/bls/mod.rs @@ -0,0 +1,9 @@ +extern crate bls_aggregates; + +pub use self::bls_aggregates::AggregateSignature; +pub use self::bls_aggregates::AggregatePublicKey; +pub use self::bls_aggregates::Signature; +pub use self::bls_aggregates::Keypair; +pub use self::bls_aggregates::PublicKey; + +pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97; diff --git a/lighthouse/main.rs b/lighthouse/main.rs index 1631cc4e74..12444f538b 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -2,6 +2,7 @@ extern crate slog; extern crate slog_term; extern crate slog_async; +extern crate ssz; extern crate clap; extern crate network_libp2p; extern crate futures; diff --git a/lighthouse/state/attestation_record.rs b/lighthouse/state/attestation_record.rs index 73ed5e543e..a1997613fc 100644 --- a/lighthouse/state/attestation_record.rs +++ b/lighthouse/state/attestation_record.rs @@ -1,5 +1,8 @@ use super::utils::types::{ Hash256, Bitfield }; -use super::utils::bls::{ AggregateSignature }; +use super::bls::{ + AggregateSignature, + BLS_AGG_SIG_BYTE_SIZE, +}; use super::ssz::{ Encodable, Decodable, @@ -16,7 +19,7 @@ pub const MIN_SSZ_ATTESTION_RECORD_LENGTH: usize = { 5 + // attester_bitfield (assuming 1 byte of bitfield) 8 + // justified_slot 32 + // justified_block_hash - 4 + (2 * 8) // aggregate sig (two 256 bit points) + 4 + BLS_AGG_SIG_BYTE_SIZE // aggregate sig (two 256 bit points) }; #[derive(Debug)] @@ -28,7 +31,7 @@ pub struct AttestationRecord { pub attester_bitfield: Bitfield, pub justified_slot: u64, pub justified_block_hash: Hash256, - pub aggregate_sig: Option, + pub aggregate_sig: AggregateSignature, } impl Encodable for AttestationRecord { @@ -40,8 +43,7 @@ impl Encodable for AttestationRecord { s.append_vec(&self.attester_bitfield.to_be_vec()); s.append(&self.justified_slot); s.append(&self.justified_block_hash); - // TODO: encode the aggregate sig correctly - s.append_vec(&vec![0_u8; 16]) + s.append_vec(&self.aggregate_sig.as_bytes()); } } @@ -57,7 +59,10 @@ impl Decodable for AttestationRecord { let (justified_slot, i) = u64::ssz_decode(bytes, i)?; let (justified_block_hash, i) = Hash256::ssz_decode(bytes, i)?; // Do aggregate sig decoding properly. - let aggregate_sig = None; let i = i + 20; + let (agg_sig_bytes, i) = decode_ssz_list(bytes, i)?; + let aggregate_sig = AggregateSignature::from_bytes(&agg_sig_bytes) + .map_err(|_| DecodeError::OutOfBounds)?; + let attestation_record = Self { slot, shard_id, @@ -82,7 +87,7 @@ impl AttestationRecord { attester_bitfield: Bitfield::new(), justified_slot: 0, justified_block_hash: Hash256::zero(), - aggregate_sig: None, + aggregate_sig: AggregateSignature::new(), } } } @@ -113,7 +118,7 @@ mod tests { attester_bitfield: Bitfield::from(&vec![17; 42][..]), justified_slot: 19, justified_block_hash: Hash256::from(&vec![15; 32][..]), - aggregate_sig: None, + aggregate_sig: AggregateSignature::new(), }; let mut ssz_stream = SszStream::new(); diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 577857a19e..3c430c251a 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -235,9 +235,9 @@ mod tests { // will tell us if the hash changes, not that it matches some // canonical reference. let expected_hash = [ - 195, 180, 208, 144, 113, 20, 129, 108, 14, 128, 166, 170, - 137, 15, 191, 186, 34, 171, 79, 214, 74, 86, 89, 202, 255, - 9, 100, 170, 149, 160, 93, 59 + 64, 176, 117, 210, 228, 229, 237, 100, 66, 66, 98, + 252, 31, 111, 218, 27, 160, 57, 164, 12, 15, 164, + 66, 102, 142, 36, 2, 196, 121, 54, 242, 3 ]; assert_eq!(hash, expected_hash); diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index fbb789490c..532d8345dc 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -4,9 +4,10 @@ extern crate blake2_rfc as blake2; extern crate bytes; extern crate ssz; +use super::bls; +use super::db; use super::Logger; use super::utils; -use super::db; pub mod active_state; pub mod attestation_record; diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation_validation.rs index 9dfa943fb6..47b6f83fbb 100644 --- a/lighthouse/state/validation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation_validation.rs @@ -6,7 +6,7 @@ use super::attestation_parent_hashes::{ use super::db::ClientDB; use super::db::stores::BlockStore; use super::ssz::SszStream; -use super::utils::bls::{ +use super::bls::{ AggregateSignature, PublicKey, }; diff --git a/lighthouse/state/validation/mod.rs b/lighthouse/state/validation/mod.rs index 39663dba11..70c3757548 100644 --- a/lighthouse/state/validation/mod.rs +++ b/lighthouse/state/validation/mod.rs @@ -6,6 +6,7 @@ use super::block::Block; use super::chain_config::ChainConfig; */ use super::block; +use super::bls; use super::Logger; use super::db; use super::attestation_record::AttestationRecord; diff --git a/lighthouse/state/validator_record.rs b/lighthouse/state/validator_record.rs index e69b169c11..9961447da6 100644 --- a/lighthouse/state/validator_record.rs +++ b/lighthouse/state/validator_record.rs @@ -1,7 +1,7 @@ extern crate rand; use super::utils::types::{ Hash256, Address, U256 }; -use super::utils::bls::{ PublicKey, Keypair }; +use super::bls::{ PublicKey, Keypair }; use self::rand::thread_rng; @@ -21,10 +21,9 @@ impl ValidatorRecord { /// /// Returns the new instance and new keypair. pub fn zero_with_thread_rand_keypair() -> (Self, Keypair) { - let mut rng = thread_rng(); - let keypair = Keypair::generate(&mut rng); + let keypair = Keypair::random(); let s = Self { - pubkey: keypair.public.clone(), + pubkey: keypair.pk.clone(), withdrawal_shard: 0, withdrawal_address: Address::zero(), randao_commitment: Hash256::zero(), diff --git a/lighthouse/utils/bls.rs b/lighthouse/utils/bls.rs deleted file mode 100644 index c6b5dae569..0000000000 --- a/lighthouse/utils/bls.rs +++ /dev/null @@ -1,13 +0,0 @@ -extern crate bls; -extern crate pairing; - -use self::bls::AggregateSignature as GenericAggregateSignature; -use self::bls::Signature as GenericSignature; -use self::bls::Keypair as GenericKeypair; -use self::bls::PublicKey as GenericPublicKey; -use self::pairing::bls12_381::Bls12; - -pub type AggregateSignature = GenericAggregateSignature; -pub type Signature = GenericSignature; -pub type Keypair = GenericKeypair; -pub type PublicKey = GenericPublicKey; diff --git a/lighthouse/utils/mod.rs b/lighthouse/utils/mod.rs index 4afe15c454..16e986e227 100644 --- a/lighthouse/utils/mod.rs +++ b/lighthouse/utils/mod.rs @@ -7,9 +7,5 @@ extern crate boolean_bitfield; pub mod macros; pub mod hash; pub mod types; -pub mod bls; -pub mod test_helpers; pub mod logging; pub mod errors; - - diff --git a/lighthouse/utils/test_helpers.rs b/lighthouse/utils/test_helpers.rs deleted file mode 100644 index 8a18496882..0000000000 --- a/lighthouse/utils/test_helpers.rs +++ /dev/null @@ -1,12 +0,0 @@ -extern crate rand; - -use super::bls::Keypair; -use self::rand::thread_rng; - -// Returns a keypair for use in testing purposes. -// It is dangerous because we provide no guarantees -// that the private key is unique or in-fact private. -pub fn get_dangerous_test_keypair() -> Keypair { - let mut rng = thread_rng(); - Keypair::generate(&mut rng) -} From 2acf141ba305c0c7ca89f3c3c859f8a6d61dda7a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 09:49:26 +1000 Subject: [PATCH 44/74] Tidy up main.rs and attestation_validation.rs --- lighthouse/main.rs | 13 ++-- .../validation/attestation_validation.rs | 77 ------------------- 2 files changed, 7 insertions(+), 83 deletions(-) diff --git a/lighthouse/main.rs b/lighthouse/main.rs index 12444f538b..603346b36d 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -8,12 +8,13 @@ extern crate network_libp2p; extern crate futures; #[macro_use] -pub mod utils; -pub mod db; -pub mod client; -pub mod state; -pub mod sync; -pub mod config; +mod utils; +mod bls; +mod db; +mod client; +mod state; +mod sync; +mod config; use std::path::PathBuf; diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation_validation.rs index 47b6f83fbb..d47d15c53b 100644 --- a/lighthouse/state/validation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation_validation.rs @@ -167,80 +167,3 @@ impl From for AttestationValidationError { } } } - -/* -// Implementation of validate_attestation in the v2.1 python reference implementation see: -// -// github.com/ethereum/beacon_chain/beacon_chain/state/state_transition.py -pub fn validate_attestation_2( - crystallized_state: &CrystallizedState, - active_state: &ActiveState, - attestation: &AttestationRecord, - block: &Block, - chain_config: &ChainConfig) - -> Result { - - if !(attestation.slot < block.slot_number) { - return Err(AttestationValidationError::SlotTooHigh); - } - - if !(attestation.slot > (block.slot_number - chain_config.cycle_length as u64)) { - return Err(AttestationValidationError::SlotTooLow(format!("Attestation slot number too low\n\tFound: {:?}, Needed greater than: {:?}", attestation.slot, block.slot_number - chain_config.cycle_length as u64))); - } - - Ok(true) - } - - -#[cfg(test)] -mod tests { - - use super::*; - // test helper functions - - fn generate_standard_state() -> ( - CrystallizedState, - ActiveState, - AttestationRecord, - Block, - ChainConfig) { - - let crystallized_state = CrystallizedState::zero(); - let active_state = ActiveState::zero(); - let attestation_record = AttestationRecord::zero(); - let block = Block::zero(); - let chain_config = ChainConfig::standard(); - - return (crystallized_state, active_state, attestation_record, block, chain_config); - } - - #[test] - fn test_attestation_validation_slot_high() { - // generate standard state - let (crystallized_state, active_state, mut attestation_record, mut block, chain_config) = generate_standard_state(); - // set slot too high - attestation_record.slot = 30; - block.slot_number = 10; - - let result = validate_attestation(&crystallized_state, &active_state, &attestation_record, &block, &chain_config); - assert_eq!(result, Err(AttestationValidationError::SlotTooHigh)); - } - - #[test] - fn test_attestation_validation_slot_low() { - // generate standard state - let (crystallized_state, active_state, mut attestation_record, mut block, chain_config) = generate_standard_state(); - // set slot too high - attestation_record.slot = 2; - block.slot_number = 10; - - let result = validate_attestation( - &crystallized_state, - &active_state, - &attestation_record, - &block, - &chain_config); - //assert_eq!(result, Err(AttestationValidationError::SlotTooLow)); - } -} -*/ From 7f01ec7c27dd0c5c7b7a95779782bea4c08574cc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 13:16:39 +1000 Subject: [PATCH 45/74] Extend validator_store --- lighthouse/db/mod.rs | 2 + lighthouse/db/stores/mod.rs | 15 +++ lighthouse/db/stores/validator_store.rs | 116 ++++++++++++++++++++++-- 3 files changed, 125 insertions(+), 8 deletions(-) diff --git a/lighthouse/db/mod.rs b/lighthouse/db/mod.rs index e030fc009a..5f8898861f 100644 --- a/lighthouse/db/mod.rs +++ b/lighthouse/db/mod.rs @@ -6,6 +6,8 @@ mod memory_db; mod traits; pub mod stores; +use super::bls; + pub use self::disk_db::DiskDB; pub use self::memory_db::MemoryDB; diff --git a/lighthouse/db/stores/mod.rs b/lighthouse/db/stores/mod.rs index a7e5c60bb8..58c0a52dbb 100644 --- a/lighthouse/db/stores/mod.rs +++ b/lighthouse/db/stores/mod.rs @@ -11,6 +11,21 @@ pub use self::block_store::BlockStore; pub use self::pow_chain_store::PoWChainStore; pub use self::validator_store::ValidatorStore; +use super::bls; + const BLOCKS_DB_COLUMN: &str = "blocks"; const POW_CHAIN_DB_COLUMN: &str = "powchain"; const VALIDATOR_DB_COLUMN: &str = "validator"; + +#[derive(Debug, PartialEq)] +pub enum StoreError { + DBError(String), + DecodeError, + EncodeError, +} + +impl From for StoreError { + fn from(error: DBError) -> Self { + StoreError::DBError(error.message) + } +} diff --git a/lighthouse/db/stores/validator_store.rs b/lighthouse/db/stores/validator_store.rs index badea16e20..a20e7503a8 100644 --- a/lighthouse/db/stores/validator_store.rs +++ b/lighthouse/db/stores/validator_store.rs @@ -1,9 +1,21 @@ +extern crate bytes; + +use self::bytes::{ + BufMut, + BytesMut, +}; use std::sync::Arc; use super::{ ClientDB, - DBError, + StoreError, }; use super::VALIDATOR_DB_COLUMN as DB_COLUMN; +use super::bls::PublicKey; + +#[derive(Debug, PartialEq)] +enum KeyPrefixes { + PublicKey, +} pub struct ValidatorStore where T: ClientDB @@ -18,17 +30,105 @@ impl ValidatorStore { } } - pub fn put_validator_record_by_index(&self, hash: &[u8], val: &[u8]) - -> Result<(), DBError> + fn prefix_bytes(&self, key_prefix: KeyPrefixes) + -> Vec { - self.db.put(DB_COLUMN, hash, val) + match key_prefix { + KeyPrefixes::PublicKey => b"pubkey".to_vec(), + } } - pub fn get_validator_record_by_index(&self, hash: &[u8]) - -> Result + fn get_db_key_for_index(&self, key_prefix: KeyPrefixes, index: usize) + -> Vec { - self.db.exists(DB_COLUMN, hash) + let mut buf = BytesMut::with_capacity(6 + 8); + buf.put(self.prefix_bytes(key_prefix)); + buf.put_u64_be(index as u64); + buf.take().to_vec() + } + + pub fn put_public_key_by_index(&self, index: usize, public_key: &PublicKey) + -> Result<(), StoreError> + { + let key = self.get_db_key_for_index(KeyPrefixes::PublicKey, index); + let val = public_key.as_bytes(); + self.db.put(DB_COLUMN, &key[..], &val[..]) + .map_err(|e| StoreError::from(e)) + } + + pub fn get_public_key_by_index(&self, index: usize) + -> Result, StoreError> + { + let key = self.get_db_key_for_index(KeyPrefixes::PublicKey, index); + let val = self.db.get(DB_COLUMN, &key[..])?; + match val { + None => Ok(None), + Some(val) => { + match PublicKey::from_bytes(&val) { + Ok(key) => Ok(Some(key)), + Err(_) => Err(StoreError::DecodeError), + } + } + } } } -// TODO: add tests +#[cfg(test)] +mod tests { + use super::*; + use super::super::super::{ + MemoryDB, + ClientDB, + }; + use super::super::bls::Keypair; + + fn open_client_db() -> MemoryDB { + let columns = vec![DB_COLUMN]; + MemoryDB::open(Some(&columns)) + } + + #[test] + fn test_validator_store_put_get() { + let db = Arc::new(open_client_db()); + let store = ValidatorStore::new(db); + + let keys = vec![ + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + ]; + + for i in 0..keys.len() { + store.put_public_key_by_index(i, &keys[i].pk).unwrap(); + } + + /* + * Check all keys are retrieved correctly. + */ + for i in 0..keys.len() { + let retrieved = store.get_public_key_by_index(i) + .unwrap().unwrap(); + assert_eq!(retrieved, keys[i].pk); + } + + /* + * Check that an index that wasn't stored returns None. + */ + assert!(store.get_public_key_by_index(keys.len() + 1) + .unwrap().is_none()); + } + + #[test] + fn test_validator_store_bad_key() { + let db = Arc::new(open_client_db()); + let store = ValidatorStore::new(db.clone()); + + let key = store.get_db_key_for_index(KeyPrefixes::PublicKey, 42); + db.put(DB_COLUMN, &key[..], "cats".as_bytes()).unwrap(); + + assert_eq!(store.get_public_key_by_index(42), + Err(StoreError::DecodeError)); + } +} From 69c97745d2d2446d828b22d7b57c2a28a7335c28 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 14:15:59 +1000 Subject: [PATCH 46/74] Set MemoryDB to use constant DB columns This is instead of needing to be passed them each time open() is called. --- lighthouse/db/memory_db.rs | 34 +++++++++++++------------ lighthouse/db/mod.rs | 2 +- lighthouse/db/stores/block_store.rs | 4 +-- lighthouse/db/stores/mod.rs | 13 +++++++--- lighthouse/db/stores/validator_store.rs | 14 +++------- 5 files changed, 32 insertions(+), 35 deletions(-) diff --git a/lighthouse/db/memory_db.rs b/lighthouse/db/memory_db.rs index 41d1f7e6de..94bd5f2906 100644 --- a/lighthouse/db/memory_db.rs +++ b/lighthouse/db/memory_db.rs @@ -1,6 +1,7 @@ use std::collections::{ HashSet, HashMap }; use std::sync::RwLock; use super::blake2::blake2b::blake2b; +use super::COLUMNS; use super::{ ClientDB, DBValue, @@ -24,13 +25,11 @@ impl MemoryDB { /// /// All columns must be supplied initially, you will get an error if you try to access a column /// that was not declared here. This condition is enforced artificially to simulate RocksDB. - pub fn open(columns: Option<&[&str]>) -> Self { + pub fn open() -> Self { let db: DBHashMap = HashMap::new(); let mut known_columns: ColumnHashSet = HashSet::new(); - if let Some(columns) = columns { - for col in columns { - known_columns.insert(col.to_string()); - } + for col in COLUMNS.iter() { + known_columns.insert(col.to_string()); } Self { db: RwLock::new(db), @@ -103,18 +102,22 @@ mod tests { use super::super::ClientDB; use std::thread; use std::sync::Arc; + use super::super::stores::{ + BLOCKS_DB_COLUMN, + VALIDATOR_DB_COLUMN, + }; #[test] fn test_memorydb_column_access() { - let col_a: &str = "ColumnA"; - let col_b: &str = "ColumnB"; + let col_a: &str = BLOCKS_DB_COLUMN; + let col_b: &str = VALIDATOR_DB_COLUMN; let column_families = vec![ col_a, col_b, ]; - let db = MemoryDB::open(Some(&column_families)); + let db = MemoryDB::open(); /* * Testing that if we write to the same key in different columns that @@ -131,7 +134,7 @@ mod tests { #[test] fn test_memorydb_unknown_column_access() { - let col_a: &str = "ColumnA"; + let col_a: &str = BLOCKS_DB_COLUMN; let col_x: &str = "ColumnX"; let column_families = vec![ @@ -139,7 +142,7 @@ mod tests { // col_x is excluded on purpose ]; - let db = MemoryDB::open(Some(&column_families)); + let db = MemoryDB::open(); /* * Test that we get errors when using undeclared columns @@ -153,15 +156,15 @@ mod tests { #[test] fn test_memorydb_exists() { - let col_a: &str = "ColumnA"; - let col_b: &str = "ColumnB"; + let col_a: &str = BLOCKS_DB_COLUMN; + let col_b: &str = VALIDATOR_DB_COLUMN; let column_families = vec![ col_a, col_b, ]; - let db = MemoryDB::open(Some(&column_families)); + let db = MemoryDB::open(); /* * Testing that if we write to the same key in different columns that @@ -178,10 +181,9 @@ mod tests { #[test] fn test_memorydb_threading() { - let col_name: &str = "TestColumn"; - let column_families = vec![col_name]; + let col_name: &str = BLOCKS_DB_COLUMN; - let db = Arc::new(MemoryDB::open(Some(&column_families))); + let db = Arc::new(MemoryDB::open()); let thread_count = 10; let write_count = 10; diff --git a/lighthouse/db/mod.rs b/lighthouse/db/mod.rs index 5f8898861f..40cab486d3 100644 --- a/lighthouse/db/mod.rs +++ b/lighthouse/db/mod.rs @@ -7,7 +7,7 @@ mod traits; pub mod stores; use super::bls; - +use self::stores::COLUMNS; pub use self::disk_db::DiskDB; pub use self::memory_db::MemoryDB; diff --git a/lighthouse/db/stores/block_store.rs b/lighthouse/db/stores/block_store.rs index fde09bec87..1836923c95 100644 --- a/lighthouse/db/stores/block_store.rs +++ b/lighthouse/db/stores/block_store.rs @@ -46,9 +46,7 @@ mod tests { #[test] fn test_block_store_on_disk_db() { - let column_families = vec![DB_COLUMN]; - - let db = Arc::new(MemoryDB::open(Some(&column_families))); + let db = Arc::new(MemoryDB::open()); let bs = Arc::new(BlockStore::new(db.clone())); let thread_count = 10; diff --git a/lighthouse/db/stores/mod.rs b/lighthouse/db/stores/mod.rs index 58c0a52dbb..93250e7dbc 100644 --- a/lighthouse/db/stores/mod.rs +++ b/lighthouse/db/stores/mod.rs @@ -13,15 +13,20 @@ pub use self::validator_store::ValidatorStore; use super::bls; -const BLOCKS_DB_COLUMN: &str = "blocks"; -const POW_CHAIN_DB_COLUMN: &str = "powchain"; -const VALIDATOR_DB_COLUMN: &str = "validator"; +pub const BLOCKS_DB_COLUMN: &str = "blocks"; +pub const POW_CHAIN_DB_COLUMN: &str = "powchain"; +pub const VALIDATOR_DB_COLUMN: &str = "validator"; + +pub const COLUMNS: [&str; 3] = [ + BLOCKS_DB_COLUMN, + POW_CHAIN_DB_COLUMN, + VALIDATOR_DB_COLUMN, +]; #[derive(Debug, PartialEq)] pub enum StoreError { DBError(String), DecodeError, - EncodeError, } impl From for StoreError { diff --git a/lighthouse/db/stores/validator_store.rs b/lighthouse/db/stores/validator_store.rs index a20e7503a8..25e5893088 100644 --- a/lighthouse/db/stores/validator_store.rs +++ b/lighthouse/db/stores/validator_store.rs @@ -76,20 +76,12 @@ impl ValidatorStore { #[cfg(test)] mod tests { use super::*; - use super::super::super::{ - MemoryDB, - ClientDB, - }; + use super::super::super::MemoryDB; use super::super::bls::Keypair; - fn open_client_db() -> MemoryDB { - let columns = vec![DB_COLUMN]; - MemoryDB::open(Some(&columns)) - } - #[test] fn test_validator_store_put_get() { - let db = Arc::new(open_client_db()); + let db = Arc::new(MemoryDB::open()); let store = ValidatorStore::new(db); let keys = vec![ @@ -122,7 +114,7 @@ mod tests { #[test] fn test_validator_store_bad_key() { - let db = Arc::new(open_client_db()); + let db = Arc::new(MemoryDB::open()); let store = ValidatorStore::new(db.clone()); let key = store.get_db_key_for_index(KeyPrefixes::PublicKey, 42); From be09fd4f295921c08e8cea1fe0b6cb78324c4ed8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 14:16:48 +1000 Subject: [PATCH 47/74] Implement attestation signature validation and test --- .../validation/attestation_validation.rs | 7 +- lighthouse/state/validation/mod.rs | 1 + lighthouse/state/validation/signatures.rs | 126 ++++++++++++++++++ 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 lighthouse/state/validation/signatures.rs diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation_validation.rs index d47d15c53b..0061f7a9f2 100644 --- a/lighthouse/state/validation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation_validation.rs @@ -4,10 +4,15 @@ use super::attestation_parent_hashes::{ ParentHashesError, }; use super::db::ClientDB; -use super::db::stores::BlockStore; +use super::db::stores::{ + BlockStore, + ValidatorStore, + StoreError, +}; use super::ssz::SszStream; use super::bls::{ AggregateSignature, + AggregatePublicKey, PublicKey, }; use super::utils::hash::canonical_hash; diff --git a/lighthouse/state/validation/mod.rs b/lighthouse/state/validation/mod.rs index 70c3757548..c11d2c3c22 100644 --- a/lighthouse/state/validation/mod.rs +++ b/lighthouse/state/validation/mod.rs @@ -15,4 +15,5 @@ use super::transition::attestation_parent_hashes; use super::utils; mod attestation_validation; +mod signatures; mod ssz_block; diff --git a/lighthouse/state/validation/signatures.rs b/lighthouse/state/validation/signatures.rs new file mode 100644 index 0000000000..a2a2893285 --- /dev/null +++ b/lighthouse/state/validation/signatures.rs @@ -0,0 +1,126 @@ +use std::collections::HashSet; +use super::bls::{ + AggregateSignature, + AggregatePublicKey, +}; +use super::db::ClientDB; +use super::db::stores::{ + ValidatorStore, + StoreError, +}; +use super::utils::types::Bitfield; + +#[derive(Debug, PartialEq)] +pub enum SignatureVerificationError { + BadValidatorIndex, + PublicKeyCorrupt, + NoPublicKeyForValidator, + DBError(String), +} + +fn verify_aggregate_signature_for_indices(message: &[u8], + agg_sig: &AggregateSignature, + attestation_indices: &[usize], + bitfield: &Bitfield, + validator_store: &ValidatorStore) + -> Result<(bool, HashSet), SignatureVerificationError> + where T: ClientDB + Sized +{ + let mut voters = HashSet::new(); + let mut agg_pub_key = AggregatePublicKey::new(); + + for i in 0..attestation_indices.len() { + let voted = bitfield.get_bit(i); + if voted { + let validator = attestation_indices[i]; + let pub_key = validator_store.get_public_key_by_index(i)? + .ok_or(SignatureVerificationError::NoPublicKeyForValidator)?; + agg_pub_key.add(&pub_key); + voters.insert(validator); + } + } + Ok((agg_sig.verify(&message, &agg_pub_key), + voters)) +} + +impl From for SignatureVerificationError { + fn from(error: StoreError) -> Self { + match error { + StoreError::DBError(s) => + SignatureVerificationError::DBError(s), + StoreError::DecodeError => + SignatureVerificationError::PublicKeyCorrupt, + } + } +} + + +#[cfg(test)] +mod tests { + use super::*; + use super::super::bls::{ + Keypair, + Signature, + }; + use super::super::db::MemoryDB; + use std::sync::Arc; + + #[test] + fn test_signature_verification() { + let message = "cats".as_bytes(); + let signing_keypairs = vec![ + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + ]; + let non_signing_keypairs = vec![ + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + ]; + /* + * Signing keypairs first, then non-signing + */ + let mut all_keypairs = signing_keypairs.clone(); + all_keypairs.append(&mut non_signing_keypairs.clone()); + + let attestation_indices: Vec = (0..all_keypairs.len()) + .collect(); + let mut bitfield = Bitfield::new(); + for i in 0..signing_keypairs.len() { + bitfield.set_bit(i, true); + } + + let db = Arc::new(MemoryDB::open()); + let store = ValidatorStore::new(db); + + for (i, keypair) in all_keypairs.iter().enumerate() { + store.put_public_key_by_index(i, &keypair.pk).unwrap(); + } + + let mut agg_sig = AggregateSignature::new(); + for keypair in &signing_keypairs { + let sig = Signature::new(&message, &keypair.sk); + agg_sig.add(&sig); + } + + let (is_valid, voters) = verify_aggregate_signature_for_indices( + &message, + &agg_sig, + &attestation_indices, + &bitfield, + &store).unwrap(); + + assert_eq!(is_valid, true); + (0..signing_keypairs.len()) + .for_each(|i| assert!(voters.contains(&i))); + (signing_keypairs.len()..non_signing_keypairs.len()) + .for_each(|i| assert!(!voters.contains(&i))); + } +} From 27743f34668dcdd5713517390ff58cdc97a366e2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 14:30:39 +1000 Subject: [PATCH 48/74] Extend tests for signature verification --- lighthouse/state/validation/signatures.rs | 28 ++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/lighthouse/state/validation/signatures.rs b/lighthouse/state/validation/signatures.rs index a2a2893285..ad00e0e3f5 100644 --- a/lighthouse/state/validation/signatures.rs +++ b/lighthouse/state/validation/signatures.rs @@ -23,7 +23,7 @@ fn verify_aggregate_signature_for_indices(message: &[u8], attestation_indices: &[usize], bitfield: &Bitfield, validator_store: &ValidatorStore) - -> Result<(bool, HashSet), SignatureVerificationError> + -> Result<(bool, Option>), SignatureVerificationError> where T: ClientDB + Sized { let mut voters = HashSet::new(); @@ -39,8 +39,11 @@ fn verify_aggregate_signature_for_indices(message: &[u8], voters.insert(validator); } } - Ok((agg_sig.verify(&message, &agg_pub_key), - voters)) + if agg_sig.verify(&message, &agg_pub_key) { + Ok((true, Some(voters))) + } else { + Ok((false, None)) + } } impl From for SignatureVerificationError { @@ -110,6 +113,9 @@ mod tests { agg_sig.add(&sig); } + /* + * Test using all valid parameters. + */ let (is_valid, voters) = verify_aggregate_signature_for_indices( &message, &agg_sig, @@ -117,10 +123,26 @@ mod tests { &bitfield, &store).unwrap(); + let voters = voters.unwrap(); assert_eq!(is_valid, true); (0..signing_keypairs.len()) .for_each(|i| assert!(voters.contains(&i))); (signing_keypairs.len()..non_signing_keypairs.len()) .for_each(|i| assert!(!voters.contains(&i))); + + /* + * Add another validator to the bitfield, run validation will all other + * parameters the same and assert that it fails. + */ + bitfield.set_bit(signing_keypairs.len() + 1, true); + let (is_valid, voters) = verify_aggregate_signature_for_indices( + &message, + &agg_sig, + &attestation_indices, + &bitfield, + &store).unwrap(); + + assert_eq!(is_valid, false); + assert_eq!(voters, None); } } From 029a69a4a56b9c7c913d93cdc5e9902024f2b3fb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 18:06:47 +1000 Subject: [PATCH 49/74] Add method to bitfield to read specific byte --- boolean-bitfield/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/boolean-bitfield/src/lib.rs b/boolean-bitfield/src/lib.rs index 8328e63509..01e8f9f799 100644 --- a/boolean-bitfield/src/lib.rs +++ b/boolean-bitfield/src/lib.rs @@ -117,6 +117,11 @@ impl BooleanBitfield { 0 } + /// Get the byte at a position, assuming big-endian encoding. + pub fn get_byte(&self, n: usize) -> Option<&u8> { + self.vec.get(n) + } + /// Clone and return the underlying byte array (`Vec`). pub fn to_be_vec(&self) -> Vec { let mut o = self.vec.clone(); From f9f5a9e49c257d969bf35fff008cc7ae85520e04 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 18:08:00 +1000 Subject: [PATCH 50/74] Implement error type specific to ValidatorStore --- lighthouse/db/stores/mod.rs | 17 ++++--------- lighthouse/db/stores/validator_store.rs | 24 ++++++++++++++----- .../validation/attestation_validation.rs | 6 +++-- lighthouse/state/validation/signatures.rs | 10 ++++---- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/lighthouse/db/stores/mod.rs b/lighthouse/db/stores/mod.rs index 93250e7dbc..7c17653d39 100644 --- a/lighthouse/db/stores/mod.rs +++ b/lighthouse/db/stores/mod.rs @@ -9,7 +9,10 @@ mod validator_store; pub use self::block_store::BlockStore; pub use self::pow_chain_store::PoWChainStore; -pub use self::validator_store::ValidatorStore; +pub use self::validator_store::{ + ValidatorStore, + ValidatorStoreError, +}; use super::bls; @@ -22,15 +25,3 @@ pub const COLUMNS: [&str; 3] = [ POW_CHAIN_DB_COLUMN, VALIDATOR_DB_COLUMN, ]; - -#[derive(Debug, PartialEq)] -pub enum StoreError { - DBError(String), - DecodeError, -} - -impl From for StoreError { - fn from(error: DBError) -> Self { - StoreError::DBError(error.message) - } -} diff --git a/lighthouse/db/stores/validator_store.rs b/lighthouse/db/stores/validator_store.rs index 25e5893088..e3c604b3ac 100644 --- a/lighthouse/db/stores/validator_store.rs +++ b/lighthouse/db/stores/validator_store.rs @@ -7,11 +7,23 @@ use self::bytes::{ use std::sync::Arc; use super::{ ClientDB, - StoreError, + DBError, }; use super::VALIDATOR_DB_COLUMN as DB_COLUMN; use super::bls::PublicKey; +#[derive(Debug, PartialEq)] +pub enum ValidatorStoreError { + DBError(String), + DecodeError, +} + +impl From for ValidatorStoreError { + fn from(error: DBError) -> Self { + ValidatorStoreError::DBError(error.message) + } +} + #[derive(Debug, PartialEq)] enum KeyPrefixes { PublicKey, @@ -48,16 +60,16 @@ impl ValidatorStore { } pub fn put_public_key_by_index(&self, index: usize, public_key: &PublicKey) - -> Result<(), StoreError> + -> Result<(), ValidatorStoreError> { let key = self.get_db_key_for_index(KeyPrefixes::PublicKey, index); let val = public_key.as_bytes(); self.db.put(DB_COLUMN, &key[..], &val[..]) - .map_err(|e| StoreError::from(e)) + .map_err(|e| ValidatorStoreError::from(e)) } pub fn get_public_key_by_index(&self, index: usize) - -> Result, StoreError> + -> Result, ValidatorStoreError> { let key = self.get_db_key_for_index(KeyPrefixes::PublicKey, index); let val = self.db.get(DB_COLUMN, &key[..])?; @@ -66,7 +78,7 @@ impl ValidatorStore { Some(val) => { match PublicKey::from_bytes(&val) { Ok(key) => Ok(Some(key)), - Err(_) => Err(StoreError::DecodeError), + Err(_) => Err(ValidatorStoreError::DecodeError), } } } @@ -121,6 +133,6 @@ mod tests { db.put(DB_COLUMN, &key[..], "cats".as_bytes()).unwrap(); assert_eq!(store.get_public_key_by_index(42), - Err(StoreError::DecodeError)); + Err(ValidatorStoreError::DecodeError)); } } diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation_validation.rs index 0061f7a9f2..694b897ab6 100644 --- a/lighthouse/state/validation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation_validation.rs @@ -3,11 +3,13 @@ use super::attestation_parent_hashes::{ attestation_parent_hashes, ParentHashesError, }; -use super::db::ClientDB; +use super::db::{ + ClientDB, + DBError +}; use super::db::stores::{ BlockStore, ValidatorStore, - StoreError, }; use super::ssz::SszStream; use super::bls::{ diff --git a/lighthouse/state/validation/signatures.rs b/lighthouse/state/validation/signatures.rs index ad00e0e3f5..30f2339b22 100644 --- a/lighthouse/state/validation/signatures.rs +++ b/lighthouse/state/validation/signatures.rs @@ -6,7 +6,7 @@ use super::bls::{ use super::db::ClientDB; use super::db::stores::{ ValidatorStore, - StoreError, + ValidatorStoreError, }; use super::utils::types::Bitfield; @@ -46,12 +46,12 @@ fn verify_aggregate_signature_for_indices(message: &[u8], } } -impl From for SignatureVerificationError { - fn from(error: StoreError) -> Self { +impl From for SignatureVerificationError { + fn from(error: ValidatorStoreError) -> Self { match error { - StoreError::DBError(s) => + ValidatorStoreError::DBError(s) => SignatureVerificationError::DBError(s), - StoreError::DecodeError => + ValidatorStoreError::DecodeError => SignatureVerificationError::PublicKeyCorrupt, } } From 26537c118466d303ccfc708695dca5c744bb1e8e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 18:08:48 +1000 Subject: [PATCH 51/74] Progress further on attestation validation --- .../validation/attestation_validation.rs | 111 ++++++++++++++---- 1 file changed, 86 insertions(+), 25 deletions(-) diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation_validation.rs index 694b897ab6..aa94a91bca 100644 --- a/lighthouse/state/validation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation_validation.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use super::AttestationRecord; use super::attestation_parent_hashes::{ attestation_parent_hashes, @@ -12,33 +13,35 @@ use super::db::stores::{ ValidatorStore, }; use super::ssz::SszStream; -use super::bls::{ - AggregateSignature, - AggregatePublicKey, - PublicKey, -}; use super::utils::hash::canonical_hash; use super::utils::types::{ Hash256, - Bitfield, }; use std::collections::HashMap; use std::sync::Arc; +use super::signatures::{ + verify_aggregate_signature_for_indices, + SignatureVerificationError, +}; #[derive(Debug,PartialEq)] pub enum AttestationValidationError { SlotTooHigh, SlotTooLow, JustifiedSlotTooHigh, + UnknownJustifiedBlock, TooManyObliqueHashes, BadCurrentHashes, BadObliqueHashes, BadAttesterMap, IntWrapping, + PublicKeyCorrupt, + NoPublicKeyForValidator, IncorrectBitField, NoSignatures, NonZeroTrailingBits, - AggregateSignatureFail + AggregateSignatureFail, + DBError(String), } type Slot = u64; @@ -55,8 +58,9 @@ pub fn validate_attestation(a: &AttestationRecord, known_last_justified_slot: u64, known_parent_hashes: Arc>, block_store: BlockStore, + validator_store: ValidatorStore, attester_map: Arc) - -> Result + -> Result<(bool, Option>), AttestationValidationError> where T: ClientDB + Sized { /* @@ -92,15 +96,46 @@ pub fn validate_attestation(a: &AttestationRecord, return Err(AttestationValidationError::TooManyObliqueHashes); } + /* + * Retrieve the set of attestation indices for this slot and shard id. + * + * This is an array mapping the order that validators will appear in the bitfield to the + * canonincal index of a validator. + */ let attestation_indices = attester_map.get(&(a.slot, a.shard_id.into())) .ok_or(AttestationValidationError::BadAttesterMap)?; + /* + * The bitfield must be no longer than the minimum required to represent each validator in the + * attestation indicies for this slot and shard id. + */ if a.attester_bitfield.num_bytes() != bytes_for_bits(attestation_indices.len()) { return Err(AttestationValidationError::IncorrectBitField); } + /* + * If there are excess bits in the bitfield because the number of a validators in not a + * multiple of 8, reject this attestation record. + * + * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to + * refer to the same AttesationRecord. + */ + let last_byte = + a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes()) + .ok_or(AttestationValidationError::IncorrectBitField)?; + if any_of_last_n_bits_are_set(last_byte, a.attester_bitfield.len() % 8) { + return Err(AttestationValidationError::IncorrectBitField) + } + + /* + * The specified justified block hash must be known to us + */ + if !block_store.block_exists(&a.justified_block_hash)? { + return Err(AttestationValidationError::UnknownJustifiedBlock) + } + let signed_message = { let parent_hashes = attestation_parent_hashes( cycle_length, @@ -116,15 +151,20 @@ pub fn validate_attestation(a: &AttestationRecord, a.justified_slot) }; - Ok(false) + let (signature_valid, voted_hashmap) = + verify_aggregate_signature_for_indices( + &signed_message, + &a.aggregate_sig, + &attestation_indices, + &a.attester_bitfield, + &validator_store)?; + + Ok((signature_valid, voted_hashmap)) } -fn collect_pub_keys(attestation_indices: &Vec, - bitfield: &Bitfield) - -> Option> -{ - // cats - None +fn any_of_last_n_bits_are_set(byte: &u8, n: usize) -> bool { + let shift = 8_u8.saturating_sub(n as u8); + ((!0 >> shift) & byte) > 0 } /// Generates the message used to validate the signature provided with an AttestationRecord. @@ -161,16 +201,37 @@ fn generate_signed_message(slot: u64, impl From for AttestationValidationError { fn from(e: ParentHashesError) -> Self { match e { - ParentHashesError::BadCurrentHashes => - AttestationValidationError::BadCurrentHashes, - ParentHashesError::BadObliqueHashes => - AttestationValidationError::BadObliqueHashes, - ParentHashesError::SlotTooLow => - AttestationValidationError::SlotTooLow, - ParentHashesError::SlotTooHigh => - AttestationValidationError::SlotTooHigh, - ParentHashesError::IntWrapping => - AttestationValidationError::IntWrapping + ParentHashesError::BadCurrentHashes + => AttestationValidationError::BadCurrentHashes, + ParentHashesError::BadObliqueHashes + => AttestationValidationError::BadObliqueHashes, + ParentHashesError::SlotTooLow + => AttestationValidationError::SlotTooLow, + ParentHashesError::SlotTooHigh + => AttestationValidationError::SlotTooHigh, + ParentHashesError::IntWrapping + => AttestationValidationError::IntWrapping + } + } +} + +impl From for AttestationValidationError { + fn from(e: DBError) -> Self { + AttestationValidationError::DBError(e.message) + } +} + +impl From for AttestationValidationError { + fn from(e: SignatureVerificationError) -> Self { + match e { + SignatureVerificationError::BadValidatorIndex + => AttestationValidationError::BadAttesterMap, + SignatureVerificationError::PublicKeyCorrupt + => AttestationValidationError::PublicKeyCorrupt, + SignatureVerificationError::NoPublicKeyForValidator + => AttestationValidationError::NoPublicKeyForValidator, + SignatureVerificationError::DBError(s) + => AttestationValidationError::DBError(s), } } } From c55b94053bc2f919a1b6189fd917f5c9c03c0400 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 18:09:01 +1000 Subject: [PATCH 52/74] Tidy validation signatures --- lighthouse/state/validation/signatures.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lighthouse/state/validation/signatures.rs b/lighthouse/state/validation/signatures.rs index 30f2339b22..302c13f630 100644 --- a/lighthouse/state/validation/signatures.rs +++ b/lighthouse/state/validation/signatures.rs @@ -18,11 +18,12 @@ pub enum SignatureVerificationError { DBError(String), } -fn verify_aggregate_signature_for_indices(message: &[u8], - agg_sig: &AggregateSignature, - attestation_indices: &[usize], - bitfield: &Bitfield, - validator_store: &ValidatorStore) +pub fn verify_aggregate_signature_for_indices( + message: &[u8], + agg_sig: &AggregateSignature, + attestation_indices: &[usize], + bitfield: &Bitfield, + validator_store: &ValidatorStore) -> Result<(bool, Option>), SignatureVerificationError> where T: ClientDB + Sized { @@ -68,6 +69,15 @@ mod tests { use super::super::db::MemoryDB; use std::sync::Arc; + /* + * Cases that still need testing: + * + * - No signatures. + * - Database failure. + * - Unknown validator index. + * - Extra validator on signature. + */ + #[test] fn test_signature_verification() { let message = "cats".as_bytes(); From 2141b8c623f55519567379e838034bd35bfc94e1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 21:53:42 +1000 Subject: [PATCH 53/74] Add attestion ssz splitter, change dir structure --- .../attestation_record.rs | 2 +- lighthouse/state/attestation_record/mod.rs | 17 +++ .../state/attestation_record/ssz_splitter.rs | 139 ++++++++++++++++++ lighthouse/state/block/ssz_block.rs | 5 +- .../attestation_validation.rs | 10 +- .../state/validation/attestation/mod.rs | 11 ++ .../{ => attestation}/signatures.rs | 0 lighthouse/state/validation/mod.rs | 3 +- lighthouse/state/validator_record.rs | 2 - 9 files changed, 175 insertions(+), 14 deletions(-) rename lighthouse/state/{ => attestation_record}/attestation_record.rs (99%) create mode 100644 lighthouse/state/attestation_record/mod.rs create mode 100644 lighthouse/state/attestation_record/ssz_splitter.rs rename lighthouse/state/validation/{ => attestation}/attestation_validation.rs (100%) create mode 100644 lighthouse/state/validation/attestation/mod.rs rename lighthouse/state/validation/{ => attestation}/signatures.rs (100%) diff --git a/lighthouse/state/attestation_record.rs b/lighthouse/state/attestation_record/attestation_record.rs similarity index 99% rename from lighthouse/state/attestation_record.rs rename to lighthouse/state/attestation_record/attestation_record.rs index a1997613fc..3fb1c8b0d9 100644 --- a/lighthouse/state/attestation_record.rs +++ b/lighthouse/state/attestation_record/attestation_record.rs @@ -22,7 +22,7 @@ pub const MIN_SSZ_ATTESTION_RECORD_LENGTH: usize = { 4 + BLS_AGG_SIG_BYTE_SIZE // aggregate sig (two 256 bit points) }; -#[derive(Debug)] +#[derive(Debug, Clone, PartialEq)] pub struct AttestationRecord { pub slot: u64, pub shard_id: u16, diff --git a/lighthouse/state/attestation_record/mod.rs b/lighthouse/state/attestation_record/mod.rs new file mode 100644 index 0000000000..74a5d8c347 --- /dev/null +++ b/lighthouse/state/attestation_record/mod.rs @@ -0,0 +1,17 @@ +use super::bls; +use super::ssz; +use super::utils; + + +mod attestation_record; +mod ssz_splitter; + +pub use self::attestation_record::{ + AttestationRecord, + MIN_SSZ_ATTESTION_RECORD_LENGTH, +}; +pub use self::ssz_splitter::{ + split_all, + split_one, + AttestationSplitError, +}; diff --git a/lighthouse/state/attestation_record/ssz_splitter.rs b/lighthouse/state/attestation_record/ssz_splitter.rs new file mode 100644 index 0000000000..4562b1fb33 --- /dev/null +++ b/lighthouse/state/attestation_record/ssz_splitter.rs @@ -0,0 +1,139 @@ +use super::MIN_SSZ_ATTESTION_RECORD_LENGTH as MIN_LENGTH; +use super::ssz::LENGTH_BYTES; +use super::ssz::decode::decode_length; + +#[derive(Debug, PartialEq)] +pub enum AttestationSplitError { + TooShort, +} + +/// Given some ssz slice, find the bounds of each serialized AttestationRecord and return a vec of +/// slices point to each. +pub fn split_all<'a>(full_ssz: &'a [u8], index: usize) + -> Result, AttestationSplitError> +{ + let mut v = vec![]; + let mut index = index; + while index < full_ssz.len() - 1 { + let (slice, i) = split_one(full_ssz, index)?; + v.push(slice); + index = i; + } + Ok(v) +} + +/// Given some ssz slice, find the bounds of one serialized AttestationRecord +/// and return a slice pointing to that. +pub fn split_one<'a>(full_ssz: &'a [u8], index: usize) + -> Result<(&'a [u8], usize), AttestationSplitError> +{ + if full_ssz.len() < MIN_LENGTH { + return Err(AttestationSplitError::TooShort); + } + + let hashes_len = decode_length(full_ssz, 10, LENGTH_BYTES) + .map_err(|_| AttestationSplitError::TooShort)?; + + let bitfield_len = decode_length( + full_ssz, hashes_len + 46, + LENGTH_BYTES) + .map_err(|_| AttestationSplitError::TooShort)?; + + // Subtract one because the min length assume 1 byte of bitfield + let len = MIN_LENGTH + + hashes_len + + bitfield_len.saturating_sub(1); + + if full_ssz.len() < len { + return Err(AttestationSplitError::TooShort); + } + + Ok((&full_ssz[index..(index + len)], index + len)) +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::AttestationRecord; + use super::super::utils::types::{ + Hash256, + Bitfield, + }; + use super::super::bls::AggregateSignature; + use super::super::ssz::{ + SszStream, + Decodable, + }; + + fn get_two_records() -> Vec { + let a = AttestationRecord { + slot: 7, + shard_id: 9, + oblique_parent_hashes: vec![Hash256::from(&vec![14; 32][..])], + shard_block_hash: Hash256::from(&vec![15; 32][..]), + attester_bitfield: Bitfield::from(&vec![17; 42][..]), + justified_slot: 19, + justified_block_hash: Hash256::from(&vec![15; 32][..]), + aggregate_sig: AggregateSignature::new(), + }; + let b = AttestationRecord { + slot: 9, + shard_id: 7, + oblique_parent_hashes: vec![Hash256::from(&vec![15; 32][..])], + shard_block_hash: Hash256::from(&vec![14; 32][..]), + attester_bitfield: Bitfield::from(&vec![19; 42][..]), + justified_slot: 15, + justified_block_hash: Hash256::from(&vec![17; 32][..]), + aggregate_sig: AggregateSignature::new(), + }; + vec![a, b] + } + + #[test] + fn test_attestation_ssz_split() { + let ars = get_two_records(); + let a = ars[0].clone(); + let b = ars[1].clone(); + + + /* + * Test split one + */ + let mut ssz_stream = SszStream::new(); + ssz_stream.append(&a); + let ssz = ssz_stream.drain(); + let (a_ssz, i) = split_one(&ssz, 0).unwrap(); + assert_eq!(i, ssz.len()); + let (decoded_a, _) = AttestationRecord::ssz_decode(a_ssz, 0) + .unwrap(); + assert_eq!(a, decoded_a); + + /* + * Test split two + */ + let mut ssz_stream = SszStream::new(); + ssz_stream.append(&a); + ssz_stream.append(&b); + let ssz = ssz_stream.drain(); + let ssz_vec = split_all(&ssz, 0).unwrap(); + let (decoded_a, _) = + AttestationRecord::ssz_decode(ssz_vec[0], 0) + .unwrap(); + let (decoded_b, _) = + AttestationRecord::ssz_decode(ssz_vec[1], 0) + .unwrap(); + assert_eq!(a, decoded_a); + assert_eq!(b, decoded_b); + + /* + * Test split two with shortened ssz + */ + let mut ssz_stream = SszStream::new(); + ssz_stream.append(&a); + ssz_stream.append(&b); + let ssz = ssz_stream.drain(); + let ssz = &ssz[0..ssz.len() - 1]; + assert!(split_all(&ssz, 0).is_err()); + } +} + diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 3c430c251a..a90ff13510 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -13,9 +13,6 @@ use super::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH; pub enum BlockValidatorError { TooShort, TooLong, - BadPowHash, - SlotTooLow, - SlotTooHigh, } const LENGTH_BYTES: usize = 4; @@ -23,7 +20,7 @@ const LENGTH_BYTES: usize = 4; /// Allows for reading of block values directly from serialized ssz bytes. /// /// The purpose of this struct is to provide the functionality to read block fields directly from -/// some serialized SSZ slice, effectively allowing us to read the block without fully +/// some serialized SSZ slice allowing us to read the block without fully /// de-serializing it. /// /// This struct should be as "zero-copy" as possible. The `ssz` field is a reference to some slice diff --git a/lighthouse/state/validation/attestation_validation.rs b/lighthouse/state/validation/attestation/attestation_validation.rs similarity index 100% rename from lighthouse/state/validation/attestation_validation.rs rename to lighthouse/state/validation/attestation/attestation_validation.rs index aa94a91bca..db19db0bca 100644 --- a/lighthouse/state/validation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation/attestation_validation.rs @@ -52,6 +52,11 @@ fn bytes_for_bits(bits: usize) -> usize { (bits.saturating_sub(1) / 8) + 1 } +fn any_of_last_n_bits_are_set(byte: &u8, n: usize) -> bool { + let shift = 8_u8.saturating_sub(n as u8); + ((!0 >> shift) & byte) > 0 +} + pub fn validate_attestation(a: &AttestationRecord, block_slot: u64, cycle_length: u8, @@ -162,11 +167,6 @@ pub fn validate_attestation(a: &AttestationRecord, Ok((signature_valid, voted_hashmap)) } -fn any_of_last_n_bits_are_set(byte: &u8, n: usize) -> bool { - let shift = 8_u8.saturating_sub(n as u8); - ((!0 >> shift) & byte) > 0 -} - /// Generates the message used to validate the signature provided with an AttestationRecord. /// /// Ensures that the signer of the message has a view of the chain that is compatible with ours. diff --git a/lighthouse/state/validation/attestation/mod.rs b/lighthouse/state/validation/attestation/mod.rs new file mode 100644 index 0000000000..b4b782a0a7 --- /dev/null +++ b/lighthouse/state/validation/attestation/mod.rs @@ -0,0 +1,11 @@ +use super::db; +use super::bls; +use super::AttestationRecord; +use super::ssz; +use super::attestation_parent_hashes; +use super::utils; + +mod attestation_validation; +mod signatures; + +pub use self::attestation_validation::validate_attestation; diff --git a/lighthouse/state/validation/signatures.rs b/lighthouse/state/validation/attestation/signatures.rs similarity index 100% rename from lighthouse/state/validation/signatures.rs rename to lighthouse/state/validation/attestation/signatures.rs diff --git a/lighthouse/state/validation/mod.rs b/lighthouse/state/validation/mod.rs index c11d2c3c22..b09805d6c9 100644 --- a/lighthouse/state/validation/mod.rs +++ b/lighthouse/state/validation/mod.rs @@ -14,6 +14,5 @@ use super::ssz; use super::transition::attestation_parent_hashes; use super::utils; -mod attestation_validation; -mod signatures; +mod attestation; mod ssz_block; diff --git a/lighthouse/state/validator_record.rs b/lighthouse/state/validator_record.rs index 9961447da6..323c9e88d4 100644 --- a/lighthouse/state/validator_record.rs +++ b/lighthouse/state/validator_record.rs @@ -3,8 +3,6 @@ extern crate rand; use super::utils::types::{ Hash256, Address, U256 }; use super::bls::{ PublicKey, Keypair }; -use self::rand::thread_rng; - pub struct ValidatorRecord { pub pubkey: PublicKey, pub withdrawal_shard: u16, From 1677b6dcc8252ffc311e9536f983a3469aefc534 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 24 Sep 2018 23:35:51 +1000 Subject: [PATCH 54/74] Progress on block validation --- .../attestation_record/attestation_record.rs | 2 +- lighthouse/state/attestation_record/mod.rs | 4 +- .../state/attestation_record/ssz_splitter.rs | 24 ++-- .../attestation/attestation_validation.rs | 20 ++- .../state/validation/attestation/mod.rs | 8 +- .../validation/attestation/signatures.rs | 12 +- lighthouse/state/validation/mod.rs | 15 +-- lighthouse/state/validation/ssz_block.rs | 125 ++++++++++++++++-- ssz/src/decode.rs | 3 +- 9 files changed, 157 insertions(+), 56 deletions(-) diff --git a/lighthouse/state/attestation_record/attestation_record.rs b/lighthouse/state/attestation_record/attestation_record.rs index 3fb1c8b0d9..9cfa535164 100644 --- a/lighthouse/state/attestation_record/attestation_record.rs +++ b/lighthouse/state/attestation_record/attestation_record.rs @@ -61,7 +61,7 @@ impl Decodable for AttestationRecord { // Do aggregate sig decoding properly. let (agg_sig_bytes, i) = decode_ssz_list(bytes, i)?; let aggregate_sig = AggregateSignature::from_bytes(&agg_sig_bytes) - .map_err(|_| DecodeError::OutOfBounds)?; + .map_err(|_| DecodeError::TooShort)?; // also could be TooLong let attestation_record = Self { slot, diff --git a/lighthouse/state/attestation_record/mod.rs b/lighthouse/state/attestation_record/mod.rs index 74a5d8c347..d96a451fdd 100644 --- a/lighthouse/state/attestation_record/mod.rs +++ b/lighthouse/state/attestation_record/mod.rs @@ -11,7 +11,7 @@ pub use self::attestation_record::{ MIN_SSZ_ATTESTION_RECORD_LENGTH, }; pub use self::ssz_splitter::{ - split_all, - split_one, + split_all_attestations, + split_one_attestation, AttestationSplitError, }; diff --git a/lighthouse/state/attestation_record/ssz_splitter.rs b/lighthouse/state/attestation_record/ssz_splitter.rs index 4562b1fb33..72b191cf30 100644 --- a/lighthouse/state/attestation_record/ssz_splitter.rs +++ b/lighthouse/state/attestation_record/ssz_splitter.rs @@ -9,13 +9,13 @@ pub enum AttestationSplitError { /// Given some ssz slice, find the bounds of each serialized AttestationRecord and return a vec of /// slices point to each. -pub fn split_all<'a>(full_ssz: &'a [u8], index: usize) +pub fn split_all_attestations<'a>(full_ssz: &'a [u8], index: usize) -> Result, AttestationSplitError> { let mut v = vec![]; let mut index = index; while index < full_ssz.len() - 1 { - let (slice, i) = split_one(full_ssz, index)?; + let (slice, i) = split_one_attestation(full_ssz, index)?; v.push(slice); index = i; } @@ -24,27 +24,27 @@ pub fn split_all<'a>(full_ssz: &'a [u8], index: usize) /// Given some ssz slice, find the bounds of one serialized AttestationRecord /// and return a slice pointing to that. -pub fn split_one<'a>(full_ssz: &'a [u8], index: usize) +pub fn split_one_attestation<'a>(full_ssz: &'a [u8], index: usize) -> Result<(&'a [u8], usize), AttestationSplitError> { if full_ssz.len() < MIN_LENGTH { return Err(AttestationSplitError::TooShort); } - let hashes_len = decode_length(full_ssz, 10, LENGTH_BYTES) + let hashes_len = decode_length(full_ssz, index + 10, LENGTH_BYTES) .map_err(|_| AttestationSplitError::TooShort)?; let bitfield_len = decode_length( - full_ssz, hashes_len + 46, + full_ssz, index + hashes_len + 46, LENGTH_BYTES) .map_err(|_| AttestationSplitError::TooShort)?; - // Subtract one because the min length assume 1 byte of bitfield - let len = MIN_LENGTH + // Subtract one because the min length assumes 1 byte of bitfield + let len = MIN_LENGTH - 1 + hashes_len - + bitfield_len.saturating_sub(1); + + bitfield_len; - if full_ssz.len() < len { + if index + full_ssz.len() < len { return Err(AttestationSplitError::TooShort); } @@ -102,7 +102,7 @@ mod tests { let mut ssz_stream = SszStream::new(); ssz_stream.append(&a); let ssz = ssz_stream.drain(); - let (a_ssz, i) = split_one(&ssz, 0).unwrap(); + let (a_ssz, i) = split_one_attestation(&ssz, 0).unwrap(); assert_eq!(i, ssz.len()); let (decoded_a, _) = AttestationRecord::ssz_decode(a_ssz, 0) .unwrap(); @@ -115,7 +115,7 @@ mod tests { ssz_stream.append(&a); ssz_stream.append(&b); let ssz = ssz_stream.drain(); - let ssz_vec = split_all(&ssz, 0).unwrap(); + let ssz_vec = split_all_attestations(&ssz, 0).unwrap(); let (decoded_a, _) = AttestationRecord::ssz_decode(ssz_vec[0], 0) .unwrap(); @@ -133,7 +133,7 @@ mod tests { ssz_stream.append(&b); let ssz = ssz_stream.drain(); let ssz = &ssz[0..ssz.len() - 1]; - assert!(split_all(&ssz, 0).is_err()); + assert!(split_all_attestations(&ssz, 0).is_err()); } } diff --git a/lighthouse/state/validation/attestation/attestation_validation.rs b/lighthouse/state/validation/attestation/attestation_validation.rs index db19db0bca..4822851250 100644 --- a/lighthouse/state/validation/attestation/attestation_validation.rs +++ b/lighthouse/state/validation/attestation/attestation_validation.rs @@ -1,9 +1,10 @@ use std::collections::HashSet; -use super::AttestationRecord; +use super::attestation_record::AttestationRecord; use super::attestation_parent_hashes::{ attestation_parent_hashes, ParentHashesError, }; +use super::AttesterMap; use super::db::{ ClientDB, DBError @@ -17,7 +18,6 @@ use super::utils::hash::canonical_hash; use super::utils::types::{ Hash256, }; -use std::collections::HashMap; use std::sync::Arc; use super::signatures::{ verify_aggregate_signature_for_indices, @@ -44,10 +44,6 @@ pub enum AttestationValidationError { DBError(String), } -type Slot = u64; -type ShardId = u64; -type AttesterMap = HashMap<(Slot, ShardId), Vec>; - fn bytes_for_bits(bits: usize) -> usize { (bits.saturating_sub(1) / 8) + 1 } @@ -62,12 +58,14 @@ pub fn validate_attestation(a: &AttestationRecord, cycle_length: u8, known_last_justified_slot: u64, known_parent_hashes: Arc>, - block_store: BlockStore, - validator_store: ValidatorStore, + block_store: Arc>, + validator_store: Arc>, attester_map: Arc) - -> Result<(bool, Option>), AttestationValidationError> + -> Result>, AttestationValidationError> where T: ClientDB + Sized { + // TODO: assert attestion isn't already known + /* * The attesation slot must not be higher than the block that contained it. */ @@ -156,7 +154,7 @@ pub fn validate_attestation(a: &AttestationRecord, a.justified_slot) }; - let (signature_valid, voted_hashmap) = + let voted_hashmap = verify_aggregate_signature_for_indices( &signed_message, &a.aggregate_sig, @@ -164,7 +162,7 @@ pub fn validate_attestation(a: &AttestationRecord, &a.attester_bitfield, &validator_store)?; - Ok((signature_valid, voted_hashmap)) + Ok(voted_hashmap) } /// Generates the message used to validate the signature provided with an AttestationRecord. diff --git a/lighthouse/state/validation/attestation/mod.rs b/lighthouse/state/validation/attestation/mod.rs index b4b782a0a7..a28f3c6acc 100644 --- a/lighthouse/state/validation/attestation/mod.rs +++ b/lighthouse/state/validation/attestation/mod.rs @@ -1,6 +1,7 @@ +use super::AttesterMap; use super::db; use super::bls; -use super::AttestationRecord; +use super::attestation_record; use super::ssz; use super::attestation_parent_hashes; use super::utils; @@ -8,4 +9,7 @@ use super::utils; mod attestation_validation; mod signatures; -pub use self::attestation_validation::validate_attestation; +pub use self::attestation_validation::{ + validate_attestation, + AttestationValidationError, +}; diff --git a/lighthouse/state/validation/attestation/signatures.rs b/lighthouse/state/validation/attestation/signatures.rs index 302c13f630..fa17eba5a4 100644 --- a/lighthouse/state/validation/attestation/signatures.rs +++ b/lighthouse/state/validation/attestation/signatures.rs @@ -24,7 +24,7 @@ pub fn verify_aggregate_signature_for_indices( attestation_indices: &[usize], bitfield: &Bitfield, validator_store: &ValidatorStore) - -> Result<(bool, Option>), SignatureVerificationError> + -> Result>, SignatureVerificationError> where T: ClientDB + Sized { let mut voters = HashSet::new(); @@ -41,9 +41,9 @@ pub fn verify_aggregate_signature_for_indices( } } if agg_sig.verify(&message, &agg_pub_key) { - Ok((true, Some(voters))) + Ok(Some(voters)) } else { - Ok((false, None)) + Ok(None) } } @@ -126,7 +126,7 @@ mod tests { /* * Test using all valid parameters. */ - let (is_valid, voters) = verify_aggregate_signature_for_indices( + let voters = verify_aggregate_signature_for_indices( &message, &agg_sig, &attestation_indices, @@ -134,7 +134,6 @@ mod tests { &store).unwrap(); let voters = voters.unwrap(); - assert_eq!(is_valid, true); (0..signing_keypairs.len()) .for_each(|i| assert!(voters.contains(&i))); (signing_keypairs.len()..non_signing_keypairs.len()) @@ -145,14 +144,13 @@ mod tests { * parameters the same and assert that it fails. */ bitfield.set_bit(signing_keypairs.len() + 1, true); - let (is_valid, voters) = verify_aggregate_signature_for_indices( + let voters = verify_aggregate_signature_for_indices( &message, &agg_sig, &attestation_indices, &bitfield, &store).unwrap(); - assert_eq!(is_valid, false); assert_eq!(voters, None); } } diff --git a/lighthouse/state/validation/mod.rs b/lighthouse/state/validation/mod.rs index b09805d6c9..8ad367f4ab 100644 --- a/lighthouse/state/validation/mod.rs +++ b/lighthouse/state/validation/mod.rs @@ -1,18 +1,17 @@ -/* -use super::crystallized_state::CrystallizedState; -use super::active_state::ActiveState; -use super::attestation_record::AttestationRecord; -use super::block::Block; -use super::chain_config::ChainConfig; -*/ +use std::collections::HashMap; use super::block; use super::bls; use super::Logger; use super::db; -use super::attestation_record::AttestationRecord; +use super::attestation_record; use super::ssz; use super::transition::attestation_parent_hashes; use super::utils; mod attestation; mod ssz_block; + +type Slot = u64; +type ShardId = u64; +type AttesterMap = HashMap<(Slot, ShardId), Vec>; +type ProposerMap = HashMap; diff --git a/lighthouse/state/validation/ssz_block.rs b/lighthouse/state/validation/ssz_block.rs index c94f252949..35c590d423 100644 --- a/lighthouse/state/validation/ssz_block.rs +++ b/lighthouse/state/validation/ssz_block.rs @@ -1,14 +1,34 @@ +use std::sync::Arc; +use super::attestation::{ + validate_attestation, + AttestationValidationError, +}; +use super::attestation_record::{ + AttestationRecord, + split_one_attestation, + split_all_attestations, + AttestationSplitError, +}; +use super::{ + AttesterMap, + ProposerMap, +}; use super::block::SszBlock; -use super::Logger; use super::db::{ ClientDB, DBError, }; +use super::Logger; use super::db::stores::{ BlockStore, PoWChainStore, ValidatorStore, }; +use super::ssz::{ + Decodable, + DecodeError, +}; +use super::utils::types::Hash256; pub enum BlockStatus { NewBlock, @@ -18,21 +38,25 @@ pub enum BlockStatus { pub enum SszBlockValidationError { FutureSlot, UnknownPoWChainRef, + BadAttestationSsz, + AttestationValidationError(AttestationValidationError), + InvalidAttestation, + NoProposerSignature, + BadProposerMap, DatabaseError(String), } -impl From for SszBlockValidationError { - fn from(e: DBError) -> SszBlockValidationError { - SszBlockValidationError::DatabaseError(e.message) - } -} - #[allow(dead_code)] pub fn validate_ssz_block(b: &SszBlock, expected_slot: u64, - block_store: &BlockStore, pow_store: &PoWChainStore, - _validator_store: &ValidatorStore, + cycle_length: u8, + last_justified_slot: u64, + parent_hashes: Arc>, + proposer_map: Arc, + attester_map: Arc, + block_store: Arc>, + validator_store: Arc>, _log: &Logger) -> Result where T: ClientDB + Sized @@ -41,7 +65,8 @@ pub fn validate_ssz_block(b: &SszBlock, return Ok(BlockStatus::KnownBlock); } - if b.slot_number() > expected_slot { + let block_slot = b.slot_number(); + if block_slot > expected_slot { return Err(SszBlockValidationError::FutureSlot); } @@ -49,6 +74,84 @@ pub fn validate_ssz_block(b: &SszBlock, return Err(SszBlockValidationError::UnknownPoWChainRef); } - // Do validation here + let attestations_ssz = &b.attestations(); + + let (first_attestation_ssz, next_index) = split_one_attestation( + &attestations_ssz, + 0)?; + let (first_attestation, _) = AttestationRecord::ssz_decode( + &first_attestation_ssz, 0)?; + + let attestation_voters = validate_attestation( + &first_attestation, + block_slot, + cycle_length, + last_justified_slot, + parent_hashes.clone(), + block_store.clone(), + validator_store.clone(), + attester_map.clone())?; + + let attestation_voters = attestation_voters + .ok_or(SszBlockValidationError::InvalidAttestation)?; + + let proposer = proposer_map.get(&block_slot) + .ok_or(SszBlockValidationError::BadProposerMap)?; + + if !attestation_voters.contains(&proposer) { + return Err(SszBlockValidationError::NoProposerSignature); + } + + let other_attestations = split_all_attestations(attestations_ssz, + next_index)?; + + for attestation in other_attestations { + let (a, _) = AttestationRecord::ssz_decode(&attestation, 0)?; + let attestation_voters = validate_attestation( + &a, + block_slot, + cycle_length, + last_justified_slot, + parent_hashes.clone(), + block_store.clone(), + validator_store.clone(), + attester_map.clone())?; + if attestation_voters.is_none() { + return Err(SszBlockValidationError::InvalidAttestation); + } + } + Ok(BlockStatus::NewBlock) } + +impl From for SszBlockValidationError { + fn from(e: DBError) -> Self { + SszBlockValidationError::DatabaseError(e.message) + } +} + +impl From for SszBlockValidationError { + fn from(e: AttestationSplitError) -> Self { + match e { + AttestationSplitError::TooShort => + SszBlockValidationError::BadAttestationSsz + } + } +} + +impl From for SszBlockValidationError { + fn from(e: DecodeError) -> Self { + match e { + DecodeError::TooShort => + SszBlockValidationError::BadAttestationSsz, + DecodeError::TooLong => + SszBlockValidationError::BadAttestationSsz, + } + } +} + +impl From for SszBlockValidationError { + fn from(e: AttestationValidationError) -> Self { + SszBlockValidationError::AttestationValidationError(e) + } +} diff --git a/ssz/src/decode.rs b/ssz/src/decode.rs index 9d4132be8d..bb51610ee0 100644 --- a/ssz/src/decode.rs +++ b/ssz/src/decode.rs @@ -4,7 +4,6 @@ use super::{ #[derive(Debug, PartialEq)] pub enum DecodeError { - OutOfBounds, TooShort, TooLong, } @@ -22,7 +21,7 @@ pub fn decode_ssz(ssz_bytes: &[u8], index: usize) where T: Decodable { if index >= ssz_bytes.len() { - return Err(DecodeError::OutOfBounds) + return Err(DecodeError::TooShort) } T::ssz_decode(ssz_bytes, index) } From 209076e371958e183677c30a4508b646553745bc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 07:17:14 +1000 Subject: [PATCH 55/74] Fix ssz_split bug --- lighthouse/state/attestation_record/ssz_splitter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse/state/attestation_record/ssz_splitter.rs b/lighthouse/state/attestation_record/ssz_splitter.rs index 72b191cf30..e1c2054586 100644 --- a/lighthouse/state/attestation_record/ssz_splitter.rs +++ b/lighthouse/state/attestation_record/ssz_splitter.rs @@ -44,7 +44,7 @@ pub fn split_one_attestation<'a>(full_ssz: &'a [u8], index: usize) + hashes_len + bitfield_len; - if index + full_ssz.len() < len { + if full_ssz.len() < index + len { return Err(AttestationSplitError::TooShort); } From bb81bacb4e448c36659df16d1ba6bc7dc67f10ed Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 09:29:56 +1000 Subject: [PATCH 56/74] Add comments to attestation fns, fix bug --- .../validation/attestation/signatures.rs | 28 +++++++- lighthouse/state/validation/ssz_block.rs | 67 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/lighthouse/state/validation/attestation/signatures.rs b/lighthouse/state/validation/attestation/signatures.rs index fa17eba5a4..8534564019 100644 --- a/lighthouse/state/validation/attestation/signatures.rs +++ b/lighthouse/state/validation/attestation/signatures.rs @@ -18,6 +18,13 @@ pub enum SignatureVerificationError { DBError(String), } +/// Verify an aggregate signature across the supplied message. +/// +/// The public keys used for verification are collected by mapping +/// each true bitfield bit to canonical ValidatorRecord index through +/// the attestation_indicies map. +/// +/// Each public key is loaded from the store on-demand. pub fn verify_aggregate_signature_for_indices( message: &[u8], agg_sig: &AggregateSignature, @@ -33,13 +40,32 @@ pub fn verify_aggregate_signature_for_indices( for i in 0..attestation_indices.len() { let voted = bitfield.get_bit(i); if voted { - let validator = attestation_indices[i]; + /* + * De-reference the attestation index into a canonical ValidatorRecord index. + */ + let validator = attestation_indices.get(i) + .ok_or(SignatureVerificationError::BadValidatorIndex)?; + /* + * Load the validators public key from our store. + */ let pub_key = validator_store.get_public_key_by_index(i)? .ok_or(SignatureVerificationError::NoPublicKeyForValidator)?; + /* + * Add the validators public key to the aggregate public key. + */ agg_pub_key.add(&pub_key); + /* + * Add to the validator to the set of voters for this attestation record. + */ voters.insert(validator); } } + /* + * Verify the aggregate public key against the aggregate signature. + * + * This verification will only succeed if the exact set of public keys + * were added to the aggregate public key as those that signed the aggregate signature. + */ if agg_sig.verify(&message, &agg_pub_key) { Ok(Some(voters)) } else { diff --git a/lighthouse/state/validation/ssz_block.rs b/lighthouse/state/validation/ssz_block.rs index 35c590d423..90c2e22618 100644 --- a/lighthouse/state/validation/ssz_block.rs +++ b/lighthouse/state/validation/ssz_block.rs @@ -46,6 +46,19 @@ pub enum SszBlockValidationError { DatabaseError(String), } +/// Validate some SszBlock. An SszBlock varies from a Block in that is a read-only structure +/// that reads directly from encoded SSZ. +/// +/// The reason to validate an SzzBlock is to avoid decoding it in its entirety if there is +/// a suspicion that the block might be invalid. Such a suspicion should be applied to +/// all blocks coming from the network. +/// +/// Of course, this function will only be more efficient if a block is already serialized. +/// Serializing a complete block and then validating with this function will be less +/// efficient than just validating the original block. +/// +/// This function will determine if the block is new, already known or invalid (either +/// intrinsically or due to some application error.) #[allow(dead_code)] pub fn validate_ssz_block(b: &SszBlock, expected_slot: u64, @@ -61,27 +74,59 @@ pub fn validate_ssz_block(b: &SszBlock, -> Result where T: ClientDB + Sized { + /* + * If this block is already known, return immediately. + */ if block_store.block_exists(&b.block_hash())? { return Ok(BlockStatus::KnownBlock); } + /* + * Copy the block slot (will be used multiple times) + */ let block_slot = b.slot_number(); + + /* + * If the block slot corresponds to a slot in the future (according to the local time), + * drop it. + */ if block_slot > expected_slot { return Err(SszBlockValidationError::FutureSlot); } + /* + * If the PoW chain hash is not known to us, drop it. + * + * We only accept blocks that reference a known PoW hash. + * + * Note: it is not clear what a "known" PoW chain ref is. Likely, + * it means "sufficienty deep in the canonical PoW chain". + */ if pow_store.block_hash_exists(b.pow_chain_ref())? == false { return Err(SszBlockValidationError::UnknownPoWChainRef); } + /* + * Store a reference to the serialized attestations from the block. + */ let attestations_ssz = &b.attestations(); + /* + * Get a slice of the first serialized attestation (the 0th) and decode it into + * a full AttestationRecord object. + */ let (first_attestation_ssz, next_index) = split_one_attestation( &attestations_ssz, 0)?; let (first_attestation, _) = AttestationRecord::ssz_decode( &first_attestation_ssz, 0)?; + /* + * Validate this first attestation. + * + * It is a requirement that the block proposer for this slot + * must have signed the 0th attestation record. + */ let attestation_voters = validate_attestation( &first_attestation, block_slot, @@ -92,19 +137,37 @@ pub fn validate_ssz_block(b: &SszBlock, validator_store.clone(), attester_map.clone())?; + /* + * If the set of voters is None, the attestation was invalid. + */ let attestation_voters = attestation_voters .ok_or(SszBlockValidationError::InvalidAttestation)?; + /* + * Read the proposer from the map of slot -> validator index. + */ let proposer = proposer_map.get(&block_slot) .ok_or(SszBlockValidationError::BadProposerMap)?; + /* + * If the proposer for this slot was not a voter, reject the block. + */ if !attestation_voters.contains(&proposer) { return Err(SszBlockValidationError::NoProposerSignature); } + /* + * Split the remaining attestations into a vector of slices, each containing + * a single serialized attestation record. + */ let other_attestations = split_all_attestations(attestations_ssz, next_index)?; + /* + * Verify each other AttestationRecord. + * + * TODO: make this parallelized. + */ for attestation in other_attestations { let (a, _) = AttestationRecord::ssz_decode(&attestation, 0)?; let attestation_voters = validate_attestation( @@ -121,6 +184,10 @@ pub fn validate_ssz_block(b: &SszBlock, } } + /* + * If we have reached this point, the block is a new valid block that is worthy of + * processing. + */ Ok(BlockStatus::NewBlock) } From 4212d8fc7ddf4717593b8f7b573d267148247ecd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 09:31:59 +1000 Subject: [PATCH 57/74] Fix bug introduced in previous commit --- lighthouse/state/validation/attestation/signatures.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse/state/validation/attestation/signatures.rs b/lighthouse/state/validation/attestation/signatures.rs index 8534564019..165b03fe71 100644 --- a/lighthouse/state/validation/attestation/signatures.rs +++ b/lighthouse/state/validation/attestation/signatures.rs @@ -43,7 +43,7 @@ pub fn verify_aggregate_signature_for_indices( /* * De-reference the attestation index into a canonical ValidatorRecord index. */ - let validator = attestation_indices.get(i) + let validator = *attestation_indices.get(i) .ok_or(SignatureVerificationError::BadValidatorIndex)?; /* * Load the validators public key from our store. From 83d95f1d34019e49362ff9410756fe50d180dc58 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 12:20:13 +1000 Subject: [PATCH 58/74] Restructure directories. Add dead_code allows --- lighthouse/main.rs | 5 +++++ lighthouse/state/attestation_record/mod.rs | 3 +++ .../validation}/attestation_validation.rs | 2 +- .../validation}/mod.rs | 4 ++-- .../validation}/signatures.rs | 0 lighthouse/state/block/mod.rs | 4 ++++ lighthouse/state/block/validation/mod.rs | 17 +++++++++++++++++ .../validation/validate_ssz_block.rs} | 4 ++-- .../attestation_parent_hashes.rs | 0 lighthouse/state/common/maps.rs | 7 +++++++ lighthouse/state/{transition => common}/mod.rs | 7 ++++--- .../{transition => common}/shuffling/README.md | 0 .../{transition => common}/shuffling/mod.rs | 0 .../{transition => common}/shuffling/rng.rs | 0 lighthouse/state/mod.rs | 12 ++++++------ lighthouse/state/validation/mod.rs | 17 ----------------- lighthouse/sync/block.rs | 0 17 files changed, 51 insertions(+), 31 deletions(-) rename lighthouse/state/{validation/attestation => attestation_record/validation}/attestation_validation.rs (100%) rename lighthouse/state/{validation/attestation => attestation_record/validation}/mod.rs (74%) rename lighthouse/state/{validation/attestation => attestation_record/validation}/signatures.rs (100%) create mode 100644 lighthouse/state/block/validation/mod.rs rename lighthouse/state/{validation/ssz_block.rs => block/validation/validate_ssz_block.rs} (99%) rename lighthouse/state/{transition => common}/attestation_parent_hashes.rs (100%) create mode 100644 lighthouse/state/common/maps.rs rename lighthouse/state/{transition => common}/mod.rs (89%) rename lighthouse/state/{transition => common}/shuffling/README.md (100%) rename lighthouse/state/{transition => common}/shuffling/mod.rs (100%) rename lighthouse/state/{transition => common}/shuffling/rng.rs (100%) delete mode 100644 lighthouse/state/validation/mod.rs create mode 100644 lighthouse/sync/block.rs diff --git a/lighthouse/main.rs b/lighthouse/main.rs index 603346b36d..0b678fa468 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -8,11 +8,16 @@ extern crate network_libp2p; extern crate futures; #[macro_use] +#[allow(dead_code)] mod utils; +#[allow(dead_code)] mod bls; +#[allow(dead_code)] mod db; mod client; +#[allow(dead_code)] mod state; +#[allow(dead_code)] mod sync; mod config; diff --git a/lighthouse/state/attestation_record/mod.rs b/lighthouse/state/attestation_record/mod.rs index d96a451fdd..447bb53829 100644 --- a/lighthouse/state/attestation_record/mod.rs +++ b/lighthouse/state/attestation_record/mod.rs @@ -1,10 +1,13 @@ use super::bls; +use super::common; +use super::db; use super::ssz; use super::utils; mod attestation_record; mod ssz_splitter; +pub mod validation; pub use self::attestation_record::{ AttestationRecord, diff --git a/lighthouse/state/validation/attestation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs similarity index 100% rename from lighthouse/state/validation/attestation/attestation_validation.rs rename to lighthouse/state/attestation_record/validation/attestation_validation.rs index 4822851250..0d20bb466c 100644 --- a/lighthouse/state/validation/attestation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -1,10 +1,10 @@ use std::collections::HashSet; use super::attestation_record::AttestationRecord; +use super::AttesterMap; use super::attestation_parent_hashes::{ attestation_parent_hashes, ParentHashesError, }; -use super::AttesterMap; use super::db::{ ClientDB, DBError diff --git a/lighthouse/state/validation/attestation/mod.rs b/lighthouse/state/attestation_record/validation/mod.rs similarity index 74% rename from lighthouse/state/validation/attestation/mod.rs rename to lighthouse/state/attestation_record/validation/mod.rs index a28f3c6acc..388685b60b 100644 --- a/lighthouse/state/validation/attestation/mod.rs +++ b/lighthouse/state/attestation_record/validation/mod.rs @@ -1,9 +1,9 @@ -use super::AttesterMap; +use super::common::maps::AttesterMap; use super::db; use super::bls; use super::attestation_record; use super::ssz; -use super::attestation_parent_hashes; +use super::common::attestation_parent_hashes; use super::utils; mod attestation_validation; diff --git a/lighthouse/state/validation/attestation/signatures.rs b/lighthouse/state/attestation_record/validation/signatures.rs similarity index 100% rename from lighthouse/state/validation/attestation/signatures.rs rename to lighthouse/state/attestation_record/validation/signatures.rs diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs index 20467eca7f..6838b2a48c 100644 --- a/lighthouse/state/block/mod.rs +++ b/lighthouse/state/block/mod.rs @@ -1,11 +1,15 @@ extern crate blake2_rfc; +use super::common; +use super::Logger; +use super::db; use super::ssz; use super::utils; use super::attestation_record; mod block; mod ssz_block; +mod validation; pub use self::block::Block; pub use self::ssz_block::SszBlock; diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs new file mode 100644 index 0000000000..ea4462ea1e --- /dev/null +++ b/lighthouse/state/block/validation/mod.rs @@ -0,0 +1,17 @@ +mod validate_ssz_block; + +use super::attestation_record; +use super::SszBlock; +use super::Logger; +use super::db; +use super::ssz; +use super::utils; + +use super::common::maps::{ + AttesterMap, + ProposerMap, +}; +pub use self::validate_ssz_block::{ + validate_ssz_block, + SszBlockValidationError +}; diff --git a/lighthouse/state/validation/ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs similarity index 99% rename from lighthouse/state/validation/ssz_block.rs rename to lighthouse/state/block/validation/validate_ssz_block.rs index 90c2e22618..0a62c7f678 100644 --- a/lighthouse/state/validation/ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -1,5 +1,5 @@ use std::sync::Arc; -use super::attestation::{ +use super::attestation_record::validation::{ validate_attestation, AttestationValidationError, }; @@ -13,7 +13,7 @@ use super::{ AttesterMap, ProposerMap, }; -use super::block::SszBlock; +use super::SszBlock; use super::db::{ ClientDB, DBError, diff --git a/lighthouse/state/transition/attestation_parent_hashes.rs b/lighthouse/state/common/attestation_parent_hashes.rs similarity index 100% rename from lighthouse/state/transition/attestation_parent_hashes.rs rename to lighthouse/state/common/attestation_parent_hashes.rs diff --git a/lighthouse/state/common/maps.rs b/lighthouse/state/common/maps.rs new file mode 100644 index 0000000000..745f474c6a --- /dev/null +++ b/lighthouse/state/common/maps.rs @@ -0,0 +1,7 @@ +use std::collections::HashMap; + +/// Maps a (slot, shard_id) to attestation_indices. +pub type AttesterMap = HashMap<(u64, u64), Vec>; + +/// Maps a slot to a block proposer. +pub type ProposerMap = HashMap; diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/common/mod.rs similarity index 89% rename from lighthouse/state/transition/mod.rs rename to lighthouse/state/common/mod.rs index 7510796716..96ff38eb95 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/common/mod.rs @@ -1,6 +1,7 @@ -use super::utils::types::Hash256; - -pub mod attestation_parent_hashes; mod shuffling; +pub mod maps; +pub mod attestation_parent_hashes; + +use super::utils::types::Hash256; pub use self::shuffling::shuffle; diff --git a/lighthouse/state/transition/shuffling/README.md b/lighthouse/state/common/shuffling/README.md similarity index 100% rename from lighthouse/state/transition/shuffling/README.md rename to lighthouse/state/common/shuffling/README.md diff --git a/lighthouse/state/transition/shuffling/mod.rs b/lighthouse/state/common/shuffling/mod.rs similarity index 100% rename from lighthouse/state/transition/shuffling/mod.rs rename to lighthouse/state/common/shuffling/mod.rs diff --git a/lighthouse/state/transition/shuffling/rng.rs b/lighthouse/state/common/shuffling/rng.rs similarity index 100% rename from lighthouse/state/transition/shuffling/rng.rs rename to lighthouse/state/common/shuffling/rng.rs diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index 532d8345dc..10d9a18468 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -4,10 +4,7 @@ extern crate blake2_rfc as blake2; extern crate bytes; extern crate ssz; -use super::bls; -use super::db; -use super::Logger; -use super::utils; +mod common; pub mod active_state; pub mod attestation_record; @@ -16,7 +13,10 @@ pub mod chain_config; pub mod block; pub mod crosslink_record; pub mod shard_and_committee; -pub mod transition; pub mod validator_record; -pub mod validation; pub mod helpers; + +use super::bls; +use super::db; +use super::Logger; +use super::utils; diff --git a/lighthouse/state/validation/mod.rs b/lighthouse/state/validation/mod.rs deleted file mode 100644 index 8ad367f4ab..0000000000 --- a/lighthouse/state/validation/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::collections::HashMap; -use super::block; -use super::bls; -use super::Logger; -use super::db; -use super::attestation_record; -use super::ssz; -use super::transition::attestation_parent_hashes; -use super::utils; - -mod attestation; -mod ssz_block; - -type Slot = u64; -type ShardId = u64; -type AttesterMap = HashMap<(Slot, ShardId), Vec>; -type ProposerMap = HashMap; diff --git a/lighthouse/sync/block.rs b/lighthouse/sync/block.rs new file mode 100644 index 0000000000..e69de29bb2 From d9046e4c1b2e9317db5ab0f17513b33d3d6d3dee Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 12:32:40 +1000 Subject: [PATCH 59/74] Fix warnings --- lighthouse/db/memory_db.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/lighthouse/db/memory_db.rs b/lighthouse/db/memory_db.rs index 94bd5f2906..2f8ffe30da 100644 --- a/lighthouse/db/memory_db.rs +++ b/lighthouse/db/memory_db.rs @@ -112,11 +112,6 @@ mod tests { let col_a: &str = BLOCKS_DB_COLUMN; let col_b: &str = VALIDATOR_DB_COLUMN; - let column_families = vec![ - col_a, - col_b, - ]; - let db = MemoryDB::open(); /* @@ -137,11 +132,6 @@ mod tests { let col_a: &str = BLOCKS_DB_COLUMN; let col_x: &str = "ColumnX"; - let column_families = vec![ - col_a, - // col_x is excluded on purpose - ]; - let db = MemoryDB::open(); /* @@ -159,11 +149,6 @@ mod tests { let col_a: &str = BLOCKS_DB_COLUMN; let col_b: &str = VALIDATOR_DB_COLUMN; - let column_families = vec![ - col_a, - col_b, - ]; - let db = MemoryDB::open(); /* From dc7d2ff150c81e85f6f2f85ed4d815fdfcac2bda Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 12:32:51 +1000 Subject: [PATCH 60/74] Move helpers.rs into common/delegation --- .../{helpers.rs => common/delegation/block_hash.rs} | 13 ++----------- lighthouse/state/common/delegation/mod.rs | 3 +++ lighthouse/state/common/mod.rs | 2 ++ lighthouse/state/mod.rs | 1 - lighthouse/sync/block.rs | 0 5 files changed, 7 insertions(+), 12 deletions(-) rename lighthouse/state/{helpers.rs => common/delegation/block_hash.rs} (86%) create mode 100644 lighthouse/state/common/delegation/mod.rs delete mode 100644 lighthouse/sync/block.rs diff --git a/lighthouse/state/helpers.rs b/lighthouse/state/common/delegation/block_hash.rs similarity index 86% rename from lighthouse/state/helpers.rs rename to lighthouse/state/common/delegation/block_hash.rs index 5ed03daf7d..092709a217 100644 --- a/lighthouse/state/helpers.rs +++ b/lighthouse/state/common/delegation/block_hash.rs @@ -1,18 +1,9 @@ -/* - * Collection of helper functions used in the state transition modules - */ use super::utils::errors::ParameterError; use super::utils::types::Hash256; /* - pub fn get_signed_parent_hashes( - active_state: &ActiveState, - block: &Block, - attestation: &AttestationRecord, - chain_config: &ChainConfig) - -> Vec { - } - */ + * Work-in-progress function: not ready for review. + */ pub fn get_block_hash( active_state_recent_block_hashes: &Vec, diff --git a/lighthouse/state/common/delegation/mod.rs b/lighthouse/state/common/delegation/mod.rs new file mode 100644 index 0000000000..da9746f635 --- /dev/null +++ b/lighthouse/state/common/delegation/mod.rs @@ -0,0 +1,3 @@ +mod block_hash; + +use super::utils; diff --git a/lighthouse/state/common/mod.rs b/lighthouse/state/common/mod.rs index 96ff38eb95..76d294b23d 100644 --- a/lighthouse/state/common/mod.rs +++ b/lighthouse/state/common/mod.rs @@ -1,7 +1,9 @@ +mod delegation; mod shuffling; pub mod maps; pub mod attestation_parent_hashes; +use super::utils; use super::utils::types::Hash256; pub use self::shuffling::shuffle; diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index 10d9a18468..e533a52a9d 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -14,7 +14,6 @@ pub mod block; pub mod crosslink_record; pub mod shard_and_committee; pub mod validator_record; -pub mod helpers; use super::bls; use super::db; diff --git a/lighthouse/sync/block.rs b/lighthouse/sync/block.rs deleted file mode 100644 index e69de29bb2..0000000000 From da1e0c8a59e289e0941f566f6d6482a99bb54c3d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 22:09:30 +1000 Subject: [PATCH 61/74] Change bitfield serialization to little-endian --- boolean-bitfield/src/lib.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/boolean-bitfield/src/lib.rs b/boolean-bitfield/src/lib.rs index 01e8f9f799..6ac8e082b4 100644 --- a/boolean-bitfield/src/lib.rs +++ b/boolean-bitfield/src/lib.rs @@ -123,6 +123,11 @@ impl BooleanBitfield { } /// Clone and return the underlying byte array (`Vec`). + pub fn to_vec(&self) -> Vec { + self.vec.clone() + } + + /// Clone and return the underlying byte array (`Vec`) in big-endinan format. pub fn to_be_vec(&self) -> Vec { let mut o = self.vec.clone(); o.reverse(); @@ -150,7 +155,7 @@ impl PartialEq for BooleanBitfield { impl ssz::Encodable for BooleanBitfield { fn ssz_append(&self, s: &mut ssz::SszStream) { - s.append_vec(&self.to_be_vec()); + s.append_vec(&self.to_vec()); } } @@ -191,7 +196,7 @@ mod tests { let mut stream = ssz::SszStream::new(); stream.append(&b); - assert_eq!(stream.drain(), vec![0, 0, 0, 2, 1, 0]); + assert_eq!(stream.drain(), vec![0, 0, 0, 2, 0, 1]); } #[test] @@ -199,7 +204,7 @@ mod tests { /* * Correct input */ - let input = vec![0, 0, 0, 2, 1, 0]; + let input = vec![0, 0, 0, 2, 0, 1]; let (b, i) = BooleanBitfield::ssz_decode(&input, 0).unwrap(); assert_eq!(i, 6); assert_eq!(b.num_true_bits(), 1); @@ -208,7 +213,7 @@ mod tests { /* * Input too long */ - let mut input = vec![0, 0, 0, 2, 1, 0]; + let mut input = vec![0, 0, 0, 2, 0, 1]; input.push(42); let (b, i) = BooleanBitfield::ssz_decode(&input, 0).unwrap(); assert_eq!(i, 6); @@ -234,17 +239,6 @@ mod tests { assert_eq!(b.to_be_vec(), vec![0]); } - #[test] - fn test_new_bitfield_len() { - let b = BooleanBitfield::new(); - assert_eq!(b.len(), 0); - assert_eq!(b.to_be_vec(), vec![0]); - - let b = BooleanBitfield::with_capacity(100); - assert_eq!(b.len(), 0); - assert_eq!(b.to_be_vec(), vec![0]); - } - #[test] fn test_bitfield_set() { let mut b = BooleanBitfield::new(); From 748b50c08cfbae835b790b27a7b439ead1768883 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 22:10:12 +1000 Subject: [PATCH 62/74] Fix bit shift function in attestation validation --- .../attestation_record/validation/attestation_validation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs index 0d20bb466c..be2885fcd9 100644 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -49,8 +49,8 @@ fn bytes_for_bits(bits: usize) -> usize { } fn any_of_last_n_bits_are_set(byte: &u8, n: usize) -> bool { - let shift = 8_u8.saturating_sub(n as u8); - ((!0 >> shift) & byte) > 0 + let shift = 8_u16.saturating_sub(n as u16); + ((!0 << shift) & u16::from(*byte)) > 0 } pub fn validate_attestation(a: &AttestationRecord, From 8d2e9f33c727634ad0f459d44b16e83c3a531a17 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 22:10:47 +1000 Subject: [PATCH 63/74] Progress further with block validation tests --- .../validation/attestation_validation.rs | 50 +---- .../validation/message_generation.rs | 34 ++++ .../attestation_record/validation/mod.rs | 3 +- ...ignatures.rs => signature_verification.rs} | 0 lighthouse/state/block/mod.rs | 1 - lighthouse/state/block/validation/mod.rs | 7 +- lighthouse/state/block/validation/tests.rs | 185 ++++++++++++++++++ .../block/validation/validate_ssz_block.rs | 6 +- lighthouse/state/chain_config.rs | 19 +- lighthouse/state/common/maps.rs | 2 +- lighthouse/utils/macros.rs | 2 - 11 files changed, 257 insertions(+), 52 deletions(-) create mode 100644 lighthouse/state/attestation_record/validation/message_generation.rs rename lighthouse/state/attestation_record/validation/{signatures.rs => signature_verification.rs} (100%) create mode 100644 lighthouse/state/block/validation/tests.rs diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs index be2885fcd9..24ffc291be 100644 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -1,4 +1,5 @@ use std::collections::HashSet; +use std::sync::Arc; use super::attestation_record::AttestationRecord; use super::AttesterMap; use super::attestation_parent_hashes::{ @@ -13,13 +14,11 @@ use super::db::stores::{ BlockStore, ValidatorStore, }; -use super::ssz::SszStream; -use super::utils::hash::canonical_hash; use super::utils::types::{ Hash256, }; -use std::sync::Arc; -use super::signatures::{ +use super::message_generation::generate_signed_message; +use super::signature_verification::{ verify_aggregate_signature_for_indices, SignatureVerificationError, }; @@ -37,7 +36,9 @@ pub enum AttestationValidationError { IntWrapping, PublicKeyCorrupt, NoPublicKeyForValidator, - IncorrectBitField, + BadBitfieldLength, + InvalidBitfield, + InvalidBitfieldEndBits, NoSignatures, NonZeroTrailingBits, AggregateSignatureFail, @@ -115,7 +116,7 @@ pub fn validate_attestation(a: &AttestationRecord, if a.attester_bitfield.num_bytes() != bytes_for_bits(attestation_indices.len()) { - return Err(AttestationValidationError::IncorrectBitField); + return Err(AttestationValidationError::BadBitfieldLength); } /* @@ -126,10 +127,10 @@ pub fn validate_attestation(a: &AttestationRecord, * refer to the same AttesationRecord. */ let last_byte = - a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes()) - .ok_or(AttestationValidationError::IncorrectBitField)?; + a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes() - 1) + .ok_or(AttestationValidationError::InvalidBitfield)?; if any_of_last_n_bits_are_set(last_byte, a.attester_bitfield.len() % 8) { - return Err(AttestationValidationError::IncorrectBitField) + return Err(AttestationValidationError::InvalidBitfieldEndBits) } /* @@ -165,37 +166,6 @@ pub fn validate_attestation(a: &AttestationRecord, Ok(voted_hashmap) } -/// Generates the message used to validate the signature provided with an AttestationRecord. -/// -/// Ensures that the signer of the message has a view of the chain that is compatible with ours. -fn generate_signed_message(slot: u64, - parent_hashes: &[Hash256], - shard_id: u16, - shard_block_hash: &Hash256, - justified_slot: u64) - -> Vec -{ - /* - * Note: it's a little risky here to use SSZ, because the encoding is not necessarily SSZ - * (for example, SSZ might change whilst this doesn't). - * - * I have suggested switching this to ssz here: - * https://github.com/ethereum/eth2.0-specs/issues/5 - * - * If this doesn't happen, it would be safer to not use SSZ at all. - */ - let mut ssz_stream = SszStream::new(); - ssz_stream.append(&slot); - for h in parent_hashes { - ssz_stream.append_encoded_raw(&h.to_vec()) - } - ssz_stream.append(&shard_id); - ssz_stream.append(shard_block_hash); - ssz_stream.append(&justified_slot); - let bytes = ssz_stream.drain(); - canonical_hash(&bytes) -} - impl From for AttestationValidationError { fn from(e: ParentHashesError) -> Self { match e { diff --git a/lighthouse/state/attestation_record/validation/message_generation.rs b/lighthouse/state/attestation_record/validation/message_generation.rs new file mode 100644 index 0000000000..da6dd2e497 --- /dev/null +++ b/lighthouse/state/attestation_record/validation/message_generation.rs @@ -0,0 +1,34 @@ +use super::ssz::SszStream; +use super::utils::hash::canonical_hash; +use super::utils::types::Hash256; + +/// Generates the message used to validate the signature provided with an AttestationRecord. +/// +/// Ensures that the signer of the message has a view of the chain that is compatible with ours. +pub fn generate_signed_message(slot: u64, + parent_hashes: &[Hash256], + shard_id: u16, + shard_block_hash: &Hash256, + justified_slot: u64) + -> Vec +{ + /* + * Note: it's a little risky here to use SSZ, because the encoding is not necessarily SSZ + * (for example, SSZ might change whilst this doesn't). + * + * I have suggested switching this to ssz here: + * https://github.com/ethereum/eth2.0-specs/issues/5 + * + * If this doesn't happen, it would be safer to not use SSZ at all. + */ + let mut ssz_stream = SszStream::new(); + ssz_stream.append(&slot); + for h in parent_hashes { + ssz_stream.append_encoded_raw(&h.to_vec()) + } + ssz_stream.append(&shard_id); + ssz_stream.append(shard_block_hash); + ssz_stream.append(&justified_slot); + let bytes = ssz_stream.drain(); + canonical_hash(&bytes) +} diff --git a/lighthouse/state/attestation_record/validation/mod.rs b/lighthouse/state/attestation_record/validation/mod.rs index 388685b60b..e70a16a2a9 100644 --- a/lighthouse/state/attestation_record/validation/mod.rs +++ b/lighthouse/state/attestation_record/validation/mod.rs @@ -7,7 +7,8 @@ use super::common::attestation_parent_hashes; use super::utils; mod attestation_validation; -mod signatures; +mod signature_verification; +mod message_generation; pub use self::attestation_validation::{ validate_attestation, diff --git a/lighthouse/state/attestation_record/validation/signatures.rs b/lighthouse/state/attestation_record/validation/signature_verification.rs similarity index 100% rename from lighthouse/state/attestation_record/validation/signatures.rs rename to lighthouse/state/attestation_record/validation/signature_verification.rs diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs index 6838b2a48c..c1279111b6 100644 --- a/lighthouse/state/block/mod.rs +++ b/lighthouse/state/block/mod.rs @@ -1,7 +1,6 @@ extern crate blake2_rfc; use super::common; -use super::Logger; use super::db; use super::ssz; use super::utils; diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index ea4462ea1e..70c37e219c 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -1,8 +1,10 @@ mod validate_ssz_block; +#[cfg(test)] +mod tests; use super::attestation_record; +use super::Block; use super::SszBlock; -use super::Logger; use super::db; use super::ssz; use super::utils; @@ -13,5 +15,6 @@ use super::common::maps::{ }; pub use self::validate_ssz_block::{ validate_ssz_block, - SszBlockValidationError + SszBlockValidationError, + BlockStatus, }; diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs new file mode 100644 index 0000000000..2d0b2d87c3 --- /dev/null +++ b/lighthouse/state/block/validation/tests.rs @@ -0,0 +1,185 @@ +extern crate ssz; + +use self::ssz::{ + SszStream, +}; +use std::sync::Arc; +use super::{ + validate_ssz_block, + BlockStatus, + AttesterMap, + ProposerMap, +}; +use super::db::stores::{ + BlockStore, + PoWChainStore, + ValidatorStore, +}; +use super::db::{ + MemoryDB, +}; +use super::utils::types::{ + Hash256, + Bitfield, +}; +use super::{ + Block, + SszBlock, +}; +use super::super::attestation_record::AttestationRecord; +use super::super::super::bls::{ + Keypair, + Signature, + AggregateSignature, +}; + +struct TestStore { + db: Arc, + block: Arc>, + pow_chain: Arc>, + validator: Arc>, +} + +impl TestStore { + pub fn new() -> Self { + let db = Arc::new(MemoryDB::open()); + let block = Arc::new(BlockStore::new(db.clone())); + let pow_chain = Arc::new(PoWChainStore::new(db.clone())); + let validator = Arc::new(ValidatorStore::new(db.clone())); + Self { + db, + block, + pow_chain, + validator, + } + } +} + +#[test] +fn test_block_validation() { + let stores = TestStore::new(); + + let cycle_length = 2; + let shard_count = 2; + let present_slot = u64::from(cycle_length) * 10000; + let justified_slot = present_slot - u64::from(cycle_length); + let justified_block_hash = Hash256::from("justified_hash".as_bytes()); + let shard_block_hash = Hash256::from("shard_hash".as_bytes()); + let parent_hashes = (0..(cycle_length * 2)) + .map(|i| Hash256::from(i as u64)) + .collect(); + let pow_chain_ref = Hash256::from("pow_chain".as_bytes()); + let active_state_root = Hash256::from("active_state".as_bytes()); + let crystallized_state_root = Hash256::from("cry_state".as_bytes()); + + stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); + stores.block.put_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); + + let validators_per_shard = 2; + + let block_slot = present_slot; + let validator_index: usize = 0; + let proposer_map = { + let mut proposer_map = ProposerMap::new(); + proposer_map.insert(present_slot, validator_index); + proposer_map + }; + let (attester_map, attestations, _keypairs) = { + let mut i = 0; + let mut attester_map = AttesterMap::new(); + let mut attestations = vec![]; + let mut keypairs = vec![]; + for shard in 0..shard_count { + println!("hello"); + let mut attesters = vec![]; + let mut attester_bitfield = Bitfield::new(); + let mut aggregate_sig = AggregateSignature::new(); + let attestation_slot = block_slot - 1; + + let attestation_message = { + let mut stream = SszStream::new(); + stream.append(&attestation_slot); + stream.append_vec(&parent_hashes); + stream.append(&shard); + stream.append(&shard_block_hash); + stream.append(&justified_slot); + stream.drain() + }; + + for _ in 0..validators_per_shard { + /* + * Add the attester to the attestation indices for this shard. + */ + attesters.push(i); + /* + * Set the voters bit on the bitfield to true. + */ + attester_bitfield.set_bit(i, true); + /* + * Generate a random keypair for this validatior and clone it into the + * list of keypairs. + */ + let keypair = Keypair::random(); + keypairs.push(keypair.clone()); + /* + * Store the validators public key in the database. + */ + stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); + /* + * Generate a new signature and aggregate it on the rolling signature. + */ + let sig = Signature::new(&attestation_message, &keypair.sk); + aggregate_sig.add(&sig); + /* + * Increment the validator counter to monotonically assign validators. + */ + i += 1; + } + + attester_map.insert((attestation_slot, shard), attesters); + let attestation = AttestationRecord { + slot: attestation_slot, + shard_id: shard, + oblique_parent_hashes: vec![], + shard_block_hash: Hash256::from("shardhash".as_bytes()), + attester_bitfield, + justified_slot, + justified_block_hash, + aggregate_sig, + }; + attestations.push(attestation); + } + (attester_map, attestations, keypairs) + }; + + let block = Block { + parent_hash: Hash256::from("parent".as_bytes()), + slot_number: block_slot, + randao_reveal: Hash256::from("randao".as_bytes()), + attestations, + pow_chain_ref, + active_state_root, + crystallized_state_root, + }; + + let mut stream = SszStream::new(); + stream.append(&block); + let serialized_block = stream.drain(); + let ssz_block = SszBlock::from_slice(&serialized_block[..]).unwrap(); + + println!("this happened"); + + let status = validate_ssz_block( + &ssz_block, + present_slot, + cycle_length, + justified_slot, + Arc::new(parent_hashes), + Arc::new(proposer_map), + Arc::new(attester_map), + stores.block.clone(), + stores.validator.clone(), + stores.pow_chain.clone()).unwrap(); + + assert_eq!(status, BlockStatus::NewBlock); +} diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs index 0a62c7f678..63198dd39d 100644 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -18,7 +18,6 @@ use super::db::{ ClientDB, DBError, }; -use super::Logger; use super::db::stores::{ BlockStore, PoWChainStore, @@ -30,11 +29,13 @@ use super::ssz::{ }; use super::utils::types::Hash256; +#[derive(Debug, PartialEq)] pub enum BlockStatus { NewBlock, KnownBlock, } +#[derive(Debug, PartialEq)] pub enum SszBlockValidationError { FutureSlot, UnknownPoWChainRef, @@ -62,7 +63,6 @@ pub enum SszBlockValidationError { #[allow(dead_code)] pub fn validate_ssz_block(b: &SszBlock, expected_slot: u64, - pow_store: &PoWChainStore, cycle_length: u8, last_justified_slot: u64, parent_hashes: Arc>, @@ -70,7 +70,7 @@ pub fn validate_ssz_block(b: &SszBlock, attester_map: Arc, block_store: Arc>, validator_store: Arc>, - _log: &Logger) + pow_store: Arc>) -> Result where T: ClientDB + Sized { diff --git a/lighthouse/state/chain_config.rs b/lighthouse/state/chain_config.rs index 857c2f8626..7dde3751cb 100644 --- a/lighthouse/state/chain_config.rs +++ b/lighthouse/state/chain_config.rs @@ -5,13 +5,28 @@ pub struct ChainConfig { pub genesis_time: u64, } +/* + * Presently this is just some arbitrary time in Sept 2018. + */ +const GENESIS_TIME: u64 = 1537488655; + impl ChainConfig { pub fn standard() -> Self { Self { - cycle_length: 8, + cycle_length: 64, shard_count: 1024, min_committee_size: 128, - genesis_time: 1537488655, // arbitrary + genesis_time: GENESIS_TIME, // arbitrary + } + } + + #[cfg(test)] + pub fn super_fast_tests() -> Self { + Self { + cycle_length: 2, + shard_count: 2, + min_committee_size: 2, + genesis_time: GENESIS_TIME, // arbitrary } } } diff --git a/lighthouse/state/common/maps.rs b/lighthouse/state/common/maps.rs index 745f474c6a..6b7c5da191 100644 --- a/lighthouse/state/common/maps.rs +++ b/lighthouse/state/common/maps.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; /// Maps a (slot, shard_id) to attestation_indices. -pub type AttesterMap = HashMap<(u64, u64), Vec>; +pub type AttesterMap = HashMap<(u64, u16), Vec>; /// Maps a slot to a block proposer. pub type ProposerMap = HashMap; diff --git a/lighthouse/utils/macros.rs b/lighthouse/utils/macros.rs index 60bb129142..e9a8c416d9 100644 --- a/lighthouse/utils/macros.rs +++ b/lighthouse/utils/macros.rs @@ -6,5 +6,3 @@ macro_rules! assert_error { } } } - - From 9207a44bcebfa1f35302a516e30441f97182bd9a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 23:31:39 +1000 Subject: [PATCH 64/74] Fix parent_hashes bug in message_generation --- .../state/attestation_record/validation/message_generation.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lighthouse/state/attestation_record/validation/message_generation.rs b/lighthouse/state/attestation_record/validation/message_generation.rs index da6dd2e497..3bcc4c16a8 100644 --- a/lighthouse/state/attestation_record/validation/message_generation.rs +++ b/lighthouse/state/attestation_record/validation/message_generation.rs @@ -23,9 +23,7 @@ pub fn generate_signed_message(slot: u64, */ let mut ssz_stream = SszStream::new(); ssz_stream.append(&slot); - for h in parent_hashes { - ssz_stream.append_encoded_raw(&h.to_vec()) - } + ssz_stream.append_vec(&parent_hashes.to_vec()); ssz_stream.append(&shard_id); ssz_stream.append(shard_block_hash); ssz_stream.append(&justified_slot); From 6529b4c09f20229645db29d363ac5ec762c94d73 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 23:31:58 +1000 Subject: [PATCH 65/74] Fix pub key getting in signature verification --- .../attestation_record/validation/signature_verification.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lighthouse/state/attestation_record/validation/signature_verification.rs b/lighthouse/state/attestation_record/validation/signature_verification.rs index 165b03fe71..c33dc28063 100644 --- a/lighthouse/state/attestation_record/validation/signature_verification.rs +++ b/lighthouse/state/attestation_record/validation/signature_verification.rs @@ -48,7 +48,8 @@ pub fn verify_aggregate_signature_for_indices( /* * Load the validators public key from our store. */ - let pub_key = validator_store.get_public_key_by_index(i)? + let pub_key = validator_store + .get_public_key_by_index(validator)? .ok_or(SignatureVerificationError::NoPublicKeyForValidator)?; /* * Add the validators public key to the aggregate public key. From 55e0e7d05d8e5605f8f944155cfa1860a8055479 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 23:32:35 +1000 Subject: [PATCH 66/74] Add/fix message hashing in block validation tests --- lighthouse/state/block/validation/tests.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index 2d0b2d87c3..627e10155c 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -18,6 +18,7 @@ use super::db::stores::{ use super::db::{ MemoryDB, }; +use super::utils::hash::canonical_hash; use super::utils::types::{ Hash256, Bitfield, @@ -59,8 +60,8 @@ impl TestStore { fn test_block_validation() { let stores = TestStore::new(); - let cycle_length = 2; - let shard_count = 2; + let cycle_length: u8 = 2; + let shard_count: u16 = 2; let present_slot = u64::from(cycle_length) * 10000; let justified_slot = present_slot - u64::from(cycle_length); let justified_block_hash = Hash256::from("justified_hash".as_bytes()); @@ -103,7 +104,8 @@ fn test_block_validation() { stream.append(&shard); stream.append(&shard_block_hash); stream.append(&justified_slot); - stream.drain() + let bytes = stream.drain(); + canonical_hash(&bytes) }; for _ in 0..validators_per_shard { From e1e494b9aa5b6826499c6392c6d29e936023aa83 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 23:33:20 +1000 Subject: [PATCH 67/74] Fix parent_hashes generation in block val. test --- lighthouse/state/block/validation/tests.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index 627e10155c..c9e512bfc8 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -66,7 +66,7 @@ fn test_block_validation() { let justified_slot = present_slot - u64::from(cycle_length); let justified_block_hash = Hash256::from("justified_hash".as_bytes()); let shard_block_hash = Hash256::from("shard_hash".as_bytes()); - let parent_hashes = (0..(cycle_length * 2)) + let parent_hashes: Vec = (0..(cycle_length * 2)) .map(|i| Hash256::from(i as u64)) .collect(); let pow_chain_ref = Hash256::from("pow_chain".as_bytes()); @@ -97,10 +97,17 @@ fn test_block_validation() { let mut aggregate_sig = AggregateSignature::new(); let attestation_slot = block_slot - 1; + let parent_hashes_slice = { + let distance: usize = (block_slot - attestation_slot) as usize; + let last: usize = parent_hashes.len() - distance; + let first: usize = last - usize::from(cycle_length); + &parent_hashes[first..last] + }; + let attestation_message = { let mut stream = SszStream::new(); stream.append(&attestation_slot); - stream.append_vec(&parent_hashes); + stream.append_vec(&parent_hashes_slice.to_vec()); stream.append(&shard); stream.append(&shard_block_hash); stream.append(&justified_slot); From 177c04dc1a21a4f6acc42ea0f3496630c3619491 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 23:34:25 +1000 Subject: [PATCH 68/74] Fix bitfield setting in block val. test --- lighthouse/state/block/validation/tests.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index c9e512bfc8..03eaf982e5 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -115,7 +115,9 @@ fn test_block_validation() { canonical_hash(&bytes) }; - for _ in 0..validators_per_shard { + + + for attestation_index in 0..validators_per_shard { /* * Add the attester to the attestation indices for this shard. */ @@ -123,7 +125,7 @@ fn test_block_validation() { /* * Set the voters bit on the bitfield to true. */ - attester_bitfield.set_bit(i, true); + attester_bitfield.set_bit(attestation_index, true); /* * Generate a random keypair for this validatior and clone it into the * list of keypairs. From 310662ee179ba1bd65d1f671ab47d9c429a638dd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 23:34:44 +1000 Subject: [PATCH 69/74] Fix shard_block_hash in block val. tests --- lighthouse/state/block/validation/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index 03eaf982e5..670c457031 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -152,7 +152,7 @@ fn test_block_validation() { slot: attestation_slot, shard_id: shard, oblique_parent_hashes: vec![], - shard_block_hash: Hash256::from("shardhash".as_bytes()), + shard_block_hash, attester_bitfield, justified_slot, justified_block_hash, From eedfa8c8ea1954f5aea87969e482a4ff0b007a09 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 23:35:10 +1000 Subject: [PATCH 70/74] Add more unique errors in validate ssz block --- lighthouse/state/block/validation/validate_ssz_block.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs index 63198dd39d..a3f015b415 100644 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -41,7 +41,8 @@ pub enum SszBlockValidationError { UnknownPoWChainRef, BadAttestationSsz, AttestationValidationError(AttestationValidationError), - InvalidAttestation, + AttestationSignatureFailed, + FirstAttestationSignatureFailed, NoProposerSignature, BadProposerMap, DatabaseError(String), @@ -141,7 +142,8 @@ pub fn validate_ssz_block(b: &SszBlock, * If the set of voters is None, the attestation was invalid. */ let attestation_voters = attestation_voters - .ok_or(SszBlockValidationError::InvalidAttestation)?; + .ok_or(SszBlockValidationError:: + FirstAttestationSignatureFailed)?; /* * Read the proposer from the map of slot -> validator index. @@ -180,7 +182,8 @@ pub fn validate_ssz_block(b: &SszBlock, validator_store.clone(), attester_map.clone())?; if attestation_voters.is_none() { - return Err(SszBlockValidationError::InvalidAttestation); + return Err(SszBlockValidationError:: + AttestationSignatureFailed); } } From 362b7b2513d54921e8d53ce2f5cf1db084ca949a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Sep 2018 23:40:01 +1000 Subject: [PATCH 71/74] Remove some stray print statements --- lighthouse/state/block/validation/tests.rs | 3 --- lighthouse/state/mod.rs | 1 - 2 files changed, 4 deletions(-) diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index 670c457031..e04f16b2ad 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -91,7 +91,6 @@ fn test_block_validation() { let mut attestations = vec![]; let mut keypairs = vec![]; for shard in 0..shard_count { - println!("hello"); let mut attesters = vec![]; let mut attester_bitfield = Bitfield::new(); let mut aggregate_sig = AggregateSignature::new(); @@ -178,8 +177,6 @@ fn test_block_validation() { let serialized_block = stream.drain(); let ssz_block = SszBlock::from_slice(&serialized_block[..]).unwrap(); - println!("this happened"); - let status = validate_ssz_block( &ssz_block, present_slot, diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index e533a52a9d..b49ab52cff 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -17,5 +17,4 @@ pub mod validator_record; use super::bls; use super::db; -use super::Logger; use super::utils; From 3188d32423c9f18ba66dcb41b42470bb84f587dc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 26 Sep 2018 11:36:07 +1000 Subject: [PATCH 72/74] Fix unused dep warning --- lighthouse/main.rs | 2 +- .../attestation_record/validation/attestation_validation.rs | 2 -- lighthouse/state/block/validation/mod.rs | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/lighthouse/main.rs b/lighthouse/main.rs index 0b678fa468..c90bcdaea5 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -23,7 +23,7 @@ mod config; use std::path::PathBuf; -use slog::{ Drain, Logger }; +use slog::Drain; use clap::{ Arg, App }; use config::LighthouseConfig; use client::Client; diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs index 24ffc291be..5070ebb448 100644 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -65,8 +65,6 @@ pub fn validate_attestation(a: &AttestationRecord, -> Result>, AttestationValidationError> where T: ClientDB + Sized { - // TODO: assert attestion isn't already known - /* * The attesation slot must not be higher than the block that contained it. */ diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index 70c37e219c..59aa3fc151 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -3,7 +3,6 @@ mod validate_ssz_block; mod tests; use super::attestation_record; -use super::Block; use super::SszBlock; use super::db; use super::ssz; From 966d891cb53afc75b67bdabfdc0aac19f5445cfa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 26 Sep 2018 11:40:28 +1000 Subject: [PATCH 73/74] Remove validation fns (will be added in a PR) --- lighthouse/state/attestation_record/mod.rs | 3 - .../validation/attestation_validation.rs | 203 ---------------- .../validation/message_generation.rs | 32 --- .../attestation_record/validation/mod.rs | 16 -- .../validation/signature_verification.rs | 183 -------------- lighthouse/state/block/mod.rs | 3 - lighthouse/state/block/validation/mod.rs | 19 -- lighthouse/state/block/validation/tests.rs | 193 --------------- .../block/validation/validate_ssz_block.rs | 227 ------------------ lighthouse/state/mod.rs | 1 - 10 files changed, 880 deletions(-) delete mode 100644 lighthouse/state/attestation_record/validation/attestation_validation.rs delete mode 100644 lighthouse/state/attestation_record/validation/message_generation.rs delete mode 100644 lighthouse/state/attestation_record/validation/mod.rs delete mode 100644 lighthouse/state/attestation_record/validation/signature_verification.rs delete mode 100644 lighthouse/state/block/validation/mod.rs delete mode 100644 lighthouse/state/block/validation/tests.rs delete mode 100644 lighthouse/state/block/validation/validate_ssz_block.rs diff --git a/lighthouse/state/attestation_record/mod.rs b/lighthouse/state/attestation_record/mod.rs index 447bb53829..d96a451fdd 100644 --- a/lighthouse/state/attestation_record/mod.rs +++ b/lighthouse/state/attestation_record/mod.rs @@ -1,13 +1,10 @@ use super::bls; -use super::common; -use super::db; use super::ssz; use super::utils; mod attestation_record; mod ssz_splitter; -pub mod validation; pub use self::attestation_record::{ AttestationRecord, diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs deleted file mode 100644 index 5070ebb448..0000000000 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ /dev/null @@ -1,203 +0,0 @@ -use std::collections::HashSet; -use std::sync::Arc; -use super::attestation_record::AttestationRecord; -use super::AttesterMap; -use super::attestation_parent_hashes::{ - attestation_parent_hashes, - ParentHashesError, -}; -use super::db::{ - ClientDB, - DBError -}; -use super::db::stores::{ - BlockStore, - ValidatorStore, -}; -use super::utils::types::{ - Hash256, -}; -use super::message_generation::generate_signed_message; -use super::signature_verification::{ - verify_aggregate_signature_for_indices, - SignatureVerificationError, -}; - -#[derive(Debug,PartialEq)] -pub enum AttestationValidationError { - SlotTooHigh, - SlotTooLow, - JustifiedSlotTooHigh, - UnknownJustifiedBlock, - TooManyObliqueHashes, - BadCurrentHashes, - BadObliqueHashes, - BadAttesterMap, - IntWrapping, - PublicKeyCorrupt, - NoPublicKeyForValidator, - BadBitfieldLength, - InvalidBitfield, - InvalidBitfieldEndBits, - NoSignatures, - NonZeroTrailingBits, - AggregateSignatureFail, - DBError(String), -} - -fn bytes_for_bits(bits: usize) -> usize { - (bits.saturating_sub(1) / 8) + 1 -} - -fn any_of_last_n_bits_are_set(byte: &u8, n: usize) -> bool { - let shift = 8_u16.saturating_sub(n as u16); - ((!0 << shift) & u16::from(*byte)) > 0 -} - -pub fn validate_attestation(a: &AttestationRecord, - block_slot: u64, - cycle_length: u8, - known_last_justified_slot: u64, - known_parent_hashes: Arc>, - block_store: Arc>, - validator_store: Arc>, - attester_map: Arc) - -> Result>, AttestationValidationError> - where T: ClientDB + Sized -{ - /* - * The attesation slot must not be higher than the block that contained it. - */ - if a.slot > block_slot { - return Err(AttestationValidationError::SlotTooHigh); - } - - /* - * The slot of this attestation must not be more than cycle_length + 1 distance - * from the block that contained it. - * - * The below code stays overflow-safe as long as cycle length is a < 64 bit integer. - */ - if a.slot < block_slot.saturating_sub(u64::from(cycle_length) + 1) { - return Err(AttestationValidationError::SlotTooLow); - } - - /* - * The attestation must indicate that its last justified slot is the same as the last justified - * slot known to us. - */ - if a.justified_slot > known_last_justified_slot { - return Err(AttestationValidationError::JustifiedSlotTooHigh); - } - - /* - * There is no need to include more oblique parents hashes than there are blocks - * in a cycle. - */ - if a.oblique_parent_hashes.len() > usize::from(cycle_length) { - return Err(AttestationValidationError::TooManyObliqueHashes); - } - - /* - * Retrieve the set of attestation indices for this slot and shard id. - * - * This is an array mapping the order that validators will appear in the bitfield to the - * canonincal index of a validator. - */ - let attestation_indices = attester_map.get(&(a.slot, a.shard_id.into())) - .ok_or(AttestationValidationError::BadAttesterMap)?; - - /* - * The bitfield must be no longer than the minimum required to represent each validator in the - * attestation indicies for this slot and shard id. - */ - if a.attester_bitfield.num_bytes() != - bytes_for_bits(attestation_indices.len()) - { - return Err(AttestationValidationError::BadBitfieldLength); - } - - /* - * If there are excess bits in the bitfield because the number of a validators in not a - * multiple of 8, reject this attestation record. - * - * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to - * refer to the same AttesationRecord. - */ - let last_byte = - a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes() - 1) - .ok_or(AttestationValidationError::InvalidBitfield)?; - if any_of_last_n_bits_are_set(last_byte, a.attester_bitfield.len() % 8) { - return Err(AttestationValidationError::InvalidBitfieldEndBits) - } - - /* - * The specified justified block hash must be known to us - */ - if !block_store.block_exists(&a.justified_block_hash)? { - return Err(AttestationValidationError::UnknownJustifiedBlock) - } - - let signed_message = { - let parent_hashes = attestation_parent_hashes( - cycle_length, - block_slot, - a.slot, - &known_parent_hashes, - &a.oblique_parent_hashes)?; - generate_signed_message( - a.slot, - &parent_hashes, - a.shard_id, - &a.shard_block_hash, - a.justified_slot) - }; - - let voted_hashmap = - verify_aggregate_signature_for_indices( - &signed_message, - &a.aggregate_sig, - &attestation_indices, - &a.attester_bitfield, - &validator_store)?; - - Ok(voted_hashmap) -} - -impl From for AttestationValidationError { - fn from(e: ParentHashesError) -> Self { - match e { - ParentHashesError::BadCurrentHashes - => AttestationValidationError::BadCurrentHashes, - ParentHashesError::BadObliqueHashes - => AttestationValidationError::BadObliqueHashes, - ParentHashesError::SlotTooLow - => AttestationValidationError::SlotTooLow, - ParentHashesError::SlotTooHigh - => AttestationValidationError::SlotTooHigh, - ParentHashesError::IntWrapping - => AttestationValidationError::IntWrapping - } - } -} - -impl From for AttestationValidationError { - fn from(e: DBError) -> Self { - AttestationValidationError::DBError(e.message) - } -} - -impl From for AttestationValidationError { - fn from(e: SignatureVerificationError) -> Self { - match e { - SignatureVerificationError::BadValidatorIndex - => AttestationValidationError::BadAttesterMap, - SignatureVerificationError::PublicKeyCorrupt - => AttestationValidationError::PublicKeyCorrupt, - SignatureVerificationError::NoPublicKeyForValidator - => AttestationValidationError::NoPublicKeyForValidator, - SignatureVerificationError::DBError(s) - => AttestationValidationError::DBError(s), - } - } -} diff --git a/lighthouse/state/attestation_record/validation/message_generation.rs b/lighthouse/state/attestation_record/validation/message_generation.rs deleted file mode 100644 index 3bcc4c16a8..0000000000 --- a/lighthouse/state/attestation_record/validation/message_generation.rs +++ /dev/null @@ -1,32 +0,0 @@ -use super::ssz::SszStream; -use super::utils::hash::canonical_hash; -use super::utils::types::Hash256; - -/// Generates the message used to validate the signature provided with an AttestationRecord. -/// -/// Ensures that the signer of the message has a view of the chain that is compatible with ours. -pub fn generate_signed_message(slot: u64, - parent_hashes: &[Hash256], - shard_id: u16, - shard_block_hash: &Hash256, - justified_slot: u64) - -> Vec -{ - /* - * Note: it's a little risky here to use SSZ, because the encoding is not necessarily SSZ - * (for example, SSZ might change whilst this doesn't). - * - * I have suggested switching this to ssz here: - * https://github.com/ethereum/eth2.0-specs/issues/5 - * - * If this doesn't happen, it would be safer to not use SSZ at all. - */ - let mut ssz_stream = SszStream::new(); - ssz_stream.append(&slot); - ssz_stream.append_vec(&parent_hashes.to_vec()); - ssz_stream.append(&shard_id); - ssz_stream.append(shard_block_hash); - ssz_stream.append(&justified_slot); - let bytes = ssz_stream.drain(); - canonical_hash(&bytes) -} diff --git a/lighthouse/state/attestation_record/validation/mod.rs b/lighthouse/state/attestation_record/validation/mod.rs deleted file mode 100644 index e70a16a2a9..0000000000 --- a/lighthouse/state/attestation_record/validation/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -use super::common::maps::AttesterMap; -use super::db; -use super::bls; -use super::attestation_record; -use super::ssz; -use super::common::attestation_parent_hashes; -use super::utils; - -mod attestation_validation; -mod signature_verification; -mod message_generation; - -pub use self::attestation_validation::{ - validate_attestation, - AttestationValidationError, -}; diff --git a/lighthouse/state/attestation_record/validation/signature_verification.rs b/lighthouse/state/attestation_record/validation/signature_verification.rs deleted file mode 100644 index c33dc28063..0000000000 --- a/lighthouse/state/attestation_record/validation/signature_verification.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::collections::HashSet; -use super::bls::{ - AggregateSignature, - AggregatePublicKey, -}; -use super::db::ClientDB; -use super::db::stores::{ - ValidatorStore, - ValidatorStoreError, -}; -use super::utils::types::Bitfield; - -#[derive(Debug, PartialEq)] -pub enum SignatureVerificationError { - BadValidatorIndex, - PublicKeyCorrupt, - NoPublicKeyForValidator, - DBError(String), -} - -/// Verify an aggregate signature across the supplied message. -/// -/// The public keys used for verification are collected by mapping -/// each true bitfield bit to canonical ValidatorRecord index through -/// the attestation_indicies map. -/// -/// Each public key is loaded from the store on-demand. -pub fn verify_aggregate_signature_for_indices( - message: &[u8], - agg_sig: &AggregateSignature, - attestation_indices: &[usize], - bitfield: &Bitfield, - validator_store: &ValidatorStore) - -> Result>, SignatureVerificationError> - where T: ClientDB + Sized -{ - let mut voters = HashSet::new(); - let mut agg_pub_key = AggregatePublicKey::new(); - - for i in 0..attestation_indices.len() { - let voted = bitfield.get_bit(i); - if voted { - /* - * De-reference the attestation index into a canonical ValidatorRecord index. - */ - let validator = *attestation_indices.get(i) - .ok_or(SignatureVerificationError::BadValidatorIndex)?; - /* - * Load the validators public key from our store. - */ - let pub_key = validator_store - .get_public_key_by_index(validator)? - .ok_or(SignatureVerificationError::NoPublicKeyForValidator)?; - /* - * Add the validators public key to the aggregate public key. - */ - agg_pub_key.add(&pub_key); - /* - * Add to the validator to the set of voters for this attestation record. - */ - voters.insert(validator); - } - } - /* - * Verify the aggregate public key against the aggregate signature. - * - * This verification will only succeed if the exact set of public keys - * were added to the aggregate public key as those that signed the aggregate signature. - */ - if agg_sig.verify(&message, &agg_pub_key) { - Ok(Some(voters)) - } else { - Ok(None) - } -} - -impl From for SignatureVerificationError { - fn from(error: ValidatorStoreError) -> Self { - match error { - ValidatorStoreError::DBError(s) => - SignatureVerificationError::DBError(s), - ValidatorStoreError::DecodeError => - SignatureVerificationError::PublicKeyCorrupt, - } - } -} - - -#[cfg(test)] -mod tests { - use super::*; - use super::super::bls::{ - Keypair, - Signature, - }; - use super::super::db::MemoryDB; - use std::sync::Arc; - - /* - * Cases that still need testing: - * - * - No signatures. - * - Database failure. - * - Unknown validator index. - * - Extra validator on signature. - */ - - #[test] - fn test_signature_verification() { - let message = "cats".as_bytes(); - let signing_keypairs = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - let non_signing_keypairs = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - /* - * Signing keypairs first, then non-signing - */ - let mut all_keypairs = signing_keypairs.clone(); - all_keypairs.append(&mut non_signing_keypairs.clone()); - - let attestation_indices: Vec = (0..all_keypairs.len()) - .collect(); - let mut bitfield = Bitfield::new(); - for i in 0..signing_keypairs.len() { - bitfield.set_bit(i, true); - } - - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db); - - for (i, keypair) in all_keypairs.iter().enumerate() { - store.put_public_key_by_index(i, &keypair.pk).unwrap(); - } - - let mut agg_sig = AggregateSignature::new(); - for keypair in &signing_keypairs { - let sig = Signature::new(&message, &keypair.sk); - agg_sig.add(&sig); - } - - /* - * Test using all valid parameters. - */ - let voters = verify_aggregate_signature_for_indices( - &message, - &agg_sig, - &attestation_indices, - &bitfield, - &store).unwrap(); - - let voters = voters.unwrap(); - (0..signing_keypairs.len()) - .for_each(|i| assert!(voters.contains(&i))); - (signing_keypairs.len()..non_signing_keypairs.len()) - .for_each(|i| assert!(!voters.contains(&i))); - - /* - * Add another validator to the bitfield, run validation will all other - * parameters the same and assert that it fails. - */ - bitfield.set_bit(signing_keypairs.len() + 1, true); - let voters = verify_aggregate_signature_for_indices( - &message, - &agg_sig, - &attestation_indices, - &bitfield, - &store).unwrap(); - - assert_eq!(voters, None); - } -} diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs index c1279111b6..20467eca7f 100644 --- a/lighthouse/state/block/mod.rs +++ b/lighthouse/state/block/mod.rs @@ -1,14 +1,11 @@ extern crate blake2_rfc; -use super::common; -use super::db; use super::ssz; use super::utils; use super::attestation_record; mod block; mod ssz_block; -mod validation; pub use self::block::Block; pub use self::ssz_block::SszBlock; diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs deleted file mode 100644 index 59aa3fc151..0000000000 --- a/lighthouse/state/block/validation/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -mod validate_ssz_block; -#[cfg(test)] -mod tests; - -use super::attestation_record; -use super::SszBlock; -use super::db; -use super::ssz; -use super::utils; - -use super::common::maps::{ - AttesterMap, - ProposerMap, -}; -pub use self::validate_ssz_block::{ - validate_ssz_block, - SszBlockValidationError, - BlockStatus, -}; diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs deleted file mode 100644 index e04f16b2ad..0000000000 --- a/lighthouse/state/block/validation/tests.rs +++ /dev/null @@ -1,193 +0,0 @@ -extern crate ssz; - -use self::ssz::{ - SszStream, -}; -use std::sync::Arc; -use super::{ - validate_ssz_block, - BlockStatus, - AttesterMap, - ProposerMap, -}; -use super::db::stores::{ - BlockStore, - PoWChainStore, - ValidatorStore, -}; -use super::db::{ - MemoryDB, -}; -use super::utils::hash::canonical_hash; -use super::utils::types::{ - Hash256, - Bitfield, -}; -use super::{ - Block, - SszBlock, -}; -use super::super::attestation_record::AttestationRecord; -use super::super::super::bls::{ - Keypair, - Signature, - AggregateSignature, -}; - -struct TestStore { - db: Arc, - block: Arc>, - pow_chain: Arc>, - validator: Arc>, -} - -impl TestStore { - pub fn new() -> Self { - let db = Arc::new(MemoryDB::open()); - let block = Arc::new(BlockStore::new(db.clone())); - let pow_chain = Arc::new(PoWChainStore::new(db.clone())); - let validator = Arc::new(ValidatorStore::new(db.clone())); - Self { - db, - block, - pow_chain, - validator, - } - } -} - -#[test] -fn test_block_validation() { - let stores = TestStore::new(); - - let cycle_length: u8 = 2; - let shard_count: u16 = 2; - let present_slot = u64::from(cycle_length) * 10000; - let justified_slot = present_slot - u64::from(cycle_length); - let justified_block_hash = Hash256::from("justified_hash".as_bytes()); - let shard_block_hash = Hash256::from("shard_hash".as_bytes()); - let parent_hashes: Vec = (0..(cycle_length * 2)) - .map(|i| Hash256::from(i as u64)) - .collect(); - let pow_chain_ref = Hash256::from("pow_chain".as_bytes()); - let active_state_root = Hash256::from("active_state".as_bytes()); - let crystallized_state_root = Hash256::from("cry_state".as_bytes()); - - stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); - stores.block.put_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); - - let validators_per_shard = 2; - - let block_slot = present_slot; - let validator_index: usize = 0; - let proposer_map = { - let mut proposer_map = ProposerMap::new(); - proposer_map.insert(present_slot, validator_index); - proposer_map - }; - let (attester_map, attestations, _keypairs) = { - let mut i = 0; - let mut attester_map = AttesterMap::new(); - let mut attestations = vec![]; - let mut keypairs = vec![]; - for shard in 0..shard_count { - let mut attesters = vec![]; - let mut attester_bitfield = Bitfield::new(); - let mut aggregate_sig = AggregateSignature::new(); - let attestation_slot = block_slot - 1; - - let parent_hashes_slice = { - let distance: usize = (block_slot - attestation_slot) as usize; - let last: usize = parent_hashes.len() - distance; - let first: usize = last - usize::from(cycle_length); - &parent_hashes[first..last] - }; - - let attestation_message = { - let mut stream = SszStream::new(); - stream.append(&attestation_slot); - stream.append_vec(&parent_hashes_slice.to_vec()); - stream.append(&shard); - stream.append(&shard_block_hash); - stream.append(&justified_slot); - let bytes = stream.drain(); - canonical_hash(&bytes) - }; - - - - for attestation_index in 0..validators_per_shard { - /* - * Add the attester to the attestation indices for this shard. - */ - attesters.push(i); - /* - * Set the voters bit on the bitfield to true. - */ - attester_bitfield.set_bit(attestation_index, true); - /* - * Generate a random keypair for this validatior and clone it into the - * list of keypairs. - */ - let keypair = Keypair::random(); - keypairs.push(keypair.clone()); - /* - * Store the validators public key in the database. - */ - stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); - /* - * Generate a new signature and aggregate it on the rolling signature. - */ - let sig = Signature::new(&attestation_message, &keypair.sk); - aggregate_sig.add(&sig); - /* - * Increment the validator counter to monotonically assign validators. - */ - i += 1; - } - - attester_map.insert((attestation_slot, shard), attesters); - let attestation = AttestationRecord { - slot: attestation_slot, - shard_id: shard, - oblique_parent_hashes: vec![], - shard_block_hash, - attester_bitfield, - justified_slot, - justified_block_hash, - aggregate_sig, - }; - attestations.push(attestation); - } - (attester_map, attestations, keypairs) - }; - - let block = Block { - parent_hash: Hash256::from("parent".as_bytes()), - slot_number: block_slot, - randao_reveal: Hash256::from("randao".as_bytes()), - attestations, - pow_chain_ref, - active_state_root, - crystallized_state_root, - }; - - let mut stream = SszStream::new(); - stream.append(&block); - let serialized_block = stream.drain(); - let ssz_block = SszBlock::from_slice(&serialized_block[..]).unwrap(); - - let status = validate_ssz_block( - &ssz_block, - present_slot, - cycle_length, - justified_slot, - Arc::new(parent_hashes), - Arc::new(proposer_map), - Arc::new(attester_map), - stores.block.clone(), - stores.validator.clone(), - stores.pow_chain.clone()).unwrap(); - - assert_eq!(status, BlockStatus::NewBlock); -} diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs deleted file mode 100644 index a3f015b415..0000000000 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ /dev/null @@ -1,227 +0,0 @@ -use std::sync::Arc; -use super::attestation_record::validation::{ - validate_attestation, - AttestationValidationError, -}; -use super::attestation_record::{ - AttestationRecord, - split_one_attestation, - split_all_attestations, - AttestationSplitError, -}; -use super::{ - AttesterMap, - ProposerMap, -}; -use super::SszBlock; -use super::db::{ - ClientDB, - DBError, -}; -use super::db::stores::{ - BlockStore, - PoWChainStore, - ValidatorStore, -}; -use super::ssz::{ - Decodable, - DecodeError, -}; -use super::utils::types::Hash256; - -#[derive(Debug, PartialEq)] -pub enum BlockStatus { - NewBlock, - KnownBlock, -} - -#[derive(Debug, PartialEq)] -pub enum SszBlockValidationError { - FutureSlot, - UnknownPoWChainRef, - BadAttestationSsz, - AttestationValidationError(AttestationValidationError), - AttestationSignatureFailed, - FirstAttestationSignatureFailed, - NoProposerSignature, - BadProposerMap, - DatabaseError(String), -} - -/// Validate some SszBlock. An SszBlock varies from a Block in that is a read-only structure -/// that reads directly from encoded SSZ. -/// -/// The reason to validate an SzzBlock is to avoid decoding it in its entirety if there is -/// a suspicion that the block might be invalid. Such a suspicion should be applied to -/// all blocks coming from the network. -/// -/// Of course, this function will only be more efficient if a block is already serialized. -/// Serializing a complete block and then validating with this function will be less -/// efficient than just validating the original block. -/// -/// This function will determine if the block is new, already known or invalid (either -/// intrinsically or due to some application error.) -#[allow(dead_code)] -pub fn validate_ssz_block(b: &SszBlock, - expected_slot: u64, - cycle_length: u8, - last_justified_slot: u64, - parent_hashes: Arc>, - proposer_map: Arc, - attester_map: Arc, - block_store: Arc>, - validator_store: Arc>, - pow_store: Arc>) - -> Result - where T: ClientDB + Sized -{ - /* - * If this block is already known, return immediately. - */ - if block_store.block_exists(&b.block_hash())? { - return Ok(BlockStatus::KnownBlock); - } - - /* - * Copy the block slot (will be used multiple times) - */ - let block_slot = b.slot_number(); - - /* - * If the block slot corresponds to a slot in the future (according to the local time), - * drop it. - */ - if block_slot > expected_slot { - return Err(SszBlockValidationError::FutureSlot); - } - - /* - * If the PoW chain hash is not known to us, drop it. - * - * We only accept blocks that reference a known PoW hash. - * - * Note: it is not clear what a "known" PoW chain ref is. Likely, - * it means "sufficienty deep in the canonical PoW chain". - */ - if pow_store.block_hash_exists(b.pow_chain_ref())? == false { - return Err(SszBlockValidationError::UnknownPoWChainRef); - } - - /* - * Store a reference to the serialized attestations from the block. - */ - let attestations_ssz = &b.attestations(); - - /* - * Get a slice of the first serialized attestation (the 0th) and decode it into - * a full AttestationRecord object. - */ - let (first_attestation_ssz, next_index) = split_one_attestation( - &attestations_ssz, - 0)?; - let (first_attestation, _) = AttestationRecord::ssz_decode( - &first_attestation_ssz, 0)?; - - /* - * Validate this first attestation. - * - * It is a requirement that the block proposer for this slot - * must have signed the 0th attestation record. - */ - let attestation_voters = validate_attestation( - &first_attestation, - block_slot, - cycle_length, - last_justified_slot, - parent_hashes.clone(), - block_store.clone(), - validator_store.clone(), - attester_map.clone())?; - - /* - * If the set of voters is None, the attestation was invalid. - */ - let attestation_voters = attestation_voters - .ok_or(SszBlockValidationError:: - FirstAttestationSignatureFailed)?; - - /* - * Read the proposer from the map of slot -> validator index. - */ - let proposer = proposer_map.get(&block_slot) - .ok_or(SszBlockValidationError::BadProposerMap)?; - - /* - * If the proposer for this slot was not a voter, reject the block. - */ - if !attestation_voters.contains(&proposer) { - return Err(SszBlockValidationError::NoProposerSignature); - } - - /* - * Split the remaining attestations into a vector of slices, each containing - * a single serialized attestation record. - */ - let other_attestations = split_all_attestations(attestations_ssz, - next_index)?; - - /* - * Verify each other AttestationRecord. - * - * TODO: make this parallelized. - */ - for attestation in other_attestations { - let (a, _) = AttestationRecord::ssz_decode(&attestation, 0)?; - let attestation_voters = validate_attestation( - &a, - block_slot, - cycle_length, - last_justified_slot, - parent_hashes.clone(), - block_store.clone(), - validator_store.clone(), - attester_map.clone())?; - if attestation_voters.is_none() { - return Err(SszBlockValidationError:: - AttestationSignatureFailed); - } - } - - /* - * If we have reached this point, the block is a new valid block that is worthy of - * processing. - */ - Ok(BlockStatus::NewBlock) -} - -impl From for SszBlockValidationError { - fn from(e: DBError) -> Self { - SszBlockValidationError::DatabaseError(e.message) - } -} - -impl From for SszBlockValidationError { - fn from(e: AttestationSplitError) -> Self { - match e { - AttestationSplitError::TooShort => - SszBlockValidationError::BadAttestationSsz - } - } -} - -impl From for SszBlockValidationError { - fn from(e: DecodeError) -> Self { - match e { - DecodeError::TooShort => - SszBlockValidationError::BadAttestationSsz, - DecodeError::TooLong => - SszBlockValidationError::BadAttestationSsz, - } - } -} - -impl From for SszBlockValidationError { - fn from(e: AttestationValidationError) -> Self { - SszBlockValidationError::AttestationValidationError(e) - } -} diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index b49ab52cff..1a8d4a2c4a 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -16,5 +16,4 @@ pub mod shard_and_committee; pub mod validator_record; use super::bls; -use super::db; use super::utils; From baa7b06b569a4a371104b78e060be7078e360498 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 26 Sep 2018 11:53:15 +1000 Subject: [PATCH 74/74] Fix clippy lints --- lighthouse/db/memory_db.rs | 2 +- lighthouse/db/stores/pow_chain_store.rs | 2 +- lighthouse/db/stores/validator_store.rs | 12 +++++----- lighthouse/state/attestation_record/mod.rs | 4 ++-- .../state/attestation_record/ssz_splitter.rs | 4 ++-- .../{attestation_record.rs => structs.rs} | 0 lighthouse/state/block/mod.rs | 4 ++-- lighthouse/state/block/ssz_block.rs | 4 ++-- .../state/block/{block.rs => structs.rs} | 0 lighthouse/state/chain_config.rs | 2 +- .../state/common/delegation/block_hash.rs | 23 +++++++++++-------- 11 files changed, 30 insertions(+), 27 deletions(-) rename lighthouse/state/attestation_record/{attestation_record.rs => structs.rs} (100%) rename lighthouse/state/block/{block.rs => structs.rs} (100%) diff --git a/lighthouse/db/memory_db.rs b/lighthouse/db/memory_db.rs index 2f8ffe30da..c912b7c683 100644 --- a/lighthouse/db/memory_db.rs +++ b/lighthouse/db/memory_db.rs @@ -28,7 +28,7 @@ impl MemoryDB { pub fn open() -> Self { let db: DBHashMap = HashMap::new(); let mut known_columns: ColumnHashSet = HashSet::new(); - for col in COLUMNS.iter() { + for col in &COLUMNS { known_columns.insert(col.to_string()); } Self { diff --git a/lighthouse/db/stores/pow_chain_store.rs b/lighthouse/db/stores/pow_chain_store.rs index d428054764..f1f050a390 100644 --- a/lighthouse/db/stores/pow_chain_store.rs +++ b/lighthouse/db/stores/pow_chain_store.rs @@ -21,7 +21,7 @@ impl PoWChainStore { pub fn put_block_hash(&self, hash: &[u8]) -> Result<(), DBError> { - self.db.put(DB_COLUMN, hash, &vec![0]) + self.db.put(DB_COLUMN, hash, &[0]) } pub fn block_hash_exists(&self, hash: &[u8]) diff --git a/lighthouse/db/stores/validator_store.rs b/lighthouse/db/stores/validator_store.rs index e3c604b3ac..173eeee96d 100644 --- a/lighthouse/db/stores/validator_store.rs +++ b/lighthouse/db/stores/validator_store.rs @@ -42,7 +42,7 @@ impl ValidatorStore { } } - fn prefix_bytes(&self, key_prefix: KeyPrefixes) + fn prefix_bytes(&self, key_prefix: &KeyPrefixes) -> Vec { match key_prefix { @@ -50,7 +50,7 @@ impl ValidatorStore { } } - fn get_db_key_for_index(&self, key_prefix: KeyPrefixes, index: usize) + fn get_db_key_for_index(&self, key_prefix: &KeyPrefixes, index: usize) -> Vec { let mut buf = BytesMut::with_capacity(6 + 8); @@ -62,16 +62,16 @@ impl ValidatorStore { pub fn put_public_key_by_index(&self, index: usize, public_key: &PublicKey) -> Result<(), ValidatorStoreError> { - let key = self.get_db_key_for_index(KeyPrefixes::PublicKey, index); + let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index); let val = public_key.as_bytes(); self.db.put(DB_COLUMN, &key[..], &val[..]) - .map_err(|e| ValidatorStoreError::from(e)) + .map_err(ValidatorStoreError::from) } pub fn get_public_key_by_index(&self, index: usize) -> Result, ValidatorStoreError> { - let key = self.get_db_key_for_index(KeyPrefixes::PublicKey, index); + let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index); let val = self.db.get(DB_COLUMN, &key[..])?; match val { None => Ok(None), @@ -129,7 +129,7 @@ mod tests { let db = Arc::new(MemoryDB::open()); let store = ValidatorStore::new(db.clone()); - let key = store.get_db_key_for_index(KeyPrefixes::PublicKey, 42); + let key = store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42); db.put(DB_COLUMN, &key[..], "cats".as_bytes()).unwrap(); assert_eq!(store.get_public_key_by_index(42), diff --git a/lighthouse/state/attestation_record/mod.rs b/lighthouse/state/attestation_record/mod.rs index d96a451fdd..5751bcd245 100644 --- a/lighthouse/state/attestation_record/mod.rs +++ b/lighthouse/state/attestation_record/mod.rs @@ -3,10 +3,10 @@ use super::ssz; use super::utils; -mod attestation_record; +mod structs; mod ssz_splitter; -pub use self::attestation_record::{ +pub use self::structs::{ AttestationRecord, MIN_SSZ_ATTESTION_RECORD_LENGTH, }; diff --git a/lighthouse/state/attestation_record/ssz_splitter.rs b/lighthouse/state/attestation_record/ssz_splitter.rs index e1c2054586..e9e1b5ff81 100644 --- a/lighthouse/state/attestation_record/ssz_splitter.rs +++ b/lighthouse/state/attestation_record/ssz_splitter.rs @@ -24,8 +24,8 @@ pub fn split_all_attestations<'a>(full_ssz: &'a [u8], index: usize) /// Given some ssz slice, find the bounds of one serialized AttestationRecord /// and return a slice pointing to that. -pub fn split_one_attestation<'a>(full_ssz: &'a [u8], index: usize) - -> Result<(&'a [u8], usize), AttestationSplitError> +pub fn split_one_attestation(full_ssz: &[u8], index: usize) + -> Result<(&[u8], usize), AttestationSplitError> { if full_ssz.len() < MIN_LENGTH { return Err(AttestationSplitError::TooShort); diff --git a/lighthouse/state/attestation_record/attestation_record.rs b/lighthouse/state/attestation_record/structs.rs similarity index 100% rename from lighthouse/state/attestation_record/attestation_record.rs rename to lighthouse/state/attestation_record/structs.rs diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs index 20467eca7f..04b70e6dd5 100644 --- a/lighthouse/state/block/mod.rs +++ b/lighthouse/state/block/mod.rs @@ -4,8 +4,8 @@ use super::ssz; use super::utils; use super::attestation_record; -mod block; +mod structs; mod ssz_block; -pub use self::block::Block; +pub use self::structs::Block; pub use self::ssz_block::SszBlock; diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index a90ff13510..1d4fb1c72a 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -3,7 +3,7 @@ use super::ssz::decode::{ Decodable, }; use super::utils::hash::canonical_hash; -use super::block::{ +use super::structs::{ MIN_SSZ_BLOCK_LENGTH, MAX_SSZ_BLOCK_LENGTH, }; @@ -146,7 +146,7 @@ impl<'a> SszBlock<'a> { #[cfg(test)] mod tests { use super::*; - use super::super::block::Block; + use super::super::structs::Block; use super::super::attestation_record::AttestationRecord; use super::super::ssz::SszStream; use super::super::utils::types::Hash256; diff --git a/lighthouse/state/block/block.rs b/lighthouse/state/block/structs.rs similarity index 100% rename from lighthouse/state/block/block.rs rename to lighthouse/state/block/structs.rs diff --git a/lighthouse/state/chain_config.rs b/lighthouse/state/chain_config.rs index 7dde3751cb..750081aad7 100644 --- a/lighthouse/state/chain_config.rs +++ b/lighthouse/state/chain_config.rs @@ -8,7 +8,7 @@ pub struct ChainConfig { /* * Presently this is just some arbitrary time in Sept 2018. */ -const GENESIS_TIME: u64 = 1537488655; +const GENESIS_TIME: u64 = 1_537_488_655; impl ChainConfig { pub fn standard() -> Self { diff --git a/lighthouse/state/common/delegation/block_hash.rs b/lighthouse/state/common/delegation/block_hash.rs index 092709a217..3d0939d29c 100644 --- a/lighthouse/state/common/delegation/block_hash.rs +++ b/lighthouse/state/common/delegation/block_hash.rs @@ -6,10 +6,10 @@ use super::utils::types::Hash256; */ pub fn get_block_hash( - active_state_recent_block_hashes: &Vec, - current_block_slot: &u64, - slot: &u64, - cycle_length: &u64, // convert from standard u8 + active_state_recent_block_hashes: &[Hash256], + current_block_slot: u64, + slot: u64, + cycle_length: u64, // convert from standard u8 ) -> Result { // active_state must have at 2*cycle_length hashes assert_error!( @@ -19,16 +19,16 @@ pub fn get_block_hash( )) ); - let state_start_slot = (*current_block_slot) + let state_start_slot = (current_block_slot) .checked_sub(cycle_length * 2) .unwrap_or(0); assert_error!( - (state_start_slot <= *slot) && (*slot < *current_block_slot), + (state_start_slot <= slot) && (slot < current_block_slot), ParameterError::InvalidInput(String::from("incorrect slot number")) ); - let index = 2 * cycle_length + (*slot) - *current_block_slot; // should always be positive + let index = 2 * cycle_length + slot - current_block_slot; // should always be positive Ok(active_state_recent_block_hashes[index as usize]) } @@ -47,13 +47,16 @@ mod tests { block_hashes.push(Hash256::random()); } - let result = get_block_hash(&block_hashes, &block_slot, &slot, &cycle_length).unwrap(); + let result = get_block_hash( + &block_hashes, + block_slot, + slot, + cycle_length) + .unwrap(); assert_eq!( result, block_hashes[(2 * cycle_length + slot - block_slot) as usize] ); - - println!("{:?}", result); } }