From 5a7903a3773ae5c640a0004ef2f2ddec77322dfb Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 19 Aug 2019 16:15:55 +1000 Subject: [PATCH 01/32] Improve BeaconState safe accessors And fix a bug in the compact committees accessor. --- .../src/per_epoch_processing.rs | 45 +++----- eth2/types/src/beacon_state.rs | 104 +++++++++++------- 2 files changed, 81 insertions(+), 68 deletions(-) diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 8d6153aeac..71d8b20dab 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -221,42 +221,29 @@ pub fn process_final_updates( // Update start shard. state.start_shard = state.next_epoch_start_shard(spec)?; - // This is a hack to allow us to update index roots and slashed balances for the next epoch. - // - // The indentation here is to make it obvious where the weird stuff happens. - { - state.slot += 1; - - // Set active index root - let index_epoch = next_epoch + spec.activation_exit_delay; - let indices_list = VariableList::::from( - state.get_active_validator_indices(index_epoch), - ); - state.set_active_index_root( - index_epoch, - Hash256::from_slice(&indices_list.tree_hash_root()), - spec, - )?; - - // Reset slashings - state.set_slashings(next_epoch, 0)?; - - // Set randao mix - state.set_randao_mix(next_epoch, *state.get_randao_mix(current_epoch)?)?; - - state.slot -= 1; - } + // Set active index root + let index_epoch = next_epoch + spec.activation_exit_delay; + let indices_list = VariableList::::from( + state.get_active_validator_indices(index_epoch), + ); + state.set_active_index_root( + index_epoch, + Hash256::from_slice(&indices_list.tree_hash_root()), + spec, + )?; // Set committees root - // Note: we do this out-of-order w.r.t. to the spec, because we don't want the slot to be - // incremented. It's safe because the updates to slashings and the RANDAO mix (above) don't - // affect this. state.set_compact_committee_root( next_epoch, get_compact_committees_root(state, RelativeEpoch::Next, spec)?, - spec, )?; + // Reset slashings + state.set_slashings(next_epoch, 0)?; + + // Set randao mix + state.set_randao_mix(next_epoch, *state.get_randao_mix(current_epoch)?)?; + // Set historical root accumulator if next_epoch.as_u64() % (T::SlotsPerHistoricalRoot::to_u64() / T::slots_per_epoch()) == 0 { let historical_batch = state.historical_batch(); diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index d312316f3a..5b00f08b78 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -60,6 +60,22 @@ pub enum Error { SszTypesError(ssz_types::Error), } +/// Control whether an epoch-indexed field can be indexed at the next epoch or not. +#[derive(Debug, PartialEq, Clone, Copy)] +enum AllowNextEpoch { + True, + False, +} + +impl AllowNextEpoch { + fn upper_bound_of(self, current_epoch: Epoch) -> Epoch { + match self { + AllowNextEpoch::True => current_epoch + 1, + AllowNextEpoch::False => current_epoch, + } + } +} + /// The state of the `BeaconChain` at some slot. /// /// Spec v0.8.0 @@ -108,12 +124,12 @@ where pub start_shard: u64, pub randao_mixes: FixedVector, #[compare_fields(as_slice)] - active_index_roots: FixedVector, + pub active_index_roots: FixedVector, #[compare_fields(as_slice)] - compact_committees_roots: FixedVector, + pub compact_committees_roots: FixedVector, // Slashings - slashings: FixedVector, + pub slashings: FixedVector, // Attestations pub previous_epoch_attestations: VariableList, T::MaxPendingAttestations>, @@ -459,12 +475,16 @@ impl BeaconState { /// Safely obtains the index for `randao_mixes` /// - /// Spec v0.8.0 - fn get_randao_mix_index(&self, epoch: Epoch) -> Result { + /// Spec v0.8.1 + fn get_randao_mix_index( + &self, + epoch: Epoch, + allow_next_epoch: AllowNextEpoch, + ) -> Result { let current_epoch = self.current_epoch(); let len = T::EpochsPerHistoricalVector::to_u64(); - if epoch + len > current_epoch && epoch <= current_epoch { + if current_epoch < epoch + len && epoch <= allow_next_epoch.upper_bound_of(current_epoch) { Ok(epoch.as_usize() % len as usize) } else { Err(Error::EpochOutOfBounds) @@ -492,7 +512,7 @@ impl BeaconState { /// /// Spec v0.8.1 pub fn get_randao_mix(&self, epoch: Epoch) -> Result<&Hash256, Error> { - let i = self.get_randao_mix_index(epoch)?; + let i = self.get_randao_mix_index(epoch, AllowNextEpoch::False)?; Ok(&self.randao_mixes[i]) } @@ -500,21 +520,29 @@ impl BeaconState { /// /// Spec v0.8.1 pub fn set_randao_mix(&mut self, epoch: Epoch, mix: Hash256) -> Result<(), Error> { - let i = self.get_randao_mix_index(epoch)?; + let i = self.get_randao_mix_index(epoch, AllowNextEpoch::True)?; self.randao_mixes[i] = mix; Ok(()) } /// Safely obtains the index for `active_index_roots`, given some `epoch`. /// + /// If `allow_next_epoch` is `True`, then we allow an _extra_ one epoch of lookahead. + /// /// Spec v0.8.1 - fn get_active_index_root_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result { + fn get_active_index_root_index( + &self, + epoch: Epoch, + spec: &ChainSpec, + allow_next_epoch: AllowNextEpoch, + ) -> Result { let current_epoch = self.current_epoch(); let lookahead = spec.activation_exit_delay; let lookback = self.active_index_roots.len() as u64 - lookahead; + let epoch_upper_bound = allow_next_epoch.upper_bound_of(current_epoch) + lookahead; - if epoch + lookback > current_epoch && current_epoch + lookahead >= epoch { + if current_epoch < epoch + lookback && epoch <= epoch_upper_bound { Ok(epoch.as_usize() % self.active_index_roots.len()) } else { Err(Error::EpochOutOfBounds) @@ -525,7 +553,7 @@ impl BeaconState { /// /// Spec v0.8.1 pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Result { - let i = self.get_active_index_root_index(epoch, spec)?; + let i = self.get_active_index_root_index(epoch, spec, AllowNextEpoch::False)?; Ok(self.active_index_roots[i]) } @@ -538,7 +566,7 @@ impl BeaconState { index_root: Hash256, spec: &ChainSpec, ) -> Result<(), Error> { - let i = self.get_active_index_root_index(epoch, spec)?; + let i = self.get_active_index_root_index(epoch, spec, AllowNextEpoch::True)?; self.active_index_roots[i] = index_root; Ok(()) } @@ -552,19 +580,17 @@ impl BeaconState { /// Safely obtains the index for `compact_committees_roots`, given some `epoch`. /// - /// Spec v0.8.0 + /// Spec v0.8.1 fn get_compact_committee_root_index( &self, epoch: Epoch, - spec: &ChainSpec, + allow_next_epoch: AllowNextEpoch, ) -> Result { let current_epoch = self.current_epoch(); + let len = T::EpochsPerHistoricalVector::to_u64(); - let lookahead = spec.activation_exit_delay; - let lookback = self.compact_committees_roots.len() as u64 - lookahead; - - if epoch + lookback > current_epoch && current_epoch + lookahead >= epoch { - Ok(epoch.as_usize() % self.compact_committees_roots.len()) + if current_epoch < epoch + len && epoch <= allow_next_epoch.upper_bound_of(current_epoch) { + Ok(epoch.as_usize() % len as usize) } else { Err(Error::EpochOutOfBounds) } @@ -572,26 +598,21 @@ impl BeaconState { /// Return the `compact_committee_root` at a recent `epoch`. /// - /// Spec v0.8.0 - pub fn get_compact_committee_root( - &self, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result { - let i = self.get_compact_committee_root_index(epoch, spec)?; + /// Spec v0.8.1 + pub fn get_compact_committee_root(&self, epoch: Epoch) -> Result { + let i = self.get_compact_committee_root_index(epoch, AllowNextEpoch::False)?; Ok(self.compact_committees_roots[i]) } /// Set the `compact_committee_root` at a recent `epoch`. /// - /// Spec v0.8.0 + /// Spec v0.8.1 pub fn set_compact_committee_root( &mut self, epoch: Epoch, index_root: Hash256, - spec: &ChainSpec, ) -> Result<(), Error> { - let i = self.get_compact_committee_root_index(epoch, spec)?; + let i = self.get_compact_committee_root_index(epoch, AllowNextEpoch::True)?; self.compact_committees_roots[i] = index_root; Ok(()) } @@ -642,14 +663,19 @@ impl BeaconState { /// Safely obtain the index for `slashings`, given some `epoch`. /// - /// Spec v0.8.0 - fn get_slashings_index(&self, epoch: Epoch) -> Result { + /// Spec v0.8.1 + fn get_slashings_index( + &self, + epoch: Epoch, + allow_next_epoch: AllowNextEpoch, + ) -> Result { // We allow the slashings vector to be accessed at any cached epoch at or before - // the current epoch. - if epoch <= self.current_epoch() - && epoch + T::EpochsPerSlashingsVector::to_u64() >= self.current_epoch() + 1 + // the current epoch, or the next epoch if `AllowNextEpoch::True` is passed. + let current_epoch = self.current_epoch(); + if current_epoch < epoch + T::EpochsPerSlashingsVector::to_u64() + && epoch <= allow_next_epoch.upper_bound_of(current_epoch) { - Ok((epoch.as_u64() % T::EpochsPerSlashingsVector::to_u64()) as usize) + Ok(epoch.as_usize() % T::EpochsPerSlashingsVector::to_usize()) } else { Err(Error::EpochOutOfBounds) } @@ -664,17 +690,17 @@ impl BeaconState { /// Get the total slashed balances for some epoch. /// - /// Spec v0.8.0 + /// Spec v0.8.1 pub fn get_slashings(&self, epoch: Epoch) -> Result { - let i = self.get_slashings_index(epoch)?; + let i = self.get_slashings_index(epoch, AllowNextEpoch::False)?; Ok(self.slashings[i]) } /// Set the total slashed balances for some epoch. /// - /// Spec v0.8.0 + /// Spec v0.8.1 pub fn set_slashings(&mut self, epoch: Epoch, value: u64) -> Result<(), Error> { - let i = self.get_slashings_index(epoch)?; + let i = self.get_slashings_index(epoch, AllowNextEpoch::True)?; self.slashings[i] = value; Ok(()) } From 4d2cdc94927279dce594510649df4cfedf30f187 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 27 Aug 2019 16:59:53 +1000 Subject: [PATCH 02/32] Update to spec v0.8.3 --- .../src/common/get_attesting_indices.rs | 4 +--- .../src/common/get_compact_committees_root.rs | 21 +++---------------- .../src/common/get_indexed_attestation.rs | 2 ++ .../src/per_epoch_processing.rs | 6 +++--- eth2/types/src/beacon_state.rs | 8 ------- 5 files changed, 9 insertions(+), 32 deletions(-) diff --git a/eth2/state_processing/src/common/get_attesting_indices.rs b/eth2/state_processing/src/common/get_attesting_indices.rs index f558909f6e..adb71801a4 100644 --- a/eth2/state_processing/src/common/get_attesting_indices.rs +++ b/eth2/state_processing/src/common/get_attesting_indices.rs @@ -17,11 +17,9 @@ pub fn get_attesting_indices( target_relative_epoch, )?; - /* TODO(freeze): re-enable this? - if bitlist.len() > committee.committee.len() { + if bitlist.len() != committee.committee.len() { return Err(BeaconStateError::InvalidBitfield); } - */ Ok(committee .committee diff --git a/eth2/state_processing/src/common/get_compact_committees_root.rs b/eth2/state_processing/src/common/get_compact_committees_root.rs index 75edb3549e..b8ab4345fd 100644 --- a/eth2/state_processing/src/common/get_compact_committees_root.rs +++ b/eth2/state_processing/src/common/get_compact_committees_root.rs @@ -3,7 +3,7 @@ use types::*; /// Return the compact committee root at `relative_epoch`. /// -/// Spec v0.8.0 +/// Spec v0.8.3 pub fn get_compact_committees_root( state: &BeaconState, relative_epoch: RelativeEpoch, @@ -11,28 +11,13 @@ pub fn get_compact_committees_root( ) -> Result { let mut committees = FixedVector::<_, T::ShardCount>::from_elem(CompactCommittee::::default()); - // FIXME: this is a spec bug, whereby the start shard for the epoch after the next epoch - // is mistakenly used. The start shard from the cache SHOULD work. - // Waiting on a release to fix https://github.com/ethereum/eth2.0-specs/issues/1315 - let start_shard = if relative_epoch == RelativeEpoch::Next { - state.next_epoch_start_shard(spec)? - } else { - state.get_epoch_start_shard(relative_epoch)? - }; + let start_shard = state.get_epoch_start_shard(relative_epoch)?; for committee_number in 0..state.get_committee_count(relative_epoch)? { let shard = (start_shard + committee_number) % T::ShardCount::to_u64(); - // FIXME: this is a partial workaround for the above, but it only works in the case - // where there's a committee for every shard in every epoch. It works for the minimal - // tests but not the mainnet ones. - let fake_shard = if relative_epoch == RelativeEpoch::Next { - (shard + 1) % T::ShardCount::to_u64() - } else { - shard - }; for &index in state - .get_crosslink_committee_for_shard(fake_shard, relative_epoch)? + .get_crosslink_committee_for_shard(shard, relative_epoch)? .committee { let validator = state diff --git a/eth2/state_processing/src/common/get_indexed_attestation.rs b/eth2/state_processing/src/common/get_indexed_attestation.rs index 7c08c8708d..82ca92eb73 100644 --- a/eth2/state_processing/src/common/get_indexed_attestation.rs +++ b/eth2/state_processing/src/common/get_indexed_attestation.rs @@ -11,6 +11,8 @@ pub fn get_indexed_attestation( state: &BeaconState, attestation: &Attestation, ) -> Result, Error> { + // Note: we rely on both calls to `get_attesting_indices` to check the bitfield lengths + // against the committee length let attesting_indices = get_attesting_indices(state, &attestation.data, &attestation.aggregation_bits)?; diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 71d8b20dab..08f42a229c 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -218,9 +218,6 @@ pub fn process_final_updates( } } - // Update start shard. - state.start_shard = state.next_epoch_start_shard(spec)?; - // Set active index root let index_epoch = next_epoch + spec.activation_exit_delay; let indices_list = VariableList::::from( @@ -252,6 +249,9 @@ pub fn process_final_updates( .push(Hash256::from_slice(&historical_batch.tree_hash_root()))?; } + // Update start shard. + state.start_shard = state.get_epoch_start_shard(RelativeEpoch::Next)?; + // Rotate current/previous epoch attestations state.previous_epoch_attestations = std::mem::replace(&mut state.current_epoch_attestations, VariableList::empty()); diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 5b00f08b78..9b623c070e 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -298,14 +298,6 @@ impl BeaconState { Ok(cache.epoch_start_shard()) } - pub fn next_epoch_start_shard(&self, spec: &ChainSpec) -> Result { - let cache = self.cache(RelativeEpoch::Current)?; - let active_validator_count = cache.active_validator_count(); - let shard_delta = T::get_shard_delta(active_validator_count, spec.target_committee_size); - - Ok((self.start_shard + shard_delta) % T::ShardCount::to_u64()) - } - /// Get the slot of an attestation. /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. From aed2f6407dac2e644825c76734ce610badf1e637 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 27 Aug 2019 17:00:21 +1000 Subject: [PATCH 03/32] Bump EF tests to v0.8.3 --- tests/ef_tests/eth2.0-spec-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ef_tests/eth2.0-spec-tests b/tests/ef_tests/eth2.0-spec-tests index aaa1673f50..ae6dd9011d 160000 --- a/tests/ef_tests/eth2.0-spec-tests +++ b/tests/ef_tests/eth2.0-spec-tests @@ -1 +1 @@ -Subproject commit aaa1673f508103e11304833e0456e4149f880065 +Subproject commit ae6dd9011df05fab8c7e651c09cf9c940973bf81 From 23a308e595cad8438aef47a1be95af2516839448 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 28 Aug 2019 18:46:16 +1000 Subject: [PATCH 04/32] BLS and SSZ static tests --- eth2/types/src/attestation_data.rs | 16 +- eth2/types/src/checkpoint.rs | 3 +- tests/ef_tests/src/cases.rs | 27 +++ .../src/cases/bls_aggregate_pubkeys.rs | 6 +- .../ef_tests/src/cases/bls_aggregate_sigs.rs | 6 +- tests/ef_tests/src/cases/bls_g2_compressed.rs | 6 +- tests/ef_tests/src/cases/bls_priv_to_pub.rs | 6 +- tests/ef_tests/src/cases/bls_sign_msg.rs | 6 +- tests/ef_tests/src/cases/ssz_static.rs | 207 +++++++++--------- tests/ef_tests/src/doc.rs | 53 +++-- tests/ef_tests/src/handler.rs | 130 +++++++++++ tests/ef_tests/src/lib.rs | 3 + tests/ef_tests/src/type_name.rs | 61 ++++++ tests/ef_tests/src/yaml_decode.rs | 14 +- tests/ef_tests/src/yaml_decode/utils.rs | 10 - tests/ef_tests/tests/tests.rs | 105 ++++++++- 16 files changed, 481 insertions(+), 178 deletions(-) create mode 100644 tests/ef_tests/src/handler.rs create mode 100644 tests/ef_tests/src/type_name.rs delete mode 100644 tests/ef_tests/src/yaml_decode/utils.rs diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index f2e63598f5..4d82ce1261 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -4,25 +4,13 @@ use crate::{Checkpoint, Crosslink, Hash256}; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; -use tree_hash::TreeHash; -use tree_hash_derive::{SignedRoot, TreeHash}; +use tree_hash_derive::TreeHash; /// The data upon which an attestation is based. /// /// Spec v0.8.0 #[derive( - Debug, - Clone, - PartialEq, - Eq, - Serialize, - Deserialize, - Hash, - Encode, - Decode, - TreeHash, - TestRandom, - SignedRoot, + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Encode, Decode, TreeHash, TestRandom, )] pub struct AttestationData { // LMD GHOST vote diff --git a/eth2/types/src/checkpoint.rs b/eth2/types/src/checkpoint.rs index dc40b336fe..0c70019215 100644 --- a/eth2/types/src/checkpoint.rs +++ b/eth2/types/src/checkpoint.rs @@ -4,7 +4,7 @@ use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; use tree_hash::TreeHash; -use tree_hash_derive::{SignedRoot, TreeHash}; +use tree_hash_derive::TreeHash; /// Casper FFG checkpoint, used in attestations. /// @@ -22,7 +22,6 @@ use tree_hash_derive::{SignedRoot, TreeHash}; Decode, TreeHash, TestRandom, - SignedRoot, )] pub struct Checkpoint { pub epoch: Epoch, diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 1ae4ea1d8d..7f6ffb0c4c 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -1,5 +1,6 @@ use super::*; use std::fmt::Debug; +use std::path::Path; mod bls_aggregate_pubkeys; mod bls_aggregate_sigs; @@ -53,6 +54,11 @@ pub use shuffling::*; pub use ssz_generic::*; pub use ssz_static::*; +pub trait LoadCase: Sized { + /// Load the test case from a test case directory. + fn load_from_dir(_path: &Path) -> Result; +} + pub trait Case: Debug { /// An optional field for implementing a custom description. /// @@ -68,6 +74,26 @@ pub trait Case: Debug { fn result(&self, case_index: usize) -> Result<(), Error>; } +pub trait BlsCase: serde::de::DeserializeOwned {} + +impl YamlDecode for T +where + T: BlsCase, +{ + fn yaml_decode(string: &str) -> Result { + serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) + } +} + +impl LoadCase for T +where + T: BlsCase, +{ + fn load_from_dir(path: &Path) -> Result { + Self::yaml_decode_file(&path.join("data.yaml")) + } +} + #[derive(Debug)] pub struct Cases { pub test_cases: Vec, @@ -86,6 +112,7 @@ where } } +// FIXME(michael): delete this impl YamlDecode for Cases { /// Decodes a YAML list of test cases fn yaml_decode(yaml: &str) -> Result { diff --git a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs index 6e38743f2f..c94e144952 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs @@ -9,11 +9,7 @@ pub struct BlsAggregatePubkeys { pub output: String, } -impl YamlDecode for BlsAggregatePubkeys { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsAggregatePubkeys {} impl Case for BlsAggregatePubkeys { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs index eeecab82cd..882ad7220a 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs @@ -9,11 +9,7 @@ pub struct BlsAggregateSigs { pub output: String, } -impl YamlDecode for BlsAggregateSigs { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsAggregateSigs {} impl Case for BlsAggregateSigs { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_g2_compressed.rs b/tests/ef_tests/src/cases/bls_g2_compressed.rs index 185cb58f35..547d8d03ae 100644 --- a/tests/ef_tests/src/cases/bls_g2_compressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_compressed.rs @@ -15,11 +15,7 @@ pub struct BlsG2Compressed { pub output: Vec, } -impl YamlDecode for BlsG2Compressed { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsG2Compressed {} impl Case for BlsG2Compressed { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_priv_to_pub.rs b/tests/ef_tests/src/cases/bls_priv_to_pub.rs index d72a43bbbc..869a0891c7 100644 --- a/tests/ef_tests/src/cases/bls_priv_to_pub.rs +++ b/tests/ef_tests/src/cases/bls_priv_to_pub.rs @@ -9,11 +9,7 @@ pub struct BlsPrivToPub { pub output: String, } -impl YamlDecode for BlsPrivToPub { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsPrivToPub {} impl Case for BlsPrivToPub { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_sign_msg.rs b/tests/ef_tests/src/cases/bls_sign_msg.rs index e62c3550fa..476ecdefb1 100644 --- a/tests/ef_tests/src/cases/bls_sign_msg.rs +++ b/tests/ef_tests/src/cases/bls_sign_msg.rs @@ -16,11 +16,7 @@ pub struct BlsSign { pub output: String, } -impl YamlDecode for BlsSign { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsSign {} impl Case for BlsSign { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index 96ba38b6ad..6a949073de 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -3,125 +3,120 @@ use crate::case_result::compare_result; use serde_derive::Deserialize; use ssz::{Decode, Encode}; use std::fmt::Debug; -use std::marker::PhantomData; -use tree_hash::TreeHash; -use types::{ - test_utils::TestRandom, Attestation, AttestationData, AttestationDataAndCustodyBit, - AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, - CompactCommittee, Crosslink, Deposit, DepositData, Eth1Data, EthSpec, Fork, Hash256, - HistoricalBatch, IndexedAttestation, PendingAttestation, ProposerSlashing, Transfer, Validator, - VoluntaryExit, -}; - -// Enum variant names are used by Serde when deserializing the test YAML -#[allow(clippy::large_enum_variant)] -#[derive(Debug, Clone, Deserialize)] -pub enum SszStatic -where - E: EthSpec, -{ - Fork(SszStaticInner), - Crosslink(SszStaticInner), - Checkpoint(SszStaticInner), - CompactCommittee(SszStaticInner, E>), - Eth1Data(SszStaticInner), - AttestationData(SszStaticInner), - AttestationDataAndCustodyBit(SszStaticInner), - IndexedAttestation(SszStaticInner, E>), - DepositData(SszStaticInner), - BeaconBlockHeader(SszStaticInner), - Validator(SszStaticInner), - PendingAttestation(SszStaticInner, E>), - HistoricalBatch(SszStaticInner, E>), - ProposerSlashing(SszStaticInner), - AttesterSlashing(SszStaticInner, E>), - Attestation(SszStaticInner, E>), - Deposit(SszStaticInner), - VoluntaryExit(SszStaticInner), - Transfer(SszStaticInner), - BeaconBlockBody(SszStaticInner, E>), - BeaconBlock(SszStaticInner, E>), - BeaconState(SszStaticInner, E>), -} +use std::fs; +use tree_hash::{SignedRoot, TreeHash}; +use types::Hash256; #[derive(Debug, Clone, Deserialize)] -pub struct SszStaticInner -where - E: EthSpec, -{ - pub value: T, - pub serialized: String, - pub root: String, - #[serde(skip, default)] - _phantom: PhantomData, +struct SszStaticRoots { + root: String, + signing_root: Option, } -impl YamlDecode for SszStatic { +impl YamlDecode for SszStaticRoots { fn yaml_decode(yaml: &str) -> Result { - serde_yaml::from_str(yaml).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) + Ok(serde_yaml::from_str(yaml).unwrap()) } } -impl Case for SszStatic { - fn result(&self, _case_index: usize) -> Result<(), Error> { - use self::SszStatic::*; - - match *self { - Fork(ref val) => ssz_static_test(val), - Crosslink(ref val) => ssz_static_test(val), - Checkpoint(ref val) => ssz_static_test(val), - CompactCommittee(ref val) => ssz_static_test(val), - Eth1Data(ref val) => ssz_static_test(val), - AttestationData(ref val) => ssz_static_test(val), - AttestationDataAndCustodyBit(ref val) => ssz_static_test(val), - IndexedAttestation(ref val) => ssz_static_test(val), - DepositData(ref val) => ssz_static_test(val), - BeaconBlockHeader(ref val) => ssz_static_test(val), - Validator(ref val) => ssz_static_test(val), - PendingAttestation(ref val) => ssz_static_test(val), - HistoricalBatch(ref val) => ssz_static_test(val), - ProposerSlashing(ref val) => ssz_static_test(val), - AttesterSlashing(ref val) => ssz_static_test(val), - Attestation(ref val) => ssz_static_test(val), - Deposit(ref val) => ssz_static_test(val), - VoluntaryExit(ref val) => ssz_static_test(val), - Transfer(ref val) => ssz_static_test(val), - BeaconBlockBody(ref val) => ssz_static_test(val), - BeaconBlock(ref val) => ssz_static_test(val), - BeaconState(ref val) => ssz_static_test(val), - } - } +#[derive(Debug, Clone)] +pub struct SszStatic { + roots: SszStaticRoots, + serialized: Vec, + value: T, } -fn ssz_static_test(tc: &SszStaticInner) -> Result<(), Error> -where - T: Clone - + Decode - + Debug - + Encode - + PartialEq - + serde::de::DeserializeOwned - + TreeHash - + TestRandom, +#[derive(Debug, Clone)] +pub struct SszStaticSR { + roots: SszStaticRoots, + serialized: Vec, + value: T, +} + +// Trait alias for all deez bounds +pub trait SszStaticType: + serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug { - // Verify we can decode SSZ in the same way we can decode YAML. - let ssz = hex::decode(&tc.serialized[2..]) - .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - let expected = tc.value.clone(); - let decode_result = T::from_ssz_bytes(&ssz); - compare_result(&decode_result, &Some(expected))?; +} - // Verify we can encode the result back into original ssz bytes - let decoded = decode_result.unwrap(); - let encoded_result = decoded.as_ssz_bytes(); - compare_result::, Error>(&Ok(encoded_result), &Some(ssz))?; +impl SszStaticType for T where + T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug +{ +} - // Verify the TreeHash root of the decoded struct matches the test. - let expected_root = - &hex::decode(&tc.root[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - let expected_root = Hash256::from_slice(&expected_root); - let tree_hash_root = Hash256::from_slice(&decoded.tree_hash_root()); - compare_result::(&Ok(tree_hash_root), &Some(expected_root))?; +fn load_from_dir(path: &Path) -> Result<(SszStaticRoots, Vec, T), Error> { + // FIXME: set description/name + let roots = SszStaticRoots::yaml_decode_file(&path.join("roots.yaml"))?; + + let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); + + let yaml = fs::read_to_string(&path.join("value.yaml")).expect("value.yaml exists"); + let value = + serde_yaml::from_str(&yaml).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; + + Ok((roots, serialized, value)) +} + +impl LoadCase for SszStatic { + fn load_from_dir(path: &Path) -> Result { + load_from_dir(path).map(|(roots, serialized, value)| Self { + roots, + serialized, + value, + }) + } +} + +impl LoadCase for SszStaticSR { + fn load_from_dir(path: &Path) -> Result { + load_from_dir(path).map(|(roots, serialized, value)| Self { + roots, + serialized, + value, + }) + } +} + +fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { + // Check serialization + let serialized_result = value.as_ssz_bytes(); + compare_result::, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?; + + // Check deserialization + let deserialized_result = T::from_ssz_bytes(serialized); + compare_result(&deserialized_result, &Some(value.clone()))?; Ok(()) } + +fn check_tree_hash(expected_str: &str, actual_root: Vec) -> Result<(), Error> { + let expected_root = hex::decode(&expected_str[2..]) + .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; + let expected_root = Hash256::from_slice(&expected_root); + let tree_hash_root = Hash256::from_slice(&actual_root); + compare_result::(&Ok(tree_hash_root), &Some(expected_root)) +} + +impl Case for SszStatic { + fn result(&self, _case_index: usize) -> Result<(), Error> { + check_serialization(&self.value, &self.serialized)?; + check_tree_hash(&self.roots.root, self.value.tree_hash_root())?; + Ok(()) + } +} + +impl Case for SszStaticSR { + fn result(&self, _case_index: usize) -> Result<(), Error> { + check_serialization(&self.value, &self.serialized)?; + check_tree_hash(&self.roots.root, self.value.tree_hash_root())?; + check_tree_hash( + &self + .roots + .signing_root + .as_ref() + .expect("signed root exists"), + self.value.signed_root(), + )?; + Ok(()) + } +} diff --git a/tests/ef_tests/src/doc.rs b/tests/ef_tests/src/doc.rs index 7dfe9954c1..f3a41697ee 100644 --- a/tests/ef_tests/src/doc.rs +++ b/tests/ef_tests/src/doc.rs @@ -2,10 +2,14 @@ use crate::case_result::CaseResult; use crate::cases::*; use crate::doc_header::DocHeader; use crate::error::Error; -use crate::yaml_decode::{yaml_split_header_and_cases, YamlDecode}; +use crate::yaml_decode::YamlDecode; use crate::EfTest; use serde_derive::Deserialize; -use std::{fs::File, io::prelude::*, path::PathBuf}; +use std::{ + fs::File, + io::prelude::*, + path::{Path, PathBuf}, +}; use types::{MainnetEthSpec, MinimalEthSpec}; #[derive(Debug, Deserialize)] @@ -19,15 +23,13 @@ impl Doc { fn from_path(path: PathBuf) -> Self { let mut file = File::open(path.clone()).unwrap(); - let mut yaml = String::new(); - file.read_to_string(&mut yaml).unwrap(); - - let (header_yaml, cases_yaml) = yaml_split_header_and_cases(yaml.clone()); + let mut cases_yaml = String::new(); + file.read_to_string(&mut cases_yaml).unwrap(); Self { - header_yaml, cases_yaml, path, + header_yaml: String::new(), } } @@ -40,8 +42,6 @@ impl Doc { header.config.as_ref(), ) { ("ssz", "uint", _) => run_test::(self), - ("ssz", "static", "minimal") => run_test::>(self), - ("ssz", "static", "mainnet") => run_test::>(self), ("sanity", "slots", "minimal") => run_test::>(self), // FIXME: skipped due to compact committees issue ("sanity", "slots", "mainnet") => vec![], // run_test::>(self), @@ -172,14 +172,36 @@ impl Doc { } } -pub fn run_test(doc: &Doc) -> Vec +pub fn assert_tests_pass(path: &Path, results: &[CaseResult]) { + let doc = Doc { + header_yaml: String::new(), + cases_yaml: String::new(), + path: path.into(), + }; + + let (failed, skipped_bls, skipped_known_failures) = categorize_results(results); + + if failed.len() + skipped_known_failures.len() > 0 { + print_results( + &doc, + &failed, + &skipped_bls, + &skipped_known_failures, + &results, + ); + if !failed.is_empty() { + panic!("Tests failed (see above)"); + } + } else { + println!("Passed {} tests in {}", results.len(), path.display()); + } +} + +pub fn run_test(_: &Doc) -> Vec where Cases: EfTest + YamlDecode, { - // Pass only the "test_cases" YAML string to `yaml_decode`. - let test_cases: Cases = Cases::yaml_decode(&doc.cases_yaml).unwrap(); - - test_cases.test_results() + panic!("FIXME(michael): delete this") } pub fn categorize_results( @@ -208,7 +230,6 @@ pub fn print_results( skipped_known_failures: &[&CaseResult], results: &[CaseResult], ) { - let header: DocHeader = serde_yaml::from_str(&doc.header_yaml).unwrap(); println!("--------------------------------------------------"); println!( "Test {}", @@ -218,7 +239,7 @@ pub fn print_results( "Failure" } ); - println!("Title: {}", header.title); + println!("Title: TODO"); println!("File: {:?}", doc.path); println!( "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs new file mode 100644 index 0000000000..1dac988aca --- /dev/null +++ b/tests/ef_tests/src/handler.rs @@ -0,0 +1,130 @@ +use crate::cases::{self, Case, Cases, LoadCase}; +use crate::type_name::TypeName; +use crate::EfTest; +use std::fs; +use std::marker::PhantomData; +use std::path::PathBuf; +use tree_hash::SignedRoot; + +pub trait Handler { + type Case: Case + LoadCase; + + fn config_name() -> &'static str { + "general" + } + + fn fork_name() -> &'static str { + "phase0" + } + + fn runner_name() -> &'static str; + + fn handler_name() -> &'static str; + + fn run() { + let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("eth2.0-spec-tests") + .join("tests") + .join(Self::config_name()) + .join(Self::fork_name()) + .join(Self::runner_name()) + .join(Self::handler_name()); + + // Iterate through test suites + // TODO: parallelism + // TODO: error handling? + let test_cases = fs::read_dir(&handler_path) + .expect("open main directory") + .flat_map(|entry| { + entry + .ok() + .filter(|e| e.file_type().map(|ty| ty.is_dir()).unwrap_or(false)) + }) + .flat_map(|suite| fs::read_dir(suite.path()).expect("open suite dir")) + .flat_map(Result::ok) + .map(|test_case_dir| Self::Case::load_from_dir(&test_case_dir.path()).expect("loads")) + .collect::>(); + + let results = Cases { test_cases }.test_results(); + + crate::doc::assert_tests_pass(&handler_path, &results); + } +} + +macro_rules! bls_handler { + ($runner_name: ident, $case_name:ident, $handler_name:expr) => { + pub struct $runner_name; + + impl Handler for $runner_name { + type Case = cases::$case_name; + + fn runner_name() -> &'static str { + "bls" + } + + fn handler_name() -> &'static str { + $handler_name + } + } + }; +} + +bls_handler!( + BlsAggregatePubkeysHandler, + BlsAggregatePubkeys, + "aggregate_pubkeys" +); +bls_handler!(BlsAggregateSigsHandler, BlsAggregateSigs, "aggregate_sigs"); +bls_handler!( + BlsG2CompressedHandler, + BlsG2Compressed, + "msg_hash_compressed" +); +bls_handler!(BlsPrivToPubHandler, BlsPrivToPub, "priv_to_pub"); +bls_handler!(BlsSignMsgHandler, BlsSign, "sign_msg"); + +/// Handler for SSZ types that do not implement `SignedRoot`. +pub struct SszStaticHandler(PhantomData<(T, E)>); + +/// Handler for SSZ types that do implement `SignedRoot`. +pub struct SszStaticSRHandler(PhantomData<(T, E)>); + +impl Handler for SszStaticHandler +where + T: cases::SszStaticType + TypeName, + E: TypeName, +{ + type Case = cases::SszStatic; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "ssz_static" + } + + fn handler_name() -> &'static str { + T::name() + } +} + +impl Handler for SszStaticSRHandler +where + T: cases::SszStaticType + SignedRoot + TypeName, + E: TypeName, +{ + type Case = cases::SszStaticSR; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "ssz_static" + } + + fn handler_name() -> &'static str { + T::name() + } +} diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index fdd4e7b859..cc17c3ea48 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -4,6 +4,7 @@ pub use case_result::CaseResult; pub use cases::Case; pub use doc::Doc; pub use error::Error; +pub use handler::*; pub use yaml_decode::YamlDecode; mod bls_setting; @@ -12,6 +13,8 @@ mod cases; mod doc; mod doc_header; mod error; +mod handler; +mod type_name; mod yaml_decode; /// Defined where an object can return the results of some test(s) adhering to the Ethereum diff --git a/tests/ef_tests/src/type_name.rs b/tests/ef_tests/src/type_name.rs new file mode 100644 index 0000000000..fe55a7f5f9 --- /dev/null +++ b/tests/ef_tests/src/type_name.rs @@ -0,0 +1,61 @@ +//! Mapping from types to canonical string identifiers used in testing. +use types::*; + +pub trait TypeName { + fn name() -> &'static str; +} + +impl TypeName for MinimalEthSpec { + fn name() -> &'static str { + "minimal" + } +} + +impl TypeName for MainnetEthSpec { + fn name() -> &'static str { + "mainnet" + } +} + +macro_rules! impl_name { + ($typ:ident) => { + impl TypeName for $typ { + fn name() -> &'static str { + stringify!($typ) + } + } + }; +} + +macro_rules! impl_name_generic { + ($typ:ident) => { + impl TypeName for $typ { + fn name() -> &'static str { + stringify!($typ) + } + } + }; +} + +impl_name_generic!(Attestation); +impl_name!(AttestationData); +impl_name!(AttestationDataAndCustodyBit); +impl_name_generic!(AttesterSlashing); +impl_name_generic!(BeaconBlock); +impl_name_generic!(BeaconBlockBody); +impl_name!(BeaconBlockHeader); +impl_name_generic!(BeaconState); +impl_name!(Checkpoint); +impl_name_generic!(CompactCommittee); +impl_name!(Crosslink); +impl_name!(Deposit); +impl_name!(DepositData); +impl_name!(Eth1Data); +impl_name!(Fork); +impl_name_generic!(HistoricalBatch); +impl_name_generic!(IndexedAttestation); +impl_name_generic!(PendingAttestation); +impl_name!(ProposerSlashing); +impl_name!(Transfer); +impl_name!(Validator); +impl_name!(VoluntaryExit); diff --git a/tests/ef_tests/src/yaml_decode.rs b/tests/ef_tests/src/yaml_decode.rs index c89dd92a9e..af122fb0cd 100644 --- a/tests/ef_tests/src/yaml_decode.rs +++ b/tests/ef_tests/src/yaml_decode.rs @@ -1,14 +1,20 @@ use super::*; use ethereum_types::{U128, U256}; +use std::fs; +use std::path::Path; use types::Fork; -mod utils; - -pub use utils::*; - pub trait YamlDecode: Sized { /// Decode an object from the test specification YAML. fn yaml_decode(string: &str) -> Result; + + fn yaml_decode_file(path: &Path) -> Result { + fs::read_to_string(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| Self::yaml_decode(&s)) + } } /// Basic types can general be decoded with the `parse` fn if they implement `str::FromStr`. diff --git a/tests/ef_tests/src/yaml_decode/utils.rs b/tests/ef_tests/src/yaml_decode/utils.rs deleted file mode 100644 index 7b6caac728..0000000000 --- a/tests/ef_tests/src/yaml_decode/utils.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub fn yaml_split_header_and_cases(mut yaml: String) -> (String, String) { - let test_cases_start = yaml.find("\ntest_cases:\n").unwrap(); - // + 1 to skip the \n we used for matching. - let mut test_cases = yaml.split_off(test_cases_start + 1); - - let end_of_first_line = test_cases.find('\n').unwrap(); - let test_cases = test_cases.split_off(end_of_first_line + 1); - - (yaml, test_cases) -} diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index deb699e786..000c533307 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,12 +1,20 @@ use ef_tests::*; use rayon::prelude::*; use std::path::{Path, PathBuf}; +use types::{ + Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, + BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink, + Deposit, DepositData, Eth1Data, Fork, HistoricalBatch, IndexedAttestation, MainnetEthSpec, + MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit, +}; use walkdir::WalkDir; fn yaml_files_in_test_dir(dir: &Path) -> Vec { let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("eth2.0-spec-tests") .join("tests") + .join("general") + .join("phase0") .join(dir); assert!( @@ -155,12 +163,107 @@ fn sanity_slots() { #[cfg(not(feature = "fake_crypto"))] fn bls() { yaml_files_in_test_dir(&Path::new("bls")) - .into_par_iter() + .into_iter() .for_each(|file| { Doc::assert_tests_pass(file); }); } +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_aggregate_pubkeys() { + BlsAggregatePubkeysHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_aggregate_sigs() { + BlsAggregateSigsHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_msg_hash_g2_compressed() { + BlsG2CompressedHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_priv_to_pub() { + BlsPrivToPubHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_sign_msg() { + BlsSignMsgHandler::run(); +} + +macro_rules! ssz_static_test { + // Signed-root + ($test_name:ident, $typ:ident$(<$generics:tt>)?, SR) => { + ssz_static_test!($test_name, SszStaticSRHandler, $typ$(<$generics>)?); + }; + // Non-signed root + ($test_name:ident, $typ:ident$(<$generics:tt>)?) => { + ssz_static_test!($test_name, SszStaticHandler, $typ$(<$generics>)?); + }; + // Generic + ($test_name:ident, $handler:ident, $typ:ident<_>) => { + ssz_static_test!( + $test_name, $handler, { + ($typ, MinimalEthSpec), + ($typ, MainnetEthSpec) + } + ); + }; + // Non-generic + ($test_name:ident, $handler:ident, $typ:ident) => { + ssz_static_test!( + $test_name, $handler, { + ($typ, MinimalEthSpec), + ($typ, MainnetEthSpec) + } + ); + }; + // Base case + ($test_name:ident, $handler:ident, { $(($typ:ty, $spec:ident)),+ }) => { + #[test] + #[cfg(feature = "fake_crypto")] + fn $test_name() { + $( + $handler::<$typ, $spec>::run(); + )+ + } + }; +} + +ssz_static_test!(ssz_static_attestation, Attestation<_>, SR); +ssz_static_test!(ssz_static_attestation_data, AttestationData); +ssz_static_test!( + ssz_static_attestation_data_and_custody_bit, + AttestationDataAndCustodyBit +); +ssz_static_test!(ssz_static_attester_slashing, AttesterSlashing<_>); +ssz_static_test!(ssz_static_beacon_block, BeaconBlock<_>, SR); +ssz_static_test!(ssz_static_beacon_block_body, BeaconBlockBody<_>); +ssz_static_test!(ssz_static_beacon_block_header, BeaconBlockHeader, SR); +ssz_static_test!(ssz_static_beacon_state, BeaconState<_>); +ssz_static_test!(ssz_static_checkpoint, Checkpoint); +ssz_static_test!(ssz_static_compact_committee, CompactCommittee<_>); +ssz_static_test!(ssz_static_crosslink, Crosslink); +ssz_static_test!(ssz_static_deposit, Deposit); +ssz_static_test!(ssz_static_deposit_data, DepositData, SR); +ssz_static_test!(ssz_static_eth1_data, Eth1Data); +ssz_static_test!(ssz_static_fork, Fork); +ssz_static_test!(ssz_static_historical_batch, HistoricalBatch<_>); +ssz_static_test!(ssz_static_indexed_attestation, IndexedAttestation<_>, SR); +ssz_static_test!(ssz_static_pending_attestation, PendingAttestation<_>); +ssz_static_test!(ssz_static_proposer_slashing, ProposerSlashing); +ssz_static_test!(ssz_static_transfer, Transfer, SR); +ssz_static_test!(ssz_static_validator, Validator); +ssz_static_test!(ssz_static_voluntary_exit, VoluntaryExit, SR); + #[test] fn epoch_processing_justification_and_finalization() { yaml_files_in_test_dir(&Path::new("epoch_processing").join("justification_and_finalization")) From 81cafdc804fb58962e08c291f24e8a2f556ded7b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 29 Aug 2019 17:41:20 +1000 Subject: [PATCH 05/32] Shuffling and sanity tests --- eth2/types/src/checkpoint.rs | 1 - eth2/utils/bls/Cargo.toml | 2 +- tests/ef_tests/src/case_result.rs | 3 + tests/ef_tests/src/cases.rs | 49 +--- tests/ef_tests/src/cases/bls_g2_compressed.rs | 13 +- tests/ef_tests/src/cases/bls_sign_msg.rs | 13 +- tests/ef_tests/src/cases/sanity_blocks.rs | 51 +++- tests/ef_tests/src/cases/sanity_slots.rs | 52 +++- tests/ef_tests/src/cases/shuffling.rs | 14 +- tests/ef_tests/src/doc.rs | 274 ------------------ tests/ef_tests/src/doc_header.rs | 12 - tests/ef_tests/src/handler.rs | 58 +++- tests/ef_tests/src/lib.rs | 4 +- tests/ef_tests/src/results.rs | 91 ++++++ tests/ef_tests/src/yaml_decode.rs | 28 ++ tests/ef_tests/tests/tests.rs | 30 +- 16 files changed, 311 insertions(+), 384 deletions(-) delete mode 100644 tests/ef_tests/src/doc.rs delete mode 100644 tests/ef_tests/src/doc_header.rs create mode 100644 tests/ef_tests/src/results.rs diff --git a/eth2/types/src/checkpoint.rs b/eth2/types/src/checkpoint.rs index 0c70019215..d5d40fa674 100644 --- a/eth2/types/src/checkpoint.rs +++ b/eth2/types/src/checkpoint.rs @@ -3,7 +3,6 @@ use crate::{Epoch, Hash256}; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; -use tree_hash::TreeHash; use tree_hash_derive::TreeHash; /// Casper FFG checkpoint, used in attestations. diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 5989dce07d..15f087b80d 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v0.9.0" } +milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian" } eth2_hashing = { path = "../eth2_hashing" } hex = "0.3" rand = "^0.5" diff --git a/tests/ef_tests/src/case_result.rs b/tests/ef_tests/src/case_result.rs index 88fd353a14..add428ec53 100644 --- a/tests/ef_tests/src/case_result.rs +++ b/tests/ef_tests/src/case_result.rs @@ -1,6 +1,7 @@ use super::*; use compare_fields::{CompareFields, Comparison, FieldComparison}; use std::fmt::Debug; +use std::path::PathBuf; use types::BeaconState; pub const MAX_VALUE_STRING_LEN: usize = 500; @@ -9,6 +10,7 @@ pub const MAX_VALUE_STRING_LEN: usize = 500; pub struct CaseResult { pub case_index: usize, pub desc: String, + pub path: PathBuf, pub result: Result<(), Error>, } @@ -17,6 +19,7 @@ impl CaseResult { CaseResult { case_index, desc: case.description(), + path: case.path().into(), result, } } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 7f6ffb0c4c..1216b8728a 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -67,6 +67,11 @@ pub trait Case: Debug { "no description".to_string() } + /// Path to the directory for this test case. + fn path(&self) -> &Path { + Path::new("") + } + /// Execute a test and return the result. /// /// `case_index` reports the index of the case in the set of test cases. It is not strictly @@ -76,19 +81,13 @@ pub trait Case: Debug { pub trait BlsCase: serde::de::DeserializeOwned {} -impl YamlDecode for T -where - T: BlsCase, -{ +impl YamlDecode for T { fn yaml_decode(string: &str) -> Result { serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) } } -impl LoadCase for T -where - T: BlsCase, -{ +impl LoadCase for T { fn load_from_dir(path: &Path) -> Result { Self::yaml_decode_file(&path.join("data.yaml")) } @@ -111,37 +110,3 @@ where .collect() } } - -// FIXME(michael): delete this -impl YamlDecode for Cases { - /// Decodes a YAML list of test cases - fn yaml_decode(yaml: &str) -> Result { - let mut p = 0; - let mut elems: Vec<&str> = yaml - .match_indices("\n- ") - // Skip the `\n` used for matching a new line - .map(|(i, _)| i + 1) - .map(|i| { - let yaml_element = &yaml[p..i]; - p = i; - - yaml_element - }) - .collect(); - - elems.push(&yaml[p..]); - - let test_cases = elems - .iter() - .map(|s| { - // Remove the `- ` prefix. - let s = &s[2..]; - // Remove a single level of indenting. - s.replace("\n ", "\n") - }) - .map(|s| T::yaml_decode(&s.to_string()).unwrap()) - .collect(); - - Ok(Self { test_cases }) - } -} diff --git a/tests/ef_tests/src/cases/bls_g2_compressed.rs b/tests/ef_tests/src/cases/bls_g2_compressed.rs index 547d8d03ae..f8381f5a7f 100644 --- a/tests/ef_tests/src/cases/bls_g2_compressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_compressed.rs @@ -41,14 +41,9 @@ impl Case for BlsG2Compressed { } } -// Converts a vector to u64 (from big endian) +// Converts a vector to u64 (from little endian) fn bytes_to_u64(array: &[u8]) -> u64 { - let mut result: u64 = 0; - for (i, value) in array.iter().rev().enumerate() { - if i == 8 { - break; - } - result += u64::pow(2, i as u32 * 8) * u64::from(*value); - } - result + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(array); + u64::from_le_bytes(bytes) } diff --git a/tests/ef_tests/src/cases/bls_sign_msg.rs b/tests/ef_tests/src/cases/bls_sign_msg.rs index 476ecdefb1..18e90896be 100644 --- a/tests/ef_tests/src/cases/bls_sign_msg.rs +++ b/tests/ef_tests/src/cases/bls_sign_msg.rs @@ -41,16 +41,11 @@ impl Case for BlsSign { } } -// Converts a vector to u64 (from big endian) +// Converts a vector to u64 (from little endian) fn bytes_to_u64(array: &[u8]) -> u64 { - let mut result: u64 = 0; - for (i, value) in array.iter().rev().enumerate() { - if i == 8 { - break; - } - result += u64::pow(2, i as u32 * 8) * u64::from(*value); - } - result + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(array); + u64::from_le_bytes(bytes) } // Increase the size of an array to 48 bytes diff --git a/tests/ef_tests/src/cases/sanity_blocks.rs b/tests/ef_tests/src/cases/sanity_blocks.rs index cd9008fdaa..d88d8f2955 100644 --- a/tests/ef_tests/src/cases/sanity_blocks.rs +++ b/tests/ef_tests/src/cases/sanity_blocks.rs @@ -1,35 +1,72 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::{ per_block_processing, per_slot_processing, BlockInvalid, BlockProcessingError, }; +use std::path::PathBuf; use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch}; +#[derive(Debug, Clone, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, + pub blocks_count: usize, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanityBlocks { - pub description: String, - pub bls_setting: Option, + pub path: PathBuf, + pub metadata: Metadata, pub pre: BeaconState, pub blocks: Vec>, pub post: Option>, } -impl YamlDecode for SanityBlocks { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for SanityBlocks { + fn load_from_dir(path: &Path) -> Result { + let metadata: Metadata = yaml_decode_file(&path.join("meta.yaml"))?; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let blocks: Vec> = (0..metadata.blocks_count) + .map(|i| { + let filename = format!("blocks_{}.ssz", i); + ssz_decode_file(&path.join(filename)) + }) + .collect::>()?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + blocks, + post, + }) } } impl Case for SanityBlocks { fn description(&self) -> String { - self.description.clone() + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; + self.metadata.bls_setting.unwrap_or_default().check()?; let mut state = self.pre.clone(); let mut expected = self.post.clone(); diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index fbce1a06af..27c6c13c32 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -1,30 +1,70 @@ use super::*; +use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_slot_processing; +use std::path::PathBuf; use types::{BeaconState, EthSpec}; +#[derive(Debug, Clone, Default, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanitySlots { - pub description: String, + pub path: PathBuf, + pub metadata: Metadata, pub pre: BeaconState, - pub slots: usize, + pub slots: u64, pub post: Option>, } -impl YamlDecode for SanitySlots { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for SanitySlots { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&path.join("meta.yaml"))? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let slots: u64 = yaml_decode_file(&path.join("slots.yaml"))?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + slots, + post, + }) } } impl Case for SanitySlots { fn description(&self) -> String { - self.description.clone() + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { + self.metadata.bls_setting.unwrap_or_default().check()?; + let mut state = self.pre.clone(); let mut expected = self.post.clone(); let spec = &E::default_spec(); diff --git a/tests/ef_tests/src/cases/shuffling.rs b/tests/ef_tests/src/cases/shuffling.rs index d7ff40e596..c0595e5843 100644 --- a/tests/ef_tests/src/cases/shuffling.rs +++ b/tests/ef_tests/src/cases/shuffling.rs @@ -8,7 +8,7 @@ use swap_or_not_shuffle::{get_permutated_index, shuffle_list}; pub struct Shuffling { pub seed: String, pub count: usize, - pub shuffled: Vec, + pub mapping: Vec, #[serde(skip)] _phantom: PhantomData, } @@ -19,10 +19,16 @@ impl YamlDecode for Shuffling { } } +impl LoadCase for Shuffling { + fn load_from_dir(path: &Path) -> Result { + Self::yaml_decode_file(&path.join("mapping.yaml")) + } +} + impl Case for Shuffling { fn result(&self, _case_index: usize) -> Result<(), Error> { if self.count == 0 { - compare_result::<_, Error>(&Ok(vec![]), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(vec![]), &Some(self.mapping.clone()))?; } else { let spec = T::default_spec(); let seed = hex::decode(&self.seed[2..]) @@ -34,12 +40,12 @@ impl Case for Shuffling { get_permutated_index(i, self.count, &seed, spec.shuffle_round_count).unwrap() }) .collect(); - compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?; // Test "shuffle_list" let input: Vec = (0..self.count).collect(); let shuffling = shuffle_list(input, spec.shuffle_round_count, &seed, false).unwrap(); - compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?; } Ok(()) diff --git a/tests/ef_tests/src/doc.rs b/tests/ef_tests/src/doc.rs deleted file mode 100644 index f3a41697ee..0000000000 --- a/tests/ef_tests/src/doc.rs +++ /dev/null @@ -1,274 +0,0 @@ -use crate::case_result::CaseResult; -use crate::cases::*; -use crate::doc_header::DocHeader; -use crate::error::Error; -use crate::yaml_decode::YamlDecode; -use crate::EfTest; -use serde_derive::Deserialize; -use std::{ - fs::File, - io::prelude::*, - path::{Path, PathBuf}, -}; -use types::{MainnetEthSpec, MinimalEthSpec}; - -#[derive(Debug, Deserialize)] -pub struct Doc { - pub header_yaml: String, - pub cases_yaml: String, - pub path: PathBuf, -} - -impl Doc { - fn from_path(path: PathBuf) -> Self { - let mut file = File::open(path.clone()).unwrap(); - - let mut cases_yaml = String::new(); - file.read_to_string(&mut cases_yaml).unwrap(); - - Self { - cases_yaml, - path, - header_yaml: String::new(), - } - } - - pub fn test_results(&self) -> Vec { - let header: DocHeader = serde_yaml::from_str(&self.header_yaml.as_str()).unwrap(); - - match ( - header.runner.as_ref(), - header.handler.as_ref(), - header.config.as_ref(), - ) { - ("ssz", "uint", _) => run_test::(self), - ("sanity", "slots", "minimal") => run_test::>(self), - // FIXME: skipped due to compact committees issue - ("sanity", "slots", "mainnet") => vec![], // run_test::>(self), - ("sanity", "blocks", "minimal") => run_test::>(self), - // FIXME: skipped due to compact committees issue - ("sanity", "blocks", "mainnet") => vec![], // run_test::>(self), - ("shuffling", "core", "minimal") => run_test::>(self), - ("shuffling", "core", "mainnet") => run_test::>(self), - ("bls", "aggregate_pubkeys", "mainnet") => run_test::(self), - ("bls", "aggregate_sigs", "mainnet") => run_test::(self), - ("bls", "msg_hash_compressed", "mainnet") => run_test::(self), - // Note this test fails due to a difference in our internal representations. It does - // not effect verification or external representation. - // - // It is skipped. - ("bls", "msg_hash_uncompressed", "mainnet") => vec![], - ("bls", "priv_to_pub", "mainnet") => run_test::(self), - ("bls", "sign_msg", "mainnet") => run_test::(self), - ("operations", "deposit", "mainnet") => { - run_test::>(self) - } - ("operations", "deposit", "minimal") => { - run_test::>(self) - } - ("operations", "transfer", "mainnet") => { - run_test::>(self) - } - ("operations", "transfer", "minimal") => { - run_test::>(self) - } - ("operations", "voluntary_exit", "mainnet") => { - run_test::>(self) - } - ("operations", "voluntary_exit", "minimal") => { - run_test::>(self) - } - ("operations", "proposer_slashing", "mainnet") => { - run_test::>(self) - } - ("operations", "proposer_slashing", "minimal") => { - run_test::>(self) - } - ("operations", "attester_slashing", "mainnet") => { - run_test::>(self) - } - ("operations", "attester_slashing", "minimal") => { - run_test::>(self) - } - ("operations", "attestation", "mainnet") => { - run_test::>(self) - } - ("operations", "attestation", "minimal") => { - run_test::>(self) - } - ("operations", "block_header", "mainnet") => { - run_test::>(self) - } - ("operations", "block_header", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "crosslinks", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "crosslinks", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "registry_updates", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "registry_updates", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "justification_and_finalization", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "justification_and_finalization", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "slashings", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "slashings", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "final_updates", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "final_updates", "mainnet") => { - vec![] - // FIXME: skipped due to compact committees issue - // run_test::>(self) - } - ("genesis", "initialization", "minimal") => { - run_test::>(self) - } - ("genesis", "initialization", "mainnet") => { - run_test::>(self) - } - ("genesis", "validity", "minimal") => run_test::>(self), - ("genesis", "validity", "mainnet") => run_test::>(self), - (runner, handler, config) => panic!( - "No implementation for runner: \"{}\", handler: \"{}\", config: \"{}\"", - runner, handler, config - ), - } - } - - pub fn assert_tests_pass(path: PathBuf) { - let doc = Self::from_path(path); - let results = doc.test_results(); - - let (failed, skipped_bls, skipped_known_failures) = categorize_results(&results); - - if failed.len() + skipped_known_failures.len() > 0 { - print_results( - &doc, - &failed, - &skipped_bls, - &skipped_known_failures, - &results, - ); - if !failed.is_empty() { - panic!("Tests failed (see above)"); - } - } else { - println!("Passed {} tests in {:?}", results.len(), doc.path); - } - } -} - -pub fn assert_tests_pass(path: &Path, results: &[CaseResult]) { - let doc = Doc { - header_yaml: String::new(), - cases_yaml: String::new(), - path: path.into(), - }; - - let (failed, skipped_bls, skipped_known_failures) = categorize_results(results); - - if failed.len() + skipped_known_failures.len() > 0 { - print_results( - &doc, - &failed, - &skipped_bls, - &skipped_known_failures, - &results, - ); - if !failed.is_empty() { - panic!("Tests failed (see above)"); - } - } else { - println!("Passed {} tests in {}", results.len(), path.display()); - } -} - -pub fn run_test(_: &Doc) -> Vec -where - Cases: EfTest + YamlDecode, -{ - panic!("FIXME(michael): delete this") -} - -pub fn categorize_results( - results: &[CaseResult], -) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { - let mut failed = vec![]; - let mut skipped_bls = vec![]; - let mut skipped_known_failures = vec![]; - - for case in results { - match case.result.as_ref().err() { - Some(Error::SkippedBls) => skipped_bls.push(case), - Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case), - Some(_) => failed.push(case), - None => (), - } - } - - (failed, skipped_bls, skipped_known_failures) -} - -pub fn print_results( - doc: &Doc, - failed: &[&CaseResult], - skipped_bls: &[&CaseResult], - skipped_known_failures: &[&CaseResult], - results: &[CaseResult], -) { - println!("--------------------------------------------------"); - println!( - "Test {}", - if failed.is_empty() { - "Result" - } else { - "Failure" - } - ); - println!("Title: TODO"); - println!("File: {:?}", doc.path); - println!( - "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", - results.len(), - failed.len(), - skipped_known_failures.len(), - skipped_bls.len(), - results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len() - ); - println!(); - - for case in skipped_known_failures { - println!("-------"); - println!( - "case[{}] ({}) skipped because it's a known failure", - case.case_index, case.desc, - ); - } - for failure in failed { - let error = failure.result.clone().unwrap_err(); - - println!("-------"); - println!( - "case[{}] ({}) failed with {}:", - failure.case_index, - failure.desc, - error.name() - ); - println!("{}", error.message()); - } - println!(); -} diff --git a/tests/ef_tests/src/doc_header.rs b/tests/ef_tests/src/doc_header.rs deleted file mode 100644 index c0d6d3276f..0000000000 --- a/tests/ef_tests/src/doc_header.rs +++ /dev/null @@ -1,12 +0,0 @@ -use serde_derive::Deserialize; - -#[derive(Debug, Deserialize)] -pub struct DocHeader { - pub title: String, - pub summary: String, - pub forks_timeline: String, - pub forks: Vec, - pub config: String, - pub runner: String, - pub handler: String, -} diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index 1dac988aca..f66dc7b187 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -5,6 +5,7 @@ use std::fs; use std::marker::PhantomData; use std::path::PathBuf; use tree_hash::SignedRoot; +use types::EthSpec; pub trait Handler { type Case: Case + LoadCase; @@ -47,7 +48,8 @@ pub trait Handler { let results = Cases { test_cases }.test_results(); - crate::doc::assert_tests_pass(&handler_path, &results); + let name = format!("{}/{}", Self::runner_name(), Self::handler_name()); + crate::results::assert_tests_pass(&name, &handler_path, &results); } } @@ -128,3 +130,57 @@ where T::name() } } + +pub struct ShufflingHandler(PhantomData); + +impl Handler for ShufflingHandler { + type Case = cases::Shuffling; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "shuffling" + } + + fn handler_name() -> &'static str { + "core" + } +} + +pub struct SanityBlocksHandler(PhantomData); + +impl Handler for SanityBlocksHandler { + type Case = cases::SanityBlocks; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "sanity" + } + + fn handler_name() -> &'static str { + "blocks" + } +} + +pub struct SanitySlotsHandler(PhantomData); + +impl Handler for SanitySlotsHandler { + type Case = cases::SanitySlots; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "sanity" + } + + fn handler_name() -> &'static str { + "slots" + } +} diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index cc17c3ea48..06e01fc03a 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -2,7 +2,6 @@ use types::EthSpec; pub use case_result::CaseResult; pub use cases::Case; -pub use doc::Doc; pub use error::Error; pub use handler::*; pub use yaml_decode::YamlDecode; @@ -10,10 +9,9 @@ pub use yaml_decode::YamlDecode; mod bls_setting; mod case_result; mod cases; -mod doc; -mod doc_header; mod error; mod handler; +mod results; mod type_name; mod yaml_decode; diff --git a/tests/ef_tests/src/results.rs b/tests/ef_tests/src/results.rs new file mode 100644 index 0000000000..20e59f7b3e --- /dev/null +++ b/tests/ef_tests/src/results.rs @@ -0,0 +1,91 @@ +use crate::case_result::CaseResult; +use crate::error::Error; +use std::path::Path; + +pub fn assert_tests_pass(handler_name: &str, path: &Path, results: &[CaseResult]) { + let (failed, skipped_bls, skipped_known_failures) = categorize_results(results); + + if failed.len() + skipped_known_failures.len() > 0 { + print_results( + handler_name, + &failed, + &skipped_bls, + &skipped_known_failures, + &results, + ); + if !failed.is_empty() { + panic!("Tests failed (see above)"); + } + } else { + println!("Passed {} tests in {}", results.len(), path.display()); + } +} + +pub fn categorize_results( + results: &[CaseResult], +) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { + let mut failed = vec![]; + let mut skipped_bls = vec![]; + let mut skipped_known_failures = vec![]; + + for case in results { + match case.result.as_ref().err() { + Some(Error::SkippedBls) => skipped_bls.push(case), + Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case), + Some(_) => failed.push(case), + None => (), + } + } + + (failed, skipped_bls, skipped_known_failures) +} + +pub fn print_results( + handler_name: &str, + failed: &[&CaseResult], + skipped_bls: &[&CaseResult], + skipped_known_failures: &[&CaseResult], + results: &[CaseResult], +) { + println!("--------------------------------------------------"); + println!( + "Test {}", + if failed.is_empty() { + "Result" + } else { + "Failure" + } + ); + println!("Title: {}", handler_name); + println!( + "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", + results.len(), + failed.len(), + skipped_known_failures.len(), + skipped_bls.len(), + results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len() + ); + println!(); + + for case in skipped_known_failures { + println!("-------"); + println!( + "case ({}) from {} skipped because it's a known failure", + case.desc, + case.path.display() + ); + } + for failure in failed { + let error = failure.result.clone().unwrap_err(); + + println!("-------"); + println!( + "case ({}) from {} failed with {}:", + failure.desc, + failure.path.display(), + error.name() + ); + println!("{}", error.message()); + } + println!(); +} diff --git a/tests/ef_tests/src/yaml_decode.rs b/tests/ef_tests/src/yaml_decode.rs index af122fb0cd..83a162930c 100644 --- a/tests/ef_tests/src/yaml_decode.rs +++ b/tests/ef_tests/src/yaml_decode.rs @@ -4,6 +4,34 @@ use std::fs; use std::path::Path; use types::Fork; +pub fn yaml_decode(string: &str) -> Result { + serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) +} + +pub fn yaml_decode_file(path: &Path) -> Result { + fs::read_to_string(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| yaml_decode(&s)) +} + +pub fn ssz_decode_file(path: &Path) -> Result { + fs::read(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| { + T::from_ssz_bytes(&s).map_err(|e| { + Error::FailedToParseTest(format!( + "Unable to parse SSZ at {}: {:?}", + path.display(), + e + )) + }) + }) +} + pub trait YamlDecode: Sized { /// Decode an object from the test specification YAML. fn yaml_decode(string: &str) -> Result; diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 000c533307..0fb751c9e8 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -48,6 +48,7 @@ fn yaml_files_in_test_dir(dir: &Path) -> Vec { paths } +/* #[test] #[cfg(feature = "fake_crypto")] fn ssz_generic() { @@ -58,6 +59,7 @@ fn ssz_generic() { }); } + #[test] #[cfg(feature = "fake_crypto")] fn ssz_static() { @@ -67,16 +69,15 @@ fn ssz_static() { Doc::assert_tests_pass(file); }); } +*/ #[test] fn shuffling() { - yaml_files_in_test_dir(&Path::new("shuffling").join("core")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + ShufflingHandler::::run(); + ShufflingHandler::::run(); } +/* #[test] fn operations_deposit() { yaml_files_in_test_dir(&Path::new("operations").join("deposit")) @@ -140,25 +141,21 @@ fn operations_block_header() { Doc::assert_tests_pass(file); }); } +*/ #[test] fn sanity_blocks() { - yaml_files_in_test_dir(&Path::new("sanity").join("blocks")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SanityBlocksHandler::::run(); + SanityBlocksHandler::::run(); } #[test] fn sanity_slots() { - yaml_files_in_test_dir(&Path::new("sanity").join("slots")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SanitySlotsHandler::::run(); + SanitySlotsHandler::::run(); } +/* #[test] #[cfg(not(feature = "fake_crypto"))] fn bls() { @@ -168,6 +165,7 @@ fn bls() { Doc::assert_tests_pass(file); }); } +*/ #[test] #[cfg(not(feature = "fake_crypto"))] @@ -264,6 +262,7 @@ ssz_static_test!(ssz_static_transfer, Transfer, SR); ssz_static_test!(ssz_static_validator, Validator); ssz_static_test!(ssz_static_voluntary_exit, VoluntaryExit, SR); +/* #[test] fn epoch_processing_justification_and_finalization() { yaml_files_in_test_dir(&Path::new("epoch_processing").join("justification_and_finalization")) @@ -326,3 +325,4 @@ fn genesis_validity() { Doc::assert_tests_pass(file); }); } +*/ From 6cf9b3c1a41e253de9cc3ef339f451f62a461035 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 30 Aug 2019 13:29:26 +1000 Subject: [PATCH 06/32] Epoch processing tests --- .../src/per_epoch_processing.rs | 7 +- tests/ef_tests/src/cases.rs | 13 +- tests/ef_tests/src/cases/epoch_processing.rs | 147 ++++++++++++++++++ .../src/cases/epoch_processing_crosslinks.rs | 37 ----- .../cases/epoch_processing_final_updates.rs | 41 ----- ...ocessing_justification_and_finalization.rs | 46 ------ .../epoch_processing_registry_updates.rs | 38 ----- .../src/cases/epoch_processing_slashings.rs | 50 ------ tests/ef_tests/src/cases/sanity_slots.rs | 2 +- tests/ef_tests/src/handler.rs | 20 ++- tests/ef_tests/src/lib.rs | 3 + tests/ef_tests/src/type_name.rs | 75 +++++---- tests/ef_tests/tests/tests.rs | 37 ++--- 13 files changed, 225 insertions(+), 291 deletions(-) create mode 100644 tests/ef_tests/src/cases/epoch_processing.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_crosslinks.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_final_updates.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_registry_updates.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_slashings.rs diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 08f42a229c..f66ce4ea28 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -1,8 +1,5 @@ use crate::common::get_compact_committees_root; -use apply_rewards::process_rewards_and_penalties; use errors::EpochProcessingError as Error; -use process_slashings::process_slashings; -use registry_updates::process_registry_updates; use std::collections::HashMap; use tree_hash::TreeHash; use types::*; @@ -17,6 +14,10 @@ pub mod tests; pub mod validator_statuses; pub mod winning_root; +pub use apply_rewards::process_rewards_and_penalties; +pub use process_slashings::process_slashings; +pub use registry_updates::process_registry_updates; + /// Maps a shard to a winning root. /// /// It is generated during crosslink processing and later used to reward/penalize validators. diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 1216b8728a..e3ec54a7f7 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -8,11 +8,7 @@ mod bls_g2_compressed; mod bls_g2_uncompressed; mod bls_priv_to_pub; mod bls_sign_msg; -mod epoch_processing_crosslinks; -mod epoch_processing_final_updates; -mod epoch_processing_justification_and_finalization; -mod epoch_processing_registry_updates; -mod epoch_processing_slashings; +mod epoch_processing; mod genesis_initialization; mod genesis_validity; mod operations_attestation; @@ -34,11 +30,7 @@ pub use bls_g2_compressed::*; pub use bls_g2_uncompressed::*; pub use bls_priv_to_pub::*; pub use bls_sign_msg::*; -pub use epoch_processing_crosslinks::*; -pub use epoch_processing_final_updates::*; -pub use epoch_processing_justification_and_finalization::*; -pub use epoch_processing_registry_updates::*; -pub use epoch_processing_slashings::*; +pub use epoch_processing::*; pub use genesis_initialization::*; pub use genesis_validity::*; pub use operations_attestation::*; @@ -69,6 +61,7 @@ pub trait Case: Debug { /// Path to the directory for this test case. fn path(&self) -> &Path { + // FIXME(michael): remove default impl Path::new("") } diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs new file mode 100644 index 0000000000..ac47ab2368 --- /dev/null +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -0,0 +1,147 @@ +use super::*; +use crate::bls_setting::BlsSetting; +use crate::case_result::compare_beacon_state_results_without_caches; +use crate::type_name; +use crate::type_name::TypeName; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use serde_derive::Deserialize; +use state_processing::per_epoch_processing::{ + errors::EpochProcessingError, process_crosslinks, process_final_updates, + process_justification_and_finalization, process_registry_updates, process_slashings, + validator_statuses::ValidatorStatuses, +}; +use std::marker::PhantomData; +use std::path::{Path, PathBuf}; +use types::{BeaconState, ChainSpec, EthSpec}; + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct EpochProcessing> { + pub path: PathBuf, + pub metadata: Metadata, + pub pre: BeaconState, + pub post: Option>, + #[serde(skip_deserializing)] + _phantom: PhantomData, +} + +pub trait EpochTransition: TypeName + Debug { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError>; +} + +#[derive(Debug)] +pub struct JustificationAndFinalization; +#[derive(Debug)] +pub struct Crosslinks; +#[derive(Debug)] +pub struct RegistryUpdates; +#[derive(Debug)] +pub struct Slashings; +#[derive(Debug)] +pub struct FinalUpdates; + +type_name!( + JustificationAndFinalization, + "justification_and_finalization" +); +type_name!(Crosslinks, "crosslinks"); +type_name!(RegistryUpdates, "registry_updates"); +type_name!(Slashings, "slashings"); +type_name!(FinalUpdates, "final_updates"); + +impl EpochTransition for JustificationAndFinalization { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + let mut validator_statuses = ValidatorStatuses::new(state, spec)?; + validator_statuses.process_attestations(state, spec)?; + process_justification_and_finalization(state, &validator_statuses.total_balances) + } +} + +impl EpochTransition for Crosslinks { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_crosslinks(state, spec)?; + Ok(()) + } +} + +impl EpochTransition for RegistryUpdates { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_registry_updates(state, spec) + } +} + +impl EpochTransition for Slashings { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; + validator_statuses.process_attestations(&state, spec)?; + process_slashings(state, validator_statuses.total_balances.current_epoch, spec)?; + Ok(()) + } +} + +impl EpochTransition for FinalUpdates { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_final_updates(state, spec) + } +} + +impl> LoadCase for EpochProcessing { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&metadata_path)? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + post, + _phantom: PhantomData, + }) + } +} + +impl> Case for EpochProcessing { + fn description(&self) -> String { + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path + } + + fn result(&self, _case_index: usize) -> Result<(), Error> { + let mut state = self.pre.clone(); + let mut expected = self.post.clone(); + + let spec = &E::default_spec(); + + let mut result = (|| { + // Processing requires the epoch cache. + state.build_all_caches(spec)?; + + T::run(&mut state, spec).map(|_| state) + })(); + + compare_beacon_state_results_without_caches(&mut result, &mut expected) + } +} diff --git a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs b/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs deleted file mode 100644 index f2676d1220..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs +++ /dev/null @@ -1,37 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::process_crosslinks; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingCrosslinks { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingCrosslinks { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingCrosslinks { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let mut result = process_crosslinks(&mut state, &E::default_spec()).map(|_| state); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_final_updates.rs b/tests/ef_tests/src/cases/epoch_processing_final_updates.rs deleted file mode 100644 index 69e6b8bd35..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_final_updates.rs +++ /dev/null @@ -1,41 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::process_final_updates; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingFinalUpdates { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingFinalUpdates { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingFinalUpdates { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - let spec = &E::default_spec(); - - let mut result = (|| { - // Processing requires the epoch cache. - state.build_all_caches(spec)?; - - process_final_updates(&mut state, spec).map(|_| state) - })(); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs b/tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs deleted file mode 100644 index 7883010865..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs +++ /dev/null @@ -1,46 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::{ - process_justification_and_finalization, validator_statuses::ValidatorStatuses, -}; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingJustificationAndFinalization { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingJustificationAndFinalization { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingJustificationAndFinalization { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - let spec = &E::default_spec(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let mut result = (|| { - let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; - validator_statuses.process_attestations(&state, spec)?; - process_justification_and_finalization(&mut state, &validator_statuses.total_balances) - .map(|_| state) - })(); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs b/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs deleted file mode 100644 index a01f895fe1..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs +++ /dev/null @@ -1,38 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::registry_updates::process_registry_updates; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingRegistryUpdates { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingRegistryUpdates { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingRegistryUpdates { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - let spec = &E::default_spec(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let mut result = process_registry_updates(&mut state, spec).map(|_| state); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_slashings.rs b/tests/ef_tests/src/cases/epoch_processing_slashings.rs deleted file mode 100644 index d2a988d92e..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_slashings.rs +++ /dev/null @@ -1,50 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::{ - process_slashings::process_slashings, validator_statuses::ValidatorStatuses, -}; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingSlashings { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingSlashings { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingSlashings { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - let spec = &E::default_spec(); - - let mut result = (|| { - // Processing requires the epoch cache. - state.build_all_caches(spec)?; - - let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; - validator_statuses.process_attestations(&state, spec)?; - process_slashings( - &mut state, - validator_statuses.total_balances.current_epoch, - spec, - ) - .map(|_| state) - })(); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index 27c6c13c32..a66f1c2c41 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -27,7 +27,7 @@ impl LoadCase for SanitySlots { fn load_from_dir(path: &Path) -> Result { let metadata_path = path.join("meta.yaml"); let metadata: Metadata = if metadata_path.is_file() { - yaml_decode_file(&path.join("meta.yaml"))? + yaml_decode_file(&metadata_path)? } else { Metadata::default() }; diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index f66dc7b187..a614afe76d 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -1,4 +1,4 @@ -use crate::cases::{self, Case, Cases, LoadCase}; +use crate::cases::{self, Case, Cases, EpochTransition, LoadCase}; use crate::type_name::TypeName; use crate::EfTest; use std::fs; @@ -184,3 +184,21 @@ impl Handler for SanitySlotsHandler { "slots" } } + +pub struct EpochProcessingHandler(PhantomData<(E, T)>); + +impl> Handler for EpochProcessingHandler { + type Case = cases::EpochProcessing; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "epoch_processing" + } + + fn handler_name() -> &'static str { + T::name() + } +} diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index 06e01fc03a..54e674d85a 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -2,6 +2,9 @@ use types::EthSpec; pub use case_result::CaseResult; pub use cases::Case; +pub use cases::{ + Crosslinks, FinalUpdates, JustificationAndFinalization, RegistryUpdates, Slashings, +}; pub use error::Error; pub use handler::*; pub use yaml_decode::YamlDecode; diff --git a/tests/ef_tests/src/type_name.rs b/tests/ef_tests/src/type_name.rs index fe55a7f5f9..5af0c52565 100644 --- a/tests/ef_tests/src/type_name.rs +++ b/tests/ef_tests/src/type_name.rs @@ -5,57 +5,56 @@ pub trait TypeName { fn name() -> &'static str; } -impl TypeName for MinimalEthSpec { - fn name() -> &'static str { - "minimal" - } -} - -impl TypeName for MainnetEthSpec { - fn name() -> &'static str { - "mainnet" - } -} - -macro_rules! impl_name { +#[macro_export] +macro_rules! type_name { ($typ:ident) => { + type_name!($typ, stringify!($typ)); + }; + ($typ:ident, $name:expr) => { impl TypeName for $typ { fn name() -> &'static str { - stringify!($typ) + $name } } }; } -macro_rules! impl_name_generic { +#[macro_export] +macro_rules! type_name_generic { ($typ:ident) => { + type_name_generic!($typ, stringify!($typ)); + }; + ($typ:ident, $name:expr) => { impl TypeName for $typ { fn name() -> &'static str { - stringify!($typ) + $name } } }; } -impl_name_generic!(Attestation); -impl_name!(AttestationData); -impl_name!(AttestationDataAndCustodyBit); -impl_name_generic!(AttesterSlashing); -impl_name_generic!(BeaconBlock); -impl_name_generic!(BeaconBlockBody); -impl_name!(BeaconBlockHeader); -impl_name_generic!(BeaconState); -impl_name!(Checkpoint); -impl_name_generic!(CompactCommittee); -impl_name!(Crosslink); -impl_name!(Deposit); -impl_name!(DepositData); -impl_name!(Eth1Data); -impl_name!(Fork); -impl_name_generic!(HistoricalBatch); -impl_name_generic!(IndexedAttestation); -impl_name_generic!(PendingAttestation); -impl_name!(ProposerSlashing); -impl_name!(Transfer); -impl_name!(Validator); -impl_name!(VoluntaryExit); +type_name!(MinimalEthSpec, "minimal"); +type_name!(MainnetEthSpec, "mainnet"); + +type_name_generic!(Attestation); +type_name!(AttestationData); +type_name!(AttestationDataAndCustodyBit); +type_name_generic!(AttesterSlashing); +type_name_generic!(BeaconBlock); +type_name_generic!(BeaconBlockBody); +type_name!(BeaconBlockHeader); +type_name_generic!(BeaconState); +type_name!(Checkpoint); +type_name_generic!(CompactCommittee); +type_name!(Crosslink); +type_name!(Deposit); +type_name!(DepositData); +type_name!(Eth1Data); +type_name!(Fork); +type_name_generic!(HistoricalBatch); +type_name_generic!(IndexedAttestation); +type_name_generic!(PendingAttestation); +type_name!(ProposerSlashing); +type_name!(Transfer); +type_name!(Validator); +type_name!(VoluntaryExit); diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 0fb751c9e8..848ae05fab 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -262,52 +262,37 @@ ssz_static_test!(ssz_static_transfer, Transfer, SR); ssz_static_test!(ssz_static_validator, Validator); ssz_static_test!(ssz_static_voluntary_exit, VoluntaryExit, SR); -/* #[test] fn epoch_processing_justification_and_finalization() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("justification_and_finalization")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_crosslinks() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("crosslinks")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_registry_updates() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("registry_updates")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_slashings() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("slashings")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_final_updates() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("final_updates")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } +/* #[test] fn genesis_initialization() { yaml_files_in_test_dir(&Path::new("genesis").join("initialization")) From c5a22b57d29e59f578affa938f1be50da0b2a539 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 30 Aug 2019 14:10:28 +1000 Subject: [PATCH 07/32] Genesis tests --- .../src/cases/genesis_initialization.rs | 39 ++++++++++--- tests/ef_tests/src/cases/genesis_validity.rs | 24 +++++--- tests/ef_tests/src/handler.rs | 36 ++++++++++++ tests/ef_tests/tests/tests.rs | 57 ++----------------- 4 files changed, 85 insertions(+), 71 deletions(-) diff --git a/tests/ef_tests/src/cases/genesis_initialization.rs b/tests/ef_tests/src/cases/genesis_initialization.rs index 7ae8eef59d..4f0fa42961 100644 --- a/tests/ef_tests/src/cases/genesis_initialization.rs +++ b/tests/ef_tests/src/cases/genesis_initialization.rs @@ -1,34 +1,55 @@ use super::*; -use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::initialize_beacon_state_from_eth1; +use std::path::PathBuf; use types::{BeaconState, Deposit, EthSpec, Hash256}; +#[derive(Debug, Clone, Deserialize)] +struct Metadata { + deposits_count: usize, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct GenesisInitialization { - pub description: String, - pub bls_setting: Option, + pub path: PathBuf, pub eth1_block_hash: Hash256, pub eth1_timestamp: u64, pub deposits: Vec, pub state: Option>, } -impl YamlDecode for GenesisInitialization { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for GenesisInitialization { + fn load_from_dir(path: &Path) -> Result { + let eth1_block_hash = ssz_decode_file(&path.join("eth1_block_hash.ssz"))?; + let eth1_timestamp = yaml_decode_file(&path.join("eth1_timestamp.yaml"))?; + let meta: Metadata = yaml_decode_file(&path.join("meta.yaml"))?; + let deposits: Vec = (0..meta.deposits_count) + .map(|i| { + let filename = format!("deposits_{}.ssz", i); + ssz_decode_file(&path.join(filename)) + }) + .collect::>()?; + let state = ssz_decode_file(&path.join("state.ssz"))?; + + Ok(Self { + path: path.into(), + eth1_block_hash, + eth1_timestamp, + deposits, + state: Some(state), + }) } } impl Case for GenesisInitialization { - fn description(&self) -> String { - self.description.clone() + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; let spec = &E::default_spec(); let mut result = initialize_beacon_state_from_eth1( diff --git a/tests/ef_tests/src/cases/genesis_validity.rs b/tests/ef_tests/src/cases/genesis_validity.rs index 7ddd3e8fd5..efebe5e110 100644 --- a/tests/ef_tests/src/cases/genesis_validity.rs +++ b/tests/ef_tests/src/cases/genesis_validity.rs @@ -1,31 +1,37 @@ use super::*; -use crate::bls_setting::BlsSetting; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::is_valid_genesis_state; +use std::path::{Path, PathBuf}; use types::{BeaconState, EthSpec}; #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct GenesisValidity { - pub description: String, - pub bls_setting: Option, + pub path: PathBuf, pub genesis: BeaconState, pub is_valid: bool, } -impl YamlDecode for GenesisValidity { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for GenesisValidity { + fn load_from_dir(path: &Path) -> Result { + let genesis = ssz_decode_file(&path.join("genesis.ssz"))?; + let is_valid = yaml_decode_file(&path.join("is_valid.yaml"))?; + + Ok(Self { + path: path.into(), + genesis, + is_valid, + }) } } impl Case for GenesisValidity { - fn description(&self) -> String { - self.description.clone() + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; let spec = &E::default_spec(); let is_valid = is_valid_genesis_state(&self.genesis, spec); diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index a614afe76d..02518a13d1 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -202,3 +202,39 @@ impl> Handler for EpochProcessingHa T::name() } } + +pub struct GenesisValidityHandler(PhantomData); + +impl Handler for GenesisValidityHandler { + type Case = cases::GenesisValidity; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "genesis" + } + + fn handler_name() -> &'static str { + "validity" + } +} + +pub struct GenesisInitializationHandler(PhantomData); + +impl Handler for GenesisInitializationHandler { + type Case = cases::GenesisInitialization; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "genesis" + } + + fn handler_name() -> &'static str { + "initialization" + } +} diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 848ae05fab..37740cec04 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,52 +1,11 @@ use ef_tests::*; use rayon::prelude::*; -use std::path::{Path, PathBuf}; use types::{ Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink, Deposit, DepositData, Eth1Data, Fork, HistoricalBatch, IndexedAttestation, MainnetEthSpec, MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit, }; -use walkdir::WalkDir; - -fn yaml_files_in_test_dir(dir: &Path) -> Vec { - let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("eth2.0-spec-tests") - .join("tests") - .join("general") - .join("phase0") - .join(dir); - - assert!( - base_path.exists(), - format!( - "Unable to locate {:?}. Did you init git submodules?", - base_path - ) - ); - - let mut paths: Vec = WalkDir::new(base_path) - .into_iter() - .filter_map(|e| e.ok()) - .filter_map(|entry| { - if entry.file_type().is_file() { - match entry.file_name().to_str() { - Some(f) if f.ends_with(".yaml") => Some(entry.path().to_path_buf()), - Some(f) if f.ends_with(".yml") => Some(entry.path().to_path_buf()), - _ => None, - } - } else { - None - } - }) - .collect(); - - // Reverse the file order. Assuming files come in lexicographical order, executing tests in - // reverse means we get the "minimal" tests before the "mainnet" tests. This makes life easier - // for debugging. - paths.reverse(); - paths -} /* #[test] @@ -292,22 +251,14 @@ fn epoch_processing_final_updates() { EpochProcessingHandler::::run(); } -/* #[test] fn genesis_initialization() { - yaml_files_in_test_dir(&Path::new("genesis").join("initialization")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + GenesisInitializationHandler::::run(); } #[test] fn genesis_validity() { - yaml_files_in_test_dir(&Path::new("genesis").join("validity")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + GenesisValidityHandler::::run(); + // TODO: mainnet tests don't exist yet + // GenesisValidityHandler::::run(); } -*/ From fcf16faad38573b6c4b655d2d589f469b6d8b051 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 30 Aug 2019 16:16:38 +1000 Subject: [PATCH 08/32] Operations tests --- tests/ef_tests/src/cases.rs | 16 +- tests/ef_tests/src/cases/operations.rs | 193 ++++++++++++++++++ .../src/cases/operations_attestation.rs | 47 ----- .../src/cases/operations_attester_slashing.rs | 48 ----- .../src/cases/operations_block_header.rs | 44 ---- .../ef_tests/src/cases/operations_deposit.rs | 42 ---- tests/ef_tests/src/cases/operations_exit.rs | 45 ---- .../src/cases/operations_proposer_slashing.rs | 46 ----- .../ef_tests/src/cases/operations_transfer.rs | 47 ----- tests/ef_tests/src/handler.rs | 58 ++++-- tests/ef_tests/tests/tests.rs | 67 ++---- 11 files changed, 248 insertions(+), 405 deletions(-) create mode 100644 tests/ef_tests/src/cases/operations.rs delete mode 100644 tests/ef_tests/src/cases/operations_attestation.rs delete mode 100644 tests/ef_tests/src/cases/operations_attester_slashing.rs delete mode 100644 tests/ef_tests/src/cases/operations_block_header.rs delete mode 100644 tests/ef_tests/src/cases/operations_deposit.rs delete mode 100644 tests/ef_tests/src/cases/operations_exit.rs delete mode 100644 tests/ef_tests/src/cases/operations_proposer_slashing.rs delete mode 100644 tests/ef_tests/src/cases/operations_transfer.rs diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index e3ec54a7f7..1192eb0a0e 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -11,13 +11,7 @@ mod bls_sign_msg; mod epoch_processing; mod genesis_initialization; mod genesis_validity; -mod operations_attestation; -mod operations_attester_slashing; -mod operations_block_header; -mod operations_deposit; -mod operations_exit; -mod operations_proposer_slashing; -mod operations_transfer; +mod operations; mod sanity_blocks; mod sanity_slots; mod shuffling; @@ -33,13 +27,7 @@ pub use bls_sign_msg::*; pub use epoch_processing::*; pub use genesis_initialization::*; pub use genesis_validity::*; -pub use operations_attestation::*; -pub use operations_attester_slashing::*; -pub use operations_block_header::*; -pub use operations_deposit::*; -pub use operations_exit::*; -pub use operations_proposer_slashing::*; -pub use operations_transfer::*; +pub use operations::*; pub use sanity_blocks::*; pub use sanity_slots::*; pub use shuffling::*; diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs new file mode 100644 index 0000000000..bcc104bad1 --- /dev/null +++ b/tests/ef_tests/src/cases/operations.rs @@ -0,0 +1,193 @@ +use super::*; +use crate::bls_setting::BlsSetting; +use crate::case_result::compare_beacon_state_results_without_caches; +use crate::type_name::TypeName; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use serde_derive::Deserialize; +use ssz::Decode; +use state_processing::per_block_processing::{ + errors::BlockProcessingError, process_attestations, process_attester_slashings, + process_block_header, process_deposits, process_exits, process_proposer_slashings, + process_transfers, +}; +use std::path::{Path, PathBuf}; +use types::{ + Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, + ProposerSlashing, Transfer, VoluntaryExit, +}; + +#[derive(Debug, Clone, Default, Deserialize)] +struct Metadata { + description: Option, + bls_setting: Option, +} + +#[derive(Debug, Clone)] +pub struct Operations> { + pub path: PathBuf, + metadata: Metadata, + pub pre: BeaconState, + pub operation: O, + pub post: Option>, +} + +pub trait Operation: Decode + TypeName + std::fmt::Debug { + fn handler_name() -> String { + Self::name().to_lowercase() + } + + fn filename() -> String { + format!("{}.ssz", Self::handler_name()) + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError>; +} + +impl Operation for Attestation { + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_attestations(state, &[self.clone()], spec) + } +} + +impl Operation for AttesterSlashing { + fn handler_name() -> String { + "attester_slashing".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_attester_slashings(state, &[self.clone()], spec) + } +} + +impl Operation for Deposit { + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_deposits(state, &[self.clone()], spec) + } +} + +impl Operation for ProposerSlashing { + fn handler_name() -> String { + "proposer_slashing".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_proposer_slashings(state, &[self.clone()], spec) + } +} + +impl Operation for Transfer { + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_transfers(state, &[self.clone()], spec) + } +} + +impl Operation for VoluntaryExit { + fn handler_name() -> String { + "voluntary_exit".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_exits(state, &[self.clone()], spec) + } +} + +impl Operation for BeaconBlock { + fn handler_name() -> String { + "block_header".into() + } + + fn filename() -> String { + "block.ssz".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_block_header(state, self, spec, true) + } +} + +impl> LoadCase for Operations { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&metadata_path)? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let operation = ssz_decode_file(&path.join(O::filename()))?; + let post_filename = path.join("post.ssz"); + let post = if post_filename.is_file() { + Some(ssz_decode_file(&post_filename)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + operation, + post, + }) + } +} + +impl> Case for Operations { + fn description(&self) -> String { + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path + } + + fn result(&self, _case_index: usize) -> Result<(), Error> { + self.metadata.bls_setting.unwrap_or_default().check()?; + + let spec = &E::default_spec(); + let mut state = self.pre.clone(); + let mut expected = self.post.clone(); + + // Processing requires the epoch cache. + state.build_all_caches(spec).unwrap(); + + let mut result = self.operation.apply_to(&mut state, spec).map(|()| state); + + compare_beacon_state_results_without_caches(&mut result, &mut expected) + } +} diff --git a/tests/ef_tests/src/cases/operations_attestation.rs b/tests/ef_tests/src/cases/operations_attestation.rs deleted file mode 100644 index 76cbe3f189..0000000000 --- a/tests/ef_tests/src/cases/operations_attestation.rs +++ /dev/null @@ -1,47 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_attestations; -use types::{Attestation, BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsAttestation { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub attestation: Attestation, - pub post: Option>, -} - -impl YamlDecode for OperationsAttestation { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(&yaml).unwrap()) - } -} - -impl Case for OperationsAttestation { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let spec = &E::default_spec(); - - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let attestation = self.attestation.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let result = process_attestations(&mut state, &[attestation], spec); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_attester_slashing.rs b/tests/ef_tests/src/cases/operations_attester_slashing.rs deleted file mode 100644 index c658b1af4f..0000000000 --- a/tests/ef_tests/src/cases/operations_attester_slashing.rs +++ /dev/null @@ -1,48 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_attester_slashings; -use types::{AttesterSlashing, BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -pub struct OperationsAttesterSlashing { - pub description: String, - pub bls_setting: Option, - #[serde(bound = "E: EthSpec")] - pub pre: BeaconState, - #[serde(bound = "E: EthSpec")] - pub attester_slashing: AttesterSlashing, - #[serde(bound = "E: EthSpec")] - pub post: Option>, -} - -impl YamlDecode for OperationsAttesterSlashing { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsAttesterSlashing { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let attester_slashing = self.attester_slashing.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let result = - process_attester_slashings(&mut state, &[attester_slashing], &E::default_spec()); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_block_header.rs b/tests/ef_tests/src/cases/operations_block_header.rs deleted file mode 100644 index 8261b16d9c..0000000000 --- a/tests/ef_tests/src/cases/operations_block_header.rs +++ /dev/null @@ -1,44 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_block_header; -use types::{BeaconBlock, BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsBlockHeader { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub block: BeaconBlock, - pub post: Option>, -} - -impl YamlDecode for OperationsBlockHeader { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsBlockHeader { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let spec = &E::default_spec(); - - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let mut result = process_block_header(&mut state, &self.block, spec, true).map(|_| state); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_deposit.rs b/tests/ef_tests/src/cases/operations_deposit.rs deleted file mode 100644 index 801c020298..0000000000 --- a/tests/ef_tests/src/cases/operations_deposit.rs +++ /dev/null @@ -1,42 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_deposits; -use types::{BeaconState, Deposit, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsDeposit { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub deposit: Deposit, - pub post: Option>, -} - -impl YamlDecode for OperationsDeposit { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsDeposit { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let deposit = self.deposit.clone(); - let mut expected = self.post.clone(); - - let result = process_deposits(&mut state, &[deposit], &E::default_spec()); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_exit.rs b/tests/ef_tests/src/cases/operations_exit.rs deleted file mode 100644 index d7e53bcb57..0000000000 --- a/tests/ef_tests/src/cases/operations_exit.rs +++ /dev/null @@ -1,45 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_exits; -use types::{BeaconState, EthSpec, VoluntaryExit}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsExit { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub voluntary_exit: VoluntaryExit, - pub post: Option>, -} - -impl YamlDecode for OperationsExit { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsExit { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let exit = self.voluntary_exit.clone(); - let mut expected = self.post.clone(); - - // Exit processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let result = process_exits(&mut state, &[exit], &E::default_spec()); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_proposer_slashing.rs b/tests/ef_tests/src/cases/operations_proposer_slashing.rs deleted file mode 100644 index e52e84f39d..0000000000 --- a/tests/ef_tests/src/cases/operations_proposer_slashing.rs +++ /dev/null @@ -1,46 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_proposer_slashings; -use types::{BeaconState, EthSpec, ProposerSlashing}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsProposerSlashing { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub proposer_slashing: ProposerSlashing, - pub post: Option>, -} - -impl YamlDecode for OperationsProposerSlashing { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsProposerSlashing { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let proposer_slashing = self.proposer_slashing.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let result = - process_proposer_slashings(&mut state, &[proposer_slashing], &E::default_spec()); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_transfer.rs b/tests/ef_tests/src/cases/operations_transfer.rs deleted file mode 100644 index 250f587693..0000000000 --- a/tests/ef_tests/src/cases/operations_transfer.rs +++ /dev/null @@ -1,47 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_transfers; -use types::{BeaconState, EthSpec, Transfer}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsTransfer { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub transfer: Transfer, - pub post: Option>, -} - -impl YamlDecode for OperationsTransfer { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsTransfer { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let transfer = self.transfer.clone(); - let mut expected = self.post.clone(); - - // Transfer processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let spec = E::default_spec(); - - let result = process_transfers(&mut state, &[transfer], &spec); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index 02518a13d1..ca1304136e 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -1,4 +1,4 @@ -use crate::cases::{self, Case, Cases, EpochTransition, LoadCase}; +use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation}; use crate::type_name::TypeName; use crate::EfTest; use std::fs; @@ -20,7 +20,7 @@ pub trait Handler { fn runner_name() -> &'static str; - fn handler_name() -> &'static str; + fn handler_name() -> String; fn run() { let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -64,8 +64,8 @@ macro_rules! bls_handler { "bls" } - fn handler_name() -> &'static str { - $handler_name + fn handler_name() -> String { + $handler_name.into() } } }; @@ -106,8 +106,8 @@ where "ssz_static" } - fn handler_name() -> &'static str { - T::name() + fn handler_name() -> String { + T::name().into() } } @@ -126,8 +126,8 @@ where "ssz_static" } - fn handler_name() -> &'static str { - T::name() + fn handler_name() -> String { + T::name().into() } } @@ -144,8 +144,8 @@ impl Handler for ShufflingHandler { "shuffling" } - fn handler_name() -> &'static str { - "core" + fn handler_name() -> String { + "core".into() } } @@ -162,8 +162,8 @@ impl Handler for SanityBlocksHandler { "sanity" } - fn handler_name() -> &'static str { - "blocks" + fn handler_name() -> String { + "blocks".into() } } @@ -180,8 +180,8 @@ impl Handler for SanitySlotsHandler { "sanity" } - fn handler_name() -> &'static str { - "slots" + fn handler_name() -> String { + "slots".into() } } @@ -198,8 +198,8 @@ impl> Handler for EpochProcessingHa "epoch_processing" } - fn handler_name() -> &'static str { - T::name() + fn handler_name() -> String { + T::name().into() } } @@ -216,8 +216,8 @@ impl Handler for GenesisValidityHandler { "genesis" } - fn handler_name() -> &'static str { - "validity" + fn handler_name() -> String { + "validity".into() } } @@ -234,7 +234,25 @@ impl Handler for GenesisInitializationHandler { "genesis" } - fn handler_name() -> &'static str { - "initialization" + fn handler_name() -> String { + "initialization".into() + } +} + +pub struct OperationsHandler(PhantomData<(E, O)>); + +impl> Handler for OperationsHandler { + type Case = cases::Operations; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "operations" + } + + fn handler_name() -> String { + O::handler_name() } } diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 37740cec04..d663eb4543 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -36,71 +36,47 @@ fn shuffling() { ShufflingHandler::::run(); } -/* #[test] fn operations_deposit() { - yaml_files_in_test_dir(&Path::new("operations").join("deposit")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + OperationsHandler::::run(); } #[test] fn operations_transfer() { - yaml_files_in_test_dir(&Path::new("operations").join("transfer")) - .into_par_iter() - .rev() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + // Note: there are no transfer tests for mainnet } #[test] fn operations_exit() { - yaml_files_in_test_dir(&Path::new("operations").join("voluntary_exit")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + OperationsHandler::::run(); } #[test] fn operations_proposer_slashing() { - yaml_files_in_test_dir(&Path::new("operations").join("proposer_slashing")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + OperationsHandler::::run(); } #[test] fn operations_attester_slashing() { - yaml_files_in_test_dir(&Path::new("operations").join("attester_slashing")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::>::run(); + OperationsHandler::>::run(); } #[test] fn operations_attestation() { - yaml_files_in_test_dir(&Path::new("operations").join("attestation")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::>::run(); + OperationsHandler::>::run(); } #[test] fn operations_block_header() { - yaml_files_in_test_dir(&Path::new("operations").join("block_header")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::>::run(); + OperationsHandler::>::run(); } -*/ #[test] fn sanity_blocks() { @@ -114,18 +90,6 @@ fn sanity_slots() { SanitySlotsHandler::::run(); } -/* -#[test] -#[cfg(not(feature = "fake_crypto"))] -fn bls() { - yaml_files_in_test_dir(&Path::new("bls")) - .into_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); -} -*/ - #[test] #[cfg(not(feature = "fake_crypto"))] fn bls_aggregate_pubkeys() { @@ -259,6 +223,5 @@ fn genesis_initialization() { #[test] fn genesis_validity() { GenesisValidityHandler::::run(); - // TODO: mainnet tests don't exist yet - // GenesisValidityHandler::::run(); + // Note: there are no genesis validity tests for mainnet } From 74baeb4d08a958890b85a53ed93c6a382623b60b Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 2 Sep 2019 05:38:11 +1000 Subject: [PATCH 09/32] WIP - Upgrade Sync algorithm --- beacon_node/network/Cargo.toml | 1 + beacon_node/network/src/sync/manager.rs | 188 +++++++++++++++++--- beacon_node/network/src/sync/simple_sync.rs | 23 ++- 3 files changed, 180 insertions(+), 32 deletions(-) diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index dc08bd311c..06fc06ddea 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -19,3 +19,4 @@ futures = "0.1.25" error-chain = "0.12.0" tokio = "0.1.16" parking_lot = "0.9.0" +smallvec = "0.6.10" diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index b81da0991f..9b2d780f45 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -1,33 +1,110 @@ +//! The `ImportManager` facilities the block syncing logic of lighthouse. The current networking +//! specification provides two methods from which to obtain blocks from peers. The `BeaconBlocks` +//! request and the `RecentBeaconBlocks` request. The former is used to obtain a large number of +//! blocks and the latter allows for searching for blocks given a block-hash. +//! +//! These two RPC methods are designed for two type of syncing. +//! - Long range (batch) sync, when a client is out of date and needs to the latest head. +//! - Parent lookup - when a peer provides us a block whose parent is unknown to us. +//! +//! Both of these syncing strategies are built into the `ImportManager`. +//! +//! +//! Currently the long-range (batch) syncing method functions by opportunistically downloading +//! batches blocks from all peers who know about a chain that we do not. When a new peer connects +//! which has a later head that is greater than `SLOT_IMPORT_TOLERANCE` from our current head slot, +//! the manager's state becomes `Syncing` and begins a batch syncing process with this peer. If +//! further peers connect, this process is run in parallel with those peers, until our head is +//! within `SLOT_IMPORT_TOLERANCE` of all connected peers. +//! +//! Batch Syncing +//! +//! This syncing process start by requesting `MAX_BLOCKS_PER_REQUEST` blocks from a peer with an +//! unknown chain (with a greater slot height) starting from our current head slot. If the earliest +//! block returned is known to us, then the group of blocks returned form part of a known chain, +//! and we process this batch of blocks, before requesting more batches forward and processing +//! those in turn until we reach the peer's chain's head. If the first batch doesn't contain a +//! block we know of, we must iteratively request blocks backwards (until our latest finalized head +//! slot) until we find a common ancestor before we can start processing the blocks. If no common +//! ancestor is found, the peer has a chain which is not part of our finalized head slot and we +//! drop the peer and the downloaded blocks. +//! Once we are fully synced with all known peers, the state of the manager becomes `Regular` which +//! then allows for parent lookups of propagated blocks. +//! +//! A schematic version of this logic with two chain variations looks like the following. +//! +//! |----------------------|---------------------------------| +//! ^finalized head ^current local head ^remotes head +//! +//! +//! An example of the remotes chain diverging before our current head. +//! |---------------------------| +//! ^---------------------------------------------| +//! ^chain diverges |initial batch| ^remotes head +//! +//! In this example, we cannot process the initial batch as it is not on a known chain. We must +//! then backwards sync until we reach a common chain to begin forwarding batch syncing. +//! +//! +//! Parent Lookup +//! +//! When a block with an unknown parent is received and we are in `Regular` sync mode, the block is +//! queued for lookup. A round-robin approach is used to request the parent from the known list of +//! fully sync'd peers. If `PARENT_FAIL_TOLERANCE` attempts at requesting the block fails, we +//! drop the propagated block and downvote the peer that sent it to us. + use super::simple_sync::{PeerSyncInfo, FUTURE_SLOT_TOLERANCE}; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::RequestId; use eth2_libp2p::PeerId; use slog::{debug, info, trace, warn, Logger}; +use smallvec::SmallVec; use std::collections::{HashMap, HashSet}; use std::ops::{Add, Sub}; -use std::sync::Arc; +use std::sync::{Arc, Weak}; use types::{BeaconBlock, EthSpec, Hash256, Slot}; +/// Blocks are downloaded in batches from peers. This constant specifies how many blocks per batch +/// is requested. Currently the value is small for testing. This will be incremented for +/// production. const MAX_BLOCKS_PER_REQUEST: u64 = 10; -/// The number of slots that we can import blocks ahead of us, before going into full Sync mode. +/// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync +/// from a peer. If a peer is within this tolerance (forwards or backwards), it is treated as a +/// fully sync'd peer. const SLOT_IMPORT_TOLERANCE: usize = 10; +/// How many attempts we try to find a parent of a block before we give up trying . const PARENT_FAIL_TOLERANCE: usize = 3; +/// The maximum depth we will search for a parent block. In principle we should have sync'd any +/// canonical chain to its head once the peer connects. A chain should not appear where it's depth +/// is further back than the most recent head slot. const PARENT_DEPTH_TOLERANCE: usize = SLOT_IMPORT_TOLERANCE * 2; #[derive(PartialEq)] +/// The current state of a block or batches lookup. enum BlockRequestsState { + /// The object is queued to be downloaded from a peer but has not yet been requested. Queued, + /// The batch or parent has been requested with the `RequestId` and we are awaiting a response. Pending(RequestId), - Complete, + /// The downloaded blocks are ready to be processed by the beacon chain. For a batch process + /// this means we have found a common chain. + ReadyToProcess, + /// A failure has occurred and we will drop and downvote the peer that caused the request. Failed, } +/// `BlockRequests` keep track of the long-range (batch) sync process per peer. struct BlockRequests { + /// The peer's head slot and the target of this batch download. target_head_slot: Slot, + /// The peer's head root, used to specify which chain of blocks we are downloading from the + /// blocks. target_head_root: Hash256, + /// The blocks that we have currently downloaded from the peer that are yet to be processed. downloaded_blocks: Vec>, + /// The current state of this batch request. state: BlockRequestsState, /// Specifies whether the current state is syncing forwards or backwards. forward_sync: bool, @@ -35,16 +112,22 @@ struct BlockRequests { current_start_slot: Slot, } +/// Maintains a sequential list of parents to lookup and the lookup's current state. struct ParentRequests { + /// The blocks that have currently been downloaded. downloaded_blocks: Vec>, + /// The number of failed attempts to retrieve a parent block. If too many attempts occur, this + /// lookup is failed and rejected. failed_attempts: usize, - last_submitted_peer: PeerId, // to downvote the submitting peer. + /// The peer who last submitted a block. If the chain ends or fails, this is the peer that is + /// downvoted. + last_submitted_peer: PeerId, + /// The current state of the parent lookup. state: BlockRequestsState, } impl BlockRequests { - // gets the start slot for next batch - // last block slot downloaded plus 1 + /// Gets the next start slot for a batch and transitions the state to a Queued state. fn update_start_slot(&mut self) { if self.forward_sync { self.current_start_slot += Slot::from(MAX_BLOCKS_PER_REQUEST); @@ -56,58 +139,104 @@ impl BlockRequests { } #[derive(PartialEq, Debug, Clone)] +/// The current state of the `ImportManager`. enum ManagerState { + /// The manager is performing a long-range (batch) sync. In this mode, parent lookups are + /// disabled. Syncing, + /// The manager is up to date with all known peers and is connected to at least one + /// fully-syncing peer. In this state, parent lookups are enabled. Regular, + /// No useful peers are connected. Long-range sync's cannot proceed and we have no useful + /// peers to download parents for. More peers need to be connected before we can proceed. Stalled, } +/// The output states that can occur from driving (polling) the manager state machine. pub(crate) enum ImportManagerOutcome { + /// There is no further work to complete. The manager is waiting for further input. Idle, + /// A `BeaconBlocks` request is required. RequestBlocks { peer_id: PeerId, request_id: RequestId, request: BeaconBlocksRequest, }, + /// A `RecentBeaconBlocks` request is required. + RecentRequest(PeerId, RecentBeaconBlocksRequest), /// Updates information with peer via requesting another HELLO handshake. Hello(PeerId), - RecentRequest(PeerId, RecentBeaconBlocksRequest), + /// A peer has caused a punishable error and should be downvoted. DownvotePeer(PeerId), } +/// The primary object for handling and driving all the current syncing logic. It maintains the +/// current state of the syncing process, the number of useful peers, downloaded blocks and +/// controls the logic behind both the long-range (batch) sync and the on-going potential parent +/// look-up of blocks. pub struct ImportManager { - /// A reference to the underlying beacon chain. - chain: Arc>, + /// A weak reference to the underlying beacon chain. + chain: Weak>, + /// The current state of the import manager. state: ManagerState, + /// A collection of `BlockRequest` per peer that is currently being downloaded. Used in the + /// long-range (batch) sync process. import_queue: HashMap>, - parent_queue: Vec>, + /// A collection of parent block lookups. + parent_queue: SmallVec<[ParentRequests; 3]>, + /// The collection of known, connected, fully-sync'd peers. full_peers: HashSet, + /// The current request Id. This is used to keep track of responses to various outbound + /// requests. This is an internal accounting mechanism, request id's are never sent to any + /// peers. current_req_id: usize, + /// The logger for the import manager. log: Logger, } impl ImportManager { + /// Generates a new `ImportManager` given a logger and an Arc reference to a beacon chain. The + /// import manager keeps a weak reference to the beacon chain, which allows the chain to be + /// dropped during the syncing process. The syncing handles this termination gracefully. pub fn new(beacon_chain: Arc>, log: &slog::Logger) -> Self { ImportManager { - chain: beacon_chain.clone(), + chain: Arc::downgrade(&beacon_chain), state: ManagerState::Regular, import_queue: HashMap::new(), - parent_queue: Vec::new(), + parent_queue: SmallVec::new(), full_peers: HashSet::new(), current_req_id: 0, log: log.clone(), } } + /// A peer has connected which has blocks that are unknown to us. + /// + /// This function handles the logic associated with the connection of a new peer. If the peer + /// is sufficiently ahead of our current head, a long-range (batch) sync is started and + /// batches of blocks are queued to download from the peer. Batched blocks begin at our + /// current head. If the resulting downloaded blocks are part of our current chain, we + /// continue with a forward sync. If not, we download blocks (in batches) backwards until we + /// reach a common ancestor. Batches are then processed and downloaded sequentially forwards. + /// + /// If the peer is within the `SLOT_IMPORT_TOLERANCE`, then it's head is sufficiently close to + /// ours that we consider it fully sync'd with respect to our current chain. pub fn add_peer(&mut self, peer_id: PeerId, remote: PeerSyncInfo) { - // TODO: Improve comments. - // initially try to download blocks from our current head - // then backwards search all the way back to our finalized epoch until we match on a chain - // has to be done sequentially to find next slot to start the batch from + // ensure the beacon chain still exists + let chain = match self.chain.upgrade() { + Some(chain) => chain, + None => { + warn!(self.log, + "Beacon chain dropped. Peer not considered for sync"; + "peer_id" => format!("{:?}", peer_id)); + return; + } + }; - let local = PeerSyncInfo::from(&self.chain); + let local = PeerSyncInfo::from(&chain); - // If a peer is within SLOT_IMPORT_TOLERANCE from our head slot, ignore a batch sync + // If a peer is within SLOT_IMPORT_TOLERANCE from our head slot, ignore a batch sync, + // consider it a fully-sync'd peer. if remote.head_slot.sub(local.head_slot).as_usize() < SLOT_IMPORT_TOLERANCE { trace!(self.log, "Ignoring full sync with peer"; "peer" => format!("{:?}", peer_id), @@ -116,34 +245,53 @@ impl ImportManager { ); // remove the peer from the queue if it exists self.import_queue.remove(&peer_id); + self.add_full_peer(peer_id); + // return; } + // Check if the peer is significantly is behind us. If within `SLOT_IMPORT_TOLERANCE` + // treat them as a fully synced peer. If not, ignore them in the sync process + if local.head_slot.sub(remote.head_slot).as_usize() < SLOT_IMPORT_TOLERANCE { + self.add_full_peer(peer_id); + } else { + debug!( + self.log, + "Out of sync peer connected"; + "peer" => format!("{:?}", peer_id), + ); + return; + } + + // Check if we are already downloading blocks from this peer, if so update, if not set up + // a new request structure if let Some(block_requests) = self.import_queue.get_mut(&peer_id) { // update the target head slot if remote.head_slot > block_requests.target_head_slot { block_requests.target_head_slot = remote.head_slot; } } else { + // not already downloading blocks from this peer let block_requests = BlockRequests { target_head_slot: remote.head_slot, // this should be larger than the current head. It is checked in the SyncManager before add_peer is called target_head_root: remote.head_root, downloaded_blocks: Vec::new(), state: BlockRequestsState::Queued, forward_sync: true, - current_start_slot: self.chain.best_slot(), + current_start_slot: chain.best_slot(), }; self.import_queue.insert(peer_id, block_requests); } } + /// A `BeaconBlocks` request has received a response. This function process the response. pub fn beacon_blocks_response( &mut self, peer_id: PeerId, request_id: RequestId, mut blocks: Vec>, ) { - // find the request + // find the request associated with this response let block_requests = match self .import_queue .get_mut(&peer_id) diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 573ac9dd1f..dd857d8c39 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -16,8 +16,6 @@ use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256, Slot}; /// Otherwise we queue it. pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1; -/// The number of slots behind our head that we still treat a peer as a fully synced peer. -const FULL_PEER_TOLERANCE: u64 = 10; const SHOULD_FORWARD_GOSSIP_BLOCK: bool = true; const SHOULD_NOT_FORWARD_GOSSIP_BLOCK: bool = false; @@ -189,18 +187,17 @@ impl SimpleSync { .exists::>(&remote.head_root) .unwrap_or_else(|_| false) { + trace!( + self.log, "Out of date or potentially sync'd peer found"; + "peer" => format!("{:?}", peer_id), + "remote_head_slot" => remote.head_slot + "remote_latest_finalized_epoch" => remote.finalized_epoch, + ); + // If the node's best-block is already known to us and they are close to our current // head, treat them as a fully sync'd peer. - if self.chain.best_slot().sub(remote.head_slot).as_u64() < FULL_PEER_TOLERANCE { - self.manager.add_full_peer(peer_id); - self.process_sync(); - } else { - debug!( - self.log, - "Out of sync peer connected"; - "peer" => format!("{:?}", peer_id), - ); - } + self.manager.add_peer(peer_id, remote); + self.process_sync(); } else { // The remote node has an equal or great finalized epoch and we don't know it's head. // @@ -218,6 +215,8 @@ impl SimpleSync { } } + /// This function drives the `ImportManager` state machine. The outcomes it provides are + /// actioned until the `ImportManager` is idle. fn process_sync(&mut self) { loop { match self.manager.poll() { From cd7b6da88ed52a604e9ce3cf1d5985aff4f94a13 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 3 Sep 2019 00:34:41 +1000 Subject: [PATCH 10/32] Updates syncing, corrects CLI variables --- beacon_node/beacon_chain/src/beacon_chain.rs | 11 +- beacon_node/eth2-libp2p/src/discovery.rs | 10 +- beacon_node/eth2-libp2p/src/service.rs | 19 +- beacon_node/network/src/sync/manager.rs | 616 +++++++++++-------- beacon_node/network/src/sync/simple_sync.rs | 5 +- beacon_node/src/main.rs | 13 +- 6 files changed, 374 insertions(+), 300 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 6380d03b3e..a142816aed 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -442,6 +442,15 @@ impl BeaconChain { None } + /// Returns the block canonical root of the current canonical chain at a given slot. + /// + /// Returns None if a block doesn't exist at the slot. + pub fn root_at_slot(&self, target_slot: Slot) -> Option { + self.rev_iter_block_roots() + .find(|(_root, slot)| *slot == target_slot) + .map(|(root, _slot)| root) + } + /// Reads the slot clock (see `self.read_slot_clock()` and returns the number of slots since /// genesis. pub fn slots_since_genesis(&self) -> Option { @@ -1006,7 +1015,7 @@ impl BeaconChain { }; // Load the parent blocks state from the database, returning an error if it is not found. - // It is an error because if know the parent block we should also know the parent state. + // It is an error because if we know the parent block we should also know the parent state. let parent_state_root = parent_block.state_root; let parent_state = self .store diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index 6c80a85968..4a8aba2b1b 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -341,13 +341,9 @@ fn save_enr_to_disc(dir: &Path, enr: &Enr, log: &slog::Logger) { } Err(e) => { warn!( - log, - <<<<<<< HEAD - "Could not write ENR to file"; "file" => format!("{:?}{:?}",dir, ENR_FILENAME), "error" => format!("{}", e) - ======= - "Could not write ENR to file"; "File" => format!("{:?}{:?}",dir, ENR_FILENAME), "Error" => format!("{}", e) - >>>>>>> interop - ); + log, + "Could not write ENR to file"; "file" => format!("{:?}{:?}",dir, ENR_FILENAME), "error" => format!("{}", e) + ); } } } diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 589106f48e..1ea1723b68 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -82,17 +82,10 @@ impl Service { // attempt to connect to user-input libp2p nodes for multiaddr in config.libp2p_nodes { match Swarm::dial_addr(&mut swarm, multiaddr.clone()) { -<<<<<<< HEAD Ok(()) => debug!(log, "Dialing libp2p peer"; "address" => format!("{}", multiaddr)), Err(err) => debug!( log, - "Could not connect to peer"; "address" => format!("{}", multiaddr), "Error" => format!("{:?}", err) -======= - Ok(()) => debug!(log, "Dialing libp2p peer"; "Address" => format!("{}", multiaddr)), - Err(err) => debug!( - log, - "Could not connect to peer"; "Address" => format!("{}", multiaddr), "Error" => format!("{:?}", err) ->>>>>>> interop + "Could not connect to peer"; "address" => format!("{}", multiaddr), "error" => format!("{:?}", err) ), }; } @@ -129,7 +122,6 @@ impl Service { let mut subscribed_topics = vec![]; for topic in topics { if swarm.subscribe(topic.clone()) { -<<<<<<< HEAD trace!(log, "Subscribed to topic"; "topic" => format!("{}", topic)); subscribed_topics.push(topic); } else { @@ -137,15 +129,6 @@ impl Service { } } info!(log, "Subscribed to topics"; "topics" => format!("{:?}", subscribed_topics.iter().map(|t| format!("{}", t)).collect::>())); -======= - trace!(log, "Subscribed to topic"; "Topic" => format!("{}", topic)); - subscribed_topics.push(topic); - } else { - warn!(log, "Could not subscribe to topic"; "Topic" => format!("{}", topic)); - } - } - info!(log, "Subscribed to topics"; "Topics" => format!("{:?}", subscribed_topics.iter().map(|t| format!("{}", t)).collect::>())); ->>>>>>> interop Ok(Service { local_peer_id, diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 9b2d780f45..a48b43ad7b 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -80,6 +80,9 @@ const PARENT_FAIL_TOLERANCE: usize = 3; /// canonical chain to its head once the peer connects. A chain should not appear where it's depth /// is further back than the most recent head slot. const PARENT_DEPTH_TOLERANCE: usize = SLOT_IMPORT_TOLERANCE * 2; +/// The number of empty batches we tolerate before dropping the peer. This prevents endless +/// requests to peers who never return blocks. +const EMPTY_BATCH_TOLERANCE: usize = 100; #[derive(PartialEq)] /// The current state of a block or batches lookup. @@ -95,6 +98,19 @@ enum BlockRequestsState { Failed, } +/// The state of batch requests. +enum SyncDirection { + /// The batch has just been initialised and we need to check to see if a backward sync is + /// required on first batch response. + Initial, + /// We are syncing forwards, the next batch should contain higher slot numbers than is + /// predecessor. + Forwards, + /// We are syncing backwards and looking for a common ancestor chain before we can start + /// processing the downloaded blocks. + Backwards, +} + /// `BlockRequests` keep track of the long-range (batch) sync process per peer. struct BlockRequests { /// The peer's head slot and the target of this batch download. @@ -104,10 +120,13 @@ struct BlockRequests { target_head_root: Hash256, /// The blocks that we have currently downloaded from the peer that are yet to be processed. downloaded_blocks: Vec>, + /// The number of empty batches we have consecutively received. If a peer returns more than + /// EMPTY_BATCHES_TOLERANCE, they are dropped. + consecutive_empty_batches: usize, /// The current state of this batch request. state: BlockRequestsState, - /// Specifies whether the current state is syncing forwards or backwards. - forward_sync: bool, + /// Specifies the current direction of this batch request. + sync_direction: SyncDirection, /// The current `start_slot` of the batched block request. current_start_slot: Slot, } @@ -129,10 +148,13 @@ struct ParentRequests { impl BlockRequests { /// Gets the next start slot for a batch and transitions the state to a Queued state. fn update_start_slot(&mut self) { - if self.forward_sync { - self.current_start_slot += Slot::from(MAX_BLOCKS_PER_REQUEST); - } else { - self.current_start_slot -= Slot::from(MAX_BLOCKS_PER_REQUEST); + match self.sync_direction { + SyncDirection::Initial | SyncDirection::Forwards => { + self.current_start_slot += Slot::from(MAX_BLOCKS_PER_REQUEST); + } + SyncDirection::Backwards => { + self.current_start_slot -= Slot::from(MAX_BLOCKS_PER_REQUEST); + } } self.state = BlockRequestsState::Queued; } @@ -175,6 +197,8 @@ pub(crate) enum ImportManagerOutcome { /// controls the logic behind both the long-range (batch) sync and the on-going potential parent /// look-up of blocks. pub struct ImportManager { + /// List of events to be processed externally. + event_queue: SmallVec<[ImportManagerOutcome; 20]>, /// A weak reference to the underlying beacon chain. chain: Weak>, /// The current state of the import manager. @@ -200,6 +224,7 @@ impl ImportManager { /// dropped during the syncing process. The syncing handles this termination gracefully. pub fn new(beacon_chain: Arc>, log: &slog::Logger) -> Self { ImportManager { + event_queue: SmallVec::new(), chain: Arc::downgrade(&beacon_chain), state: ManagerState::Regular, import_queue: HashMap::new(), @@ -253,7 +278,7 @@ impl ImportManager { // Check if the peer is significantly is behind us. If within `SLOT_IMPORT_TOLERANCE` // treat them as a fully synced peer. If not, ignore them in the sync process if local.head_slot.sub(remote.head_slot).as_usize() < SLOT_IMPORT_TOLERANCE { - self.add_full_peer(peer_id); + self.add_full_peer(peer_id.clone()); } else { debug!( self.log, @@ -275,9 +300,10 @@ impl ImportManager { let block_requests = BlockRequests { target_head_slot: remote.head_slot, // this should be larger than the current head. It is checked in the SyncManager before add_peer is called target_head_root: remote.head_root, + consecutive_empty_batches: 0, downloaded_blocks: Vec::new(), state: BlockRequestsState::Queued, - forward_sync: true, + sync_direction: SyncDirection::Initial, current_start_slot: chain.best_slot(), }; self.import_queue.insert(peer_id, block_requests); @@ -291,6 +317,16 @@ impl ImportManager { request_id: RequestId, mut blocks: Vec>, ) { + // ensure the underlying chain still exists + let chain = match self.chain.upgrade() { + Some(chain) => chain, + None => { + debug!(self.log, "Chain dropped. Sync terminating"); + self.event_queue.clear(); + return; + } + }; + // find the request associated with this response let block_requests = match self .import_queue @@ -315,10 +351,19 @@ impl ImportManager { if blocks.is_empty() { debug!(self.log, "BeaconBlocks response was empty"; "request_id" => request_id); - block_requests.update_start_slot(); + block_requests.consecutive_empty_batches += 1; + if block_requests.consecutive_empty_batches >= EMPTY_BATCH_TOLERANCE { + warn!(self.log, "Peer returned too many empty block batches"; + "peer" => format!("{:?}", peer_id)); + block_requests.state = BlockRequestsState::Failed; + } else { + block_requests.update_start_slot(); + } return; } + block_requests.consecutive_empty_batches = 0; + // verify the range of received blocks // Note that the order of blocks is verified in block processing let last_sent_slot = blocks[blocks.len() - 1].slot; @@ -328,83 +373,90 @@ impl ImportManager { .add(MAX_BLOCKS_PER_REQUEST) < last_sent_slot { - //TODO: Downvote peer - add a reason to failed - dbg!(&blocks); warn!(self.log, "BeaconBlocks response returned out of range blocks"; "request_id" => request_id, "response_initial_slot" => blocks[0].slot, "requested_initial_slot" => block_requests.current_start_slot); + self.event_queue + .push(ImportManagerOutcome::DownvotePeer(peer_id)); // consider this sync failed block_requests.state = BlockRequestsState::Failed; return; } // Determine if more blocks need to be downloaded. There are a few cases: - // - We have downloaded a batch from our head_slot, which has not reached the remotes head - // (target head). Therefore we need to download another sequential batch. - // - The latest batch includes blocks that greater than or equal to the target_head slot, - // which means we have caught up to their head. We then check to see if the first - // block downloaded matches our head. If so, we are on the same chain and can process - // the blocks. If not we need to sync back further until we are on the same chain. So - // request more blocks. - // - We are syncing backwards (from our head slot) and need to check if we are on the same - // chain. If so, process the blocks, if not, request more blocks all the way up to - // our last finalized slot. + // - We are in initial sync mode - We have requested blocks and need to determine if this + // is part of a known chain to determine the whether to start syncing backwards or continue + // syncing forwards. + // - We are syncing backwards and need to verify if we have found a common ancestor in + // order to start processing the downloaded blocks. + // - We are syncing forwards. We mark this as complete and check if any further blocks are + // required to download when processing the batch. - if block_requests.forward_sync { - // append blocks if syncing forward - block_requests.downloaded_blocks.append(&mut blocks); - } else { - // prepend blocks if syncing backwards - block_requests.downloaded_blocks.splice(..0, blocks); - } + match block_requests.sync_direction { + SyncDirection::Initial => { + block_requests.downloaded_blocks.append(&mut blocks); - // does the batch contain the target_head_slot - let last_element_index = block_requests.downloaded_blocks.len() - 1; - if block_requests.downloaded_blocks[last_element_index].slot - >= block_requests.target_head_slot - || !block_requests.forward_sync - { - // if the batch is on our chain, this is complete and we can then process. - // Otherwise start backwards syncing until we reach a common chain. - let earliest_slot = block_requests.downloaded_blocks[0].slot; - //TODO: Decide which is faster. Reading block from db and comparing or calculating - //the hash tree root and comparing. - if Some(block_requests.downloaded_blocks[0].canonical_root()) - == root_at_slot(&self.chain, earliest_slot) - { - block_requests.state = BlockRequestsState::Complete; - return; + // this batch is the first batch downloaded. Check if we can process or if we need + // to backwards search. + + //TODO: Decide which is faster. Reading block from db and comparing or calculating + //the hash tree root and comparing. + let earliest_slot = block_requests.downloaded_blocks[0].slot; + if Some(block_requests.downloaded_blocks[0].canonical_root()) + == chain.root_at_slot(earliest_slot) + { + // we have a common head, start processing and begin a forwards sync + block_requests.sync_direction = SyncDirection::Forwards; + block_requests.state = BlockRequestsState::ReadyToProcess; + return; + } + // no common head, begin a backwards search + block_requests.sync_direction = SyncDirection::Backwards; + block_requests.current_start_slot = + std::cmp::min(chain.best_slot(), block_requests.downloaded_blocks[0].slot); + block_requests.update_start_slot(); } - - // not on the same chain, request blocks backwards - let state = &self.chain.head().beacon_state; - let local_finalized_slot = state - .finalized_checkpoint - .epoch - .start_slot(T::EthSpec::slots_per_epoch()); - - // check that the request hasn't failed by having no common chain - if local_finalized_slot >= block_requests.current_start_slot { - warn!(self.log, "Peer returned an unknown chain."; "request_id" => request_id); - block_requests.state = BlockRequestsState::Failed; - return; + SyncDirection::Forwards => { + // continue processing all blocks forwards, verify the end in the processing + block_requests.downloaded_blocks.append(&mut blocks); + block_requests.state = BlockRequestsState::ReadyToProcess; } + SyncDirection::Backwards => { + block_requests.downloaded_blocks.splice(..0, blocks); - // if this is a forward sync, then we have reached the head without a common chain - // and we need to start syncing backwards. - if block_requests.forward_sync { - // Start a backwards sync by requesting earlier blocks - block_requests.forward_sync = false; - block_requests.current_start_slot = std::cmp::min( - self.chain.best_slot(), - block_requests.downloaded_blocks[0].slot, - ); + // verify the request hasn't failed by having no common ancestor chain + // get our local finalized_slot + let local_finalized_slot = { + let state = &chain.head().beacon_state; + state + .finalized_checkpoint + .epoch + .start_slot(T::EthSpec::slots_per_epoch()) + }; + + if local_finalized_slot >= block_requests.current_start_slot { + warn!(self.log, "Peer returned an unknown chain."; "request_id" => request_id); + block_requests.state = BlockRequestsState::Failed; + return; + } + + // check if we have reached a common chain ancestor + let earliest_slot = block_requests.downloaded_blocks[0].slot; + if Some(block_requests.downloaded_blocks[0].canonical_root()) + == chain.root_at_slot(earliest_slot) + { + // we have a common head, start processing and begin a forwards sync + block_requests.sync_direction = SyncDirection::Forwards; + block_requests.state = BlockRequestsState::ReadyToProcess; + return; + } + + // no common chain, haven't passed last_finalized_head, so continue backwards + // search + block_requests.update_start_slot(); } } - - // update the start slot and re-queue the batch - block_requests.update_start_slot(); } pub fn recent_blocks_response( @@ -447,7 +499,7 @@ impl ImportManager { } // queue for processing - parent_request.state = BlockRequestsState::Complete; + parent_request.state = BlockRequestsState::ReadyToProcess; } pub fn _inject_error(_peer_id: PeerId, _id: RequestId) { @@ -500,29 +552,41 @@ impl ImportManager { pub(crate) fn poll(&mut self) -> ImportManagerOutcome { loop { + //TODO: Optimize the lookups. Potentially keep state of whether each of these functions + //need to be called. + + // only break once everything has been processed + let mut re_run = false; + + // only process batch requests if there are any + if !self.import_queue.is_empty() { + // process potential block requests + self.process_potential_block_requests(); + + // process any complete long-range batches + re_run = self.process_complete_batches(); + } + + // only process parent objects if we are in regular sync + if let ManagerState::Regular = self.state { + // process any parent block lookup-requests + self.process_parent_requests(); + + // process any complete parent lookups + re_run = self.process_complete_parent_requests(); + } + + // return any queued events + if !self.event_queue.is_empty() { + let event = self.event_queue.remove(0); + self.event_queue.shrink_to_fit(); + return event; + } + // update the state of the manager self.update_state(); - // process potential block requests - if let Some(outcome) = self.process_potential_block_requests() { - return outcome; - } - - // process any complete long-range batches - if let Some(outcome) = self.process_complete_batches() { - return outcome; - } - - // process any parent block lookup-requests - if let Some(outcome) = self.process_parent_requests() { - return outcome; - } - - // process any complete parent lookups - let (re_run, outcome) = self.process_complete_parent_requests(); - if let Some(outcome) = outcome { - return outcome; - } else if !re_run { + if !re_run { break; } } @@ -549,11 +613,11 @@ impl ImportManager { } } - fn process_potential_block_requests(&mut self) -> Option { + fn process_potential_block_requests(&mut self) { // check if an outbound request is required // Managing a fixed number of outbound requests is maintained at the RPC protocol libp2p - // layer and not needed here. - // If any in queued state we submit a request. + // layer and not needed here. Therefore we create many outbound requests and let the RPC + // handle the number of simultaneous requests. Request all queued objects. // remove any failed batches let debug_log = &self.log; @@ -585,56 +649,84 @@ impl ImportManager { count: MAX_BLOCKS_PER_REQUEST, step: 0, }; - return Some(ImportManagerOutcome::RequestBlocks { + self.event_queue.push(ImportManagerOutcome::RequestBlocks { peer_id: peer_id.clone(), request, request_id, }); } - - None } - fn process_complete_batches(&mut self) -> Option { - let completed_batches = self - .import_queue - .iter() - .filter(|(_peer, block_requests)| block_requests.state == BlockRequestsState::Complete) - .map(|(peer, _)| peer) - .cloned() - .collect::>(); - for peer_id in completed_batches { - let block_requests = self.import_queue.remove(&peer_id).expect("key exists"); - match self.process_blocks(block_requests.downloaded_blocks.clone()) { - Ok(()) => { - //TODO: Verify it's impossible to have empty downloaded_blocks - let last_element = block_requests.downloaded_blocks.len() - 1; - debug!(self.log, "Blocks processed successfully"; - "peer" => format!("{:?}", peer_id), - "start_slot" => block_requests.downloaded_blocks[0].slot, - "end_slot" => block_requests.downloaded_blocks[last_element].slot, - "no_blocks" => last_element + 1, - ); - // Re-HELLO to ensure we are up to the latest head - return Some(ImportManagerOutcome::Hello(peer_id)); - } - Err(e) => { - let last_element = block_requests.downloaded_blocks.len() - 1; - warn!(self.log, "Block processing failed"; + fn process_complete_batches(&mut self) -> bool { + // flag to indicate if the manager can be switched to idle or not + let mut re_run = false; + + // create reference variables to be moved into subsequent closure + let chain_ref = self.chain.clone(); + let log_ref = &self.log; + let event_queue_ref = &mut self.event_queue; + + self.import_queue.retain(|peer_id, block_requests| { + // check that the chain still exists + if let Some(chain) = chain_ref.upgrade() { + let downloaded_blocks = + std::mem::replace(&mut block_requests.downloaded_blocks, Vec::new()); + let last_element = block_requests.downloaded_blocks.len() - 1; + let start_slot = block_requests.downloaded_blocks[0].slot; + let end_slot = block_requests.downloaded_blocks[last_element].slot; + + match process_blocks(chain, downloaded_blocks, log_ref) { + Ok(()) => { + debug!(log_ref, "Blocks processed successfully"; "peer" => format!("{:?}", peer_id), - "start_slot" => block_requests.downloaded_blocks[0].slot, - "end_slot" => block_requests.downloaded_blocks[last_element].slot, + "start_slot" => start_slot, + "end_slot" => end_slot, "no_blocks" => last_element + 1, - "error" => format!("{:?}", e), - ); - return Some(ImportManagerOutcome::DownvotePeer(peer_id)); + ); + + // check if the batch is complete, by verifying if we have reached the + // target head + if end_slot >= block_requests.target_head_slot { + // Completed, re-hello the peer to ensure we are up to the latest head + event_queue_ref.push(ImportManagerOutcome::Hello(peer_id.clone())); + // remove the request + false + } else { + // have not reached the end, queue another batch + block_requests.update_start_slot(); + re_run = true; + // keep the batch + true + } + } + Err(e) => { + warn!(log_ref, "Block processing failed"; + "peer" => format!("{:?}", peer_id), + "start_slot" => start_slot, + "end_slot" => end_slot, + "no_blocks" => last_element + 1, + "error" => format!("{:?}", e), + ); + event_queue_ref.push(ImportManagerOutcome::DownvotePeer(peer_id.clone())); + false + } } + } else { + // chain no longer exists, empty the queue and return + event_queue_ref.clear(); + return false; } - } - None + }); + + re_run } - fn process_parent_requests(&mut self) -> Option { + fn process_parent_requests(&mut self) { + // check to make sure there are peers to search for the parent from + if self.full_peers.is_empty() { + return; + } + // remove any failed requests let debug_log = &self.log; self.parent_queue.retain(|parent_request| { @@ -649,11 +741,6 @@ impl ImportManager { } }); - // check to make sure there are peers to search for the parent from - if self.full_peers.is_empty() { - return None; - } - // check if parents need to be searched for for parent_request in self.parent_queue.iter_mut() { if parent_request.failed_attempts >= PARENT_FAIL_TOLERANCE { @@ -677,23 +764,21 @@ impl ImportManager { // select a random fully synced peer to attempt to download the parent block let peer_id = self.full_peers.iter().next().expect("List is not empty"); - return Some(ImportManagerOutcome::RecentRequest(peer_id.clone(), req)); + self.event_queue + .push(ImportManagerOutcome::RecentRequest(peer_id.clone(), req)); } } - - None } - fn process_complete_parent_requests(&mut self) -> (bool, Option) { - // flag to determine if there is more process to drive or if the manager can be switched to - // an idle state + fn process_complete_parent_requests(&mut self) -> bool { + // returned value indicating whether the manager can be switched to idle or not let mut re_run = false; // Find any parent_requests ready to be processed for completed_request in self .parent_queue .iter_mut() - .filter(|req| req.state == BlockRequestsState::Complete) + .filter(|req| req.state == BlockRequestsState::ReadyToProcess) { // verify the last added block is the parent of the last requested block let last_index = completed_request.downloaded_blocks.len() - 1; @@ -711,7 +796,9 @@ impl ImportManager { "received_block" => format!("{}", block_hash), "expected_parent" => format!("{}", expected_hash), ); - return (true, Some(ImportManagerOutcome::DownvotePeer(peer))); + re_run = true; + self.event_queue + .push(ImportManagerOutcome::DownvotePeer(peer)); } // try and process the list of blocks up to the requested block @@ -720,154 +807,153 @@ impl ImportManager { .downloaded_blocks .pop() .expect("Block must exist exist"); - match self.chain.process_block(block.clone()) { - Ok(BlockProcessingOutcome::ParentUnknown { parent: _ }) => { - // need to keep looking for parents - completed_request.downloaded_blocks.push(block); - completed_request.state = BlockRequestsState::Queued; - re_run = true; - break; - } - Ok(BlockProcessingOutcome::Processed { block_root: _ }) => {} - Ok(outcome) => { - // it's a future slot or an invalid block, remove it and try again - completed_request.failed_attempts += 1; - trace!( - self.log, "Invalid parent block"; - "outcome" => format!("{:?}", outcome), - "peer" => format!("{:?}", completed_request.last_submitted_peer), - ); - completed_request.state = BlockRequestsState::Queued; - re_run = true; - return ( - re_run, - Some(ImportManagerOutcome::DownvotePeer( + + // check if the chain exists + if let Some(chain) = self.chain.upgrade() { + match chain.process_block(block.clone()) { + Ok(BlockProcessingOutcome::ParentUnknown { parent: _ }) => { + // need to keep looking for parents + completed_request.downloaded_blocks.push(block); + completed_request.state = BlockRequestsState::Queued; + re_run = true; + break; + } + Ok(BlockProcessingOutcome::Processed { block_root: _ }) => {} + Ok(outcome) => { + // it's a future slot or an invalid block, remove it and try again + completed_request.failed_attempts += 1; + trace!( + self.log, "Invalid parent block"; + "outcome" => format!("{:?}", outcome), + "peer" => format!("{:?}", completed_request.last_submitted_peer), + ); + completed_request.state = BlockRequestsState::Queued; + re_run = true; + self.event_queue.push(ImportManagerOutcome::DownvotePeer( completed_request.last_submitted_peer.clone(), - )), - ); - } - Err(e) => { - completed_request.failed_attempts += 1; - warn!( - self.log, "Parent processing error"; - "error" => format!("{:?}", e) - ); - completed_request.state = BlockRequestsState::Queued; - re_run = true; - return ( - re_run, - Some(ImportManagerOutcome::DownvotePeer( + )); + return re_run; + } + Err(e) => { + completed_request.failed_attempts += 1; + warn!( + self.log, "Parent processing error"; + "error" => format!("{:?}", e) + ); + completed_request.state = BlockRequestsState::Queued; + re_run = true; + self.event_queue.push(ImportManagerOutcome::DownvotePeer( completed_request.last_submitted_peer.clone(), - )), - ); + )); + return re_run; + } } + } else { + // chain doesn't exist - clear the event queue and return + self.event_queue.clear(); + return false; } } } - // remove any full completed and processed parent chains + // remove any fully processed parent chains self.parent_queue.retain(|req| { - if req.state == BlockRequestsState::Complete { + if req.state == BlockRequestsState::ReadyToProcess { false } else { true } }); - (re_run, None) + re_run } +} - fn process_blocks(&mut self, blocks: Vec>) -> Result<(), String> { - for block in blocks { - let processing_result = self.chain.process_block(block.clone()); +// Helper function to process blocks +fn process_blocks( + chain: Arc>, + blocks: Vec>, + log: &Logger, +) -> Result<(), String> { + for block in blocks { + let processing_result = chain.process_block(block.clone()); - if let Ok(outcome) = processing_result { - match outcome { - BlockProcessingOutcome::Processed { block_root } => { - // The block was valid and we processed it successfully. + if let Ok(outcome) = processing_result { + match outcome { + BlockProcessingOutcome::Processed { block_root } => { + // The block was valid and we processed it successfully. + trace!( + log, "Imported block from network"; + "slot" => block.slot, + "block_root" => format!("{}", block_root), + ); + } + BlockProcessingOutcome::ParentUnknown { parent } => { + // blocks should be sequential and all parents should exist + trace!( + log, "ParentBlockUnknown"; + "parent_root" => format!("{}", parent), + "baby_block_slot" => block.slot, + ); + return Err(format!( + "Block at slot {} has an unknown parent.", + block.slot + )); + } + BlockProcessingOutcome::FutureSlot { + present_slot, + block_slot, + } => { + if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot { + // The block is too far in the future, drop it. trace!( - self.log, "Imported block from network"; - "slot" => block.slot, - "block_root" => format!("{}", block_root), - ); - } - BlockProcessingOutcome::ParentUnknown { parent } => { - // blocks should be sequential and all parents should exist - trace!( - self.log, "ParentBlockUnknown"; - "parent_root" => format!("{}", parent), - "baby_block_slot" => block.slot, + log, "FutureBlock"; + "msg" => "block for future slot rejected, check your time", + "present_slot" => present_slot, + "block_slot" => block_slot, + "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, ); return Err(format!( - "Block at slot {} has an unknown parent.", + "Block at slot {} is too far in the future", block.slot )); - } - BlockProcessingOutcome::FutureSlot { - present_slot, - block_slot, - } => { - if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot { - // The block is too far in the future, drop it. - trace!( - self.log, "FutureBlock"; - "msg" => "block for future slot rejected, check your time", - "present_slot" => present_slot, - "block_slot" => block_slot, - "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, - ); - return Err(format!( - "Block at slot {} is too far in the future", - block.slot - )); - } else { - // The block is in the future, but not too far. - trace!( - self.log, "QueuedFutureBlock"; - "msg" => "queuing future block, check your time", - "present_slot" => present_slot, - "block_slot" => block_slot, - "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, - ); - } - } - BlockProcessingOutcome::FinalizedSlot => { + } else { + // The block is in the future, but not too far. trace!( - self.log, "Finalized or earlier block processed"; - "outcome" => format!("{:?}", outcome), + log, "QueuedFutureBlock"; + "msg" => "queuing future block, check your time", + "present_slot" => present_slot, + "block_slot" => block_slot, + "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, ); - // block reached our finalized slot or was earlier, move to the next block - } - _ => { - trace!( - self.log, "InvalidBlock"; - "msg" => "peer sent invalid block", - "outcome" => format!("{:?}", outcome), - ); - return Err(format!("Invalid block at slot {}", block.slot)); } } - } else { - trace!( - self.log, "BlockProcessingFailure"; - "msg" => "unexpected condition in processing block.", - "outcome" => format!("{:?}", processing_result) - ); - return Err(format!( - "Unexpected block processing error: {:?}", - processing_result - )); + BlockProcessingOutcome::FinalizedSlot => { + trace!( + log, "Finalized or earlier block processed"; + "outcome" => format!("{:?}", outcome), + ); + // block reached our finalized slot or was earlier, move to the next block + } + _ => { + trace!( + log, "InvalidBlock"; + "msg" => "peer sent invalid block", + "outcome" => format!("{:?}", outcome), + ); + return Err(format!("Invalid block at slot {}", block.slot)); + } } + } else { + trace!( + log, "BlockProcessingFailure"; + "msg" => "unexpected condition in processing block.", + "outcome" => format!("{:?}", processing_result) + ); + return Err(format!( + "Unexpected block processing error: {:?}", + processing_result + )); } - Ok(()) } -} - -fn root_at_slot( - chain: &Arc>, - target_slot: Slot, -) -> Option { - chain - .rev_iter_block_roots() - .find(|(_root, slot)| *slot == target_slot) - .map(|(root, _slot)| root) + Ok(()) } diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index dd857d8c39..36947082eb 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -6,7 +6,6 @@ use eth2_libp2p::rpc::{RPCEvent, RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; use slog::{debug, info, o, trace, warn}; use ssz::Encode; -use std::ops::Sub; use std::sync::Arc; use store::Store; use tokio::sync::mpsc; @@ -190,7 +189,7 @@ impl SimpleSync { trace!( self.log, "Out of date or potentially sync'd peer found"; "peer" => format!("{:?}", peer_id), - "remote_head_slot" => remote.head_slot + "remote_head_slot" => remote.head_slot, "remote_latest_finalized_epoch" => remote.finalized_epoch, ); @@ -386,7 +385,7 @@ impl SimpleSync { "peer" => format!("{:?}", peer_id), "msg" => "Failed to return all requested hashes", "start_slot" => req.start_slot, - "current_slot" => self.chain.present_slot(), + "current_slot" => self.chain.best_slot(), "requested" => req.count, "returned" => blocks.len(), ); diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 26537c6f76..ea801cd8bc 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -33,14 +33,14 @@ fn main() { .arg( Arg::with_name("logfile") .long("logfile") - .value_name("logfile") + .value_name("FILE") .help("File path where output will be written.") .takes_value(true), ) .arg( Arg::with_name("network-dir") .long("network-dir") - .value_name("NETWORK-DIR") + .value_name("DIR") .help("Data directory for network keys.") .takes_value(true) .global(true) @@ -83,7 +83,7 @@ fn main() { Arg::with_name("boot-nodes") .long("boot-nodes") .allow_hyphen_values(true) - .value_name("BOOTNODES") + .value_name("ENR-LIST") .help("One or more comma-delimited base64-encoded ENR's to bootstrap the p2p network.") .takes_value(true), ) @@ -128,13 +128,14 @@ fn main() { .arg( Arg::with_name("rpc-address") .long("rpc-address") - .value_name("Address") + .value_name("ADDRESS") .help("Listen address for RPC endpoint.") .takes_value(true), ) .arg( Arg::with_name("rpc-port") .long("rpc-port") + .value_name("PORT") .help("Listen port for RPC endpoint.") .conflicts_with("port-bump") .takes_value(true), @@ -149,14 +150,14 @@ fn main() { .arg( Arg::with_name("api-address") .long("api-address") - .value_name("APIADDRESS") + .value_name("ADDRESS") .help("Set the listen address for the RESTful HTTP API server.") .takes_value(true), ) .arg( Arg::with_name("api-port") .long("api-port") - .value_name("APIPORT") + .value_name("PORT") .help("Set the listen TCP port for the RESTful HTTP API server.") .conflicts_with("port-bump") .takes_value(true), From 13b5df56b3f4e81238733943e23bb3e11d602c5a Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 3 Sep 2019 07:50:44 +1000 Subject: [PATCH 11/32] Account manager, bootnodes, RPC display and sync fixes --- account_manager/src/main.rs | 10 +- beacon_node/eth2-libp2p/src/discovery.rs | 2 +- beacon_node/eth2-libp2p/src/rpc/methods.rs | 48 ++++++++ beacon_node/eth2-libp2p/src/rpc/mod.rs | 10 ++ beacon_node/eth2-libp2p/src/rpc/protocol.rs | 11 ++ beacon_node/eth2-libp2p/src/service.rs | 16 ++- beacon_node/network/src/service.rs | 4 +- beacon_node/network/src/sync/manager.rs | 124 ++++++++++++-------- beacon_node/network/src/sync/simple_sync.rs | 2 +- beacon_node/src/main.rs | 14 --- 10 files changed, 168 insertions(+), 73 deletions(-) diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index b7448ddf25..ae3823049d 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -125,9 +125,13 @@ fn main() { } } } - _ => panic!( - "The account manager must be run with a subcommand. See help for more information." - ), + _ => { + crit!( + log, + "The account manager must be run with a subcommand. See help for more information." + ); + return; + } } } diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index 4a8aba2b1b..c3f2522d85 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -114,7 +114,7 @@ impl Discovery { self.find_peers(); } - /// Add an Enr to the routing table of the discovery mechanism. + /// Add an ENR to the routing table of the discovery mechanism. pub fn add_enr(&mut self, enr: Enr) { self.discovery.add_enr(enr); } diff --git a/beacon_node/eth2-libp2p/src/rpc/methods.rs b/beacon_node/eth2-libp2p/src/rpc/methods.rs index d912bcfa1e..c9610b000e 100644 --- a/beacon_node/eth2-libp2p/src/rpc/methods.rs +++ b/beacon_node/eth2-libp2p/src/rpc/methods.rs @@ -157,3 +157,51 @@ impl ErrorMessage { String::from_utf8(self.error_message.clone()).unwrap_or_else(|_| "".into()) } } + +impl std::fmt::Display for HelloMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Hello Message: Fork Version: {:?}, Finalized Root: {}, Finalized Epoch: {}, Head Root: {}, Head Slot: {}", self.fork_version, self.finalized_root, self.finalized_epoch, self.head_root, self.head_slot) + } +} + +impl std::fmt::Display for RPCResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RPCResponse::Hello(hello) => write!(f, "{}", hello), + RPCResponse::BeaconBlocks(_) => write!(f, ""), + RPCResponse::RecentBeaconBlocks(_) => write!(f, ""), + } + } +} + +impl std::fmt::Display for RPCErrorResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RPCErrorResponse::Success(res) => write!(f, "{}", res), + RPCErrorResponse::InvalidRequest(err) => write!(f, "Invalid Request: {:?}", err), + RPCErrorResponse::ServerError(err) => write!(f, "Server Error: {:?}", err), + RPCErrorResponse::Unknown(err) => write!(f, "Unknown Error: {:?}", err), + } + } +} + +impl std::fmt::Display for GoodbyeReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + GoodbyeReason::ClientShutdown => write!(f, "Client Shutdown"), + GoodbyeReason::IrrelevantNetwork => write!(f, "Irrelevant Network"), + GoodbyeReason::Fault => write!(f, "Fault"), + GoodbyeReason::Unknown => write!(f, "Unknown Reason"), + } + } +} + +impl std::fmt::Display for BeaconBlocksRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Head Block Root: {}, Start Slot: {}, Count: {}, Step: {}", + self.head_block_root, self.start_slot, self.count, self.step + ) + } +} diff --git a/beacon_node/eth2-libp2p/src/rpc/mod.rs b/beacon_node/eth2-libp2p/src/rpc/mod.rs index 756a62e71b..2076615a9c 100644 --- a/beacon_node/eth2-libp2p/src/rpc/mod.rs +++ b/beacon_node/eth2-libp2p/src/rpc/mod.rs @@ -47,6 +47,16 @@ impl RPCEvent { } } +impl std::fmt::Display for RPCEvent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RPCEvent::Request(id, req) => write!(f, "RPC Request(Id: {}, {})", id, req), + RPCEvent::Response(id, res) => write!(f, "RPC Response(Id: {}, {})", id, res), + RPCEvent::Error(id, err) => write!(f, "RPC Request(Id: {}, Error: {:?})", id, err), + } + } +} + /// Implements the libp2p `NetworkBehaviour` trait and therefore manages network-level /// logic. pub struct RPC { diff --git a/beacon_node/eth2-libp2p/src/rpc/protocol.rs b/beacon_node/eth2-libp2p/src/rpc/protocol.rs index be1efdf5d4..401fa8b9ea 100644 --- a/beacon_node/eth2-libp2p/src/rpc/protocol.rs +++ b/beacon_node/eth2-libp2p/src/rpc/protocol.rs @@ -288,3 +288,14 @@ impl std::error::Error for RPCError { } } } + +impl std::fmt::Display for RPCRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RPCRequest::Hello(hello) => write!(f, "Hello Message: {}", hello), + RPCRequest::Goodbye(reason) => write!(f, "Goodbye: {}", reason), + RPCRequest::BeaconBlocks(req) => write!(f, "Beacon Blocks: {}", req), + RPCRequest::RecentBeaconBlocks(req) => write!(f, "Recent Beacon Blocks: {:?}", req), + } + } +} diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 1ea1723b68..96b5a276ea 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -79,8 +79,8 @@ impl Service { } }; - // attempt to connect to user-input libp2p nodes - for multiaddr in config.libp2p_nodes { + // helper closure for dialing peers + let mut dial_addr = |multiaddr: Multiaddr| { match Swarm::dial_addr(&mut swarm, multiaddr.clone()) { Ok(()) => debug!(log, "Dialing libp2p peer"; "address" => format!("{}", multiaddr)), Err(err) => debug!( @@ -88,6 +88,18 @@ impl Service { "Could not connect to peer"; "address" => format!("{}", multiaddr), "error" => format!("{:?}", err) ), }; + }; + + // attempt to connect to user-input libp2p nodes + for multiaddr in config.libp2p_nodes { + dial_addr(multiaddr); + } + + // attempt to connect to any specified boot-nodes + for bootnode_enr in config.boot_nodes { + for multiaddr in bootnode_enr.multiaddr() { + dial_addr(multiaddr); + } } // subscribe to default gossipsub topics diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index a8b3c74b6a..ae75620331 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -161,7 +161,7 @@ fn network_service( Ok(Async::Ready(Some(message))) => match message { NetworkMessage::Send(peer_id, outgoing_message) => match outgoing_message { OutgoingMessage::RPC(rpc_event) => { - trace!(log, "Sending RPC Event: {:?}", rpc_event); + trace!(log, "{}", rpc_event); libp2p_service.lock().swarm.send_rpc(peer_id, rpc_event); } }, @@ -185,7 +185,7 @@ fn network_service( match libp2p_service.lock().poll() { Ok(Async::Ready(Some(event))) => match event { Libp2pEvent::RPC(peer_id, rpc_event) => { - trace!(log, "RPC Event: RPC message received: {:?}", rpc_event); + trace!(log, "{}", rpc_event); message_handler_send .try_send(HandlerMessage::RPC(peer_id, rpc_event)) .map_err(|_| "Failed to send RPC to handler")?; diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index a48b43ad7b..8ba7486a51 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -68,7 +68,7 @@ use types::{BeaconBlock, EthSpec, Hash256, Slot}; /// Blocks are downloaded in batches from peers. This constant specifies how many blocks per batch /// is requested. Currently the value is small for testing. This will be incremented for /// production. -const MAX_BLOCKS_PER_REQUEST: u64 = 10; +const MAX_BLOCKS_PER_REQUEST: u64 = 100; /// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync /// from a peer. If a peer is within this tolerance (forwards or backwards), it is treated as a @@ -120,6 +120,8 @@ struct BlockRequests { target_head_root: Hash256, /// The blocks that we have currently downloaded from the peer that are yet to be processed. downloaded_blocks: Vec>, + /// The number of blocks successfully processed in this request. + blocks_processed: usize, /// The number of empty batches we have consecutively received. If a peer returns more than /// EMPTY_BATCHES_TOLERANCE, they are dropped. consecutive_empty_batches: usize, @@ -302,6 +304,7 @@ impl ImportManager { target_head_root: remote.head_root, consecutive_empty_batches: 0, downloaded_blocks: Vec::new(), + blocks_processed: 0, state: BlockRequestsState::Queued, sync_direction: SyncDirection::Initial, current_start_slot: chain.best_slot(), @@ -356,6 +359,10 @@ impl ImportManager { warn!(self.log, "Peer returned too many empty block batches"; "peer" => format!("{:?}", peer_id)); block_requests.state = BlockRequestsState::Failed; + } else if block_requests.current_start_slot >= block_requests.target_head_slot { + warn!(self.log, "Peer did not return blocks it claimed to possess"; + "peer" => format!("{:?}", peer_id)); + block_requests.state = BlockRequestsState::Failed; } else { block_requests.update_start_slot(); } @@ -561,19 +568,19 @@ impl ImportManager { // only process batch requests if there are any if !self.import_queue.is_empty() { // process potential block requests - self.process_potential_block_requests(); + re_run = re_run || self.process_potential_block_requests(); // process any complete long-range batches - re_run = self.process_complete_batches(); + re_run = re_run || self.process_complete_batches(); } // only process parent objects if we are in regular sync - if let ManagerState::Regular = self.state { + if !self.parent_queue.is_empty() { // process any parent block lookup-requests - self.process_parent_requests(); + re_run = re_run || self.process_parent_requests(); // process any complete parent lookups - re_run = self.process_complete_parent_requests(); + re_run = re_run || self.process_complete_parent_requests(); } // return any queued events @@ -613,20 +620,23 @@ impl ImportManager { } } - fn process_potential_block_requests(&mut self) { + fn process_potential_block_requests(&mut self) -> bool { // check if an outbound request is required // Managing a fixed number of outbound requests is maintained at the RPC protocol libp2p // layer and not needed here. Therefore we create many outbound requests and let the RPC // handle the number of simultaneous requests. Request all queued objects. + let mut re_run = false; // remove any failed batches let debug_log = &self.log; + let full_peer_ref = &mut self.full_peers; self.import_queue.retain(|peer_id, block_request| { if let BlockRequestsState::Failed = block_request.state { debug!(debug_log, "Block import from peer failed"; "peer_id" => format!("{:?}", peer_id), - "downloaded_blocks" => block_request.downloaded_blocks.len() + "downloaded_blocks" => block_request.blocks_processed ); + full_peer_ref.remove(peer_id); false } else { true @@ -654,7 +664,10 @@ impl ImportManager { request, request_id, }); + re_run = true; } + + re_run } fn process_complete_batches(&mut self) -> bool { @@ -667,66 +680,75 @@ impl ImportManager { let event_queue_ref = &mut self.event_queue; self.import_queue.retain(|peer_id, block_requests| { - // check that the chain still exists - if let Some(chain) = chain_ref.upgrade() { - let downloaded_blocks = - std::mem::replace(&mut block_requests.downloaded_blocks, Vec::new()); - let last_element = block_requests.downloaded_blocks.len() - 1; - let start_slot = block_requests.downloaded_blocks[0].slot; - let end_slot = block_requests.downloaded_blocks[last_element].slot; + if block_requests.state == BlockRequestsState::ReadyToProcess { + // check that the chain still exists + if let Some(chain) = chain_ref.upgrade() { + let downloaded_blocks = + std::mem::replace(&mut block_requests.downloaded_blocks, Vec::new()); + let last_element = downloaded_blocks.len() - 1; + let start_slot = downloaded_blocks[0].slot; + let end_slot = downloaded_blocks[last_element].slot; - match process_blocks(chain, downloaded_blocks, log_ref) { - Ok(()) => { - debug!(log_ref, "Blocks processed successfully"; - "peer" => format!("{:?}", peer_id), - "start_slot" => start_slot, - "end_slot" => end_slot, - "no_blocks" => last_element + 1, - ); - - // check if the batch is complete, by verifying if we have reached the - // target head - if end_slot >= block_requests.target_head_slot { - // Completed, re-hello the peer to ensure we are up to the latest head - event_queue_ref.push(ImportManagerOutcome::Hello(peer_id.clone())); - // remove the request - false - } else { - // have not reached the end, queue another batch - block_requests.update_start_slot(); - re_run = true; - // keep the batch - true - } - } - Err(e) => { - warn!(log_ref, "Block processing failed"; + match process_blocks(chain, downloaded_blocks, log_ref) { + Ok(()) => { + debug!(log_ref, "Blocks processed successfully"; "peer" => format!("{:?}", peer_id), "start_slot" => start_slot, "end_slot" => end_slot, "no_blocks" => last_element + 1, - "error" => format!("{:?}", e), - ); - event_queue_ref.push(ImportManagerOutcome::DownvotePeer(peer_id.clone())); - false + ); + block_requests.blocks_processed += last_element + 1; + + // check if the batch is complete, by verifying if we have reached the + // target head + if end_slot >= block_requests.target_head_slot { + // Completed, re-hello the peer to ensure we are up to the latest head + event_queue_ref.push(ImportManagerOutcome::Hello(peer_id.clone())); + // remove the request + false + } else { + // have not reached the end, queue another batch + block_requests.update_start_slot(); + re_run = true; + // keep the batch + true + } + } + Err(e) => { + warn!(log_ref, "Block processing failed"; + "peer" => format!("{:?}", peer_id), + "start_slot" => start_slot, + "end_slot" => end_slot, + "no_blocks" => last_element + 1, + "error" => format!("{:?}", e), + ); + event_queue_ref + .push(ImportManagerOutcome::DownvotePeer(peer_id.clone())); + false + } } + } else { + // chain no longer exists, empty the queue and return + event_queue_ref.clear(); + return false; } } else { - // chain no longer exists, empty the queue and return - event_queue_ref.clear(); - return false; + // not ready to process + true } }); re_run } - fn process_parent_requests(&mut self) { + fn process_parent_requests(&mut self) -> bool { // check to make sure there are peers to search for the parent from if self.full_peers.is_empty() { - return; + return false; } + let mut re_run = false; + // remove any failed requests let debug_log = &self.log; self.parent_queue.retain(|parent_request| { @@ -766,8 +788,10 @@ impl ImportManager { self.event_queue .push(ImportManagerOutcome::RecentRequest(peer_id.clone(), req)); + re_run = true; } } + re_run } fn process_complete_parent_requests(&mut self) -> bool { diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 36947082eb..e1ca30b0ac 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -453,7 +453,7 @@ impl SimpleSync { } BlockProcessingOutcome::ParentUnknown { parent: _ } => { // Inform the sync manager to find parents for this block - trace!(self.log, "Unknown parent gossip"; + trace!(self.log, "Block with unknown parent received"; "peer_id" => format!("{:?}",peer_id)); self.manager.add_unknown_block(block.clone(), peer_id); SHOULD_FORWARD_GOSSIP_BLOCK diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index ea801cd8bc..ab9803ebaf 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -187,13 +187,6 @@ fn main() { .possible_values(&["info", "debug", "trace", "warn", "error", "crit"]) .default_value("trace"), ) - .arg( - Arg::with_name("verbosity") - .short("v") - .multiple(true) - .help("Sets the verbosity level") - .takes_value(true), - ) /* * The "testnet" sub-command. * @@ -332,13 +325,6 @@ fn main() { _ => unreachable!("guarded by clap"), }; - let drain = match matches.occurrences_of("verbosity") { - 0 => drain.filter_level(Level::Info), - 1 => drain.filter_level(Level::Debug), - 2 => drain.filter_level(Level::Trace), - _ => drain.filter_level(Level::Trace), - }; - let log = slog::Logger::root(drain.fuse(), o!()); warn!( From 8d5a579aa64a4ba53c010f12913c9f45adedd56c Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 23 Aug 2019 18:33:27 +1000 Subject: [PATCH 12/32] Fix BeaconChain tests --- eth2/types/src/beacon_state/tests.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 67adccdda8..0363e5848d 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -90,11 +90,11 @@ fn test_active_index(state_slot: Slot) { // Test the start and end of the range. assert_eq!( - state.get_active_index_root_index(*range.start(), &spec), + state.get_active_index_root_index(*range.start(), &spec, AllowNextEpoch::False), Ok(modulo(*range.start())) ); assert_eq!( - state.get_active_index_root_index(*range.end(), &spec), + state.get_active_index_root_index(*range.end(), &spec, AllowNextEpoch::False), Ok(modulo(*range.end())) ); @@ -102,12 +102,12 @@ fn test_active_index(state_slot: Slot) { if state.current_epoch() > 0 { // Test is invalid on epoch zero, cannot subtract from zero. assert_eq!( - state.get_active_index_root_index(*range.start() - 1, &spec), + state.get_active_index_root_index(*range.start() - 1, &spec, AllowNextEpoch::False), Err(Error::EpochOutOfBounds) ); } assert_eq!( - state.get_active_index_root_index(*range.end() + 1, &spec), + state.get_active_index_root_index(*range.end() + 1, &spec, AllowNextEpoch::False), Err(Error::EpochOutOfBounds) ); } From f47eaf57304d627e552408aae8003197f0318f5d Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 3 Sep 2019 16:46:10 +1000 Subject: [PATCH 13/32] Parallel tests and SSZ generic --- eth2/utils/ssz_types/src/fixed_vector.rs | 17 +- tests/ef_tests/src/cases.rs | 12 +- tests/ef_tests/src/cases/epoch_processing.rs | 2 +- tests/ef_tests/src/cases/operations.rs | 3 +- tests/ef_tests/src/cases/ssz_generic.rs | 229 +++++++++++++++---- tests/ef_tests/src/cases/ssz_static.rs | 8 +- tests/ef_tests/src/handler.rs | 32 ++- tests/ef_tests/src/lib.rs | 7 - tests/ef_tests/tests/tests.rs | 25 +- 9 files changed, 246 insertions(+), 89 deletions(-) diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index edac77f0d0..edf499adf5 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -220,13 +220,26 @@ where fn from_ssz_bytes(bytes: &[u8]) -> Result { if bytes.is_empty() { - Ok(FixedVector::from(vec![])) + Err(ssz::DecodeError::InvalidByteLength { + len: 0, + expected: 1, + }) } else if T::is_ssz_fixed_len() { bytes .chunks(T::ssz_fixed_len()) .map(|chunk| T::from_ssz_bytes(chunk)) .collect::, _>>() - .and_then(|vec| Ok(vec.into())) + .and_then(|vec| { + if vec.len() == N::to_usize() { + Ok(vec.into()) + } else { + Err(ssz::DecodeError::BytesInvalid(format!( + "wrong number of vec elements, got: {}, expected: {}", + vec.len(), + N::to_usize() + ))) + } + }) } else { ssz::decode_list_of_variable_length_items(bytes).and_then(|vec| Ok(vec.into())) } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 1192eb0a0e..ed00f0ffee 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -1,4 +1,5 @@ use super::*; +use rayon::prelude::*; use std::fmt::Debug; use std::path::Path; @@ -39,7 +40,7 @@ pub trait LoadCase: Sized { fn load_from_dir(_path: &Path) -> Result; } -pub trait Case: Debug { +pub trait Case: Debug + Sync { /// An optional field for implementing a custom description. /// /// Defaults to "no description". @@ -79,13 +80,10 @@ pub struct Cases { pub test_cases: Vec, } -impl EfTest for Cases -where - T: Case + Debug, -{ - fn test_results(&self) -> Vec { +impl Cases { + pub fn test_results(&self) -> Vec { self.test_cases - .iter() + .into_par_iter() .enumerate() .map(|(i, tc)| CaseResult::new(i, tc, tc.result(i))) .collect() diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs index ac47ab2368..d79b5fc48f 100644 --- a/tests/ef_tests/src/cases/epoch_processing.rs +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -31,7 +31,7 @@ pub struct EpochProcessing> { _phantom: PhantomData, } -pub trait EpochTransition: TypeName + Debug { +pub trait EpochTransition: TypeName + Debug + Sync { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError>; } diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs index bcc104bad1..e86e6f5986 100644 --- a/tests/ef_tests/src/cases/operations.rs +++ b/tests/ef_tests/src/cases/operations.rs @@ -10,6 +10,7 @@ use state_processing::per_block_processing::{ process_block_header, process_deposits, process_exits, process_proposer_slashings, process_transfers, }; +use std::fmt::Debug; use std::path::{Path, PathBuf}; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, @@ -31,7 +32,7 @@ pub struct Operations> { pub post: Option>, } -pub trait Operation: Decode + TypeName + std::fmt::Debug { +pub trait Operation: Decode + TypeName + Debug + Sync { fn handler_name() -> String { Self::name().to_lowercase() } diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index ca49d21060..05b96ad7db 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -1,68 +1,205 @@ use super::*; -use crate::case_result::compare_result; -use ethereum_types::{U128, U256}; +use crate::cases::ssz_static::{check_serialization, check_tree_hash, SszStaticType}; +use crate::yaml_decode::yaml_decode_file; use serde_derive::Deserialize; -use ssz::Decode; -use std::fmt::Debug; +use std::fs; +use std::path::{Path, PathBuf}; +use types::typenum::*; +use types::{BitList, BitVector, FixedVector, VariableList}; #[derive(Debug, Clone, Deserialize)] -pub struct SszGeneric { - #[serde(alias = "type")] - pub type_name: String, - pub valid: bool, - pub value: Option, - pub ssz: Option, +struct Metadata { + root: String, + signing_root: Option, } -impl YamlDecode for SszGeneric { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +#[derive(Debug, Clone)] +pub struct SszGeneric { + path: PathBuf, + handler_name: String, + case_name: String, +} + +impl LoadCase for SszGeneric { + fn load_from_dir(path: &Path) -> Result { + let components = path + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .rev() + .collect::>(); + // Test case name is last + let case_name = components[0].clone(); + // Handler name is third last, before suite name and case name + let handler_name = components[2].clone(); + Ok(Self { + path: path.into(), + handler_name, + case_name, + }) + } +} + +macro_rules! type_dispatch { + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => primitive_type ] $($rest:tt)*) => { + match $value { + "bool" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* bool>, $($rest)*), + "uint8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u8>, $($rest)*), + "uint16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u16>, $($rest)*), + "uint32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u32>, $($rest)*), + "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*), + // FIXME(michael): implement tree hash for big ints + // "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* etherum_types::U128>, $($rest)*), + // "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ethereum_types::U256>, $($rest)*), + _ => { println!("unsupported: {}", $value); Ok(()) }, + } + }; + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => typenum ] $($rest:tt)*) => { + match $value { + // DO YOU LIKE NUMBERS? + "0" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U0>, $($rest)*), + "1" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1>, $($rest)*), + "2" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2>, $($rest)*), + "3" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U3>, $($rest)*), + "4" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4>, $($rest)*), + "5" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U5>, $($rest)*), + "6" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U6>, $($rest)*), + "7" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U7>, $($rest)*), + "8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8>, $($rest)*), + "9" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U9>, $($rest)*), + "16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U16>, $($rest)*), + "31" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U31>, $($rest)*), + "32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U32>, $($rest)*), + "64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U64>, $($rest)*), + "128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U128>, $($rest)*), + "256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U256>, $($rest)*), + "512" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U512>, $($rest)*), + "513" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U513>, $($rest)*), + "1024" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1024>, $($rest)*), + "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*), + "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*), + "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*), + _ => { println!("unsupported: {}", $value); Ok(()) }, + } + }; + // No base type: apply type params to function + ($function:ident, ($($arg:expr),*), _, <$($param_ty:ty),*>,) => { + $function::<$($param_ty),*>($($arg),*) + }; + ($function:ident, ($($arg:expr),*), $base_type_name:ident, <$($param_ty:ty),*>,) => { + $function::<$base_type_name<$($param_ty),*>>($($arg),*) } } impl Case for SszGeneric { + fn path(&self) -> &Path { + &self.path + } + fn result(&self, _case_index: usize) -> Result<(), Error> { - if let Some(ssz) = &self.ssz { - match self.type_name.as_ref() { - "uint8" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint16" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint32" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint64" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint128" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint256" => ssz_generic_test::(self.valid, ssz, &self.value), - _ => Err(Error::FailedToParseTest(format!( - "Unknown type: {}", - self.type_name - ))), + let parts = self.case_name.split('_').collect::>(); + + match self.handler_name.as_str() { + "basic_vector" => { + let elem_ty = parts[1]; + let length = parts[2]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + FixedVector, + <>, + [elem_ty => primitive_type] + [length => typenum] + )?; } - } else { - // Skip tests that do not have an ssz field. - // - // See: https://github.com/ethereum/eth2.0-specs/issues/1079 - Ok(()) + "bitlist" => { + let limit = parts[1]; + + // FIXME(michael): mark length "no" cases as known failures + + type_dispatch!( + ssz_generic_test, + (&self.path), + BitList, + <>, + [limit => typenum] + )?; + } + "bitvector" => { + let length = parts[1]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + BitVector, + <>, + [length => typenum] + )?; + } + "boolean" => { + ssz_generic_test::(&self.path)?; + } + "uints" => { + let type_name = "uint".to_owned() + parts[1]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + _, + <>, + [type_name.as_str() => primitive_type] + )?; + } + // FIXME(michael): support for the containers tests + _ => panic!("unsupported handler: {}", self.handler_name), } + Ok(()) } } -/// Execute a `ssz_generic` test case. -fn ssz_generic_test(should_be_ok: bool, ssz: &str, value: &Option) -> Result<(), Error> -where - T: Decode + YamlDecode + Debug + PartialEq, -{ - let ssz = hex::decode(&ssz[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - - // We do not cater for the scenario where the test is valid but we are not passed any SSZ. - if should_be_ok && value.is_none() { - panic!("Unexpected test input. Cannot pass without value.") - } - - let expected = if let Some(string) = value { - Some(T::yaml_decode(string)?) +fn ssz_generic_test(path: &Path) -> Result<(), Error> { + let meta_path = path.join("meta.yaml"); + let meta: Option = if meta_path.is_file() { + Some(yaml_decode_file(&meta_path)?) } else { None }; - let decoded = T::from_ssz_bytes(&ssz); + let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); - compare_result(&decoded, &expected) + let value_path = path.join("value.yaml"); + let value: Option = if value_path.is_file() { + Some(yaml_decode_file(&value_path)?) + } else { + None + }; + + // Valid + // TODO: signing root + if let Some(value) = value { + check_serialization(&value, &serialized)?; + + if let Some(ref meta) = meta { + check_tree_hash(&meta.root, value.tree_hash_root())?; + } + } + // Invalid + else { + if let Ok(decoded) = T::from_ssz_bytes(&serialized) { + return Err(Error::DidntFail(format!( + "Decoded invalid bytes into: {:?}", + decoded + ))); + } + } + + Ok(()) } diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index 6a949073de..f9f59cc4b1 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -35,12 +35,12 @@ pub struct SszStaticSR { // Trait alias for all deez bounds pub trait SszStaticType: - serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync { } impl SszStaticType for T where - T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync { } @@ -77,7 +77,7 @@ impl LoadCase for SszStaticSR { } } -fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { +pub fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { // Check serialization let serialized_result = value.as_ssz_bytes(); compare_result::, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?; @@ -89,7 +89,7 @@ fn check_serialization(value: &T, serialized: &[u8]) -> Result Ok(()) } -fn check_tree_hash(expected_str: &str, actual_root: Vec) -> Result<(), Error> { +pub fn check_tree_hash(expected_str: &str, actual_root: Vec) -> Result<(), Error> { let expected_root = hex::decode(&expected_str[2..]) .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; let expected_root = Hash256::from_slice(&expected_root); diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index ca1304136e..b6334c3839 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -1,6 +1,6 @@ use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation}; +use crate::type_name; use crate::type_name::TypeName; -use crate::EfTest; use std::fs; use std::marker::PhantomData; use std::path::PathBuf; @@ -256,3 +256,33 @@ impl> Handler for OperationsHandler O::handler_name() } } + +pub struct SszGenericHandler(PhantomData); + +impl Handler for SszGenericHandler { + type Case = cases::SszGeneric; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "ssz_generic" + } + + fn handler_name() -> String { + H::name().into() + } +} + +// Supported SSZ generic handlers +pub struct BasicVector; +type_name!(BasicVector, "basic_vector"); +pub struct Bitlist; +type_name!(Bitlist, "bitlist"); +pub struct Bitvector; +type_name!(Bitvector, "bitvector"); +pub struct Boolean; +type_name!(Boolean, "boolean"); +pub struct Uints; +type_name!(Uints, "uints"); diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index 54e674d85a..bcf7c77a04 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -17,10 +17,3 @@ mod handler; mod results; mod type_name; mod yaml_decode; - -/// Defined where an object can return the results of some test(s) adhering to the Ethereum -/// Foundation testing format. -pub trait EfTest { - /// Returns the results of executing one or more tests. - fn test_results(&self) -> Vec; -} diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index d663eb4543..71fa53c66c 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,5 +1,4 @@ use ef_tests::*; -use rayon::prelude::*; use types::{ Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink, @@ -7,29 +6,15 @@ use types::{ MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit, }; -/* #[test] -#[cfg(feature = "fake_crypto")] fn ssz_generic() { - yaml_files_in_test_dir(&Path::new("ssz_generic")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); } - -#[test] -#[cfg(feature = "fake_crypto")] -fn ssz_static() { - yaml_files_in_test_dir(&Path::new("ssz_static")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); -} -*/ - #[test] fn shuffling() { ShufflingHandler::::run(); From d511c939eb4015796573dc3466b40d5a4811735a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 4 Sep 2019 13:03:44 +1000 Subject: [PATCH 14/32] SSZ generic tests for big uints --- eth2/utils/tree_hash/src/impls.rs | 42 ++++++++- tests/ef_tests/Cargo.toml | 2 + tests/ef_tests/src/cases.rs | 16 +--- .../src/cases/bls_aggregate_pubkeys.rs | 1 + .../ef_tests/src/cases/bls_aggregate_sigs.rs | 1 + tests/ef_tests/src/cases/bls_g2_compressed.rs | 1 + .../ef_tests/src/cases/bls_g2_uncompressed.rs | 9 +- tests/ef_tests/src/cases/bls_priv_to_pub.rs | 1 + tests/ef_tests/src/cases/bls_sign_msg.rs | 1 + tests/ef_tests/src/cases/common.rs | 72 ++++++++++++++ tests/ef_tests/src/cases/epoch_processing.rs | 2 +- .../src/cases/genesis_initialization.rs | 2 +- tests/ef_tests/src/cases/genesis_validity.rs | 2 +- tests/ef_tests/src/cases/operations.rs | 2 +- tests/ef_tests/src/cases/sanity_blocks.rs | 2 +- tests/ef_tests/src/cases/sanity_slots.rs | 2 +- tests/ef_tests/src/cases/shuffling.rs | 9 +- tests/ef_tests/src/cases/ssz_generic.rs | 20 ++-- tests/ef_tests/src/cases/ssz_static.rs | 33 ++----- tests/ef_tests/src/decode.rs | 31 +++++++ tests/ef_tests/src/lib.rs | 3 +- tests/ef_tests/src/results.rs | 3 +- tests/ef_tests/src/yaml_decode.rs | 93 ------------------- 23 files changed, 185 insertions(+), 165 deletions(-) create mode 100644 tests/ef_tests/src/cases/common.rs create mode 100644 tests/ef_tests/src/decode.rs delete mode 100644 tests/ef_tests/src/yaml_decode.rs diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs index 88293196e4..9f09f50ce7 100644 --- a/eth2/utils/tree_hash/src/impls.rs +++ b/eth2/utils/tree_hash/src/impls.rs @@ -1,5 +1,5 @@ use super::*; -use ethereum_types::H256; +use ethereum_types::{H256, U128, U256}; macro_rules! impl_for_bitsize { ($type: ident, $bit_size: expr) => { @@ -73,6 +73,46 @@ macro_rules! impl_for_u8_array { impl_for_u8_array!(4); impl_for_u8_array!(32); +impl TreeHash for U128 { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + let mut result = vec![0; 16]; + self.to_little_endian(&mut result); + result + } + + fn tree_hash_packing_factor() -> usize { + 2 + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self.tree_hash_packed_encoding(), 0) + } +} + +impl TreeHash for U256 { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + let mut result = vec![0; 32]; + self.to_little_endian(&mut result); + result + } + + fn tree_hash_packing_factor() -> usize { + 1 + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self.tree_hash_packed_encoding(), 0) + } +} + impl TreeHash for H256 { fn tree_hash_type() -> TreeHashType { TreeHashType::Vector diff --git a/tests/ef_tests/Cargo.toml b/tests/ef_tests/Cargo.toml index ba6aca259d..2f1dea11d1 100644 --- a/tests/ef_tests/Cargo.toml +++ b/tests/ef_tests/Cargo.toml @@ -18,7 +18,9 @@ serde_derive = "1.0" serde_repr = "0.1" serde_yaml = "0.8" eth2_ssz = "0.1" +eth2_ssz_derive = "0.1" tree_hash = "0.1" +tree_hash_derive = "0.2" state_processing = { path = "../../eth2/state_processing" } swap_or_not_shuffle = { path = "../../eth2/utils/swap_or_not_shuffle" } types = { path = "../../eth2/types" } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index ed00f0ffee..279086b682 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -9,6 +9,7 @@ mod bls_g2_compressed; mod bls_g2_uncompressed; mod bls_priv_to_pub; mod bls_sign_msg; +mod common; mod epoch_processing; mod genesis_initialization; mod genesis_validity; @@ -25,6 +26,7 @@ pub use bls_g2_compressed::*; pub use bls_g2_uncompressed::*; pub use bls_priv_to_pub::*; pub use bls_sign_msg::*; +pub use common::SszStaticType; pub use epoch_processing::*; pub use genesis_initialization::*; pub use genesis_validity::*; @@ -61,20 +63,6 @@ pub trait Case: Debug + Sync { fn result(&self, case_index: usize) -> Result<(), Error>; } -pub trait BlsCase: serde::de::DeserializeOwned {} - -impl YamlDecode for T { - fn yaml_decode(string: &str) -> Result { - serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } -} - -impl LoadCase for T { - fn load_from_dir(path: &Path) -> Result { - Self::yaml_decode_file(&path.join("data.yaml")) - } -} - #[derive(Debug)] pub struct Cases { pub test_cases: Vec, diff --git a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs index c94e144952..13c2fea179 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{AggregatePublicKey, PublicKey}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs index 882ad7220a..22fa197df6 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{AggregateSignature, Signature}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/bls_g2_compressed.rs b/tests/ef_tests/src/cases/bls_g2_compressed.rs index f8381f5a7f..1a9f1d5617 100644 --- a/tests/ef_tests/src/cases/bls_g2_compressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_compressed.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{compress_g2, hash_on_g2}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/bls_g2_uncompressed.rs b/tests/ef_tests/src/cases/bls_g2_uncompressed.rs index 962b6aac39..3eae29967b 100644 --- a/tests/ef_tests/src/cases/bls_g2_uncompressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_uncompressed.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::hash_on_g2; use serde_derive::Deserialize; @@ -9,18 +10,14 @@ pub struct BlsG2UncompressedInput { pub domain: String, } +impl BlsCase for BlsG2UncompressedInput {} + #[derive(Debug, Clone, Deserialize)] pub struct BlsG2Uncompressed { pub input: BlsG2UncompressedInput, pub output: Vec>, } -impl YamlDecode for BlsG2Uncompressed { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - impl Case for BlsG2Uncompressed { fn result(&self, _case_index: usize) -> Result<(), Error> { // Convert message and domain to required types diff --git a/tests/ef_tests/src/cases/bls_priv_to_pub.rs b/tests/ef_tests/src/cases/bls_priv_to_pub.rs index 869a0891c7..016e04dd14 100644 --- a/tests/ef_tests/src/cases/bls_priv_to_pub.rs +++ b/tests/ef_tests/src/cases/bls_priv_to_pub.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{PublicKey, SecretKey}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/bls_sign_msg.rs b/tests/ef_tests/src/cases/bls_sign_msg.rs index 18e90896be..7ee109f812 100644 --- a/tests/ef_tests/src/cases/bls_sign_msg.rs +++ b/tests/ef_tests/src/cases/bls_sign_msg.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{SecretKey, Signature}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/common.rs b/tests/ef_tests/src/cases/common.rs new file mode 100644 index 0000000000..8e787f157c --- /dev/null +++ b/tests/ef_tests/src/cases/common.rs @@ -0,0 +1,72 @@ +use crate::cases::LoadCase; +use crate::decode::yaml_decode_file; +use crate::error::Error; +use serde_derive::Deserialize; +use ssz::{Decode, Encode}; +use ssz_derive::{Decode, Encode}; +use std::convert::TryFrom; +use std::fmt::Debug; +use std::path::Path; +use tree_hash::TreeHash; + +/// Trait for all BLS cases to eliminate some boilerplate. +pub trait BlsCase: serde::de::DeserializeOwned {} + +impl LoadCase for T { + fn load_from_dir(path: &Path) -> Result { + yaml_decode_file(&path.join("data.yaml")) + } +} + +/// Macro to wrap U128 and U256 so they deserialize correctly. +macro_rules! uint_wrapper { + ($wrapper_name:ident, $wrapped_type:ty) => { + #[derive(Debug, Clone, Copy, Default, PartialEq, Decode, Encode, Deserialize)] + #[serde(try_from = "String")] + pub struct $wrapper_name { + pub x: $wrapped_type, + } + + impl TryFrom for $wrapper_name { + type Error = String; + + fn try_from(s: String) -> Result { + <$wrapped_type>::from_dec_str(&s) + .map(|x| Self { x }) + .map_err(|e| format!("{:?}", e)) + } + } + + impl tree_hash::TreeHash for $wrapper_name { + fn tree_hash_type() -> tree_hash::TreeHashType { + <$wrapped_type>::tree_hash_type() + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.x.tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + <$wrapped_type>::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> Vec { + self.x.tree_hash_root() + } + } + }; +} + +uint_wrapper!(TestU128, ethereum_types::U128); +uint_wrapper!(TestU256, ethereum_types::U256); + +/// Trait alias for all deez bounds +pub trait SszStaticType: + serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync +{ +} + +impl SszStaticType for T where + T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync +{ +} diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs index d79b5fc48f..2a2dde629b 100644 --- a/tests/ef_tests/src/cases/epoch_processing.rs +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -1,9 +1,9 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use crate::type_name; use crate::type_name::TypeName; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_epoch_processing::{ errors::EpochProcessingError, process_crosslinks, process_final_updates, diff --git a/tests/ef_tests/src/cases/genesis_initialization.rs b/tests/ef_tests/src/cases/genesis_initialization.rs index 4f0fa42961..bd0507b9dd 100644 --- a/tests/ef_tests/src/cases/genesis_initialization.rs +++ b/tests/ef_tests/src/cases/genesis_initialization.rs @@ -1,6 +1,6 @@ use super::*; use crate::case_result::compare_beacon_state_results_without_caches; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::initialize_beacon_state_from_eth1; use std::path::PathBuf; diff --git a/tests/ef_tests/src/cases/genesis_validity.rs b/tests/ef_tests/src/cases/genesis_validity.rs index efebe5e110..3a1b9e2677 100644 --- a/tests/ef_tests/src/cases/genesis_validity.rs +++ b/tests/ef_tests/src/cases/genesis_validity.rs @@ -1,5 +1,5 @@ use super::*; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::is_valid_genesis_state; use std::path::{Path, PathBuf}; diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs index e86e6f5986..7b4ffff987 100644 --- a/tests/ef_tests/src/cases/operations.rs +++ b/tests/ef_tests/src/cases/operations.rs @@ -1,8 +1,8 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use crate::type_name::TypeName; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use ssz::Decode; use state_processing::per_block_processing::{ diff --git a/tests/ef_tests/src/cases/sanity_blocks.rs b/tests/ef_tests/src/cases/sanity_blocks.rs index d88d8f2955..9fadea42e2 100644 --- a/tests/ef_tests/src/cases/sanity_blocks.rs +++ b/tests/ef_tests/src/cases/sanity_blocks.rs @@ -1,7 +1,7 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::{ per_block_processing, per_slot_processing, BlockInvalid, BlockProcessingError, diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index a66f1c2c41..34acb1105a 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -1,7 +1,7 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_slot_processing; use std::path::PathBuf; diff --git a/tests/ef_tests/src/cases/shuffling.rs b/tests/ef_tests/src/cases/shuffling.rs index c0595e5843..2fe632e84d 100644 --- a/tests/ef_tests/src/cases/shuffling.rs +++ b/tests/ef_tests/src/cases/shuffling.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::decode::yaml_decode_file; use serde_derive::Deserialize; use std::marker::PhantomData; use swap_or_not_shuffle::{get_permutated_index, shuffle_list}; @@ -13,15 +14,9 @@ pub struct Shuffling { _phantom: PhantomData, } -impl YamlDecode for Shuffling { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - impl LoadCase for Shuffling { fn load_from_dir(path: &Path) -> Result { - Self::yaml_decode_file(&path.join("mapping.yaml")) + yaml_decode_file(&path.join("mapping.yaml")) } } diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index 05b96ad7db..5f9cd3faf0 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -1,11 +1,12 @@ use super::*; -use crate::cases::ssz_static::{check_serialization, check_tree_hash, SszStaticType}; -use crate::yaml_decode::yaml_decode_file; +use crate::cases::common::{SszStaticType, TestU128, TestU256}; +use crate::cases::ssz_static::{check_serialization, check_tree_hash}; +use crate::decode::yaml_decode_file; use serde_derive::Deserialize; use std::fs; use std::path::{Path, PathBuf}; use types::typenum::*; -use types::{BitList, BitVector, FixedVector, VariableList}; +use types::{BitList, BitVector, FixedVector}; #[derive(Debug, Clone, Deserialize)] struct Metadata { @@ -51,9 +52,8 @@ macro_rules! type_dispatch { "uint16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u16>, $($rest)*), "uint32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u32>, $($rest)*), "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*), - // FIXME(michael): implement tree hash for big ints - // "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* etherum_types::U128>, $($rest)*), - // "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ethereum_types::U256>, $($rest)*), + "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU128>, $($rest)*), + "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU256>, $($rest)*), _ => { println!("unsupported: {}", $value); Ok(()) }, } }; @@ -121,9 +121,13 @@ impl Case for SszGeneric { )?; } "bitlist" => { - let limit = parts[1]; + let mut limit = parts[1]; - // FIXME(michael): mark length "no" cases as known failures + // Test format is inconsistent, pretend the limit is 32 (arbitrary) + // https://github.com/ethereum/eth2.0-spec-tests + if limit == "no" { + limit = "32"; + } type_dispatch!( ssz_generic_test, diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index f9f59cc4b1..d1c9b1048c 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -1,10 +1,10 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::SszStaticType; +use crate::decode::yaml_decode_file; use serde_derive::Deserialize; -use ssz::{Decode, Encode}; -use std::fmt::Debug; use std::fs; -use tree_hash::{SignedRoot, TreeHash}; +use tree_hash::SignedRoot; use types::Hash256; #[derive(Debug, Clone, Deserialize)] @@ -13,12 +13,6 @@ struct SszStaticRoots { signing_root: Option, } -impl YamlDecode for SszStaticRoots { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - #[derive(Debug, Clone)] pub struct SszStatic { roots: SszStaticRoots, @@ -33,26 +27,11 @@ pub struct SszStaticSR { value: T, } -// Trait alias for all deez bounds -pub trait SszStaticType: - serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync -{ -} - -impl SszStaticType for T where - T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync -{ -} - fn load_from_dir(path: &Path) -> Result<(SszStaticRoots, Vec, T), Error> { - // FIXME: set description/name - let roots = SszStaticRoots::yaml_decode_file(&path.join("roots.yaml"))?; - + // FIXME(michael): set description/name + let roots = yaml_decode_file(&path.join("roots.yaml"))?; let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); - - let yaml = fs::read_to_string(&path.join("value.yaml")).expect("value.yaml exists"); - let value = - serde_yaml::from_str(&yaml).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; + let value = yaml_decode_file(&path.join("value.yaml"))?; Ok((roots, serialized, value)) } diff --git a/tests/ef_tests/src/decode.rs b/tests/ef_tests/src/decode.rs new file mode 100644 index 0000000000..c1ea6fb3b9 --- /dev/null +++ b/tests/ef_tests/src/decode.rs @@ -0,0 +1,31 @@ +use super::*; +use std::fs; +use std::path::Path; + +pub fn yaml_decode(string: &str) -> Result { + serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) +} + +pub fn yaml_decode_file(path: &Path) -> Result { + fs::read_to_string(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| yaml_decode(&s)) +} + +pub fn ssz_decode_file(path: &Path) -> Result { + fs::read(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| { + T::from_ssz_bytes(&s).map_err(|e| { + Error::FailedToParseTest(format!( + "Unable to parse SSZ at {}: {:?}", + path.display(), + e + )) + }) + }) +} diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index bcf7c77a04..719bfc1aaf 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -7,13 +7,12 @@ pub use cases::{ }; pub use error::Error; pub use handler::*; -pub use yaml_decode::YamlDecode; mod bls_setting; mod case_result; mod cases; +mod decode; mod error; mod handler; mod results; mod type_name; -mod yaml_decode; diff --git a/tests/ef_tests/src/results.rs b/tests/ef_tests/src/results.rs index 20e59f7b3e..4f5513a9ae 100644 --- a/tests/ef_tests/src/results.rs +++ b/tests/ef_tests/src/results.rs @@ -80,7 +80,8 @@ pub fn print_results( println!("-------"); println!( - "case ({}) from {} failed with {}:", + "case {} ({}) from {} failed with {}:", + failure.case_index, failure.desc, failure.path.display(), error.name() diff --git a/tests/ef_tests/src/yaml_decode.rs b/tests/ef_tests/src/yaml_decode.rs deleted file mode 100644 index 83a162930c..0000000000 --- a/tests/ef_tests/src/yaml_decode.rs +++ /dev/null @@ -1,93 +0,0 @@ -use super::*; -use ethereum_types::{U128, U256}; -use std::fs; -use std::path::Path; -use types::Fork; - -pub fn yaml_decode(string: &str) -> Result { - serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) -} - -pub fn yaml_decode_file(path: &Path) -> Result { - fs::read_to_string(path) - .map_err(|e| { - Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) - }) - .and_then(|s| yaml_decode(&s)) -} - -pub fn ssz_decode_file(path: &Path) -> Result { - fs::read(path) - .map_err(|e| { - Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) - }) - .and_then(|s| { - T::from_ssz_bytes(&s).map_err(|e| { - Error::FailedToParseTest(format!( - "Unable to parse SSZ at {}: {:?}", - path.display(), - e - )) - }) - }) -} - -pub trait YamlDecode: Sized { - /// Decode an object from the test specification YAML. - fn yaml_decode(string: &str) -> Result; - - fn yaml_decode_file(path: &Path) -> Result { - fs::read_to_string(path) - .map_err(|e| { - Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) - }) - .and_then(|s| Self::yaml_decode(&s)) - } -} - -/// Basic types can general be decoded with the `parse` fn if they implement `str::FromStr`. -macro_rules! impl_via_parse { - ($ty: ty) => { - impl YamlDecode for $ty { - fn yaml_decode(string: &str) -> Result { - string - .parse::() - .map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } - } - }; -} - -impl_via_parse!(u8); -impl_via_parse!(u16); -impl_via_parse!(u32); -impl_via_parse!(u64); - -/// Some `ethereum-types` methods have a `str::FromStr` implementation that expects `0x`-prefixed: -/// hex, so we use `from_dec_str` instead. -macro_rules! impl_via_from_dec_str { - ($ty: ty) => { - impl YamlDecode for $ty { - fn yaml_decode(string: &str) -> Result { - Self::from_dec_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } - } - }; -} - -impl_via_from_dec_str!(U128); -impl_via_from_dec_str!(U256); - -/// Types that already implement `serde::Deserialize` can be decoded using `serde_yaml`. -macro_rules! impl_via_serde_yaml { - ($ty: ty) => { - impl YamlDecode for $ty { - fn yaml_decode(string: &str) -> Result { - serde_yaml::from_str(string) - .map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } - } - }; -} - -impl_via_serde_yaml!(Fork); From 3b40b691ab31e6c49d047bb8c705fcfa96b79310 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 4 Sep 2019 15:58:51 +1000 Subject: [PATCH 15/32] Download ENR during bootstrap --- beacon_node/src/config.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 6a13a9aaec..cf56169384 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -137,6 +137,7 @@ fn process_testnet_subcommand( .and_then(|s| s.parse::().ok()); builder.import_bootstrap_libp2p_address(server, port)?; + builder.import_bootstrap_enr_address(server)?; builder.import_bootstrap_eth2_config(server)?; builder.set_beacon_chain_start_method(BeaconChainStartMethod::HttpBootstrap { @@ -301,7 +302,7 @@ impl<'a> ConfigBuilder<'a> { self.client_config.eth1_backend_method = method; } - /// Import the libp2p address for `server` into the list of bootnodes in `self`. + /// Import the libp2p address for `server` into the list of libp2p nodes to connect with. /// /// If `port` is `Some`, it is used as the port for the `Multiaddr`. If `port` is `None`, /// attempts to connect to the `server` via HTTP and retrieve it's libp2p listen port. @@ -333,6 +334,28 @@ impl<'a> ConfigBuilder<'a> { Ok(()) } + /// Import the enr address for `server` into the list of initial enrs (boot nodes). + pub fn import_bootstrap_enr_address(&mut self, server: &str) -> Result<()> { + let bootstrapper = Bootstrapper::connect(server.to_string(), &self.log)?; + + if let Ok(enr) = bootstrapper.enr() { + info!( + self.log, + "Loaded bootstrapper libp2p address"; + "enr" => format!("{:?}", enr) + ); + + self.client_config.network.boot_nodes.push(enr); + } else { + warn!( + self.log, + "Unable to estimate a bootstrapper enr address, this node may not find any peers." + ); + }; + + Ok(()) + } + /// Set the config data_dir to be an random directory. /// /// Useful for easily spinning up ephemeral testnets. From e7ab89a783f02b13a242d7ec528520c12a6e65d1 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 5 Sep 2019 02:06:39 +1000 Subject: [PATCH 16/32] Adds gossipsub object validation and verification --- beacon_node/eth2-libp2p/Cargo.toml | 4 +- beacon_node/eth2-libp2p/src/behaviour.rs | 28 +++++++++-- beacon_node/eth2-libp2p/src/config.rs | 3 +- beacon_node/eth2-libp2p/src/discovery.rs | 1 + beacon_node/eth2-libp2p/src/service.rs | 5 +- beacon_node/network/src/message_handler.rs | 49 +++++++++++++++---- beacon_node/network/src/service.rs | 53 ++++++++++++--------- beacon_node/network/src/sync/simple_sync.rs | 10 ++-- 8 files changed, 106 insertions(+), 47 deletions(-) diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index caa5b28e43..59c7991053 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -7,8 +7,8 @@ edition = "2018" [dependencies] clap = "2.32.0" #SigP repository -libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "61036890d574f5b46573952b20def2baafd6a6e9" } -enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "61036890d574f5b46573952b20def2baafd6a6e9", features = ["serde"] } +libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "76f7475e4b7063e663ad03c7524cf091f9961968" } +enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "76f7475e4b7063e663ad03c7524cf091f9961968", features = ["serde"] } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index 2c574e46ad..a47d32ec2a 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -15,7 +15,7 @@ use libp2p::{ tokio_io::{AsyncRead, AsyncWrite}, NetworkBehaviour, PeerId, }; -use slog::{debug, o, trace}; +use slog::{debug, o}; use std::num::NonZeroU32; use std::time::Duration; @@ -90,13 +90,15 @@ impl NetworkBehaviourEventProcess { - trace!(self.log, "Received GossipEvent"); - + GossipsubEvent::Message(propagation_source, gs_msg) => { + let id = gs_msg.id(); let msg = PubsubMessage::from_topics(&gs_msg.topics, gs_msg.data); + // Note: We are keeping track here of the peer that sent us the message, not the + // peer that originally published the message. self.events.push(BehaviourEvent::GossipMessage { - source: gs_msg.source, + id, + source: propagation_source, topics: gs_msg.topics, message: msg, }); @@ -199,6 +201,13 @@ impl Behaviour { } } + /// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated + /// once validated by the beacon chain. + pub fn propagate_message(&mut self, propagation_source: &PeerId, message_id: String) { + self.gossipsub + .propagate_message(&message_id, propagation_source); + } + /* Eth2 RPC behaviour functions */ /// Sends an RPC Request/Response via the RPC protocol. @@ -214,12 +223,21 @@ impl Behaviour { /// The types of events than can be obtained from polling the behaviour. pub enum BehaviourEvent { + /// A received RPC event and the peer that it was received from. RPC(PeerId, RPCEvent), + /// We have completed an initial connection to a new peer. PeerDialed(PeerId), + /// A peer has disconnected. PeerDisconnected(PeerId), + /// A gossipsub message has been received. GossipMessage { + /// The gossipsub message id. Used when propagating blocks after validation. + id: String, + /// The peer from which we received this message, not the peer that published it. source: PeerId, + /// The topics that this message was sent on. topics: Vec, + /// The message itself. message: PubsubMessage, }, } diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index 7cb501c1f0..fd44b99af1 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -74,7 +74,8 @@ impl Default for Config { // parameter. gs_config: GossipsubConfigBuilder::new() .max_transmit_size(1_048_576) - .heartbeat_interval(Duration::from_secs(20)) + .heartbeat_interval(Duration::from_secs(20)) // TODO: Reduce for mainnet + .propagate_messages(false) // require validation before propagation .build(), boot_nodes: vec![], libp2p_nodes: vec![], diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index 4a8aba2b1b..759adc482a 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -169,6 +169,7 @@ where fn inject_connected(&mut self, peer_id: PeerId, _endpoint: ConnectedPoint) { self.connected_peers.insert(peer_id); + // TODO: Drop peers if over max_peer limit metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); metrics::set_gauge(&metrics::PEERS_CONNECTED, self.connected_peers() as i64); diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 34781927ca..3559fb8502 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -145,16 +145,16 @@ impl Stream for Service { fn poll(&mut self) -> Poll, Self::Error> { loop { match self.swarm.poll() { - //Behaviour events Ok(Async::Ready(Some(event))) => match event { - // TODO: Stub here for debugging BehaviourEvent::GossipMessage { + id, source, topics, message, } => { trace!(self.log, "Gossipsub message received"; "service" => "Swarm"); return Ok(Async::Ready(Some(Libp2pEvent::PubsubMessage { + id, source, topics, message, @@ -222,6 +222,7 @@ pub enum Libp2pEvent { PeerDisconnected(PeerId), /// Received pubsub message. PubsubMessage { + id: String, source: PeerId, topics: Vec, message: PubsubMessage, diff --git a/beacon_node/network/src/message_handler.rs b/beacon_node/network/src/message_handler.rs index c14fc970d7..d6e9f8be87 100644 --- a/beacon_node/network/src/message_handler.rs +++ b/beacon_node/network/src/message_handler.rs @@ -21,6 +21,8 @@ pub struct MessageHandler { _chain: Arc>, /// The syncing framework. sync: SimpleSync, + /// A channel to the network service to allow for gossip propagation. + network_send: mpsc::UnboundedSender, /// The `MessageHandler` logger. log: slog::Logger, } @@ -34,8 +36,9 @@ pub enum HandlerMessage { PeerDisconnected(PeerId), /// An RPC response/request has been received. RPC(PeerId, RPCEvent), - /// A gossip message has been received. - PubsubMessage(PeerId, PubsubMessage), + /// A gossip message has been received. The fields are: message id, the peer that sent us this + /// message and the message itself. + PubsubMessage(String, PeerId, PubsubMessage), } impl MessageHandler { @@ -50,12 +53,13 @@ impl MessageHandler { let (handler_send, handler_recv) = mpsc::unbounded_channel(); // Initialise sync and begin processing in thread - let sync = SimpleSync::new(beacon_chain.clone(), network_send, &log); + let sync = SimpleSync::new(beacon_chain.clone(), network_send.clone(), &log); // generate the Message handler let mut handler = MessageHandler { _chain: beacon_chain.clone(), sync, + network_send, log: log.clone(), }; @@ -87,8 +91,8 @@ impl MessageHandler { self.handle_rpc_message(peer_id, rpc_event); } // An RPC message request/response has been received - HandlerMessage::PubsubMessage(peer_id, gossip) => { - self.handle_gossip(peer_id, gossip); + HandlerMessage::PubsubMessage(id, peer_id, gossip) => { + self.handle_gossip(id, peer_id, gossip); } } } @@ -194,24 +198,34 @@ impl MessageHandler { } /// Handle RPC messages - fn handle_gossip(&mut self, peer_id: PeerId, gossip_message: PubsubMessage) { + fn handle_gossip(&mut self, id: String, peer_id: PeerId, gossip_message: PubsubMessage) { match gossip_message { PubsubMessage::Block(message) => match self.decode_gossip_block(message) { Ok(block) => { - let _should_forward_on = self.sync.on_block_gossip(peer_id, block); + let should_forward_on = self.sync.on_block_gossip(peer_id.clone(), block); + // TODO: Apply more sophisticated validation and decoding logic + if should_forward_on { + self.propagate_message(id, peer_id.clone()); + } } Err(e) => { debug!(self.log, "Invalid gossiped beacon block"; "peer_id" => format!("{}", peer_id), "Error" => format!("{:?}", e)); } }, PubsubMessage::Attestation(message) => match self.decode_gossip_attestation(message) { - Ok(attestation) => self.sync.on_attestation_gossip(peer_id, attestation), + Ok(attestation) => { + // TODO: Apply more sophisticated validation and decoding logic + self.propagate_message(id, peer_id.clone()); + self.sync.on_attestation_gossip(peer_id, attestation); + } Err(e) => { debug!(self.log, "Invalid gossiped attestation"; "peer_id" => format!("{}", peer_id), "Error" => format!("{:?}", e)); } }, PubsubMessage::VoluntaryExit(message) => match self.decode_gossip_exit(message) { Ok(_exit) => { + // TODO: Apply more sophisticated validation and decoding logic + self.propagate_message(id, peer_id.clone()); // TODO: Handle exits debug!(self.log, "Received a voluntary exit"; "peer_id" => format!("{}", peer_id) ); } @@ -222,6 +236,8 @@ impl MessageHandler { PubsubMessage::ProposerSlashing(message) => { match self.decode_gossip_proposer_slashing(message) { Ok(_slashing) => { + // TODO: Apply more sophisticated validation and decoding logic + self.propagate_message(id, peer_id.clone()); // TODO: Handle proposer slashings debug!(self.log, "Received a proposer slashing"; "peer_id" => format!("{}", peer_id) ); } @@ -233,6 +249,8 @@ impl MessageHandler { PubsubMessage::AttesterSlashing(message) => { match self.decode_gossip_attestation_slashing(message) { Ok(_slashing) => { + // TODO: Apply more sophisticated validation and decoding logic + self.propagate_message(id, peer_id.clone()); // TODO: Handle attester slashings debug!(self.log, "Received an attester slashing"; "peer_id" => format!("{}", peer_id) ); } @@ -248,6 +266,21 @@ impl MessageHandler { } } + /// Informs the network service that the message should be forwarded to other peers. + fn propagate_message(&mut self, message_id: String, propagation_source: PeerId) { + self.network_send + .try_send(NetworkMessage::Propagate { + propagation_source, + message_id, + }) + .unwrap_or_else(|_| { + warn!( + self.log, + "Could not send propagation request to the network service" + ) + }); + } + /* Decoding of gossipsub objects from the network. * * The decoding is done in the message handler as it has access to to a `BeaconChain` and can diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index a8b3c74b6a..5336c71189 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -159,12 +159,23 @@ fn network_service( // poll the network channel match network_recv.poll() { Ok(Async::Ready(Some(message))) => match message { - NetworkMessage::Send(peer_id, outgoing_message) => match outgoing_message { - OutgoingMessage::RPC(rpc_event) => { - trace!(log, "Sending RPC Event: {:?}", rpc_event); - libp2p_service.lock().swarm.send_rpc(peer_id, rpc_event); - } - }, + NetworkMessage::RPC(peer_id, rpc_event) => { + trace!(log, "Sending RPC Event: {:?}", rpc_event); + libp2p_service.lock().swarm.send_rpc(peer_id, rpc_event); + } + NetworkMessage::Propagate { + propagation_source, + message_id, + } => { + trace!(log, "Propagating gossipsub message"; + "propagation_peer" => format!("{:?}", propagation_source), + "message_id" => format!("{}", message_id), + ); + libp2p_service + .lock() + .swarm + .propagate_message(&propagation_source, message_id); + } NetworkMessage::Publish { topics, message } => { debug!(log, "Sending pubsub message"; "topics" => format!("{:?}",topics)); libp2p_service.lock().swarm.publish(&topics, message); @@ -203,13 +214,14 @@ fn network_service( .map_err(|_| "Failed to send PeerDisconnected to handler")?; } Libp2pEvent::PubsubMessage { - source, message, .. + id, + source, + message, + .. } => { - //TODO: Decide if we need to propagate the topic upwards. (Potentially for - //attestations) message_handler_send - .try_send(HandlerMessage::PubsubMessage(source, message)) - .map_err(|_| " failed to send pubsub message to handler")?; + .try_send(HandlerMessage::PubsubMessage(id, source, message)) + .map_err(|_| "Failed to send pubsub message to handler")?; } }, Ok(Async::Ready(None)) => unreachable!("Stream never ends"), @@ -225,19 +237,16 @@ fn network_service( /// Types of messages that the network service can receive. #[derive(Debug)] pub enum NetworkMessage { - /// Send a message to libp2p service. - //TODO: Define typing for messages across the wire - Send(PeerId, OutgoingMessage), - /// Publish a message to pubsub mechanism. + /// Send an RPC message to the libp2p service. + RPC(PeerId, RPCEvent), + /// Publish a message to gossipsub. Publish { topics: Vec, message: PubsubMessage, }, -} - -/// Type of outgoing messages that can be sent through the network service. -#[derive(Debug)] -pub enum OutgoingMessage { - /// Send an RPC request/response. - RPC(RPCEvent), + /// Propagate a received gossipsub message + Propagate { + propagation_source: PeerId, + message_id: String, + }, } diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 573ac9dd1f..789f5b6be5 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -1,5 +1,5 @@ use super::manager::{ImportManager, ImportManagerOutcome}; -use crate::service::{NetworkMessage, OutgoingMessage}; +use crate::service::NetworkMessage; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCEvent, RPCRequest, RPCResponse, RequestId}; @@ -468,7 +468,7 @@ impl SimpleSync { SHOULD_FORWARD_GOSSIP_BLOCK } BlockProcessingOutcome::BlockIsAlreadyKnown => SHOULD_FORWARD_GOSSIP_BLOCK, - _ => SHOULD_NOT_FORWARD_GOSSIP_BLOCK, + _ => SHOULD_NOT_FORWARD_GOSSIP_BLOCK, //TODO: Decide if we want to forward these } } else { SHOULD_NOT_FORWARD_GOSSIP_BLOCK @@ -554,12 +554,8 @@ impl NetworkContext { } fn send_rpc_event(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { - self.send(peer_id, OutgoingMessage::RPC(rpc_event)) - } - - fn send(&mut self, peer_id: PeerId, outgoing_message: OutgoingMessage) { self.network_send - .try_send(NetworkMessage::Send(peer_id, outgoing_message)) + .try_send(NetworkMessage::RPC(peer_id, rpc_event)) .unwrap_or_else(|_| { warn!( self.log, From a3877b6135272a29a0d0e43f5a36f4c43d73a5ab Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 5 Sep 2019 08:07:57 +1000 Subject: [PATCH 17/32] Updates syncing stability, fixes large RPC message codec, corrects beacon chain referencing --- beacon_node/client/src/notifier.rs | 4 +- beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs | 36 ++- beacon_node/eth2-libp2p/src/rpc/methods.rs | 6 +- beacon_node/eth2-libp2p/src/service.rs | 5 + beacon_node/network/src/message_handler.rs | 7 +- beacon_node/network/src/sync/manager.rs | 211 +++++++------ beacon_node/network/src/sync/simple_sync.rs | 304 +++++++++++-------- 7 files changed, 312 insertions(+), 261 deletions(-) diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index d705637cba..343918d4d5 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -34,10 +34,10 @@ pub fn run(client: &Client, executor: TaskExecutor, exit // Panics if libp2p is poisoned. let connected_peer_count = libp2p.lock().swarm.connected_peers(); - debug!(log, "Libp2p connected peer status"; "peer_count" => connected_peer_count); + debug!(log, "Connected peer status"; "peer_count" => connected_peer_count); if connected_peer_count <= WARN_PEER_COUNT { - warn!(log, "Low libp2p peer count"; "peer_count" => connected_peer_count); + warn!(log, "Low peer count"; "peer_count" => connected_peer_count); } Ok(()) diff --git a/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs b/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs index 260a00346c..1966bab628 100644 --- a/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs +++ b/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs @@ -152,45 +152,49 @@ impl Decoder for SSZOutboundCodec { type Error = RPCError; fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { - match self.inner.decode(src).map_err(RPCError::from) { - Ok(Some(packet)) => match self.protocol.message_name.as_str() { + if src.is_empty() { + // the object sent could be empty. We return the empty object if this is the case + match self.protocol.message_name.as_str() { "hello" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::Hello(HelloMessage::from_ssz_bytes( - &packet, - )?))), + "1" => Err(RPCError::Custom( + "Hello stream terminated unexpectedly".into(), + )), // cannot have an empty HELLO message. The stream has terminated unexpectedly _ => unreachable!("Cannot negotiate an unknown version"), }, "goodbye" => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")), "beacon_blocks" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::BeaconBlocks(packet.to_vec()))), + "1" => Ok(Some(RPCResponse::BeaconBlocks(Vec::new()))), _ => unreachable!("Cannot negotiate an unknown version"), }, "recent_beacon_blocks" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(packet.to_vec()))), + "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(Vec::new()))), _ => unreachable!("Cannot negotiate an unknown version"), }, _ => unreachable!("Cannot negotiate an unknown protocol"), - }, - Ok(None) => { - // the object sent could be a empty. We return the empty object if this is the case - match self.protocol.message_name.as_str() { + } + } else { + match self.inner.decode(src).map_err(RPCError::from) { + Ok(Some(packet)) => match self.protocol.message_name.as_str() { "hello" => match self.protocol.version.as_str() { - "1" => Ok(None), // cannot have an empty HELLO message. The stream has terminated unexpectedly + "1" => Ok(Some(RPCResponse::Hello(HelloMessage::from_ssz_bytes( + &packet, + )?))), _ => unreachable!("Cannot negotiate an unknown version"), }, "goodbye" => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")), "beacon_blocks" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::BeaconBlocks(Vec::new()))), + "1" => Ok(Some(RPCResponse::BeaconBlocks(packet.to_vec()))), _ => unreachable!("Cannot negotiate an unknown version"), }, "recent_beacon_blocks" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(Vec::new()))), + "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(packet.to_vec()))), _ => unreachable!("Cannot negotiate an unknown version"), }, _ => unreachable!("Cannot negotiate an unknown protocol"), - } + }, + Ok(None) => Ok(None), // waiting for more bytes + Err(e) => Err(e), } - Err(e) => Err(e), } } } diff --git a/beacon_node/eth2-libp2p/src/rpc/methods.rs b/beacon_node/eth2-libp2p/src/rpc/methods.rs index c9610b000e..49813abe9a 100644 --- a/beacon_node/eth2-libp2p/src/rpc/methods.rs +++ b/beacon_node/eth2-libp2p/src/rpc/methods.rs @@ -168,8 +168,10 @@ impl std::fmt::Display for RPCResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { RPCResponse::Hello(hello) => write!(f, "{}", hello), - RPCResponse::BeaconBlocks(_) => write!(f, ""), - RPCResponse::RecentBeaconBlocks(_) => write!(f, ""), + RPCResponse::BeaconBlocks(data) => write!(f, ", len: {}", data.len()), + RPCResponse::RecentBeaconBlocks(data) => { + write!(f, ", len: {}", data.len()) + } } } } diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 9f08b1eda6..dac0117528 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -98,6 +98,11 @@ impl Service { // attempt to connect to any specified boot-nodes for bootnode_enr in config.boot_nodes { for multiaddr in bootnode_enr.multiaddr() { + // ignore udp multiaddr if it exists + let components = multiaddr.iter().collect::>(); + if let Protocol::Udp(_) = components[1] { + continue; + } dial_addr(multiaddr); } } diff --git a/beacon_node/network/src/message_handler.rs b/beacon_node/network/src/message_handler.rs index d6e9f8be87..cade65d63a 100644 --- a/beacon_node/network/src/message_handler.rs +++ b/beacon_node/network/src/message_handler.rs @@ -17,8 +17,6 @@ use types::{Attestation, AttesterSlashing, BeaconBlock, ProposerSlashing, Volunt /// Handles messages received from the network and client and organises syncing. pub struct MessageHandler { - /// Currently loaded and initialised beacon chain. - _chain: Arc>, /// The syncing framework. sync: SimpleSync, /// A channel to the network service to allow for gossip propagation. @@ -53,13 +51,12 @@ impl MessageHandler { let (handler_send, handler_recv) = mpsc::unbounded_channel(); // Initialise sync and begin processing in thread - let sync = SimpleSync::new(beacon_chain.clone(), network_send.clone(), &log); + let sync = SimpleSync::new(Arc::downgrade(&beacon_chain), network_send.clone(), &log); // generate the Message handler let mut handler = MessageHandler { - _chain: beacon_chain.clone(), - sync, network_send, + sync, log: log.clone(), }; diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 1eec518430..2b2ed9dcae 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -62,13 +62,13 @@ use slog::{debug, info, trace, warn, Logger}; use smallvec::SmallVec; use std::collections::{HashMap, HashSet}; use std::ops::{Add, Sub}; -use std::sync::{Arc, Weak}; +use std::sync::Weak; use types::{BeaconBlock, EthSpec, Hash256, Slot}; /// Blocks are downloaded in batches from peers. This constant specifies how many blocks per batch /// is requested. Currently the value is small for testing. This will be incremented for /// production. -const MAX_BLOCKS_PER_REQUEST: u64 = 100; +const MAX_BLOCKS_PER_REQUEST: u64 = 50; /// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync /// from a peer. If a peer is within this tolerance (forwards or backwards), it is treated as a @@ -224,10 +224,10 @@ impl ImportManager { /// Generates a new `ImportManager` given a logger and an Arc reference to a beacon chain. The /// import manager keeps a weak reference to the beacon chain, which allows the chain to be /// dropped during the syncing process. The syncing handles this termination gracefully. - pub fn new(beacon_chain: Arc>, log: &slog::Logger) -> Self { + pub fn new(beacon_chain: Weak>, log: &slog::Logger) -> Self { ImportManager { event_queue: SmallVec::new(), - chain: Arc::downgrade(&beacon_chain), + chain: beacon_chain, state: ManagerState::Regular, import_queue: HashMap::new(), parent_queue: SmallVec::new(), @@ -359,7 +359,9 @@ impl ImportManager { warn!(self.log, "Peer returned too many empty block batches"; "peer" => format!("{:?}", peer_id)); block_requests.state = BlockRequestsState::Failed; - } else if block_requests.current_start_slot >= block_requests.target_head_slot { + } else if block_requests.current_start_slot + MAX_BLOCKS_PER_REQUEST + >= block_requests.target_head_slot + { warn!(self.log, "Peer did not return blocks it claimed to possess"; "peer" => format!("{:?}", peer_id)); block_requests.state = BlockRequestsState::Failed; @@ -583,6 +585,11 @@ impl ImportManager { re_run = re_run || self.process_complete_parent_requests(); } + // exit early if the beacon chain is dropped + if let None = self.chain.upgrade() { + return ImportManagerOutcome::Idle; + } + // return any queued events if !self.event_queue.is_empty() { let event = self.event_queue.remove(0); @@ -681,56 +688,48 @@ impl ImportManager { self.import_queue.retain(|peer_id, block_requests| { if block_requests.state == BlockRequestsState::ReadyToProcess { - // check that the chain still exists - if let Some(chain) = chain_ref.upgrade() { - let downloaded_blocks = - std::mem::replace(&mut block_requests.downloaded_blocks, Vec::new()); - let last_element = downloaded_blocks.len() - 1; - let start_slot = downloaded_blocks[0].slot; - let end_slot = downloaded_blocks[last_element].slot; + let downloaded_blocks = + std::mem::replace(&mut block_requests.downloaded_blocks, Vec::new()); + let last_element = downloaded_blocks.len() - 1; + let start_slot = downloaded_blocks[0].slot; + let end_slot = downloaded_blocks[last_element].slot; - match process_blocks(chain, downloaded_blocks, log_ref) { - Ok(()) => { - debug!(log_ref, "Blocks processed successfully"; + match process_blocks(chain_ref.clone(), downloaded_blocks, log_ref) { + Ok(()) => { + debug!(log_ref, "Blocks processed successfully"; + "peer" => format!("{:?}", peer_id), + "start_slot" => start_slot, + "end_slot" => end_slot, + "no_blocks" => last_element + 1, + ); + block_requests.blocks_processed += last_element + 1; + + // check if the batch is complete, by verifying if we have reached the + // target head + if end_slot >= block_requests.target_head_slot { + // Completed, re-hello the peer to ensure we are up to the latest head + event_queue_ref.push(ImportManagerOutcome::Hello(peer_id.clone())); + // remove the request + false + } else { + // have not reached the end, queue another batch + block_requests.update_start_slot(); + re_run = true; + // keep the batch + true + } + } + Err(e) => { + warn!(log_ref, "Block processing failed"; "peer" => format!("{:?}", peer_id), "start_slot" => start_slot, "end_slot" => end_slot, "no_blocks" => last_element + 1, - ); - block_requests.blocks_processed += last_element + 1; - - // check if the batch is complete, by verifying if we have reached the - // target head - if end_slot >= block_requests.target_head_slot { - // Completed, re-hello the peer to ensure we are up to the latest head - event_queue_ref.push(ImportManagerOutcome::Hello(peer_id.clone())); - // remove the request - false - } else { - // have not reached the end, queue another batch - block_requests.update_start_slot(); - re_run = true; - // keep the batch - true - } - } - Err(e) => { - warn!(log_ref, "Block processing failed"; - "peer" => format!("{:?}", peer_id), - "start_slot" => start_slot, - "end_slot" => end_slot, - "no_blocks" => last_element + 1, - "error" => format!("{:?}", e), - ); - event_queue_ref - .push(ImportManagerOutcome::DownvotePeer(peer_id.clone())); - false - } + "error" => format!("{:?}", e), + ); + event_queue_ref.push(ImportManagerOutcome::DownvotePeer(peer_id.clone())); + false } - } else { - // chain no longer exists, empty the queue and return - event_queue_ref.clear(); - return false; } } else { // not ready to process @@ -894,42 +893,43 @@ impl ImportManager { // Helper function to process blocks fn process_blocks( - chain: Arc>, + weak_chain: Weak>, blocks: Vec>, log: &Logger, ) -> Result<(), String> { for block in blocks { - let processing_result = chain.process_block(block.clone()); + if let Some(chain) = weak_chain.upgrade() { + let processing_result = chain.process_block(block.clone()); - if let Ok(outcome) = processing_result { - match outcome { - BlockProcessingOutcome::Processed { block_root } => { - // The block was valid and we processed it successfully. - trace!( - log, "Imported block from network"; - "slot" => block.slot, - "block_root" => format!("{}", block_root), - ); - } - BlockProcessingOutcome::ParentUnknown { parent } => { - // blocks should be sequential and all parents should exist - trace!( - log, "Parent block is unknown"; - "parent_root" => format!("{}", parent), - "baby_block_slot" => block.slot, - ); - return Err(format!( - "Block at slot {} has an unknown parent.", - block.slot - )); - } - BlockProcessingOutcome::BlockIsAlreadyKnown => { - // this block is already known to us, move to the next - debug!( - log, "Imported a block that is already known"; - "parent_root" => format!("{}", parent), - "baby_block_slot" => block.slot, - ); + if let Ok(outcome) = processing_result { + match outcome { + BlockProcessingOutcome::Processed { block_root } => { + // The block was valid and we processed it successfully. + trace!( + log, "Imported block from network"; + "slot" => block.slot, + "block_root" => format!("{}", block_root), + ); + } + BlockProcessingOutcome::ParentUnknown { parent } => { + // blocks should be sequential and all parents should exist + trace!( + log, "Parent block is unknown"; + "parent_root" => format!("{}", parent), + "baby_block_slot" => block.slot, + ); + return Err(format!( + "Block at slot {} has an unknown parent.", + block.slot + )); + } + BlockProcessingOutcome::BlockIsAlreadyKnown => { + // this block is already known to us, move to the next + debug!( + log, "Imported a block that is already known"; + "block_slot" => block.slot, + ); + } BlockProcessingOutcome::FutureSlot { present_slot, block_slot, @@ -937,7 +937,7 @@ fn process_blocks( if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot { // The block is too far in the future, drop it. trace!( - self.log, "Block is ahead of our slot clock"; + log, "Block is ahead of our slot clock"; "msg" => "block for future slot rejected, check your time", "present_slot" => present_slot, "block_slot" => block_slot, @@ -950,7 +950,7 @@ fn process_blocks( } else { // The block is in the future, but not too far. trace!( - self.log, "Block is slightly ahead of our slot clock, ignoring."; + log, "Block is slightly ahead of our slot clock, ignoring."; "present_slot" => present_slot, "block_slot" => block_slot, "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, @@ -959,44 +959,41 @@ fn process_blocks( } BlockProcessingOutcome::WouldRevertFinalizedSlot { .. } => { trace!( - self.log, "Finalized or earlier block processed"; + log, "Finalized or earlier block processed"; "outcome" => format!("{:?}", outcome), ); // block reached our finalized slot or was earlier, move to the next block } BlockProcessingOutcome::GenesisBlock => { trace!( - self.log, "Genesis block was processed"; + log, "Genesis block was processed"; "outcome" => format!("{:?}", outcome), ); } - BlockProcessingOutcome::FinalizedSlot => { - trace!( - log, "Finalized or earlier block processed"; - "outcome" => format!("{:?}", outcome), - ); - // block reached our finalized slot or was earlier, move to the next block - } - _ => { - warn!( - log, "Invalid block received"; - "msg" => "peer sent invalid block", - "outcome" => format!("{:?}", outcome), - ); - return Err(format!("Invalid block at slot {}", block.slot)); + _ => { + warn!( + log, "Invalid block received"; + "msg" => "peer sent invalid block", + "outcome" => format!("{:?}", outcome), + ); + return Err(format!("Invalid block at slot {}", block.slot)); + } } + } else { + warn!( + log, "BlockProcessingFailure"; + "msg" => "unexpected condition in processing block.", + "outcome" => format!("{:?}", processing_result) + ); + return Err(format!( + "Unexpected block processing error: {:?}", + processing_result + )); } } else { - warn!( - log, "BlockProcessingFailure"; - "msg" => "unexpected condition in processing block.", - "outcome" => format!("{:?}", processing_result) - ); - return Err(format!( - "Unexpected block processing error: {:?}", - processing_result - )); + return Ok(()); // terminate early due to dropped beacon chain } } + Ok(()) } diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 9d05b312b5..a8b271700c 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -6,7 +6,7 @@ use eth2_libp2p::rpc::{RPCEvent, RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; use slog::{debug, info, o, trace, warn}; use ssz::Encode; -use std::sync::Arc; +use std::sync::{Arc, Weak}; use store::Store; use tokio::sync::mpsc; use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256, Slot}; @@ -57,7 +57,7 @@ pub enum SyncState { /// Simple Syncing protocol. pub struct SimpleSync { /// A reference to the underlying beacon chain. - chain: Arc>, + chain: Weak>, manager: ImportManager, network: NetworkContext, log: slog::Logger, @@ -66,7 +66,7 @@ pub struct SimpleSync { impl SimpleSync { /// Instantiate a `SimpleSync` instance, with no peers and an empty queue. pub fn new( - beacon_chain: Arc>, + beacon_chain: Weak>, network_send: mpsc::UnboundedSender, log: &slog::Logger, ) -> Self { @@ -91,8 +91,10 @@ impl SimpleSync { /// /// Sends a `Hello` message to the peer. pub fn on_connect(&mut self, peer_id: PeerId) { - self.network - .send_rpc_request(None, peer_id, RPCRequest::Hello(hello_message(&self.chain))); + if let Some(chain) = self.chain.upgrade() { + self.network + .send_rpc_request(None, peer_id, RPCRequest::Hello(hello_message(&chain))); + } } /// Handle a `Hello` request. @@ -104,16 +106,19 @@ impl SimpleSync { request_id: RequestId, hello: HelloMessage, ) { - trace!(self.log, "HelloRequest"; "peer" => format!("{:?}", peer_id)); + // ignore hello responses if we are shutting down + if let Some(chain) = self.chain.upgrade() { + trace!(self.log, "HelloRequest"; "peer" => format!("{:?}", peer_id)); - // Say hello back. - self.network.send_rpc_response( - peer_id.clone(), - request_id, - RPCResponse::Hello(hello_message(&self.chain)), - ); + // Say hello back. + self.network.send_rpc_response( + peer_id.clone(), + request_id, + RPCResponse::Hello(hello_message(&chain)), + ); - self.process_hello(peer_id, hello); + self.process_hello(peer_id, hello); + } } /// Process a `Hello` response from a peer. @@ -128,88 +133,107 @@ impl SimpleSync { /// /// Disconnects the peer if required. fn process_hello(&mut self, peer_id: PeerId, hello: HelloMessage) { - let remote = PeerSyncInfo::from(hello); - let local = PeerSyncInfo::from(&self.chain); - - let start_slot = |epoch: Epoch| epoch.start_slot(T::EthSpec::slots_per_epoch()); - - if local.fork_version != remote.fork_version { - // The node is on a different network/fork, disconnect them. - debug!( - self.log, "HandshakeFailure"; - "peer" => format!("{:?}", peer_id), - "reason" => "network_id" - ); - - self.network - .disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork); - } else if remote.finalized_epoch <= local.finalized_epoch - && remote.finalized_root != Hash256::zero() - && local.finalized_root != Hash256::zero() - && (self.root_at_slot(start_slot(remote.finalized_epoch)) - != Some(remote.finalized_root)) + // If we update the manager we may need to drive the sync. This flag lies out of scope of + // the beacon chain so that the process sync command has no long-lived beacon chain + // references. + let mut process_sync = false; { - // The remotes finalized epoch is less than or greater than ours, but the block root is - // different to the one in our chain. - // - // Therefore, the node is on a different chain and we should not communicate with them. - debug!( - self.log, "HandshakeFailure"; - "peer" => format!("{:?}", peer_id), - "reason" => "different finalized chain" - ); - self.network - .disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork); - } else if remote.finalized_epoch < local.finalized_epoch { - // The node has a lower finalized epoch, their chain is not useful to us. There are two - // cases where a node can have a lower finalized epoch: - // - // ## The node is on the same chain - // - // If a node is on the same chain but has a lower finalized epoch, their head must be - // lower than ours. Therefore, we have nothing to request from them. - // - // ## The node is on a fork - // - // If a node is on a fork that has a lower finalized epoch, switching to that fork would - // cause us to revert a finalized block. This is not permitted, therefore we have no - // interest in their blocks. - debug!( - self.log, - "NaivePeer"; - "peer" => format!("{:?}", peer_id), - "reason" => "lower finalized epoch" - ); - } else if self - .chain - .store - .exists::>(&remote.head_root) - .unwrap_or_else(|_| false) - { - trace!( - self.log, "Out of date or potentially sync'd peer found"; - "peer" => format!("{:?}", peer_id), - "remote_head_slot" => remote.head_slot, - "remote_latest_finalized_epoch" => remote.finalized_epoch, - ); + // scope of beacon chain reference + let chain = match self.chain.upgrade() { + Some(chain) => chain, + None => { + info!(self.log, "Sync shutting down"; + "reason" => "Beacon chain dropped"); + return; + } + }; - // If the node's best-block is already known to us and they are close to our current - // head, treat them as a fully sync'd peer. - self.manager.add_peer(peer_id, remote); - self.process_sync(); - } else { - // The remote node has an equal or great finalized epoch and we don't know it's head. - // - // Therefore, there are some blocks between the local finalized epoch and the remote - // head that are worth downloading. - debug!( - self.log, "UsefulPeer"; - "peer" => format!("{:?}", peer_id), - "local_finalized_epoch" => local.finalized_epoch, - "remote_latest_finalized_epoch" => remote.finalized_epoch, - ); + let remote = PeerSyncInfo::from(hello); + let local = PeerSyncInfo::from(&chain); - self.manager.add_peer(peer_id, remote); + let start_slot = |epoch: Epoch| epoch.start_slot(T::EthSpec::slots_per_epoch()); + + if local.fork_version != remote.fork_version { + // The node is on a different network/fork, disconnect them. + debug!( + self.log, "HandshakeFailure"; + "peer" => format!("{:?}", peer_id), + "reason" => "network_id" + ); + + self.network + .disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork); + } else if remote.finalized_epoch <= local.finalized_epoch + && remote.finalized_root != Hash256::zero() + && local.finalized_root != Hash256::zero() + && (chain.root_at_slot(start_slot(remote.finalized_epoch)) + != Some(remote.finalized_root)) + { + // The remotes finalized epoch is less than or greater than ours, but the block root is + // different to the one in our chain. + // + // Therefore, the node is on a different chain and we should not communicate with them. + debug!( + self.log, "HandshakeFailure"; + "peer" => format!("{:?}", peer_id), + "reason" => "different finalized chain" + ); + self.network + .disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork); + } else if remote.finalized_epoch < local.finalized_epoch { + // The node has a lower finalized epoch, their chain is not useful to us. There are two + // cases where a node can have a lower finalized epoch: + // + // ## The node is on the same chain + // + // If a node is on the same chain but has a lower finalized epoch, their head must be + // lower than ours. Therefore, we have nothing to request from them. + // + // ## The node is on a fork + // + // If a node is on a fork that has a lower finalized epoch, switching to that fork would + // cause us to revert a finalized block. This is not permitted, therefore we have no + // interest in their blocks. + debug!( + self.log, + "NaivePeer"; + "peer" => format!("{:?}", peer_id), + "reason" => "lower finalized epoch" + ); + } else if chain + .store + .exists::>(&remote.head_root) + .unwrap_or_else(|_| false) + { + trace!( + self.log, "Peer with known chain found"; + "peer" => format!("{:?}", peer_id), + "remote_head_slot" => remote.head_slot, + "remote_latest_finalized_epoch" => remote.finalized_epoch, + ); + + // If the node's best-block is already known to us and they are close to our current + // head, treat them as a fully sync'd peer. + self.manager.add_peer(peer_id, remote); + process_sync = true; + } else { + // The remote node has an equal or great finalized epoch and we don't know it's head. + // + // Therefore, there are some blocks between the local finalized epoch and the remote + // head that are worth downloading. + debug!( + self.log, "UsefulPeer"; + "peer" => format!("{:?}", peer_id), + "local_finalized_epoch" => local.finalized_epoch, + "remote_latest_finalized_epoch" => remote.finalized_epoch, + ); + + self.manager.add_peer(peer_id, remote); + process_sync = true + } + } // end beacon chain reference scope + + if process_sync { self.process_sync(); } } @@ -226,11 +250,13 @@ impl SimpleSync { "method" => "HELLO", "peer" => format!("{:?}", peer_id) ); - self.network.send_rpc_request( - None, - peer_id, - RPCRequest::Hello(hello_message(&self.chain)), - ); + if let Some(chain) = self.chain.upgrade() { + self.network.send_rpc_request( + None, + peer_id, + RPCRequest::Hello(hello_message(&chain)), + ); + } } ImportManagerOutcome::RequestBlocks { peer_id, @@ -283,14 +309,6 @@ impl SimpleSync { } } - //TODO: Move to beacon chain - fn root_at_slot(&self, target_slot: Slot) -> Option { - self.chain - .rev_iter_block_roots() - .find(|(_root, slot)| *slot == target_slot) - .map(|(root, _slot)| root) - } - /// Handle a `RecentBeaconBlocks` request from the peer. pub fn on_recent_beacon_blocks_request( &mut self, @@ -298,11 +316,20 @@ impl SimpleSync { request_id: RequestId, request: RecentBeaconBlocksRequest, ) { + let chain = match self.chain.upgrade() { + Some(chain) => chain, + None => { + info!(self.log, "Sync shutting down"; + "reason" => "Beacon chain dropped"); + return; + } + }; + let blocks: Vec> = request .block_roots .iter() .filter_map(|root| { - if let Ok(Some(block)) = self.chain.store.get::>(root) { + if let Ok(Some(block)) = chain.store.get::>(root) { Some(block) } else { debug!( @@ -319,7 +346,7 @@ impl SimpleSync { debug!( self.log, - "BlockBodiesRequest"; + "RecentBeaconBlocksRequest"; "peer" => format!("{:?}", peer_id), "requested" => request.block_roots.len(), "returned" => blocks.len(), @@ -339,6 +366,15 @@ impl SimpleSync { request_id: RequestId, req: BeaconBlocksRequest, ) { + let chain = match self.chain.upgrade() { + Some(chain) => chain, + None => { + info!(self.log, "Sync shutting down"; + "reason" => "Beacon chain dropped"); + return; + } + }; + debug!( self.log, "BeaconBlocksRequest"; @@ -352,15 +388,14 @@ impl SimpleSync { // In the current implementation we read from the db then filter out out-of-range blocks. // Improving the db schema to prevent this would be ideal. - let mut blocks: Vec> = self - .chain + let mut blocks: Vec> = chain .rev_iter_block_roots() .filter(|(_root, slot)| { req.start_slot <= slot.as_u64() && req.start_slot + req.count > slot.as_u64() }) .take_while(|(_root, slot)| req.start_slot <= slot.as_u64()) .filter_map(|(root, _slot)| { - if let Ok(Some(block)) = self.chain.store.get::>(&root) { + if let Ok(Some(block)) = chain.store.get::>(&root) { Some(block) } else { warn!( @@ -378,18 +413,16 @@ impl SimpleSync { blocks.reverse(); blocks.dedup_by_key(|brs| brs.slot); - if blocks.len() as u64 != req.count { - debug!( - self.log, - "BeaconBlocksRequest response"; - "peer" => format!("{:?}", peer_id), - "msg" => "Failed to return all requested hashes", - "start_slot" => req.start_slot, - "current_slot" => format!("{:?}", self.chain.slot()), - "requested" => req.count, - "returned" => blocks.len(), - ); - } + debug!( + self.log, + "BeaconBlocksRequest response"; + "peer" => format!("{:?}", peer_id), + "msg" => "Failed to return all requested hashes", + "start_slot" => req.start_slot, + "current_slot" => chain.slot().unwrap_or_else(|_| Slot::from(0_u64)).as_u64(), + "requested" => req.count, + "returned" => blocks.len(), + ); self.network.send_rpc_response( peer_id, @@ -444,7 +477,16 @@ impl SimpleSync { /// /// Returns a `bool` which, if `true`, indicates we should forward the block to our peers. pub fn on_block_gossip(&mut self, peer_id: PeerId, block: BeaconBlock) -> bool { - if let Ok(outcome) = self.chain.process_block(block.clone()) { + let chain = match self.chain.upgrade() { + Some(chain) => chain, + None => { + info!(self.log, "Sync shutting down"; + "reason" => "Beacon chain dropped"); + return false; + } + }; + + if let Ok(outcome) = chain.process_block(block.clone()) { match outcome { BlockProcessingOutcome::Processed { .. } => { trace!(self.log, "Gossipsub block processed"; @@ -477,7 +519,16 @@ impl SimpleSync { /// /// Not currently implemented. pub fn on_attestation_gossip(&mut self, _peer_id: PeerId, msg: Attestation) { - match self.chain.process_attestation(msg) { + let chain = match self.chain.upgrade() { + Some(chain) => chain, + None => { + info!(self.log, "Sync shutting down"; + "reason" => "Beacon chain dropped"); + return; + } + }; + + match chain.process_attestation(msg) { Ok(outcome) => info!( self.log, "Processed attestation"; @@ -489,11 +540,6 @@ impl SimpleSync { } } } - - /// Generates our current state in the form of a HELLO RPC message. - pub fn generate_hello(&self) -> HelloMessage { - hello_message(&self.chain) - } } /// Build a `HelloMessage` representing the state of the given `beacon_chain`. From 289f8d13b00c4984ab61324f989eff028e5cc207 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 5 Sep 2019 10:19:52 +1000 Subject: [PATCH 18/32] Cleanups and SSZ generic container tests --- tests/ef_tests/src/bls_setting.rs | 1 - tests/ef_tests/src/case_result.rs | 11 ++- tests/ef_tests/src/cases.rs | 12 +-- tests/ef_tests/src/cases/epoch_processing.rs | 4 - .../src/cases/genesis_initialization.rs | 4 - tests/ef_tests/src/cases/genesis_validity.rs | 13 +-- tests/ef_tests/src/cases/operations.rs | 8 +- tests/ef_tests/src/cases/sanity_blocks.rs | 7 -- tests/ef_tests/src/cases/sanity_slots.rs | 7 -- tests/ef_tests/src/cases/ssz_generic.rs | 79 +++++++++++++++-- tests/ef_tests/src/cases/ssz_static.rs | 1 - tests/ef_tests/src/handler.rs | 16 ++-- tests/ef_tests/tests/tests.rs | 84 ++++++++++--------- 13 files changed, 137 insertions(+), 110 deletions(-) diff --git a/tests/ef_tests/src/bls_setting.rs b/tests/ef_tests/src/bls_setting.rs index 79990c8eec..add7d8b7bd 100644 --- a/tests/ef_tests/src/bls_setting.rs +++ b/tests/ef_tests/src/bls_setting.rs @@ -2,7 +2,6 @@ use self::BlsSetting::*; use crate::error::Error; use serde_repr::Deserialize_repr; -// TODO: use this in every test case #[derive(Deserialize_repr, Debug, Clone, Copy)] #[repr(u8)] pub enum BlsSetting { diff --git a/tests/ef_tests/src/case_result.rs b/tests/ef_tests/src/case_result.rs index add428ec53..9df60f402c 100644 --- a/tests/ef_tests/src/case_result.rs +++ b/tests/ef_tests/src/case_result.rs @@ -1,7 +1,7 @@ use super::*; use compare_fields::{CompareFields, Comparison, FieldComparison}; use std::fmt::Debug; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use types::BeaconState; pub const MAX_VALUE_STRING_LEN: usize = 500; @@ -15,11 +15,16 @@ pub struct CaseResult { } impl CaseResult { - pub fn new(case_index: usize, case: &impl Case, result: Result<(), Error>) -> Self { + pub fn new( + case_index: usize, + path: &Path, + case: &impl Case, + result: Result<(), Error>, + ) -> Self { CaseResult { case_index, desc: case.description(), - path: case.path().into(), + path: path.into(), result, } } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 279086b682..c5b0d8c4f1 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -1,7 +1,7 @@ use super::*; use rayon::prelude::*; use std::fmt::Debug; -use std::path::Path; +use std::path::{Path, PathBuf}; mod bls_aggregate_pubkeys; mod bls_aggregate_sigs; @@ -50,12 +50,6 @@ pub trait Case: Debug + Sync { "no description".to_string() } - /// Path to the directory for this test case. - fn path(&self) -> &Path { - // FIXME(michael): remove default impl - Path::new("") - } - /// Execute a test and return the result. /// /// `case_index` reports the index of the case in the set of test cases. It is not strictly @@ -65,7 +59,7 @@ pub trait Case: Debug + Sync { #[derive(Debug)] pub struct Cases { - pub test_cases: Vec, + pub test_cases: Vec<(PathBuf, T)>, } impl Cases { @@ -73,7 +67,7 @@ impl Cases { self.test_cases .into_par_iter() .enumerate() - .map(|(i, tc)| CaseResult::new(i, tc, tc.result(i))) + .map(|(i, (ref path, ref tc))| CaseResult::new(i, path, tc, tc.result(i))) .collect() } } diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs index 2a2dde629b..ece69b3fe2 100644 --- a/tests/ef_tests/src/cases/epoch_processing.rs +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -125,10 +125,6 @@ impl> Case for EpochProcessing { .unwrap_or_else(String::new) } - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { let mut state = self.pre.clone(); let mut expected = self.post.clone(); diff --git a/tests/ef_tests/src/cases/genesis_initialization.rs b/tests/ef_tests/src/cases/genesis_initialization.rs index bd0507b9dd..0fb64ccb37 100644 --- a/tests/ef_tests/src/cases/genesis_initialization.rs +++ b/tests/ef_tests/src/cases/genesis_initialization.rs @@ -45,10 +45,6 @@ impl LoadCase for GenesisInitialization { } impl Case for GenesisInitialization { - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { let spec = &E::default_spec(); diff --git a/tests/ef_tests/src/cases/genesis_validity.rs b/tests/ef_tests/src/cases/genesis_validity.rs index 3a1b9e2677..f72ac4c3e6 100644 --- a/tests/ef_tests/src/cases/genesis_validity.rs +++ b/tests/ef_tests/src/cases/genesis_validity.rs @@ -2,13 +2,12 @@ use super::*; use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::is_valid_genesis_state; -use std::path::{Path, PathBuf}; +use std::path::Path; use types::{BeaconState, EthSpec}; #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct GenesisValidity { - pub path: PathBuf, pub genesis: BeaconState, pub is_valid: bool, } @@ -18,19 +17,11 @@ impl LoadCase for GenesisValidity { let genesis = ssz_decode_file(&path.join("genesis.ssz"))?; let is_valid = yaml_decode_file(&path.join("is_valid.yaml"))?; - Ok(Self { - path: path.into(), - genesis, - is_valid, - }) + Ok(Self { genesis, is_valid }) } } impl Case for GenesisValidity { - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { let spec = &E::default_spec(); diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs index 7b4ffff987..89fa3cccad 100644 --- a/tests/ef_tests/src/cases/operations.rs +++ b/tests/ef_tests/src/cases/operations.rs @@ -11,7 +11,7 @@ use state_processing::per_block_processing::{ process_transfers, }; use std::fmt::Debug; -use std::path::{Path, PathBuf}; +use std::path::Path; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, ProposerSlashing, Transfer, VoluntaryExit, @@ -25,7 +25,6 @@ struct Metadata { #[derive(Debug, Clone)] pub struct Operations> { - pub path: PathBuf, metadata: Metadata, pub pre: BeaconState, pub operation: O, @@ -156,7 +155,6 @@ impl> LoadCase for Operations { }; Ok(Self { - path: path.into(), metadata, pre, operation, @@ -173,10 +171,6 @@ impl> Case for Operations { .unwrap_or_else(String::new) } - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { self.metadata.bls_setting.unwrap_or_default().check()?; diff --git a/tests/ef_tests/src/cases/sanity_blocks.rs b/tests/ef_tests/src/cases/sanity_blocks.rs index 9fadea42e2..292f47415d 100644 --- a/tests/ef_tests/src/cases/sanity_blocks.rs +++ b/tests/ef_tests/src/cases/sanity_blocks.rs @@ -6,7 +6,6 @@ use serde_derive::Deserialize; use state_processing::{ per_block_processing, per_slot_processing, BlockInvalid, BlockProcessingError, }; -use std::path::PathBuf; use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch}; #[derive(Debug, Clone, Deserialize)] @@ -19,7 +18,6 @@ pub struct Metadata { #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanityBlocks { - pub path: PathBuf, pub metadata: Metadata, pub pre: BeaconState, pub blocks: Vec>, @@ -44,7 +42,6 @@ impl LoadCase for SanityBlocks { }; Ok(Self { - path: path.into(), metadata, pre, blocks, @@ -61,10 +58,6 @@ impl Case for SanityBlocks { .unwrap_or_else(String::new) } - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { self.metadata.bls_setting.unwrap_or_default().check()?; diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index 34acb1105a..e9b80a252d 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -4,7 +4,6 @@ use crate::case_result::compare_beacon_state_results_without_caches; use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_slot_processing; -use std::path::PathBuf; use types::{BeaconState, EthSpec}; #[derive(Debug, Clone, Default, Deserialize)] @@ -16,7 +15,6 @@ pub struct Metadata { #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanitySlots { - pub path: PathBuf, pub metadata: Metadata, pub pre: BeaconState, pub slots: u64, @@ -41,7 +39,6 @@ impl LoadCase for SanitySlots { }; Ok(Self { - path: path.into(), metadata, pre, slots, @@ -58,10 +55,6 @@ impl Case for SanitySlots { .unwrap_or_else(String::new) } - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { self.metadata.bls_setting.unwrap_or_default().check()?; diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index 5f9cd3faf0..ce43f3c50f 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -1,12 +1,16 @@ +#![allow(non_snake_case)] + use super::*; use crate::cases::common::{SszStaticType, TestU128, TestU256}; use crate::cases::ssz_static::{check_serialization, check_tree_hash}; use crate::decode::yaml_decode_file; use serde_derive::Deserialize; +use ssz_derive::{Decode, Encode}; use std::fs; use std::path::{Path, PathBuf}; +use tree_hash_derive::TreeHash; use types::typenum::*; -use types::{BitList, BitVector, FixedVector}; +use types::{BitList, BitVector, FixedVector, VariableList}; #[derive(Debug, Clone, Deserialize)] struct Metadata { @@ -54,7 +58,7 @@ macro_rules! type_dispatch { "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*), "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU128>, $($rest)*), "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU256>, $($rest)*), - _ => { println!("unsupported: {}", $value); Ok(()) }, + _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), } }; ($function:ident, @@ -86,7 +90,23 @@ macro_rules! type_dispatch { "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*), "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*), "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*), - _ => { println!("unsupported: {}", $value); Ok(()) }, + _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), + } + }; + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => test_container ] $($rest:tt)*) => { + match $value { + "SingleFieldTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SingleFieldTestStruct>, $($rest)*), + "SmallTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SmallTestStruct>, $($rest)*), + "FixedTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* FixedTestStruct>, $($rest)*), + "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*), + "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*), + // TODO: enable ComplexTestStruct + "ComplexTestStruct" => Err(Error::SkippedKnownFailure), + _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), } }; // No base type: apply type params to function @@ -99,10 +119,6 @@ macro_rules! type_dispatch { } impl Case for SszGeneric { - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { let parts = self.case_name.split('_').collect::>(); @@ -162,7 +178,17 @@ impl Case for SszGeneric { [type_name.as_str() => primitive_type] )?; } - // FIXME(michael): support for the containers tests + "containers" => { + let type_name = parts[0]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + _, + <>, + [type_name => test_container] + )?; + } _ => panic!("unsupported handler: {}", self.handler_name), } Ok(()) @@ -187,7 +213,7 @@ fn ssz_generic_test(path: &Path) -> Result<(), Error> { }; // Valid - // TODO: signing root + // TODO: signing root (annoying because of traits) if let Some(value) = value { check_serialization(&value, &serialized)?; @@ -207,3 +233,38 @@ fn ssz_generic_test(path: &Path) -> Result<(), Error> { Ok(()) } + +// Containers for SSZ generic tests +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct SingleFieldTestStruct { + A: u8, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct SmallTestStruct { + A: u16, + B: u16, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct FixedTestStruct { + A: u8, + B: u64, + C: u32, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct VarTestStruct { + A: u16, + B: VariableList, + C: u8, +} + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct BitsStruct { + A: BitList, + B: BitVector, + C: BitVector, + D: BitList, + E: BitVector, +} diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index d1c9b1048c..6e4a672cb5 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -28,7 +28,6 @@ pub struct SszStaticSR { } fn load_from_dir(path: &Path) -> Result<(SszStaticRoots, Vec, T), Error> { - // FIXME(michael): set description/name let roots = yaml_decode_file(&path.join("roots.yaml"))?; let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); let value = yaml_decode_file(&path.join("value.yaml"))?; diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index b6334c3839..e5d175e115 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -32,19 +32,21 @@ pub trait Handler { .join(Self::handler_name()); // Iterate through test suites - // TODO: parallelism - // TODO: error handling? let test_cases = fs::read_dir(&handler_path) - .expect("open main directory") + .expect("handler dir exists") .flat_map(|entry| { entry .ok() .filter(|e| e.file_type().map(|ty| ty.is_dir()).unwrap_or(false)) }) - .flat_map(|suite| fs::read_dir(suite.path()).expect("open suite dir")) + .flat_map(|suite| fs::read_dir(suite.path()).expect("suite dir exists")) .flat_map(Result::ok) - .map(|test_case_dir| Self::Case::load_from_dir(&test_case_dir.path()).expect("loads")) - .collect::>(); + .map(|test_case_dir| { + let path = test_case_dir.path(); + let case = Self::Case::load_from_dir(&path).expect("test should load"); + (path, case) + }) + .collect(); let results = Cases { test_cases }.test_results(); @@ -286,3 +288,5 @@ pub struct Boolean; type_name!(Boolean, "boolean"); pub struct Uints; type_name!(Uints, "uints"); +pub struct Containers; +type_name!(Containers, "containers"); diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 71fa53c66c..337c54b461 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,19 +1,5 @@ use ef_tests::*; -use types::{ - Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, - BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink, - Deposit, DepositData, Eth1Data, Fork, HistoricalBatch, IndexedAttestation, MainnetEthSpec, - MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit, -}; - -#[test] -fn ssz_generic() { - SszGenericHandler::::run(); - SszGenericHandler::::run(); - SszGenericHandler::::run(); - SszGenericHandler::::run(); - SszGenericHandler::::run(); -} +use types::*; #[test] fn shuffling() { @@ -105,6 +91,7 @@ fn bls_sign_msg() { BlsSignMsgHandler::run(); } +#[cfg(feature = "fake_crypto")] macro_rules! ssz_static_test { // Signed-root ($test_name:ident, $typ:ident$(<$generics:tt>)?, SR) => { @@ -135,7 +122,6 @@ macro_rules! ssz_static_test { // Base case ($test_name:ident, $handler:ident, { $(($typ:ty, $spec:ident)),+ }) => { #[test] - #[cfg(feature = "fake_crypto")] fn $test_name() { $( $handler::<$typ, $spec>::run(); @@ -144,31 +130,47 @@ macro_rules! ssz_static_test { }; } -ssz_static_test!(ssz_static_attestation, Attestation<_>, SR); -ssz_static_test!(ssz_static_attestation_data, AttestationData); -ssz_static_test!( - ssz_static_attestation_data_and_custody_bit, - AttestationDataAndCustodyBit -); -ssz_static_test!(ssz_static_attester_slashing, AttesterSlashing<_>); -ssz_static_test!(ssz_static_beacon_block, BeaconBlock<_>, SR); -ssz_static_test!(ssz_static_beacon_block_body, BeaconBlockBody<_>); -ssz_static_test!(ssz_static_beacon_block_header, BeaconBlockHeader, SR); -ssz_static_test!(ssz_static_beacon_state, BeaconState<_>); -ssz_static_test!(ssz_static_checkpoint, Checkpoint); -ssz_static_test!(ssz_static_compact_committee, CompactCommittee<_>); -ssz_static_test!(ssz_static_crosslink, Crosslink); -ssz_static_test!(ssz_static_deposit, Deposit); -ssz_static_test!(ssz_static_deposit_data, DepositData, SR); -ssz_static_test!(ssz_static_eth1_data, Eth1Data); -ssz_static_test!(ssz_static_fork, Fork); -ssz_static_test!(ssz_static_historical_batch, HistoricalBatch<_>); -ssz_static_test!(ssz_static_indexed_attestation, IndexedAttestation<_>, SR); -ssz_static_test!(ssz_static_pending_attestation, PendingAttestation<_>); -ssz_static_test!(ssz_static_proposer_slashing, ProposerSlashing); -ssz_static_test!(ssz_static_transfer, Transfer, SR); -ssz_static_test!(ssz_static_validator, Validator); -ssz_static_test!(ssz_static_voluntary_exit, VoluntaryExit, SR); +#[cfg(feature = "fake_crypto")] +mod ssz_static { + use ef_tests::{Handler, SszStaticHandler, SszStaticSRHandler}; + use types::*; + + ssz_static_test!(attestation, Attestation<_>, SR); + ssz_static_test!(attestation_data, AttestationData); + ssz_static_test!( + attestation_data_and_custody_bit, + AttestationDataAndCustodyBit + ); + ssz_static_test!(attester_slashing, AttesterSlashing<_>); + ssz_static_test!(beacon_block, BeaconBlock<_>, SR); + ssz_static_test!(beacon_block_body, BeaconBlockBody<_>); + ssz_static_test!(beacon_block_header, BeaconBlockHeader, SR); + ssz_static_test!(beacon_state, BeaconState<_>); + ssz_static_test!(checkpoint, Checkpoint); + ssz_static_test!(compact_committee, CompactCommittee<_>); + ssz_static_test!(crosslink, Crosslink); + ssz_static_test!(deposit, Deposit); + ssz_static_test!(deposit_data, DepositData, SR); + ssz_static_test!(eth1_data, Eth1Data); + ssz_static_test!(fork, Fork); + ssz_static_test!(historical_batch, HistoricalBatch<_>); + ssz_static_test!(indexed_attestation, IndexedAttestation<_>, SR); + ssz_static_test!(pending_attestation, PendingAttestation<_>); + ssz_static_test!(proposer_slashing, ProposerSlashing); + ssz_static_test!(transfer, Transfer, SR); + ssz_static_test!(validator, Validator); + ssz_static_test!(voluntary_exit, VoluntaryExit, SR); +} + +#[test] +fn ssz_generic() { + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); +} #[test] fn epoch_processing_justification_and_finalization() { From a074d8f09b83dcb4b3a52464c1113d76c2139574 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 5 Sep 2019 16:10:57 +1000 Subject: [PATCH 19/32] Update book --- book/src/interop-cheat-sheet.md | 10 ++++++++++ book/src/interop-scenarios.md | 5 ++++- book/src/interop-tips.md | 1 - 3 files changed, 14 insertions(+), 2 deletions(-) delete mode 100644 book/src/interop-tips.md diff --git a/book/src/interop-cheat-sheet.md b/book/src/interop-cheat-sheet.md index ea7794c338..7fea539ead 100644 --- a/book/src/interop-cheat-sheet.md +++ b/book/src/interop-cheat-sheet.md @@ -9,6 +9,7 @@ interop testing. - [Avoid port clashes when starting multiple nodes](#port-bump) - [Specify a custom slot time](#slot-time) - Using the beacon node HTTP API: + - [Pretty-print the genesis state and state root](#http-state) - [Curl a node's ENR](#http-enr) - [Curl a node's connected peers](#http-peer-ids) - [Curl a node's local peer id](#http-peer-id) @@ -82,6 +83,15 @@ $ ./beacon_node testnet -t 500 recent 8 Examples assume there is a Lighthouse node exposing a HTTP API on `localhost:5052`. Responses are JSON. + +### Pretty-print the genesis state and state root + +Returns the genesis state and state root in your terminal, in YAML. + +``` +$ curl --header "Content-Type: application/yaml" "localhost:5052/beacon/state?slot=0" +``` + ### Get the node's ENR diff --git a/book/src/interop-scenarios.md b/book/src/interop-scenarios.md index dc87893622..5e44d822a9 100644 --- a/book/src/interop-scenarios.md +++ b/book/src/interop-scenarios.md @@ -25,8 +25,11 @@ cheat-sheet](./interop-cheat-sheet.md). To start a brand-new beacon node (with no history) use: ``` -$ ./beacon_node testnet -f quick 8 1567222226 +$ ./beacon_node testnet -f quick 8 ``` + +Where `GENESIS_TIME` is in [unix time](https://duckduckgo.com/?q=unix+time&t=ffab&ia=answer). + > Notes: > > - This method conforms the ["Quick-start diff --git a/book/src/interop-tips.md b/book/src/interop-tips.md deleted file mode 100644 index 0d52e896ac..0000000000 --- a/book/src/interop-tips.md +++ /dev/null @@ -1 +0,0 @@ -# Interop Tips & Tricks From 940ddd0d13b9b7ecdd322d63989b14bef88eedbd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 5 Sep 2019 20:57:48 +1000 Subject: [PATCH 20/32] Use michael's milagro in interop keypairs --- eth2/utils/eth2_interop_keypairs/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/utils/eth2_interop_keypairs/Cargo.toml b/eth2/utils/eth2_interop_keypairs/Cargo.toml index 31f9718cd4..d8a1118553 100644 --- a/eth2/utils/eth2_interop_keypairs/Cargo.toml +++ b/eth2/utils/eth2_interop_keypairs/Cargo.toml @@ -10,7 +10,7 @@ edition = "2018" lazy_static = "1.4" num-bigint = "0.2" eth2_hashing = "0.1" -milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v0.10.0" } +milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian-v0.10" } [dev-dependencies] base64 = "0.10" From ee25766caea02711fceb71950f847678ccadc6fc Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 5 Sep 2019 22:18:17 +1000 Subject: [PATCH 21/32] Correct recent beacon block request bug --- beacon_node/network/src/sync/manager.rs | 20 ++++++++++++++------ beacon_node/network/src/sync/simple_sync.rs | 12 ++++++++---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 2b2ed9dcae..fa1315c39b 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -187,7 +187,11 @@ pub(crate) enum ImportManagerOutcome { request: BeaconBlocksRequest, }, /// A `RecentBeaconBlocks` request is required. - RecentRequest(PeerId, RecentBeaconBlocksRequest), + RecentRequest { + peer_id: PeerId, + request_id: RequestId, + request: RecentBeaconBlocksRequest, + }, /// Updates information with peer via requesting another HELLO handshake. Hello(PeerId), /// A peer has caused a punishable error and should be downvoted. @@ -532,7 +536,7 @@ impl ImportManager { pub fn add_unknown_block(&mut self, block: BeaconBlock, peer_id: PeerId) { // if we are not in regular sync mode, ignore this block - if let ManagerState::Regular = self.state { + if self.state != ManagerState::Regular { return; } @@ -774,19 +778,23 @@ impl ImportManager { continue; } - parent_request.state = BlockRequestsState::Pending(self.current_req_id); + let request_id = self.current_req_id; + parent_request.state = BlockRequestsState::Pending(request_id); self.current_req_id += 1; let last_element_index = parent_request.downloaded_blocks.len() - 1; let parent_hash = parent_request.downloaded_blocks[last_element_index].parent_root; - let req = RecentBeaconBlocksRequest { + let request = RecentBeaconBlocksRequest { block_roots: vec![parent_hash], }; // select a random fully synced peer to attempt to download the parent block let peer_id = self.full_peers.iter().next().expect("List is not empty"); - self.event_queue - .push(ImportManagerOutcome::RecentRequest(peer_id.clone(), req)); + self.event_queue.push(ImportManagerOutcome::RecentRequest { + peer_id: peer_id.clone(), + request_id, + request, + }); re_run = true; } } diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index a8b271700c..4a853f05d0 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -277,18 +277,22 @@ impl SimpleSync { RPCRequest::BeaconBlocks(request), ); } - ImportManagerOutcome::RecentRequest(peer_id, req) => { + ImportManagerOutcome::RecentRequest { + peer_id, + request_id, + request, + } => { trace!( self.log, "RPC Request"; "method" => "RecentBeaconBlocks", - "count" => req.block_roots.len(), + "count" => request.block_roots.len(), "peer" => format!("{:?}", peer_id) ); self.network.send_rpc_request( - None, + Some(request_id), peer_id.clone(), - RPCRequest::RecentBeaconBlocks(req), + RPCRequest::RecentBeaconBlocks(request), ); } ImportManagerOutcome::DownvotePeer(peer_id) => { From 8b69a48fc5824bfad353bc87c861d8001ad60f00 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 6 Sep 2019 10:03:45 +1000 Subject: [PATCH 22/32] Allow validator client to start before genesis --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 +- beacon_node/client/src/lib.rs | 16 ++++++-- eth2/utils/slot_clock/src/lib.rs | 23 ++++++----- .../slot_clock/src/system_time_slot_clock.rs | 2 +- validator_client/src/service.rs | 40 +++++++++---------- 5 files changed, 48 insertions(+), 37 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5409d37287..0f76507feb 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -178,7 +178,7 @@ impl BeaconChain { genesis_state.genesis_time, Duration::from_millis(spec.milliseconds_per_slot), ) - .ok_or_else(|| Error::SlotClockDidNotStart)?; + .map_err(|_| Error::SlotClockDidNotStart)?; info!(log, "Beacon chain initialized from genesis"; "validator_count" => genesis_state.validators.len(), @@ -220,7 +220,7 @@ impl BeaconChain { state.genesis_time, Duration::from_millis(spec.milliseconds_per_slot), ) - .ok_or_else(|| Error::SlotClockDidNotStart)?; + .map_err(|_| Error::SlotClockDidNotStart)?; let last_finalized_root = p.canonical_head.beacon_state.finalized_checkpoint.root; let last_finalized_block = &p.canonical_head.beacon_block; diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 1d3cb40ecf..afcd538b58 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -16,7 +16,7 @@ use slog::{crit, error, info, o}; use slot_clock::SlotClock; use std::marker::PhantomData; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::runtime::TaskExecutor; use tokio::timer::Interval; use types::EthSpec; @@ -177,8 +177,18 @@ where .map_err(error::Error::from)?, ); - if beacon_chain.slot().is_err() { - panic!("Cannot start client before genesis!") + let since_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| format!("Unable to read system time: {}", e))?; + let since_genesis = Duration::from_secs(beacon_chain.head().beacon_state.genesis_time); + + if since_genesis > since_epoch { + info!( + log, + "Starting node prior to genesis"; + "now" => since_epoch.as_secs(), + "genesis_seconds" => since_genesis.as_secs(), + ); } let network_config = &client_config.network; diff --git a/eth2/utils/slot_clock/src/lib.rs b/eth2/utils/slot_clock/src/lib.rs index fd3bf029be..6192d1b6f1 100644 --- a/eth2/utils/slot_clock/src/lib.rs +++ b/eth2/utils/slot_clock/src/lib.rs @@ -5,7 +5,7 @@ mod metrics; mod system_time_slot_clock; mod testing_slot_clock; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, SystemTimeError, UNIX_EPOCH}; pub use crate::system_time_slot_clock::SystemTimeSlotClock; pub use crate::testing_slot_clock::TestingSlotClock; @@ -17,18 +17,21 @@ pub trait SlotClock: Send + Sync + Sized { genesis_slot: Slot, genesis_seconds: u64, slot_duration: Duration, - ) -> Option { - let duration_between_now_and_unix_epoch = - SystemTime::now().duration_since(UNIX_EPOCH).ok()?; + ) -> Result { + let duration_between_now_and_unix_epoch = SystemTime::now().duration_since(UNIX_EPOCH)?; let duration_between_unix_epoch_and_genesis = Duration::from_secs(genesis_seconds); - if duration_between_now_and_unix_epoch < duration_between_unix_epoch_and_genesis { - None + let genesis_instant = if duration_between_now_and_unix_epoch + < duration_between_unix_epoch_and_genesis + { + Instant::now() + + (duration_between_unix_epoch_and_genesis - duration_between_now_and_unix_epoch) } else { - let genesis_instant = Instant::now() - - (duration_between_now_and_unix_epoch - duration_between_unix_epoch_and_genesis); - Some(Self::new(genesis_slot, genesis_instant, slot_duration)) - } + Instant::now() + - (duration_between_now_and_unix_epoch - duration_between_unix_epoch_and_genesis) + }; + + Ok(Self::new(genesis_slot, genesis_instant, slot_duration)) } fn new(genesis_slot: Slot, genesis: Instant, slot_duration: Duration) -> Self; diff --git a/eth2/utils/slot_clock/src/system_time_slot_clock.rs b/eth2/utils/slot_clock/src/system_time_slot_clock.rs index 0d4a52ef64..aae12c18c4 100644 --- a/eth2/utils/slot_clock/src/system_time_slot_clock.rs +++ b/eth2/utils/slot_clock/src/system_time_slot_clock.rs @@ -42,7 +42,7 @@ impl SlotClock for SystemTimeSlotClock { fn duration_to_next_slot(&self) -> Option { let now = Instant::now(); if now < self.genesis { - None + Some(self.genesis - now) } else { let duration_since_genesis = now - self.genesis; let millis_since_genesis = duration_since_genesis.as_millis(); diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index 8adc79b91e..8cdba537a3 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -27,7 +27,7 @@ use slot_clock::{SlotClock, SystemTimeSlotClock}; use std::marker::PhantomData; use std::sync::Arc; use std::sync::RwLock; -use std::time::{Duration, Instant, SystemTime}; +use std::time::{Duration, Instant}; use tokio::prelude::*; use tokio::runtime::Builder; use tokio::timer::Interval; @@ -100,19 +100,6 @@ impl Service { - // verify the node's genesis time - if SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() - < info.genesis_time - { - error!( - log, - "Beacon Node's genesis time is in the future. No work to do.\n Exiting" - ); - return Err("Genesis time in the future".into()); - } // verify the node's network id if eth2_config.spec.network_id != info.network_id as u8 { error!( @@ -177,13 +164,11 @@ impl Service(|| { - "Unable to start slot clock. Genesis may not have occurred yet. Exiting.".into() + .map_err::(|e| { + format!("Unable to start slot clock: {}.", e).into() })?; - let current_slot = slot_clock.now().ok_or_else::(|| { - "Genesis has not yet occurred. Exiting.".into() - })?; + let current_slot = slot_clock.now().unwrap_or_else(|| Slot::new(0)); /* Generate the duties manager */ @@ -237,7 +222,7 @@ impl Service::initialize_service( client_config, eth2_config, - log, + log.clone(), )?; // we have connected to a node and established its parameters. Spin up the core service @@ -253,7 +238,7 @@ impl Service(|| { - "Genesis is not in the past. Exiting.".into() + "Unable to determine duration to next slot. Exiting.".into() })?; // set up the validator work interval - start at next slot and proceed every slot @@ -264,6 +249,19 @@ impl Service duration_to_next_slot.as_secs() + ); + /* kick off the core service */ runtime.block_on( interval From 14cf6b0118bc722402858333d194cf7b4b699d62 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 6 Sep 2019 10:17:23 +1000 Subject: [PATCH 23/32] Add option to validator service to fix bug With the previous setup it would never produce on the 0 slot. --- validator_client/src/service.rs | 44 +++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index 8cdba537a3..5169f67f87 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -46,8 +46,8 @@ pub struct Service, slots_per_epoch: u64, /// The chain specification for this clients instance. spec: Arc, @@ -168,8 +168,6 @@ impl Service Service Service error_chain::Result<()> { - let current_slot = self + let wall_clock_slot = self .slot_clock .now() .ok_or_else::(|| { "Genesis is not in the past. Exiting.".into() })?; - let current_epoch = current_slot.epoch(self.slots_per_epoch); + let wall_clock_epoch = wall_clock_slot.epoch(self.slots_per_epoch); // this is a non-fatal error. If the slot clock repeats, the node could // have been slow to process the previous slot and is now duplicating tasks. // We ignore duplicated but raise a critical error. - if current_slot <= self.current_slot { - crit!( - self.log, - "The validator tried to duplicate a slot. Likely missed the previous slot" - ); - return Err("Duplicate slot".into()); + if let Some(current_slot) = self.current_slot { + if wall_clock_slot <= current_slot { + crit!( + self.log, + "The validator tried to duplicate a slot. Likely missed the previous slot" + ); + return Err("Duplicate slot".into()); + } } - self.current_slot = current_slot; - info!(self.log, "Processing"; "slot" => current_slot.as_u64(), "epoch" => current_epoch.as_u64()); + self.current_slot = Some(wall_clock_slot); + info!(self.log, "Processing"; "slot" => wall_clock_slot.as_u64(), "epoch" => wall_clock_epoch.as_u64()); Ok(()) } @@ -324,7 +324,10 @@ impl Service Service Date: Fri, 6 Sep 2019 17:05:40 +1000 Subject: [PATCH 24/32] Fix bug in SSZ encoding of FixedVector --- eth2/utils/ssz_types/src/fixed_vector.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index edf499adf5..090d04d84b 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -172,7 +172,7 @@ where T: ssz::Encode, { fn is_ssz_fixed_len() -> bool { - true + T::is_ssz_fixed_len() } fn ssz_fixed_len() -> usize { From 23a4fdabe44a266f05b0e92491e6617d7d35cecc Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 6 Sep 2019 17:06:11 +1000 Subject: [PATCH 25/32] Enable remaining SSZ generic tests --- tests/ef_tests/src/cases/ssz_generic.rs | 36 +++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index ce43f3c50f..420f596798 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -4,6 +4,7 @@ use super::*; use crate::cases::common::{SszStaticType, TestU128, TestU256}; use crate::cases::ssz_static::{check_serialization, check_tree_hash}; use crate::decode::yaml_decode_file; +use serde::{de::Error as SerdeError, Deserialize, Deserializer}; use serde_derive::Deserialize; use ssz_derive::{Decode, Encode}; use std::fs; @@ -103,9 +104,8 @@ macro_rules! type_dispatch { "SmallTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SmallTestStruct>, $($rest)*), "FixedTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* FixedTestStruct>, $($rest)*), "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*), + "ComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ComplexTestStruct>, $($rest)*), "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*), - // TODO: enable ComplexTestStruct - "ComplexTestStruct" => Err(Error::SkippedKnownFailure), _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), } }; @@ -260,6 +260,18 @@ struct VarTestStruct { C: u8, } +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct ComplexTestStruct { + A: u16, + B: VariableList, + C: u8, + #[serde(deserialize_with = "byte_list_from_hex_str")] + D: VariableList, + E: VarTestStruct, + F: FixedVector, + G: FixedVector, +} + #[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] struct BitsStruct { A: BitList, @@ -268,3 +280,23 @@ struct BitsStruct { D: BitList, E: BitVector, } + +fn byte_list_from_hex_str<'de, D, N: Unsigned>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s: String = Deserialize::deserialize(deserializer)?; + let decoded: Vec = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?; + + if decoded.len() > N::to_usize() { + return Err(D::Error::custom(format!( + "Too many values for list, got: {}, limit: {}", + decoded.len(), + N::to_usize() + ))); + } else { + Ok(decoded.into()) + } +} From 9b062e052328559b962f0fcabb142713be61deda Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 6 Sep 2019 22:45:05 +1000 Subject: [PATCH 26/32] Fix compile error in ef_tests --- tests/ef_tests/src/cases/ssz_generic.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index 420f596798..fc62e66fc0 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -4,7 +4,7 @@ use super::*; use crate::cases::common::{SszStaticType, TestU128, TestU256}; use crate::cases::ssz_static::{check_serialization, check_tree_hash}; use crate::decode::yaml_decode_file; -use serde::{de::Error as SerdeError, Deserialize, Deserializer}; +use serde::{de::Error as SerdeError, Deserializer}; use serde_derive::Deserialize; use ssz_derive::{Decode, Encode}; use std::fs; @@ -287,7 +287,7 @@ fn byte_list_from_hex_str<'de, D, N: Unsigned>( where D: Deserializer<'de>, { - let s: String = Deserialize::deserialize(deserializer)?; + let s: String = serde::de::Deserialize::deserialize(deserializer)?; let decoded: Vec = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?; if decoded.len() > N::to_usize() { From 812e1fbe2691bcbebd623ffdeb2adc1101e2aa0e Mon Sep 17 00:00:00 2001 From: Age Manning Date: Sat, 7 Sep 2019 00:28:54 +1000 Subject: [PATCH 27/32] Implements a new thread dedicated for syncing --- beacon_node/network/src/message_handler.rs | 52 +- beacon_node/network/src/sync/manager.rs | 502 +++++++++++++------- beacon_node/network/src/sync/mod.rs | 2 +- beacon_node/network/src/sync/simple_sync.rs | 415 ++++++---------- 4 files changed, 514 insertions(+), 457 deletions(-) diff --git a/beacon_node/network/src/message_handler.rs b/beacon_node/network/src/message_handler.rs index cade65d63a..be8fa21f89 100644 --- a/beacon_node/network/src/message_handler.rs +++ b/beacon_node/network/src/message_handler.rs @@ -1,6 +1,6 @@ use crate::error; use crate::service::NetworkMessage; -use crate::sync::SimpleSync; +use crate::sync::MessageProcessor; use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2_libp2p::{ behaviour::PubsubMessage, @@ -15,12 +15,16 @@ use std::sync::Arc; use tokio::sync::mpsc; use types::{Attestation, AttesterSlashing, BeaconBlock, ProposerSlashing, VoluntaryExit}; -/// Handles messages received from the network and client and organises syncing. +/// Handles messages received from the network and client and organises syncing. This +/// functionality of this struct is to validate an decode messages from the network before +/// passing them to the internal message processor. The message processor spawns a syncing thread +/// which manages which blocks need to be requested and processed. pub struct MessageHandler { - /// The syncing framework. - sync: SimpleSync, /// A channel to the network service to allow for gossip propagation. network_send: mpsc::UnboundedSender, + /// Processes validated and decoded messages from the network. Has direct access to the + /// sync manager. + message_processor: MessageProcessor, /// The `MessageHandler` logger. log: slog::Logger, } @@ -50,13 +54,15 @@ impl MessageHandler { trace!(log, "Service starting"); let (handler_send, handler_recv) = mpsc::unbounded_channel(); - // Initialise sync and begin processing in thread - let sync = SimpleSync::new(Arc::downgrade(&beacon_chain), network_send.clone(), &log); + + // Initialise a message instance, which itself spawns the syncing thread. + let message_processor = + MessageProcessor::new(executor, beacon_chain, network_send.clone(), &log); // generate the Message handler let mut handler = MessageHandler { network_send, - sync, + message_processor, log: log.clone(), }; @@ -66,7 +72,11 @@ impl MessageHandler { .for_each(move |msg| Ok(handler.handle_message(msg))) .map_err(move |_| { debug!(log, "Network message handler terminated."); - }), + }), /* + .then(move |_| { + debug!(log.clone(), "Message handler shutdown"); + }), + */ ); Ok(handler_send) @@ -77,11 +87,11 @@ impl MessageHandler { match message { // we have initiated a connection to a peer HandlerMessage::PeerDialed(peer_id) => { - self.sync.on_connect(peer_id); + self.message_processor.on_connect(peer_id); } // A peer has disconnected HandlerMessage::PeerDisconnected(peer_id) => { - self.sync.on_disconnect(peer_id); + self.message_processor.on_disconnect(peer_id); } // An RPC message request/response has been received HandlerMessage::RPC(peer_id, rpc_event) => { @@ -109,7 +119,7 @@ impl MessageHandler { fn handle_rpc_request(&mut self, peer_id: PeerId, request_id: RequestId, request: RPCRequest) { match request { RPCRequest::Hello(hello_message) => { - self.sync + self.message_processor .on_hello_request(peer_id, request_id, hello_message) } RPCRequest::Goodbye(goodbye_reason) => { @@ -118,13 +128,13 @@ impl MessageHandler { "peer" => format!("{:?}", peer_id), "reason" => format!("{:?}", goodbye_reason), ); - self.sync.on_disconnect(peer_id); + self.message_processor.on_disconnect(peer_id); } RPCRequest::BeaconBlocks(request) => self - .sync + .message_processor .on_beacon_blocks_request(peer_id, request_id, request), RPCRequest::RecentBeaconBlocks(request) => self - .sync + .message_processor .on_recent_beacon_blocks_request(peer_id, request_id, request), } } @@ -151,12 +161,13 @@ impl MessageHandler { RPCErrorResponse::Success(response) => { match response { RPCResponse::Hello(hello_message) => { - self.sync.on_hello_response(peer_id, hello_message); + self.message_processor + .on_hello_response(peer_id, hello_message); } RPCResponse::BeaconBlocks(response) => { match self.decode_beacon_blocks(&response) { Ok(beacon_blocks) => { - self.sync.on_beacon_blocks_response( + self.message_processor.on_beacon_blocks_response( peer_id, request_id, beacon_blocks, @@ -171,7 +182,7 @@ impl MessageHandler { RPCResponse::RecentBeaconBlocks(response) => { match self.decode_beacon_blocks(&response) { Ok(beacon_blocks) => { - self.sync.on_recent_beacon_blocks_response( + self.message_processor.on_recent_beacon_blocks_response( peer_id, request_id, beacon_blocks, @@ -199,7 +210,9 @@ impl MessageHandler { match gossip_message { PubsubMessage::Block(message) => match self.decode_gossip_block(message) { Ok(block) => { - let should_forward_on = self.sync.on_block_gossip(peer_id.clone(), block); + let should_forward_on = self + .message_processor + .on_block_gossip(peer_id.clone(), block); // TODO: Apply more sophisticated validation and decoding logic if should_forward_on { self.propagate_message(id, peer_id.clone()); @@ -213,7 +226,8 @@ impl MessageHandler { Ok(attestation) => { // TODO: Apply more sophisticated validation and decoding logic self.propagate_message(id, peer_id.clone()); - self.sync.on_attestation_gossip(peer_id, attestation); + self.message_processor + .on_attestation_gossip(peer_id, attestation); } Err(e) => { debug!(self.log, "Invalid gossiped attestation"; "peer_id" => format!("{}", peer_id), "Error" => format!("{:?}", e)); diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index fa1315c39b..12bef95fa5 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -1,4 +1,4 @@ -//! The `ImportManager` facilities the block syncing logic of lighthouse. The current networking +//! The `SyncManager` facilities the block syncing logic of lighthouse. The current networking //! specification provides two methods from which to obtain blocks from peers. The `BeaconBlocks` //! request and the `RecentBeaconBlocks` request. The former is used to obtain a large number of //! blocks and the latter allows for searching for blocks given a block-hash. @@ -7,7 +7,7 @@ //! - Long range (batch) sync, when a client is out of date and needs to the latest head. //! - Parent lookup - when a peer provides us a block whose parent is unknown to us. //! -//! Both of these syncing strategies are built into the `ImportManager`. +//! Both of these syncing strategies are built into the `SyncManager`. //! //! //! Currently the long-range (batch) syncing method functions by opportunistically downloading @@ -53,16 +53,18 @@ //! fully sync'd peers. If `PARENT_FAIL_TOLERANCE` attempts at requesting the block fails, we //! drop the propagated block and downvote the peer that sent it to us. -use super::simple_sync::{PeerSyncInfo, FUTURE_SLOT_TOLERANCE}; +use super::simple_sync::{hello_message, NetworkContext, PeerSyncInfo, FUTURE_SLOT_TOLERANCE}; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use eth2_libp2p::rpc::methods::*; -use eth2_libp2p::rpc::RequestId; +use eth2_libp2p::rpc::{RPCRequest, RequestId}; use eth2_libp2p::PeerId; +use futures::prelude::*; use slog::{debug, info, trace, warn, Logger}; use smallvec::SmallVec; use std::collections::{HashMap, HashSet}; use std::ops::{Add, Sub}; use std::sync::Weak; +use tokio::sync::{mpsc, oneshot}; use types::{BeaconBlock, EthSpec, Hash256, Slot}; /// Blocks are downloaded in batches from peers. This constant specifies how many blocks per batch @@ -84,6 +86,31 @@ const PARENT_DEPTH_TOLERANCE: usize = SLOT_IMPORT_TOLERANCE * 2; /// requests to peers who never return blocks. const EMPTY_BATCH_TOLERANCE: usize = 100; +#[derive(Debug)] +/// A message than can be sent to the sync manager thread. +pub enum SyncMessage { + /// A useful peer has been discovered. + AddPeer(PeerId, PeerSyncInfo), + /// A `BeaconBlocks` response has been received. + BeaconBlocksResponse { + peer_id: PeerId, + request_id: RequestId, + beacon_blocks: Vec>, + }, + /// A `RecentBeaconBlocks` response has been received. + RecentBeaconBlocksResponse { + peer_id: PeerId, + request_id: RequestId, + beacon_blocks: Vec>, + }, + /// A block with an unknown parent has been received. + UnknownBlock(PeerId, BeaconBlock), + /// A peer has disconnected. + Disconnect(PeerId), + /// An RPC Error has occurred on a request. + _RPCError(RequestId), +} + #[derive(PartialEq)] /// The current state of a block or batches lookup. enum BlockRequestsState { @@ -176,39 +203,19 @@ enum ManagerState { Stalled, } -/// The output states that can occur from driving (polling) the manager state machine. -pub(crate) enum ImportManagerOutcome { - /// There is no further work to complete. The manager is waiting for further input. - Idle, - /// A `BeaconBlocks` request is required. - RequestBlocks { - peer_id: PeerId, - request_id: RequestId, - request: BeaconBlocksRequest, - }, - /// A `RecentBeaconBlocks` request is required. - RecentRequest { - peer_id: PeerId, - request_id: RequestId, - request: RecentBeaconBlocksRequest, - }, - /// Updates information with peer via requesting another HELLO handshake. - Hello(PeerId), - /// A peer has caused a punishable error and should be downvoted. - DownvotePeer(PeerId), -} - /// The primary object for handling and driving all the current syncing logic. It maintains the /// current state of the syncing process, the number of useful peers, downloaded blocks and /// controls the logic behind both the long-range (batch) sync and the on-going potential parent /// look-up of blocks. -pub struct ImportManager { - /// List of events to be processed externally. - event_queue: SmallVec<[ImportManagerOutcome; 20]>, +pub struct SyncManager { /// A weak reference to the underlying beacon chain. chain: Weak>, /// The current state of the import manager. state: ManagerState, + /// A receiving channel sent by the message processor thread. + input_channel: mpsc::UnboundedReceiver>, + /// A network context to contact the network service. + network: NetworkContext, /// A collection of `BlockRequest` per peer that is currently being downloaded. Used in the /// long-range (batch) sync process. import_queue: HashMap>, @@ -224,22 +231,51 @@ pub struct ImportManager { log: Logger, } -impl ImportManager { - /// Generates a new `ImportManager` given a logger and an Arc reference to a beacon chain. The - /// import manager keeps a weak reference to the beacon chain, which allows the chain to be - /// dropped during the syncing process. The syncing handles this termination gracefully. - pub fn new(beacon_chain: Weak>, log: &slog::Logger) -> Self { - ImportManager { - event_queue: SmallVec::new(), - chain: beacon_chain, - state: ManagerState::Regular, - import_queue: HashMap::new(), - parent_queue: SmallVec::new(), - full_peers: HashSet::new(), - current_req_id: 0, - log: log.clone(), - } - } +/// Spawns a new `SyncManager` thread which has a weak reference to underlying beacon +/// chain. This allows the chain to be +/// dropped during the syncing process which will gracefully end the `SyncManager`. +pub fn spawn( + executor: &tokio::runtime::TaskExecutor, + beacon_chain: Weak>, + network: NetworkContext, + log: slog::Logger, +) -> ( + mpsc::UnboundedSender>, + oneshot::Sender<()>, +) { + // generate the exit channel + let (sync_exit, exit_rx) = tokio::sync::oneshot::channel(); + // generate the message channel + let (sync_send, sync_recv) = mpsc::unbounded_channel::>(); + + // create an instance of the SyncManager + let sync_manager = SyncManager { + chain: beacon_chain, + state: ManagerState::Regular, + input_channel: sync_recv, + network, + import_queue: HashMap::new(), + parent_queue: SmallVec::new(), + full_peers: HashSet::new(), + current_req_id: 0, + log: log.clone(), + }; + + // spawn the sync manager thread + debug!(log, "Sync Manager started"); + executor.spawn( + sync_manager + .select(exit_rx.then(|_| Ok(()))) + .then(move |_| { + info!(log.clone(), "Sync Manager shutdown"); + Ok(()) + }), + ); + (sync_send, sync_exit) +} + +impl SyncManager { + /* Input Handling Functions */ /// A peer has connected which has blocks that are unknown to us. /// @@ -281,7 +317,7 @@ impl ImportManager { return; } - // Check if the peer is significantly is behind us. If within `SLOT_IMPORT_TOLERANCE` + // Check if the peer is significantly behind us. If within `SLOT_IMPORT_TOLERANCE` // treat them as a fully synced peer. If not, ignore them in the sync process if local.head_slot.sub(remote.head_slot).as_usize() < SLOT_IMPORT_TOLERANCE { self.add_full_peer(peer_id.clone()); @@ -328,8 +364,7 @@ impl ImportManager { let chain = match self.chain.upgrade() { Some(chain) => chain, None => { - debug!(self.log, "Chain dropped. Sync terminating"); - self.event_queue.clear(); + trace!(self.log, "Chain dropped. Sync terminating"); return; } }; @@ -390,8 +425,7 @@ impl ImportManager { "request_id" => request_id, "response_initial_slot" => blocks[0].slot, "requested_initial_slot" => block_requests.current_start_slot); - self.event_queue - .push(ImportManagerOutcome::DownvotePeer(peer_id)); + downvote_peer(&mut self.network, &self.log, peer_id); // consider this sync failed block_requests.state = BlockRequestsState::Failed; return; @@ -515,26 +549,7 @@ impl ImportManager { parent_request.state = BlockRequestsState::ReadyToProcess; } - pub fn _inject_error(_peer_id: PeerId, _id: RequestId) { - //TODO: Remove block state from pending - } - - pub fn peer_disconnect(&mut self, peer_id: &PeerId) { - self.import_queue.remove(peer_id); - self.full_peers.remove(peer_id); - self.update_state(); - } - - pub fn add_full_peer(&mut self, peer_id: PeerId) { - debug!( - self.log, "Fully synced peer added"; - "peer" => format!("{:?}", peer_id), - ); - self.full_peers.insert(peer_id); - self.update_state(); - } - - pub fn add_unknown_block(&mut self, block: BeaconBlock, peer_id: PeerId) { + fn add_unknown_block(&mut self, peer_id: PeerId, block: BeaconBlock) { // if we are not in regular sync mode, ignore this block if self.state != ManagerState::Regular { return; @@ -563,55 +578,29 @@ impl ImportManager { self.parent_queue.push(req); } - pub(crate) fn poll(&mut self) -> ImportManagerOutcome { - loop { - //TODO: Optimize the lookups. Potentially keep state of whether each of these functions - //need to be called. - - // only break once everything has been processed - let mut re_run = false; - - // only process batch requests if there are any - if !self.import_queue.is_empty() { - // process potential block requests - re_run = re_run || self.process_potential_block_requests(); - - // process any complete long-range batches - re_run = re_run || self.process_complete_batches(); - } - - // only process parent objects if we are in regular sync - if !self.parent_queue.is_empty() { - // process any parent block lookup-requests - re_run = re_run || self.process_parent_requests(); - - // process any complete parent lookups - re_run = re_run || self.process_complete_parent_requests(); - } - - // exit early if the beacon chain is dropped - if let None = self.chain.upgrade() { - return ImportManagerOutcome::Idle; - } - - // return any queued events - if !self.event_queue.is_empty() { - let event = self.event_queue.remove(0); - self.event_queue.shrink_to_fit(); - return event; - } - - // update the state of the manager - self.update_state(); - - if !re_run { - break; - } - } - - return ImportManagerOutcome::Idle; + fn inject_error(&mut self, _id: RequestId) { + //TODO: Remove block state from pending } + fn peer_disconnect(&mut self, peer_id: &PeerId) { + self.import_queue.remove(peer_id); + self.full_peers.remove(peer_id); + self.update_state(); + } + + fn add_full_peer(&mut self, peer_id: PeerId) { + debug!( + self.log, "Fully synced peer added"; + "peer" => format!("{:?}", peer_id), + ); + self.full_peers.insert(peer_id); + self.update_state(); + } + + /* Processing State Functions */ + // These functions are called in the main poll function to transition the state of the sync + // manager + fn update_state(&mut self) { let previous_state = self.state.clone(); self.state = { @@ -631,13 +620,12 @@ impl ImportManager { } } - fn process_potential_block_requests(&mut self) -> bool { + fn process_potential_block_requests(&mut self) { // check if an outbound request is required // Managing a fixed number of outbound requests is maintained at the RPC protocol libp2p // layer and not needed here. Therefore we create many outbound requests and let the RPC // handle the number of simultaneous requests. Request all queued objects. - let mut re_run = false; // remove any failed batches let debug_log = &self.log; let full_peer_ref = &mut self.full_peers; @@ -655,40 +643,40 @@ impl ImportManager { }); // process queued block requests - for (peer_id, block_requests) in self - .import_queue - .iter_mut() - .find(|(_peer_id, req)| req.state == BlockRequestsState::Queued) - { - let request_id = self.current_req_id; - block_requests.state = BlockRequestsState::Pending(request_id); - self.current_req_id += 1; + for (peer_id, block_requests) in self.import_queue.iter_mut() { + { + if block_requests.state == BlockRequestsState::Queued { + let request_id = self.current_req_id; + block_requests.state = BlockRequestsState::Pending(request_id); + self.current_req_id += 1; - let request = BeaconBlocksRequest { - head_block_root: block_requests.target_head_root, - start_slot: block_requests.current_start_slot.as_u64(), - count: MAX_BLOCKS_PER_REQUEST, - step: 0, - }; - self.event_queue.push(ImportManagerOutcome::RequestBlocks { - peer_id: peer_id.clone(), - request, - request_id, - }); - re_run = true; + let request = BeaconBlocksRequest { + head_block_root: block_requests.target_head_root, + start_slot: block_requests.current_start_slot.as_u64(), + count: MAX_BLOCKS_PER_REQUEST, + step: 0, + }; + request_blocks( + &mut self.network, + &self.log, + peer_id.clone(), + request_id, + request, + ); + } + } } - - re_run } fn process_complete_batches(&mut self) -> bool { - // flag to indicate if the manager can be switched to idle or not - let mut re_run = false; + // This function can queue extra blocks and the main poll loop will need to be re-executed + // to process these. This flag indicates that the main poll loop has to continue. + let mut re_run_poll = false; // create reference variables to be moved into subsequent closure let chain_ref = self.chain.clone(); let log_ref = &self.log; - let event_queue_ref = &mut self.event_queue; + let network_ref = &mut self.network; self.import_queue.retain(|peer_id, block_requests| { if block_requests.state == BlockRequestsState::ReadyToProcess { @@ -712,13 +700,13 @@ impl ImportManager { // target head if end_slot >= block_requests.target_head_slot { // Completed, re-hello the peer to ensure we are up to the latest head - event_queue_ref.push(ImportManagerOutcome::Hello(peer_id.clone())); + hello_peer(network_ref, log_ref, chain_ref.clone(), peer_id.clone()); // remove the request false } else { // have not reached the end, queue another batch block_requests.update_start_slot(); - re_run = true; + re_run_poll = true; // keep the batch true } @@ -731,7 +719,7 @@ impl ImportManager { "no_blocks" => last_element + 1, "error" => format!("{:?}", e), ); - event_queue_ref.push(ImportManagerOutcome::DownvotePeer(peer_id.clone())); + downvote_peer(network_ref, log_ref, peer_id.clone()); false } } @@ -741,17 +729,15 @@ impl ImportManager { } }); - re_run + re_run_poll } - fn process_parent_requests(&mut self) -> bool { + fn process_parent_requests(&mut self) { // check to make sure there are peers to search for the parent from if self.full_peers.is_empty() { - return false; + return; } - let mut re_run = false; - // remove any failed requests let debug_log = &self.log; self.parent_queue.retain(|parent_request| { @@ -790,20 +776,20 @@ impl ImportManager { // select a random fully synced peer to attempt to download the parent block let peer_id = self.full_peers.iter().next().expect("List is not empty"); - self.event_queue.push(ImportManagerOutcome::RecentRequest { - peer_id: peer_id.clone(), + recent_blocks_request( + &mut self.network, + &self.log, + peer_id.clone(), request_id, request, - }); - re_run = true; + ); } } - re_run } fn process_complete_parent_requests(&mut self) -> bool { // returned value indicating whether the manager can be switched to idle or not - let mut re_run = false; + let mut re_run_poll = false; // Find any parent_requests ready to be processed for completed_request in self @@ -827,9 +813,8 @@ impl ImportManager { "received_block" => format!("{}", block_hash), "expected_parent" => format!("{}", expected_hash), ); - re_run = true; - self.event_queue - .push(ImportManagerOutcome::DownvotePeer(peer)); + re_run_poll = true; + downvote_peer(&mut self.network, &self.log, peer); } // try and process the list of blocks up to the requested block @@ -846,7 +831,7 @@ impl ImportManager { // need to keep looking for parents completed_request.downloaded_blocks.push(block); completed_request.state = BlockRequestsState::Queued; - re_run = true; + re_run_poll = true; break; } Ok(BlockProcessingOutcome::Processed { block_root: _ }) => {} @@ -859,11 +844,13 @@ impl ImportManager { "peer" => format!("{:?}", completed_request.last_submitted_peer), ); completed_request.state = BlockRequestsState::Queued; - re_run = true; - self.event_queue.push(ImportManagerOutcome::DownvotePeer( + re_run_poll = true; + downvote_peer( + &mut self.network, + &self.log, completed_request.last_submitted_peer.clone(), - )); - return re_run; + ); + return re_run_poll; } Err(e) => { completed_request.failed_attempts += 1; @@ -872,16 +859,17 @@ impl ImportManager { "error" => format!("{:?}", e) ); completed_request.state = BlockRequestsState::Queued; - re_run = true; - self.event_queue.push(ImportManagerOutcome::DownvotePeer( + re_run_poll = true; + downvote_peer( + &mut self.network, + &self.log, completed_request.last_submitted_peer.clone(), - )); - return re_run; + ); + return re_run_poll; } } } else { // chain doesn't exist - clear the event queue and return - self.event_queue.clear(); return false; } } @@ -895,11 +883,83 @@ impl ImportManager { true } }); - re_run + re_run_poll } } -// Helper function to process blocks +/* Network Context Helper Functions */ + +fn hello_peer( + network: &mut NetworkContext, + log: &slog::Logger, + chain: Weak>, + peer_id: PeerId, +) { + trace!( + log, + "RPC Request"; + "method" => "HELLO", + "peer" => format!("{:?}", peer_id) + ); + if let Some(chain) = chain.upgrade() { + network.send_rpc_request(None, peer_id, RPCRequest::Hello(hello_message(&chain))); + } +} + +fn request_blocks( + network: &mut NetworkContext, + log: &slog::Logger, + peer_id: PeerId, + request_id: RequestId, + request: BeaconBlocksRequest, +) { + trace!( + log, + "RPC Request"; + "method" => "BeaconBlocks", + "id" => request_id, + "count" => request.count, + "peer" => format!("{:?}", peer_id) + ); + network.send_rpc_request( + Some(request_id), + peer_id.clone(), + RPCRequest::BeaconBlocks(request), + ); +} + +fn recent_blocks_request( + network: &mut NetworkContext, + log: &slog::Logger, + peer_id: PeerId, + request_id: RequestId, + request: RecentBeaconBlocksRequest, +) { + trace!( + log, + "RPC Request"; + "method" => "RecentBeaconBlocks", + "count" => request.block_roots.len(), + "peer" => format!("{:?}", peer_id) + ); + network.send_rpc_request( + Some(request_id), + peer_id.clone(), + RPCRequest::RecentBeaconBlocks(request), + ); +} + +fn downvote_peer(network: &mut NetworkContext, log: &slog::Logger, peer_id: PeerId) { + trace!( + log, + "Peer downvoted"; + "peer" => format!("{:?}", peer_id) + ); + // TODO: Implement reputation + network.disconnect(peer_id.clone(), GoodbyeReason::Fault); +} + +// Helper function to process blocks which only consumes the chain and blocks to process fn process_blocks( weak_chain: Weak>, blocks: Vec>, @@ -1005,3 +1065,99 @@ fn process_blocks( Ok(()) } + +impl Future for SyncManager { + type Item = (); + type Error = String; + + fn poll(&mut self) -> Result, Self::Error> { + // process any inbound messages + loop { + match self.input_channel.poll() { + Ok(Async::Ready(Some(message))) => match message { + SyncMessage::AddPeer(peer_id, info) => { + self.add_peer(peer_id, info); + dbg!("add peer"); + } + SyncMessage::BeaconBlocksResponse { + peer_id, + request_id, + beacon_blocks, + } => { + self.beacon_blocks_response(peer_id, request_id, beacon_blocks); + } + SyncMessage::RecentBeaconBlocksResponse { + peer_id, + request_id, + beacon_blocks, + } => { + self.recent_blocks_response(peer_id, request_id, beacon_blocks); + } + SyncMessage::UnknownBlock(peer_id, block) => { + self.add_unknown_block(peer_id, block); + } + SyncMessage::Disconnect(peer_id) => { + self.peer_disconnect(&peer_id); + } + SyncMessage::_RPCError(request_id) => { + self.inject_error(request_id); + } + }, + Ok(Async::NotReady) => break, + Ok(Async::Ready(None)) => { + return Err("Sync manager channel closed".into()); + } + Err(e) => { + return Err(format!("Sync Manager channel error: {:?}", e)); + } + } + } + + loop { + //TODO: Optimize the lookups. Potentially keep state of whether each of these functions + //need to be called. + let mut re_run = false; + + dbg!(self.import_queue.len()); + // only process batch requests if there are any + if !self.import_queue.is_empty() { + // process potential block requests + self.process_potential_block_requests(); + + dbg!(self.import_queue.len()); + // process any complete long-range batches + re_run = re_run || self.process_complete_batches(); + dbg!(self.import_queue.len()); + dbg!(&self.state); + } + + // only process parent objects if we are in regular sync + if !self.parent_queue.is_empty() { + // process any parent block lookup-requests + self.process_parent_requests(); + + // process any complete parent lookups + re_run = re_run || self.process_complete_parent_requests(); + } + + dbg!(self.import_queue.len()); + dbg!(&self.state); + + // Shutdown the thread if the chain has termined + if let None = self.chain.upgrade() { + return Ok(Async::Ready(())); + } + + if !re_run { + break; + } + } + dbg!(self.import_queue.len()); + dbg!(&self.state); + + // update the state of the manager + self.update_state(); + + return Ok(Async::NotReady); + } +} diff --git a/beacon_node/network/src/sync/mod.rs b/beacon_node/network/src/sync/mod.rs index b26d78c147..58ec386aac 100644 --- a/beacon_node/network/src/sync/mod.rs +++ b/beacon_node/network/src/sync/mod.rs @@ -4,7 +4,7 @@ mod manager; /// Stores the various syncing methods for the beacon chain. mod simple_sync; -pub use simple_sync::SimpleSync; +pub use simple_sync::MessageProcessor; /// Currently implemented sync methods. pub enum SyncMethod { diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 4a853f05d0..d8b5f2dbf3 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -1,4 +1,4 @@ -use super::manager::{ImportManager, ImportManagerOutcome}; +use super::manager::SyncMessage; use crate::service::NetworkMessage; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use eth2_libp2p::rpc::methods::*; @@ -6,11 +6,14 @@ use eth2_libp2p::rpc::{RPCEvent, RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; use slog::{debug, info, o, trace, warn}; use ssz::Encode; -use std::sync::{Arc, Weak}; +use std::sync::Arc; use store::Store; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot}; use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256, Slot}; +//TODO: Put a maximum limit on the number of block that can be requested. +//TODO: Rate limit requests + /// If a block is more than `FUTURE_SLOT_TOLERANCE` slots ahead of our slot clock, we drop it. /// Otherwise we queue it. pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1; @@ -46,55 +49,71 @@ impl From<&Arc>> for PeerSyncInfo { } } -/// The current syncing state. -#[derive(PartialEq)] -pub enum SyncState { - _Idle, - _Downloading, - _Stopped, -} - -/// Simple Syncing protocol. -pub struct SimpleSync { +/// Processes validated messages from the network. It relays necessary data to the syncing thread +/// and processes blocks from the pubsub network. +pub struct MessageProcessor { /// A reference to the underlying beacon chain. - chain: Weak>, - manager: ImportManager, + chain: Arc>, + /// A channel to the syncing thread. + sync_send: mpsc::UnboundedSender>, + /// A oneshot channel for destroying the sync thread. + _sync_exit: oneshot::Sender<()>, + /// A nextwork context to return and handle RPC requests. network: NetworkContext, + /// The `RPCHandler` logger. log: slog::Logger, } -impl SimpleSync { - /// Instantiate a `SimpleSync` instance, with no peers and an empty queue. +impl MessageProcessor { + /// Instantiate a `MessageProcessor` instance pub fn new( - beacon_chain: Weak>, + executor: &tokio::runtime::TaskExecutor, + beacon_chain: Arc>, network_send: mpsc::UnboundedSender, log: &slog::Logger, ) -> Self { let sync_logger = log.new(o!("Service"=> "Sync")); + let sync_network_context = NetworkContext::new(network_send.clone(), sync_logger.clone()); - SimpleSync { - chain: beacon_chain.clone(), - manager: ImportManager::new(beacon_chain, log), + // spawn the sync thread + let (sync_send, _sync_exit) = super::manager::spawn( + executor, + Arc::downgrade(&beacon_chain), + sync_network_context, + sync_logger, + ); + + MessageProcessor { + chain: beacon_chain, + sync_send, + _sync_exit, network: NetworkContext::new(network_send, log.clone()), - log: sync_logger, + log: log.clone(), } } + fn send_to_sync(&mut self, message: SyncMessage) { + self.sync_send.try_send(message).unwrap_or_else(|_| { + warn!( + self.log, + "Could not send message to the sync service"; + ) + }); + } + /// Handle a peer disconnect. /// /// Removes the peer from the manager. pub fn on_disconnect(&mut self, peer_id: PeerId) { - self.manager.peer_disconnect(&peer_id); + self.send_to_sync(SyncMessage::Disconnect(peer_id)); } /// Handle the connection of a new peer. /// /// Sends a `Hello` message to the peer. pub fn on_connect(&mut self, peer_id: PeerId) { - if let Some(chain) = self.chain.upgrade() { - self.network - .send_rpc_request(None, peer_id, RPCRequest::Hello(hello_message(&chain))); - } + self.network + .send_rpc_request(None, peer_id, RPCRequest::Hello(hello_message(&self.chain))); } /// Handle a `Hello` request. @@ -107,18 +126,16 @@ impl SimpleSync { hello: HelloMessage, ) { // ignore hello responses if we are shutting down - if let Some(chain) = self.chain.upgrade() { - trace!(self.log, "HelloRequest"; "peer" => format!("{:?}", peer_id)); + trace!(self.log, "HelloRequest"; "peer" => format!("{:?}", peer_id)); - // Say hello back. - self.network.send_rpc_response( - peer_id.clone(), - request_id, - RPCResponse::Hello(hello_message(&chain)), - ); + // Say hello back. + self.network.send_rpc_response( + peer_id.clone(), + request_id, + RPCResponse::Hello(hello_message(&self.chain)), + ); - self.process_hello(peer_id, hello); - } + self.process_hello(peer_id, hello); } /// Process a `Hello` response from a peer. @@ -133,183 +150,86 @@ impl SimpleSync { /// /// Disconnects the peer if required. fn process_hello(&mut self, peer_id: PeerId, hello: HelloMessage) { - // If we update the manager we may need to drive the sync. This flag lies out of scope of - // the beacon chain so that the process sync command has no long-lived beacon chain - // references. - let mut process_sync = false; + let remote = PeerSyncInfo::from(hello); + let local = PeerSyncInfo::from(&self.chain); + + let start_slot = |epoch: Epoch| epoch.start_slot(T::EthSpec::slots_per_epoch()); + + if local.fork_version != remote.fork_version { + // The node is on a different network/fork, disconnect them. + debug!( + self.log, "HandshakeFailure"; + "peer" => format!("{:?}", peer_id), + "reason" => "network_id" + ); + + self.network + .disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork); + } else if remote.finalized_epoch <= local.finalized_epoch + && remote.finalized_root != Hash256::zero() + && local.finalized_root != Hash256::zero() + && (self.chain.root_at_slot(start_slot(remote.finalized_epoch)) + != Some(remote.finalized_root)) { - // scope of beacon chain reference - let chain = match self.chain.upgrade() { - Some(chain) => chain, - None => { - info!(self.log, "Sync shutting down"; - "reason" => "Beacon chain dropped"); - return; - } - }; + // The remotes finalized epoch is less than or greater than ours, but the block root is + // different to the one in our chain. + // + // Therefore, the node is on a different chain and we should not communicate with them. + debug!( + self.log, "HandshakeFailure"; + "peer" => format!("{:?}", peer_id), + "reason" => "different finalized chain" + ); + self.network + .disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork); + } else if remote.finalized_epoch < local.finalized_epoch { + // The node has a lower finalized epoch, their chain is not useful to us. There are two + // cases where a node can have a lower finalized epoch: + // + // ## The node is on the same chain + // + // If a node is on the same chain but has a lower finalized epoch, their head must be + // lower than ours. Therefore, we have nothing to request from them. + // + // ## The node is on a fork + // + // If a node is on a fork that has a lower finalized epoch, switching to that fork would + // cause us to revert a finalized block. This is not permitted, therefore we have no + // interest in their blocks. + debug!( + self.log, + "NaivePeer"; + "peer" => format!("{:?}", peer_id), + "reason" => "lower finalized epoch" + ); + } else if self + .chain + .store + .exists::>(&remote.head_root) + .unwrap_or_else(|_| false) + { + trace!( + self.log, "Peer with known chain found"; + "peer" => format!("{:?}", peer_id), + "remote_head_slot" => remote.head_slot, + "remote_latest_finalized_epoch" => remote.finalized_epoch, + ); - let remote = PeerSyncInfo::from(hello); - let local = PeerSyncInfo::from(&chain); - - let start_slot = |epoch: Epoch| epoch.start_slot(T::EthSpec::slots_per_epoch()); - - if local.fork_version != remote.fork_version { - // The node is on a different network/fork, disconnect them. - debug!( - self.log, "HandshakeFailure"; - "peer" => format!("{:?}", peer_id), - "reason" => "network_id" - ); - - self.network - .disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork); - } else if remote.finalized_epoch <= local.finalized_epoch - && remote.finalized_root != Hash256::zero() - && local.finalized_root != Hash256::zero() - && (chain.root_at_slot(start_slot(remote.finalized_epoch)) - != Some(remote.finalized_root)) - { - // The remotes finalized epoch is less than or greater than ours, but the block root is - // different to the one in our chain. - // - // Therefore, the node is on a different chain and we should not communicate with them. - debug!( - self.log, "HandshakeFailure"; - "peer" => format!("{:?}", peer_id), - "reason" => "different finalized chain" - ); - self.network - .disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork); - } else if remote.finalized_epoch < local.finalized_epoch { - // The node has a lower finalized epoch, their chain is not useful to us. There are two - // cases where a node can have a lower finalized epoch: - // - // ## The node is on the same chain - // - // If a node is on the same chain but has a lower finalized epoch, their head must be - // lower than ours. Therefore, we have nothing to request from them. - // - // ## The node is on a fork - // - // If a node is on a fork that has a lower finalized epoch, switching to that fork would - // cause us to revert a finalized block. This is not permitted, therefore we have no - // interest in their blocks. - debug!( - self.log, - "NaivePeer"; - "peer" => format!("{:?}", peer_id), - "reason" => "lower finalized epoch" - ); - } else if chain - .store - .exists::>(&remote.head_root) - .unwrap_or_else(|_| false) - { - trace!( - self.log, "Peer with known chain found"; - "peer" => format!("{:?}", peer_id), - "remote_head_slot" => remote.head_slot, - "remote_latest_finalized_epoch" => remote.finalized_epoch, - ); - - // If the node's best-block is already known to us and they are close to our current - // head, treat them as a fully sync'd peer. - self.manager.add_peer(peer_id, remote); - process_sync = true; - } else { - // The remote node has an equal or great finalized epoch and we don't know it's head. - // - // Therefore, there are some blocks between the local finalized epoch and the remote - // head that are worth downloading. - debug!( - self.log, "UsefulPeer"; - "peer" => format!("{:?}", peer_id), - "local_finalized_epoch" => local.finalized_epoch, - "remote_latest_finalized_epoch" => remote.finalized_epoch, - ); - - self.manager.add_peer(peer_id, remote); - process_sync = true - } - } // end beacon chain reference scope - - if process_sync { - self.process_sync(); - } - } - - /// This function drives the `ImportManager` state machine. The outcomes it provides are - /// actioned until the `ImportManager` is idle. - fn process_sync(&mut self) { - loop { - match self.manager.poll() { - ImportManagerOutcome::Hello(peer_id) => { - trace!( - self.log, - "RPC Request"; - "method" => "HELLO", - "peer" => format!("{:?}", peer_id) - ); - if let Some(chain) = self.chain.upgrade() { - self.network.send_rpc_request( - None, - peer_id, - RPCRequest::Hello(hello_message(&chain)), - ); - } - } - ImportManagerOutcome::RequestBlocks { - peer_id, - request_id, - request, - } => { - trace!( - self.log, - "RPC Request"; - "method" => "BeaconBlocks", - "id" => request_id, - "count" => request.count, - "peer" => format!("{:?}", peer_id) - ); - self.network.send_rpc_request( - Some(request_id), - peer_id.clone(), - RPCRequest::BeaconBlocks(request), - ); - } - ImportManagerOutcome::RecentRequest { - peer_id, - request_id, - request, - } => { - trace!( - self.log, - "RPC Request"; - "method" => "RecentBeaconBlocks", - "count" => request.block_roots.len(), - "peer" => format!("{:?}", peer_id) - ); - self.network.send_rpc_request( - Some(request_id), - peer_id.clone(), - RPCRequest::RecentBeaconBlocks(request), - ); - } - ImportManagerOutcome::DownvotePeer(peer_id) => { - trace!( - self.log, - "Peer downvoted"; - "peer" => format!("{:?}", peer_id) - ); - // TODO: Implement reputation - self.network - .disconnect(peer_id.clone(), GoodbyeReason::Fault); - } - ImportManagerOutcome::Idle => { - // nothing to do - return; - } - } + // If the node's best-block is already known to us and they are close to our current + // head, treat them as a fully sync'd peer. + self.send_to_sync(SyncMessage::AddPeer(peer_id, remote)); + } else { + // The remote node has an equal or great finalized epoch and we don't know it's head. + // + // Therefore, there are some blocks between the local finalized epoch and the remote + // head that are worth downloading. + debug!( + self.log, "UsefulPeer"; + "peer" => format!("{:?}", peer_id), + "local_finalized_epoch" => local.finalized_epoch, + "remote_latest_finalized_epoch" => remote.finalized_epoch, + ); + self.send_to_sync(SyncMessage::AddPeer(peer_id, remote)); } } @@ -320,20 +240,11 @@ impl SimpleSync { request_id: RequestId, request: RecentBeaconBlocksRequest, ) { - let chain = match self.chain.upgrade() { - Some(chain) => chain, - None => { - info!(self.log, "Sync shutting down"; - "reason" => "Beacon chain dropped"); - return; - } - }; - let blocks: Vec> = request .block_roots .iter() .filter_map(|root| { - if let Ok(Some(block)) = chain.store.get::>(root) { + if let Ok(Some(block)) = self.chain.store.get::>(root) { Some(block) } else { debug!( @@ -370,15 +281,6 @@ impl SimpleSync { request_id: RequestId, req: BeaconBlocksRequest, ) { - let chain = match self.chain.upgrade() { - Some(chain) => chain, - None => { - info!(self.log, "Sync shutting down"; - "reason" => "Beacon chain dropped"); - return; - } - }; - debug!( self.log, "BeaconBlocksRequest"; @@ -392,14 +294,15 @@ impl SimpleSync { // In the current implementation we read from the db then filter out out-of-range blocks. // Improving the db schema to prevent this would be ideal. - let mut blocks: Vec> = chain + let mut blocks: Vec> = self + .chain .rev_iter_block_roots() .filter(|(_root, slot)| { req.start_slot <= slot.as_u64() && req.start_slot + req.count > slot.as_u64() }) .take_while(|(_root, slot)| req.start_slot <= slot.as_u64()) .filter_map(|(root, _slot)| { - if let Ok(Some(block)) = chain.store.get::>(&root) { + if let Ok(Some(block)) = self.chain.store.get::>(&root) { Some(block) } else { warn!( @@ -423,7 +326,7 @@ impl SimpleSync { "peer" => format!("{:?}", peer_id), "msg" => "Failed to return all requested hashes", "start_slot" => req.start_slot, - "current_slot" => chain.slot().unwrap_or_else(|_| Slot::from(0_u64)).as_u64(), + "current_slot" => self.chain.slot().unwrap_or_else(|_| Slot::from(0_u64)).as_u64(), "requested" => req.count, "returned" => blocks.len(), ); @@ -449,10 +352,11 @@ impl SimpleSync { "count" => beacon_blocks.len(), ); - self.manager - .beacon_blocks_response(peer_id, request_id, beacon_blocks); - - self.process_sync(); + self.send_to_sync(SyncMessage::RecentBeaconBlocksResponse { + peer_id, + request_id, + beacon_blocks, + }); } /// Handle a `RecentBeaconBlocks` response from the peer. @@ -469,10 +373,11 @@ impl SimpleSync { "count" => beacon_blocks.len(), ); - self.manager - .recent_blocks_response(peer_id, request_id, beacon_blocks); - - self.process_sync(); + self.send_to_sync(SyncMessage::BeaconBlocksResponse { + peer_id, + request_id, + beacon_blocks, + }); } /// Process a gossip message declaring a new block. @@ -481,16 +386,7 @@ impl SimpleSync { /// /// Returns a `bool` which, if `true`, indicates we should forward the block to our peers. pub fn on_block_gossip(&mut self, peer_id: PeerId, block: BeaconBlock) -> bool { - let chain = match self.chain.upgrade() { - Some(chain) => chain, - None => { - info!(self.log, "Sync shutting down"; - "reason" => "Beacon chain dropped"); - return false; - } - }; - - if let Ok(outcome) = chain.process_block(block.clone()) { + if let Ok(outcome) = self.chain.process_block(block.clone()) { match outcome { BlockProcessingOutcome::Processed { .. } => { trace!(self.log, "Gossipsub block processed"; @@ -501,7 +397,7 @@ impl SimpleSync { // Inform the sync manager to find parents for this block trace!(self.log, "Block with unknown parent received"; "peer_id" => format!("{:?}",peer_id)); - self.manager.add_unknown_block(block.clone(), peer_id); + self.send_to_sync(SyncMessage::UnknownBlock(peer_id, block.clone())); SHOULD_FORWARD_GOSSIP_BLOCK } BlockProcessingOutcome::FutureSlot { @@ -523,16 +419,7 @@ impl SimpleSync { /// /// Not currently implemented. pub fn on_attestation_gossip(&mut self, _peer_id: PeerId, msg: Attestation) { - let chain = match self.chain.upgrade() { - Some(chain) => chain, - None => { - info!(self.log, "Sync shutting down"; - "reason" => "Beacon chain dropped"); - return; - } - }; - - match chain.process_attestation(msg) { + match self.chain.process_attestation(msg) { Ok(outcome) => info!( self.log, "Processed attestation"; @@ -547,7 +434,7 @@ impl SimpleSync { } /// Build a `HelloMessage` representing the state of the given `beacon_chain`. -fn hello_message(beacon_chain: &BeaconChain) -> HelloMessage { +pub(crate) fn hello_message(beacon_chain: &BeaconChain) -> HelloMessage { let state = &beacon_chain.head().beacon_state; HelloMessage { From 04b47a357b997e8da7bf37c4830c2a48090e0720 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Sat, 7 Sep 2019 09:31:05 +1000 Subject: [PATCH 28/32] Correct bugs in new sync threading --- beacon_node/network/src/message_handler.rs | 7 ++++--- beacon_node/network/src/service.rs | 9 ++------- beacon_node/network/src/sync/manager.rs | 20 +++++++------------- beacon_node/network/src/sync/simple_sync.rs | 6 +++--- tests/ef_tests/eth2.0-spec-tests | 2 +- 5 files changed, 17 insertions(+), 27 deletions(-) diff --git a/beacon_node/network/src/message_handler.rs b/beacon_node/network/src/message_handler.rs index be8fa21f89..782d2129ef 100644 --- a/beacon_node/network/src/message_handler.rs +++ b/beacon_node/network/src/message_handler.rs @@ -9,7 +9,7 @@ use eth2_libp2p::{ }; use futures::future::Future; use futures::stream::Stream; -use slog::{debug, trace, warn}; +use slog::{debug, o, trace, warn}; use ssz::{Decode, DecodeError}; use std::sync::Arc; use tokio::sync::mpsc; @@ -51,7 +51,8 @@ impl MessageHandler { executor: &tokio::runtime::TaskExecutor, log: slog::Logger, ) -> error::Result> { - trace!(log, "Service starting"); + let message_handler_log = log.new(o!("Service"=> "Message Handler")); + trace!(message_handler_log, "Service starting"); let (handler_send, handler_recv) = mpsc::unbounded_channel(); @@ -63,7 +64,7 @@ impl MessageHandler { let mut handler = MessageHandler { network_send, message_processor, - log: log.clone(), + log: message_handler_log, }; // spawn handler task and move the message handler instance into the spawned thread diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index f546306157..1357b54951 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -34,13 +34,8 @@ impl Service { // build the network channel let (network_send, network_recv) = mpsc::unbounded_channel::(); // launch message handler thread - let message_handler_log = log.new(o!("Service" => "MessageHandler")); - let message_handler_send = MessageHandler::spawn( - beacon_chain, - network_send.clone(), - executor, - message_handler_log, - )?; + let message_handler_send = + MessageHandler::spawn(beacon_chain, network_send.clone(), executor, log.clone())?; let network_log = log.new(o!("Service" => "Network")); // launch libp2p service diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 12bef95fa5..171d0fdf0b 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -251,7 +251,7 @@ pub fn spawn( // create an instance of the SyncManager let sync_manager = SyncManager { chain: beacon_chain, - state: ManagerState::Regular, + state: ManagerState::Stalled, input_channel: sync_recv, network, import_queue: HashMap::new(), @@ -510,7 +510,7 @@ impl SyncManager { &mut self, peer_id: PeerId, request_id: RequestId, - blocks: Vec>, + mut blocks: Vec>, ) { // find the request let parent_request = match self @@ -545,6 +545,11 @@ impl SyncManager { return; } + // add the block to response + parent_request + .downloaded_blocks + .push(blocks.pop().expect("must exist")); + // queue for processing parent_request.state = BlockRequestsState::ReadyToProcess; } @@ -594,7 +599,6 @@ impl SyncManager { "peer" => format!("{:?}", peer_id), ); self.full_peers.insert(peer_id); - self.update_state(); } /* Processing State Functions */ @@ -1077,7 +1081,6 @@ impl Future for SyncManager { Ok(Async::Ready(Some(message))) => match message { SyncMessage::AddPeer(peer_id, info) => { self.add_peer(peer_id, info); - dbg!("add peer"); } SyncMessage::BeaconBlocksResponse { peer_id, @@ -1118,17 +1121,13 @@ impl Future for SyncManager { //need to be called. let mut re_run = false; - dbg!(self.import_queue.len()); // only process batch requests if there are any if !self.import_queue.is_empty() { // process potential block requests self.process_potential_block_requests(); - dbg!(self.import_queue.len()); // process any complete long-range batches re_run = re_run || self.process_complete_batches(); - dbg!(self.import_queue.len()); - dbg!(&self.state); } // only process parent objects if we are in regular sync @@ -1140,9 +1139,6 @@ impl Future for SyncManager { re_run = re_run || self.process_complete_parent_requests(); } - dbg!(self.import_queue.len()); - dbg!(&self.state); - // Shutdown the thread if the chain has termined if let None = self.chain.upgrade() { return Ok(Async::Ready(())); @@ -1152,8 +1148,6 @@ impl Future for SyncManager { break; } } - dbg!(self.import_queue.len()); - dbg!(&self.state); // update the state of the manager self.update_state(); diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index d8b5f2dbf3..c54c481c73 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -352,7 +352,7 @@ impl MessageProcessor { "count" => beacon_blocks.len(), ); - self.send_to_sync(SyncMessage::RecentBeaconBlocksResponse { + self.send_to_sync(SyncMessage::BeaconBlocksResponse { peer_id, request_id, beacon_blocks, @@ -368,12 +368,12 @@ impl MessageProcessor { ) { debug!( self.log, - "BeaconBlocksResponse"; + "RecentBeaconBlocksResponse"; "peer" => format!("{:?}", peer_id), "count" => beacon_blocks.len(), ); - self.send_to_sync(SyncMessage::BeaconBlocksResponse { + self.send_to_sync(SyncMessage::RecentBeaconBlocksResponse { peer_id, request_id, beacon_blocks, diff --git a/tests/ef_tests/eth2.0-spec-tests b/tests/ef_tests/eth2.0-spec-tests index aaa1673f50..ae6dd9011d 160000 --- a/tests/ef_tests/eth2.0-spec-tests +++ b/tests/ef_tests/eth2.0-spec-tests @@ -1 +1 @@ -Subproject commit aaa1673f508103e11304833e0456e4149f880065 +Subproject commit ae6dd9011df05fab8c7e651c09cf9c940973bf81 From 69442a2ab302ed04ecbbc86009f2179803a6dd00 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Sun, 8 Sep 2019 01:57:56 +1000 Subject: [PATCH 29/32] Correct warnings --- beacon_node/rest_api/src/helpers.rs | 4 +--- beacon_node/rest_api/src/url_query.rs | 2 +- beacon_node/rest_api/src/validator.rs | 5 ++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 6b0211662a..a21f1831eb 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -2,9 +2,7 @@ use crate::{ApiError, ApiResult}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use bls::PublicKey; use hex; -use hyper::{Body, Request, StatusCode}; -use serde::de::value::StringDeserializer; -use serde_json::Deserializer; +use hyper::{Body, Request}; use store::{iter::AncestorIter, Store}; use types::{BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; diff --git a/beacon_node/rest_api/src/url_query.rs b/beacon_node/rest_api/src/url_query.rs index e39a9a4499..3802ff8317 100644 --- a/beacon_node/rest_api/src/url_query.rs +++ b/beacon_node/rest_api/src/url_query.rs @@ -64,7 +64,7 @@ impl<'a> UrlQuery<'a> { /// Returns a vector of all values present where `key` is in `keys /// /// If no match is found, an `InvalidQueryParams` error is returned. - pub fn all_of(mut self, key: &str) -> Result, ApiError> { + pub fn all_of(self, key: &str) -> Result, ApiError> { let queries: Vec<_> = self .0 .filter_map(|(k, v)| { diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 365b7e5521..0440a73684 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -5,9 +5,8 @@ use bls::PublicKey; use hyper::{Body, Request}; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use store::Store; use types::beacon_state::EthSpec; -use types::{BeaconBlock, BeaconState, Epoch, RelativeEpoch, Shard, Slot}; +use types::{Epoch, RelativeEpoch, Shard, Slot}; #[derive(Debug, Serialize, Deserialize)] pub struct ValidatorDuty { @@ -61,7 +60,7 @@ pub fn get_validator_duties(req: Request) - )) })?; //TODO: Handle an array of validators, currently only takes one - let mut validators: Vec = match query.all_of("validator_pubkeys") { + let validators: Vec = match query.all_of("validator_pubkeys") { Ok(v) => v .iter() .map(|pk| parse_pubkey(pk)) From 9461b5063b3ea9e0af05527af7dacf3d9d13f438 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 8 Sep 2019 04:19:54 +1000 Subject: [PATCH 30/32] Replace EF tests submodule with a makefile --- .gitmodules | 3 --- Makefile | 31 +++++++++++++++++++++++++ book/src/setup.md | 40 ++++++++++++++++++++++---------- tests/ef_tests/.gitignore | 1 + tests/ef_tests/eth2.0-spec-tests | 1 - tests/ef_tests/src/handler.rs | 5 ++++ 6 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 Makefile create mode 100644 tests/ef_tests/.gitignore delete mode 160000 tests/ef_tests/eth2.0-spec-tests diff --git a/.gitmodules b/.gitmodules index 1b0e150ce6..e69de29bb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "tests/ef_tests/eth2.0-spec-tests"] - path = tests/ef_tests/eth2.0-spec-tests - url = https://github.com/ethereum/eth2.0-spec-tests diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..d5517ed231 --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +TESTS_TAG := v0.8.3 +TESTS = general minimal mainnet + +TESTS_BASE_DIR := ./tests/ef_tests +REPO_NAME := eth2.0-spec-tests +OUTPUT_DIR := $(TESTS_BASE_DIR)/$(REPO_NAME) + +BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(SPEC_VERSION) + +release: + cargo build --all --release + +clean_ef_tests: + rm -r $(OUTPUT_DIR) + +ef_tests: download_tests extract_tests + mkdir $(OUTPUT_DIR) + for test in $(TESTS); do \ + tar -C $(OUTPUT_DIR) -xvf $(TESTS_BASE_DIR)/$$test.tar ;\ + rm $(TESTS_BASE_DIR)/$$test.tar ;\ + done + +extract_tests: + for test in $(TESTS); do \ + gzip -df $(TESTS_BASE_DIR)/$$test.tar.gz ;\ + done + +download_tests: + for test in $(TESTS); do \ + wget -P $(TESTS_BASE_DIR) $(BASE_URL)/$$test.tar.gz; \ + done diff --git a/book/src/setup.md b/book/src/setup.md index e53ca93d83..532de3fc04 100644 --- a/book/src/setup.md +++ b/book/src/setup.md @@ -9,11 +9,8 @@ See the [Quick instructions](#quick-instructions) for a summary or the 1. Install Rust + Cargo with [rustup](https://rustup.rs/). 1. Install build dependencies using your package manager. - - `$ clang protobuf libssl-dev cmake git-lfs` - - Ensure [git-lfs](https://git-lfs.github.com/) is installed with `git lfs - install`. -1. Clone the [sigp/lighthouse](https://github.com/sigp/lighthouse), ensuring to - **initialize submodules**. + - `$ clang protobuf libssl-dev cmake` +1. Clone the [sigp/lighthouse](https://github.com/sigp/lighthouse). 1. In the root of the repo, run the tests with `cargo test --all --release`. 1. Then, build the binaries with `cargo build --all --release`. 1. Lighthouse is now fully built and tested. @@ -37,13 +34,8 @@ steps: - `protobuf`: required for protobuf serialization (gRPC) - `libssl-dev`: also gRPC - `cmake`: required for building protobuf - - `git-lfs`: The Git extension for [Large File - Support](https://git-lfs.github.com/) (required for Ethereum Foundation - test vectors). - 1. Clone the repository with submodules: `git clone --recursive - https://github.com/sigp/lighthouse`. If you're already cloned the repo, - ensure testing submodules are present: `$ git submodule init; git - submodule update` + 1. Clone the repository with submodules: `git clone + https://github.com/sigp/lighthouse`. 1. Change directory to the root of the repository. 1. Run the test suite with `cargo test --all --release`. The build and test process can take several minutes. If you experience any failures on @@ -63,3 +55,27 @@ Perl](http://strawberryperl.com/), or alternatively use a choco install command Additionally, the dependency `protoc-grpcio v0.3.1` is reported to have issues compiling in Windows. You can specify a known working version by editing version in `protos/Cargo.toml` section to `protoc-grpcio = "<=0.3.0"`. + +## eth2.0-spec-tests + +The +[ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/) +repository contains a large set of tests that verify Lighthouse behaviour +against the Ethereum Foundation specifications. + +The `tests/ef_tests` crate runs these tests and it has some interesting +behaviours: + +- If the `tests/ef_tests/eth2.0-spec-tests` directory is not present, all tests + indicate a `pass` when they did not actually run. +- If that directory _is_ present, the tests are executed faithfully, failing if + a discrepancy is found. + +The `tests/ef_tests/eth2.0-spec-tests` directory is not present by default. To +obtain it, use the Makefile in the root of the repository: + +``` +make ef_tests +``` + +_Note: this will download 100+ MB of test files from the [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/)._ diff --git a/tests/ef_tests/.gitignore b/tests/ef_tests/.gitignore new file mode 100644 index 0000000000..a83c5aa961 --- /dev/null +++ b/tests/ef_tests/.gitignore @@ -0,0 +1 @@ +/eth2.0-spec-tests diff --git a/tests/ef_tests/eth2.0-spec-tests b/tests/ef_tests/eth2.0-spec-tests deleted file mode 160000 index ae6dd9011d..0000000000 --- a/tests/ef_tests/eth2.0-spec-tests +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ae6dd9011df05fab8c7e651c09cf9c940973bf81 diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index e5d175e115..e8c83e1f84 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -31,6 +31,11 @@ pub trait Handler { .join(Self::runner_name()) .join(Self::handler_name()); + // If the directory containing the tests does not exist, just let all tests pass. + if !handler_path.exists() { + return; + } + // Iterate through test suites let test_cases = fs::read_dir(&handler_path) .expect("handler dir exists") From e8619399f254ac301db7591ddb6c5905347960e7 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Sun, 8 Sep 2019 07:10:36 +1000 Subject: [PATCH 31/32] Patch to correct for single byte RPC responses --- beacon_node/eth2-libp2p/src/rpc/codec/base.rs | 8 +++++--- beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs | 10 +++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/beacon_node/eth2-libp2p/src/rpc/codec/base.rs b/beacon_node/eth2-libp2p/src/rpc/codec/base.rs index a8a2398673..973567473e 100644 --- a/beacon_node/eth2-libp2p/src/rpc/codec/base.rs +++ b/beacon_node/eth2-libp2p/src/rpc/codec/base.rs @@ -101,13 +101,15 @@ where type Error = ::Error; fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { + // if we have only received the response code, wait for more bytes + if src.len() == 1 { + return Ok(None); + } + // using the response code determine which kind of payload needs to be decoded. let response_code = { if let Some(resp_code) = self.response_code { resp_code } else { - // buffer should not be empty - debug_assert!(!src.is_empty()); - let resp_byte = src.split_to(1); let mut resp_code_byte = [0; 1]; resp_code_byte.copy_from_slice(&resp_byte); diff --git a/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs b/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs index 1966bab628..d0e4d01cf3 100644 --- a/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs +++ b/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs @@ -4,7 +4,7 @@ use crate::rpc::{ protocol::{ProtocolId, RPCError}, }; use crate::rpc::{ErrorMessage, RPCErrorResponse, RPCRequest, RPCResponse}; -use bytes::{Bytes, BytesMut}; +use bytes::{BufMut, Bytes, BytesMut}; use ssz::{Decode, Encode}; use tokio::codec::{Decoder, Encoder}; use unsigned_varint::codec::UviBytes; @@ -56,6 +56,10 @@ impl Encoder for SSZInboundCodec { .inner .encode(Bytes::from(bytes), dst) .map_err(RPCError::from); + } else { + // payload is empty, add a 0-byte length prefix + dst.reserve(1); + dst.put_u8(0); } Ok(()) } @@ -152,8 +156,8 @@ impl Decoder for SSZOutboundCodec { type Error = RPCError; fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { - if src.is_empty() { - // the object sent could be empty. We return the empty object if this is the case + if src.len() == 1 && src[0] == 0_u8 { + // the object is empty. We return the empty object if this is the case match self.protocol.message_name.as_str() { "hello" => match self.protocol.version.as_str() { "1" => Err(RPCError::Custom( From 6a870d468cbae4da17db45fc9861309d0972c7c2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 8 Sep 2019 12:23:37 -0400 Subject: [PATCH 32/32] Add ssz_fixed_len method to ssz::Encode --- beacon_node/eth2-libp2p/src/rpc/methods.rs | 35 +++++++- eth2/types/src/lib.rs | 5 +- eth2/types/src/slot_epoch_macros.rs | 4 + eth2/utils/bls/src/macros.rs | 4 + eth2/utils/ssz/examples/struct_definition.rs | 7 ++ eth2/utils/ssz/src/encode.rs | 2 + eth2/utils/ssz/src/encode/impls.rs | 68 ++++++++++++++ eth2/utils/ssz/src/lib.rs | 1 - eth2/utils/ssz/src/macros.rs | 95 -------------------- eth2/utils/ssz/tests/tests.rs | 1 + eth2/utils/ssz_derive/src/lib.rs | 25 +++++- eth2/utils/ssz_types/src/bitfield.rs | 35 ++++++++ eth2/utils/ssz_types/src/fixed_vector.rs | 5 ++ eth2/utils/ssz_types/src/variable_list.rs | 5 ++ tests/ef_tests/src/cases/ssz_static.rs | 1 + 15 files changed, 191 insertions(+), 102 deletions(-) diff --git a/beacon_node/eth2-libp2p/src/rpc/methods.rs b/beacon_node/eth2-libp2p/src/rpc/methods.rs index 49813abe9a..ee8ad4860b 100644 --- a/beacon_node/eth2-libp2p/src/rpc/methods.rs +++ b/beacon_node/eth2-libp2p/src/rpc/methods.rs @@ -1,6 +1,5 @@ //!Available RPC methods types and ids. -use ssz::{impl_decode_via_from, impl_encode_via_from}; use ssz_derive::{Decode, Encode}; use types::{Epoch, Hash256, Slot}; @@ -66,8 +65,38 @@ impl Into for GoodbyeReason { } } -impl_encode_via_from!(GoodbyeReason, u64); -impl_decode_via_from!(GoodbyeReason, u64); +impl ssz::Encode for GoodbyeReason { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + } + + fn ssz_bytes_len(&self) -> usize { + 0_u64.ssz_bytes_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + let conv: u64 = self.clone().into(); + conv.ssz_append(buf) + } +} + +impl ssz::Decode for GoodbyeReason { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + u64::from_ssz_bytes(bytes).and_then(|n| Ok(n.into())) + } +} /// Request a number of beacon block roots from a peer. #[derive(Encode, Decode, Clone, Debug, PartialEq)] diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 3edf8b36ba..d1eaa393f8 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -86,5 +86,8 @@ pub type AttesterMap = HashMap<(u64, u64), Vec>; /// Maps a slot to a block proposer. pub type ProposerMap = HashMap; -pub use bls::{AggregatePublicKey, AggregateSignature, Keypair, PublicKey, SecretKey, Signature}; +pub use bls::{ + AggregatePublicKey, AggregateSignature, Keypair, PublicKey, PublicKeyBytes, SecretKey, + Signature, SignatureBytes, +}; pub use ssz_types::{typenum, typenum::Unsigned, BitList, BitVector, FixedVector, VariableList}; diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 62ca6b3af3..3bd54ee2da 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -201,6 +201,10 @@ macro_rules! impl_ssz { ::ssz_fixed_len() } + fn ssz_bytes_len(&self) -> usize { + 0_u64.ssz_bytes_len() + } + fn ssz_append(&self, buf: &mut Vec) { self.0.ssz_append(buf) } diff --git a/eth2/utils/bls/src/macros.rs b/eth2/utils/bls/src/macros.rs index 09838b73ed..e8bd3dd048 100644 --- a/eth2/utils/bls/src/macros.rs +++ b/eth2/utils/bls/src/macros.rs @@ -9,6 +9,10 @@ macro_rules! impl_ssz { $byte_size } + fn ssz_bytes_len(&self) -> usize { + $byte_size + } + fn ssz_append(&self, buf: &mut Vec) { buf.append(&mut self.as_bytes()) } diff --git a/eth2/utils/ssz/examples/struct_definition.rs b/eth2/utils/ssz/examples/struct_definition.rs index fa3ed2a642..0971e21da9 100644 --- a/eth2/utils/ssz/examples/struct_definition.rs +++ b/eth2/utils/ssz/examples/struct_definition.rs @@ -12,6 +12,13 @@ impl Encode for Foo { ::is_ssz_fixed_len() && as Encode>::is_ssz_fixed_len() } + fn ssz_bytes_len(&self) -> usize { + ::ssz_fixed_len() + + ssz::BYTES_PER_LENGTH_OFFSET + + ::ssz_fixed_len() + + self.b.ssz_bytes_len() + } + fn ssz_append(&self, buf: &mut Vec) { let offset = ::ssz_fixed_len() + as Encode>::ssz_fixed_len() diff --git a/eth2/utils/ssz/src/encode.rs b/eth2/utils/ssz/src/encode.rs index 6ceb08debc..5113fb71a1 100644 --- a/eth2/utils/ssz/src/encode.rs +++ b/eth2/utils/ssz/src/encode.rs @@ -27,6 +27,8 @@ pub trait Encode { BYTES_PER_LENGTH_OFFSET } + fn ssz_bytes_len(&self) -> usize; + /// Returns the full-form encoding of this object. /// /// The default implementation of this method should suffice for most cases. diff --git a/eth2/utils/ssz/src/encode/impls.rs b/eth2/utils/ssz/src/encode/impls.rs index 3d68d8911a..d25e79370e 100644 --- a/eth2/utils/ssz/src/encode/impls.rs +++ b/eth2/utils/ssz/src/encode/impls.rs @@ -13,6 +13,10 @@ macro_rules! impl_encodable_for_uint { $bit_size / 8 } + fn ssz_bytes_len(&self) -> usize { + $bit_size / 8 + } + fn ssz_append(&self, buf: &mut Vec) { buf.extend_from_slice(&self.to_le_bytes()); } @@ -58,6 +62,23 @@ macro_rules! impl_encode_for_tuples { } } + fn ssz_bytes_len(&self) -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() + } else { + let mut len = 0; + $( + len += if <$T as Encode>::is_ssz_fixed_len() { + <$T as Encode>::ssz_fixed_len() + } else { + BYTES_PER_LENGTH_OFFSET + + self.$idx.ssz_bytes_len() + }; + )* + len + } + } + fn ssz_append(&self, buf: &mut Vec) { let offset = $( <$T as Encode>::ssz_fixed_len() + @@ -185,6 +206,19 @@ impl Encode for Option { false } + fn ssz_bytes_len(&self) -> usize { + if let Some(some) = self { + let len = if ::is_ssz_fixed_len() { + ::ssz_fixed_len() + } else { + some.ssz_bytes_len() + }; + len + BYTES_PER_LENGTH_OFFSET + } else { + BYTES_PER_LENGTH_OFFSET + } + } + fn ssz_append(&self, buf: &mut Vec) { match self { None => buf.append(&mut encode_union_index(0)), @@ -201,6 +235,16 @@ impl Encode for Vec { false } + fn ssz_bytes_len(&self) -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() * self.len() + } else { + let mut len = self.into_iter().map(|item| item.ssz_bytes_len()).sum(); + len += BYTES_PER_LENGTH_OFFSET * self.len(); + len + } + } + fn ssz_append(&self, buf: &mut Vec) { if T::is_ssz_fixed_len() { buf.reserve(T::ssz_fixed_len() * self.len()); @@ -229,6 +273,10 @@ impl Encode for bool { 1 } + fn ssz_bytes_len(&self) -> usize { + 1 + } + fn ssz_append(&self, buf: &mut Vec) { buf.extend_from_slice(&(*self as u8).to_le_bytes()); } @@ -243,6 +291,10 @@ impl Encode for NonZeroUsize { ::ssz_fixed_len() } + fn ssz_bytes_len(&self) -> usize { + std::mem::size_of::() + } + fn ssz_append(&self, buf: &mut Vec) { self.get().ssz_append(buf) } @@ -257,6 +309,10 @@ impl Encode for H256 { 32 } + fn ssz_bytes_len(&self) -> usize { + 32 + } + fn ssz_append(&self, buf: &mut Vec) { buf.extend_from_slice(self.as_bytes()); } @@ -271,6 +327,10 @@ impl Encode for U256 { 32 } + fn ssz_bytes_len(&self) -> usize { + 32 + } + fn ssz_append(&self, buf: &mut Vec) { let n = ::ssz_fixed_len(); let s = buf.len(); @@ -289,6 +349,10 @@ impl Encode for U128 { 16 } + fn ssz_bytes_len(&self) -> usize { + 16 + } + fn ssz_append(&self, buf: &mut Vec) { let n = ::ssz_fixed_len(); let s = buf.len(); @@ -309,6 +373,10 @@ macro_rules! impl_encodable_for_u8_array { $len } + fn ssz_bytes_len(&self) -> usize { + $len + } + fn ssz_append(&self, buf: &mut Vec) { buf.extend_from_slice(&self[..]); } diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index 696d36cbf7..115633889e 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -36,7 +36,6 @@ mod decode; mod encode; -mod macros; pub use decode::{ impls::decode_list_of_variable_length_items, Decode, DecodeError, SszDecoder, SszDecoderBuilder, diff --git a/eth2/utils/ssz/src/macros.rs b/eth2/utils/ssz/src/macros.rs index 04147a8058..8b13789179 100644 --- a/eth2/utils/ssz/src/macros.rs +++ b/eth2/utils/ssz/src/macros.rs @@ -1,96 +1 @@ -/// Implements `Encode` for `$impl_type` using an implementation of `From<$impl_type> for -/// $from_type`. -/// -/// In effect, this allows for easy implementation of `Encode` for some type that implements a -/// `From` conversion into another type that already has `Encode` implemented. -#[macro_export] -macro_rules! impl_encode_via_from { - ($impl_type: ty, $from_type: ty) => { - impl ssz::Encode for $impl_type { - fn is_ssz_fixed_len() -> bool { - <$from_type as ssz::Encode>::is_ssz_fixed_len() - } - fn ssz_fixed_len() -> usize { - <$from_type as ssz::Encode>::ssz_fixed_len() - } - - fn ssz_append(&self, buf: &mut Vec) { - let conv: $from_type = self.clone().into(); - - conv.ssz_append(buf) - } - } - }; -} - -/// Implements `Decode` for `$impl_type` using an implementation of `From<$impl_type> for -/// $from_type`. -/// -/// In effect, this allows for easy implementation of `Decode` for some type that implements a -/// `From` conversion into another type that already has `Decode` implemented. -#[macro_export] -macro_rules! impl_decode_via_from { - ($impl_type: ty, $from_type: tt) => { - impl ssz::Decode for $impl_type { - fn is_ssz_fixed_len() -> bool { - <$from_type as ssz::Decode>::is_ssz_fixed_len() - } - - fn ssz_fixed_len() -> usize { - <$from_type as ssz::Decode>::ssz_fixed_len() - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - $from_type::from_ssz_bytes(bytes).and_then(|dec| Ok(dec.into())) - } - } - }; -} - -#[cfg(test)] -mod tests { - use self::ssz::{Decode, Encode}; - use crate as ssz; - - #[derive(PartialEq, Debug, Clone, Copy)] - struct Wrapper(u64); - - impl From for Wrapper { - fn from(x: u64) -> Wrapper { - Wrapper(x) - } - } - - impl From for u64 { - fn from(x: Wrapper) -> u64 { - x.0 - } - } - - impl_encode_via_from!(Wrapper, u64); - impl_decode_via_from!(Wrapper, u64); - - #[test] - fn impl_encode_via_from() { - let check_encode = |a: u64, b: Wrapper| assert_eq!(a.as_ssz_bytes(), b.as_ssz_bytes()); - - check_encode(0, Wrapper(0)); - check_encode(1, Wrapper(1)); - check_encode(42, Wrapper(42)); - } - - #[test] - fn impl_decode_via_from() { - let check_decode = |bytes: Vec| { - let a = u64::from_ssz_bytes(&bytes).unwrap(); - let b = Wrapper::from_ssz_bytes(&bytes).unwrap(); - - assert_eq!(a, b.into()) - }; - - check_decode(vec![0, 0, 0, 0, 0, 0, 0, 0]); - check_decode(vec![1, 0, 0, 0, 0, 0, 0, 0]); - check_decode(vec![1, 0, 0, 0, 2, 0, 0, 0]); - } -} diff --git a/eth2/utils/ssz/tests/tests.rs b/eth2/utils/ssz/tests/tests.rs index c19e366622..26f2f53efe 100644 --- a/eth2/utils/ssz/tests/tests.rs +++ b/eth2/utils/ssz/tests/tests.rs @@ -8,6 +8,7 @@ mod round_trip { fn round_trip(items: Vec) { for item in items { let encoded = &item.as_ssz_bytes(); + assert_eq!(item.ssz_bytes_len(), encoded.len()); assert_eq!(T::from_ssz_bytes(&encoded), Ok(item)); } } diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index 47d96859e0..5bdb9ca9dd 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -81,9 +81,12 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { }; let field_idents = get_serializable_named_field_idents(&struct_data); + let field_idents_a = get_serializable_named_field_idents(&struct_data); let field_types_a = get_serializable_field_types(&struct_data); let field_types_b = field_types_a.clone(); - let field_types_c = field_types_a.clone(); + let field_types_d = field_types_a.clone(); + let field_types_e = field_types_a.clone(); + let field_types_f = field_types_a.clone(); let output = quote! { impl #impl_generics ssz::Encode for #name #ty_generics #where_clause { @@ -105,9 +108,27 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { } } + fn ssz_bytes_len(&self) -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() + } else { + let mut len = 0; + #( + if <#field_types_d as ssz::Encode>::is_ssz_fixed_len() { + len += <#field_types_e as ssz::Encode>::ssz_fixed_len(); + } else { + len += ssz::BYTES_PER_LENGTH_OFFSET; + len += self.#field_idents_a.ssz_bytes_len(); + } + )* + + len + } + } + fn ssz_append(&self, buf: &mut Vec) { let offset = #( - <#field_types_c as ssz::Encode>::ssz_fixed_len() + + <#field_types_f as ssz::Encode>::ssz_fixed_len() + )* 0; diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 197426046c..dbe1addbea 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -476,6 +476,12 @@ impl Encode for Bitfield> { false } + fn ssz_bytes_len(&self) -> usize { + // We could likely do better than turning this into bytes and reading the length, however + // it is kept this way for simplicity. + self.clone().into_bytes().len() + } + fn ssz_append(&self, buf: &mut Vec) { buf.append(&mut self.clone().into_bytes()) } @@ -498,6 +504,10 @@ impl Encode for Bitfield> { true } + fn ssz_bytes_len(&self) -> usize { + self.as_slice().len() + } + fn ssz_fixed_len() -> usize { bytes_for_bit_len(N::to_usize()) } @@ -616,6 +626,7 @@ mod bitvector { pub type BitVector4 = BitVector; pub type BitVector8 = BitVector; pub type BitVector16 = BitVector; + pub type BitVector64 = BitVector; #[test] fn ssz_encode() { @@ -706,6 +717,18 @@ mod bitvector { fn assert_round_trip(t: T) { assert_eq!(T::from_ssz_bytes(&t.as_ssz_bytes()).unwrap(), t); } + + #[test] + fn ssz_bytes_len() { + for i in 0..64 { + let mut bitfield = BitVector64::new(); + for j in 0..i { + bitfield.set(j, true).expect("should set bit in bounds"); + } + let bytes = bitfield.as_ssz_bytes(); + assert_eq!(bitfield.ssz_bytes_len(), bytes.len(), "i = {}", i); + } + } } #[cfg(test)] @@ -1152,4 +1175,16 @@ mod bitlist { vec![false, false, true, false, false, false, false, false, true] ); } + + #[test] + fn ssz_bytes_len() { + for i in 1..64 { + let mut bitfield = BitList1024::with_capacity(i).unwrap(); + for j in 0..i { + bitfield.set(j, true).expect("should set bit in bounds"); + } + let bytes = bitfield.as_ssz_bytes(); + assert_eq!(bitfield.ssz_bytes_len(), bytes.len(), "i = {}", i); + } + } } diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index 090d04d84b..f9c8963313 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -183,6 +183,10 @@ where } } + fn ssz_bytes_len(&self) -> usize { + self.vec.ssz_bytes_len() + } + fn ssz_append(&self, buf: &mut Vec) { if T::is_ssz_fixed_len() { buf.reserve(T::ssz_fixed_len() * self.len()); @@ -318,6 +322,7 @@ mod test { fn ssz_round_trip(item: T) { let encoded = &item.as_ssz_bytes(); + assert_eq!(item.ssz_bytes_len(), encoded.len()); assert_eq!(T::from_ssz_bytes(&encoded), Ok(item)); } diff --git a/eth2/utils/ssz_types/src/variable_list.rs b/eth2/utils/ssz_types/src/variable_list.rs index beb7e6a938..feb656745b 100644 --- a/eth2/utils/ssz_types/src/variable_list.rs +++ b/eth2/utils/ssz_types/src/variable_list.rs @@ -208,6 +208,10 @@ where >::ssz_fixed_len() } + fn ssz_bytes_len(&self) -> usize { + self.vec.ssz_bytes_len() + } + fn ssz_append(&self, buf: &mut Vec) { self.vec.ssz_append(buf) } @@ -304,6 +308,7 @@ mod test { fn round_trip(item: T) { let encoded = &item.as_ssz_bytes(); + assert_eq!(item.ssz_bytes_len(), encoded.len()); assert_eq!(T::from_ssz_bytes(&encoded), Ok(item)); } diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index 6e4a672cb5..62f285d580 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -58,6 +58,7 @@ impl LoadCase for SszStaticSR { pub fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { // Check serialization let serialized_result = value.as_ssz_bytes(); + compare_result::(&Ok(value.ssz_bytes_len()), &Some(serialized.len()))?; compare_result::, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?; // Check deserialization