From f47eaf57304d627e552408aae8003197f0318f5d Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 3 Sep 2019 16:46:10 +1000 Subject: [PATCH] Parallel tests and SSZ generic --- eth2/utils/ssz_types/src/fixed_vector.rs | 17 +- tests/ef_tests/src/cases.rs | 12 +- tests/ef_tests/src/cases/epoch_processing.rs | 2 +- tests/ef_tests/src/cases/operations.rs | 3 +- tests/ef_tests/src/cases/ssz_generic.rs | 229 +++++++++++++++---- tests/ef_tests/src/cases/ssz_static.rs | 8 +- tests/ef_tests/src/handler.rs | 32 ++- tests/ef_tests/src/lib.rs | 7 - tests/ef_tests/tests/tests.rs | 25 +- 9 files changed, 246 insertions(+), 89 deletions(-) diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index edac77f0d0..edf499adf5 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -220,13 +220,26 @@ where fn from_ssz_bytes(bytes: &[u8]) -> Result { if bytes.is_empty() { - Ok(FixedVector::from(vec![])) + Err(ssz::DecodeError::InvalidByteLength { + len: 0, + expected: 1, + }) } else if T::is_ssz_fixed_len() { bytes .chunks(T::ssz_fixed_len()) .map(|chunk| T::from_ssz_bytes(chunk)) .collect::, _>>() - .and_then(|vec| Ok(vec.into())) + .and_then(|vec| { + if vec.len() == N::to_usize() { + Ok(vec.into()) + } else { + Err(ssz::DecodeError::BytesInvalid(format!( + "wrong number of vec elements, got: {}, expected: {}", + vec.len(), + N::to_usize() + ))) + } + }) } else { ssz::decode_list_of_variable_length_items(bytes).and_then(|vec| Ok(vec.into())) } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 1192eb0a0e..ed00f0ffee 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -1,4 +1,5 @@ use super::*; +use rayon::prelude::*; use std::fmt::Debug; use std::path::Path; @@ -39,7 +40,7 @@ pub trait LoadCase: Sized { fn load_from_dir(_path: &Path) -> Result; } -pub trait Case: Debug { +pub trait Case: Debug + Sync { /// An optional field for implementing a custom description. /// /// Defaults to "no description". @@ -79,13 +80,10 @@ pub struct Cases { pub test_cases: Vec, } -impl EfTest for Cases -where - T: Case + Debug, -{ - fn test_results(&self) -> Vec { +impl Cases { + pub fn test_results(&self) -> Vec { self.test_cases - .iter() + .into_par_iter() .enumerate() .map(|(i, tc)| CaseResult::new(i, tc, tc.result(i))) .collect() diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs index ac47ab2368..d79b5fc48f 100644 --- a/tests/ef_tests/src/cases/epoch_processing.rs +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -31,7 +31,7 @@ pub struct EpochProcessing> { _phantom: PhantomData, } -pub trait EpochTransition: TypeName + Debug { +pub trait EpochTransition: TypeName + Debug + Sync { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError>; } diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs index bcc104bad1..e86e6f5986 100644 --- a/tests/ef_tests/src/cases/operations.rs +++ b/tests/ef_tests/src/cases/operations.rs @@ -10,6 +10,7 @@ use state_processing::per_block_processing::{ process_block_header, process_deposits, process_exits, process_proposer_slashings, process_transfers, }; +use std::fmt::Debug; use std::path::{Path, PathBuf}; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, @@ -31,7 +32,7 @@ pub struct Operations> { pub post: Option>, } -pub trait Operation: Decode + TypeName + std::fmt::Debug { +pub trait Operation: Decode + TypeName + Debug + Sync { fn handler_name() -> String { Self::name().to_lowercase() } diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index ca49d21060..05b96ad7db 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -1,68 +1,205 @@ use super::*; -use crate::case_result::compare_result; -use ethereum_types::{U128, U256}; +use crate::cases::ssz_static::{check_serialization, check_tree_hash, SszStaticType}; +use crate::yaml_decode::yaml_decode_file; use serde_derive::Deserialize; -use ssz::Decode; -use std::fmt::Debug; +use std::fs; +use std::path::{Path, PathBuf}; +use types::typenum::*; +use types::{BitList, BitVector, FixedVector, VariableList}; #[derive(Debug, Clone, Deserialize)] -pub struct SszGeneric { - #[serde(alias = "type")] - pub type_name: String, - pub valid: bool, - pub value: Option, - pub ssz: Option, +struct Metadata { + root: String, + signing_root: Option, } -impl YamlDecode for SszGeneric { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +#[derive(Debug, Clone)] +pub struct SszGeneric { + path: PathBuf, + handler_name: String, + case_name: String, +} + +impl LoadCase for SszGeneric { + fn load_from_dir(path: &Path) -> Result { + let components = path + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .rev() + .collect::>(); + // Test case name is last + let case_name = components[0].clone(); + // Handler name is third last, before suite name and case name + let handler_name = components[2].clone(); + Ok(Self { + path: path.into(), + handler_name, + case_name, + }) + } +} + +macro_rules! type_dispatch { + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => primitive_type ] $($rest:tt)*) => { + match $value { + "bool" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* bool>, $($rest)*), + "uint8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u8>, $($rest)*), + "uint16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u16>, $($rest)*), + "uint32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u32>, $($rest)*), + "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*), + // FIXME(michael): implement tree hash for big ints + // "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* etherum_types::U128>, $($rest)*), + // "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ethereum_types::U256>, $($rest)*), + _ => { println!("unsupported: {}", $value); Ok(()) }, + } + }; + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => typenum ] $($rest:tt)*) => { + match $value { + // DO YOU LIKE NUMBERS? + "0" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U0>, $($rest)*), + "1" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1>, $($rest)*), + "2" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2>, $($rest)*), + "3" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U3>, $($rest)*), + "4" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4>, $($rest)*), + "5" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U5>, $($rest)*), + "6" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U6>, $($rest)*), + "7" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U7>, $($rest)*), + "8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8>, $($rest)*), + "9" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U9>, $($rest)*), + "16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U16>, $($rest)*), + "31" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U31>, $($rest)*), + "32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U32>, $($rest)*), + "64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U64>, $($rest)*), + "128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U128>, $($rest)*), + "256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U256>, $($rest)*), + "512" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U512>, $($rest)*), + "513" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U513>, $($rest)*), + "1024" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1024>, $($rest)*), + "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*), + "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*), + "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*), + _ => { println!("unsupported: {}", $value); Ok(()) }, + } + }; + // No base type: apply type params to function + ($function:ident, ($($arg:expr),*), _, <$($param_ty:ty),*>,) => { + $function::<$($param_ty),*>($($arg),*) + }; + ($function:ident, ($($arg:expr),*), $base_type_name:ident, <$($param_ty:ty),*>,) => { + $function::<$base_type_name<$($param_ty),*>>($($arg),*) } } impl Case for SszGeneric { + fn path(&self) -> &Path { + &self.path + } + fn result(&self, _case_index: usize) -> Result<(), Error> { - if let Some(ssz) = &self.ssz { - match self.type_name.as_ref() { - "uint8" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint16" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint32" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint64" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint128" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint256" => ssz_generic_test::(self.valid, ssz, &self.value), - _ => Err(Error::FailedToParseTest(format!( - "Unknown type: {}", - self.type_name - ))), + let parts = self.case_name.split('_').collect::>(); + + match self.handler_name.as_str() { + "basic_vector" => { + let elem_ty = parts[1]; + let length = parts[2]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + FixedVector, + <>, + [elem_ty => primitive_type] + [length => typenum] + )?; } - } else { - // Skip tests that do not have an ssz field. - // - // See: https://github.com/ethereum/eth2.0-specs/issues/1079 - Ok(()) + "bitlist" => { + let limit = parts[1]; + + // FIXME(michael): mark length "no" cases as known failures + + type_dispatch!( + ssz_generic_test, + (&self.path), + BitList, + <>, + [limit => typenum] + )?; + } + "bitvector" => { + let length = parts[1]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + BitVector, + <>, + [length => typenum] + )?; + } + "boolean" => { + ssz_generic_test::(&self.path)?; + } + "uints" => { + let type_name = "uint".to_owned() + parts[1]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + _, + <>, + [type_name.as_str() => primitive_type] + )?; + } + // FIXME(michael): support for the containers tests + _ => panic!("unsupported handler: {}", self.handler_name), } + Ok(()) } } -/// Execute a `ssz_generic` test case. -fn ssz_generic_test(should_be_ok: bool, ssz: &str, value: &Option) -> Result<(), Error> -where - T: Decode + YamlDecode + Debug + PartialEq, -{ - let ssz = hex::decode(&ssz[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - - // We do not cater for the scenario where the test is valid but we are not passed any SSZ. - if should_be_ok && value.is_none() { - panic!("Unexpected test input. Cannot pass without value.") - } - - let expected = if let Some(string) = value { - Some(T::yaml_decode(string)?) +fn ssz_generic_test(path: &Path) -> Result<(), Error> { + let meta_path = path.join("meta.yaml"); + let meta: Option = if meta_path.is_file() { + Some(yaml_decode_file(&meta_path)?) } else { None }; - let decoded = T::from_ssz_bytes(&ssz); + let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); - compare_result(&decoded, &expected) + let value_path = path.join("value.yaml"); + let value: Option = if value_path.is_file() { + Some(yaml_decode_file(&value_path)?) + } else { + None + }; + + // Valid + // TODO: signing root + if let Some(value) = value { + check_serialization(&value, &serialized)?; + + if let Some(ref meta) = meta { + check_tree_hash(&meta.root, value.tree_hash_root())?; + } + } + // Invalid + else { + if let Ok(decoded) = T::from_ssz_bytes(&serialized) { + return Err(Error::DidntFail(format!( + "Decoded invalid bytes into: {:?}", + decoded + ))); + } + } + + Ok(()) } diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index 6a949073de..f9f59cc4b1 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -35,12 +35,12 @@ pub struct SszStaticSR { // Trait alias for all deez bounds pub trait SszStaticType: - serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync { } impl SszStaticType for T where - T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync { } @@ -77,7 +77,7 @@ impl LoadCase for SszStaticSR { } } -fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { +pub fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { // Check serialization let serialized_result = value.as_ssz_bytes(); compare_result::, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?; @@ -89,7 +89,7 @@ fn check_serialization(value: &T, serialized: &[u8]) -> Result Ok(()) } -fn check_tree_hash(expected_str: &str, actual_root: Vec) -> Result<(), Error> { +pub fn check_tree_hash(expected_str: &str, actual_root: Vec) -> Result<(), Error> { let expected_root = hex::decode(&expected_str[2..]) .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; let expected_root = Hash256::from_slice(&expected_root); diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index ca1304136e..b6334c3839 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -1,6 +1,6 @@ use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation}; +use crate::type_name; use crate::type_name::TypeName; -use crate::EfTest; use std::fs; use std::marker::PhantomData; use std::path::PathBuf; @@ -256,3 +256,33 @@ impl> Handler for OperationsHandler O::handler_name() } } + +pub struct SszGenericHandler(PhantomData); + +impl Handler for SszGenericHandler { + type Case = cases::SszGeneric; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "ssz_generic" + } + + fn handler_name() -> String { + H::name().into() + } +} + +// Supported SSZ generic handlers +pub struct BasicVector; +type_name!(BasicVector, "basic_vector"); +pub struct Bitlist; +type_name!(Bitlist, "bitlist"); +pub struct Bitvector; +type_name!(Bitvector, "bitvector"); +pub struct Boolean; +type_name!(Boolean, "boolean"); +pub struct Uints; +type_name!(Uints, "uints"); diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index 54e674d85a..bcf7c77a04 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -17,10 +17,3 @@ mod handler; mod results; mod type_name; mod yaml_decode; - -/// Defined where an object can return the results of some test(s) adhering to the Ethereum -/// Foundation testing format. -pub trait EfTest { - /// Returns the results of executing one or more tests. - fn test_results(&self) -> Vec; -} diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index d663eb4543..71fa53c66c 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,5 +1,4 @@ use ef_tests::*; -use rayon::prelude::*; use types::{ Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink, @@ -7,29 +6,15 @@ use types::{ MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit, }; -/* #[test] -#[cfg(feature = "fake_crypto")] fn ssz_generic() { - yaml_files_in_test_dir(&Path::new("ssz_generic")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); } - -#[test] -#[cfg(feature = "fake_crypto")] -fn ssz_static() { - yaml_files_in_test_dir(&Path::new("ssz_static")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); -} -*/ - #[test] fn shuffling() { ShufflingHandler::::run();