Cleanups and SSZ generic container tests

tests/ef_tests/src/bls_setting.rs

@@ -2,7 +2,6 @@ use self::BlsSetting::*;
 use crate::error::Error;
 use serde_repr::Deserialize_repr;
 
-// TODO: use this in every test case
 #[derive(Deserialize_repr, Debug, Clone, Copy)]
 #[repr(u8)]
 pub enum BlsSetting {

tests/ef_tests/src/case_result.rs

@@ -1,7 +1,7 @@
 use super::*;
 use compare_fields::{CompareFields, Comparison, FieldComparison};
 use std::fmt::Debug;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use types::BeaconState;
 
 pub const MAX_VALUE_STRING_LEN: usize = 500;
@@ -15,11 +15,16 @@ pub struct CaseResult {
 }
 
 impl CaseResult {
-    pub fn new(case_index: usize, case: &impl Case, result: Result<(), Error>) -> Self {
+    pub fn new(
+        case_index: usize,
+        path: &Path,
+        case: &impl Case,
+        result: Result<(), Error>,
+    ) -> Self {
         CaseResult {
             case_index,
             desc: case.description(),
-            path: case.path().into(),
+            path: path.into(),
             result,
         }
     }

tests/ef_tests/src/cases.rs

@@ -1,7 +1,7 @@
 use super::*;
 use rayon::prelude::*;
 use std::fmt::Debug;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 mod bls_aggregate_pubkeys;
 mod bls_aggregate_sigs;
@@ -50,12 +50,6 @@ pub trait Case: Debug + Sync {
         "no description".to_string()
     }
 
-    /// Path to the directory for this test case.
-    fn path(&self) -> &Path {
-        // FIXME(michael): remove default impl
-        Path::new("")
-    }
-
     /// Execute a test and return the result.
     ///
     /// `case_index` reports the index of the case in the set of test cases. It is not strictly
@@ -65,7 +59,7 @@ pub trait Case: Debug + Sync {
 
 #[derive(Debug)]
 pub struct Cases<T> {
-    pub test_cases: Vec<T>,
+    pub test_cases: Vec<(PathBuf, T)>,
 }
 
 impl<T: Case> Cases<T> {
@@ -73,7 +67,7 @@ impl<T: Case> Cases<T> {
         self.test_cases
             .into_par_iter()
             .enumerate()
-            .map(|(i, tc)| CaseResult::new(i, tc, tc.result(i)))
+            .map(|(i, (ref path, ref tc))| CaseResult::new(i, path, tc, tc.result(i)))
             .collect()
     }
 }

tests/ef_tests/src/cases/epoch_processing.rs

@@ -125,10 +125,6 @@ impl<E: EthSpec, T: EpochTransition<E>> Case for EpochProcessing<E, T> {
             .unwrap_or_else(String::new)
     }
 
-    fn path(&self) -> &Path {
-        &self.path
-    }
-
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         let mut state = self.pre.clone();
         let mut expected = self.post.clone();

tests/ef_tests/src/cases/genesis_initialization.rs

@@ -45,10 +45,6 @@ impl<E: EthSpec> LoadCase for GenesisInitialization<E> {
 }
 
 impl<E: EthSpec> Case for GenesisInitialization<E> {
-    fn path(&self) -> &Path {
-        &self.path
-    }
-
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         let spec = &E::default_spec();
 

tests/ef_tests/src/cases/genesis_validity.rs

@@ -2,13 +2,12 @@ use super::*;
 use crate::decode::{ssz_decode_file, yaml_decode_file};
 use serde_derive::Deserialize;
 use state_processing::is_valid_genesis_state;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use types::{BeaconState, EthSpec};
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
 pub struct GenesisValidity<E: EthSpec> {
-    pub path: PathBuf,
     pub genesis: BeaconState<E>,
     pub is_valid: bool,
 }
@@ -18,19 +17,11 @@ impl<E: EthSpec> LoadCase for GenesisValidity<E> {
         let genesis = ssz_decode_file(&path.join("genesis.ssz"))?;
         let is_valid = yaml_decode_file(&path.join("is_valid.yaml"))?;
 
-        Ok(Self {
-            path: path.into(),
-            genesis,
-            is_valid,
-        })
+        Ok(Self { genesis, is_valid })
     }
 }
 
 impl<E: EthSpec> Case for GenesisValidity<E> {
-    fn path(&self) -> &Path {
-        &self.path
-    }
-
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         let spec = &E::default_spec();
 

tests/ef_tests/src/cases/operations.rs

@@ -11,7 +11,7 @@ use state_processing::per_block_processing::{
     process_transfers,
 };
 use std::fmt::Debug;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use types::{
     Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec,
     ProposerSlashing, Transfer, VoluntaryExit,
@@ -25,7 +25,6 @@ struct Metadata {
 
 #[derive(Debug, Clone)]
 pub struct Operations<E: EthSpec, O: Operation<E>> {
-    pub path: PathBuf,
     metadata: Metadata,
     pub pre: BeaconState<E>,
     pub operation: O,
@@ -156,7 +155,6 @@ impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
         };
 
         Ok(Self {
-            path: path.into(),
             metadata,
             pre,
             operation,
@@ -173,10 +171,6 @@ impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
             .unwrap_or_else(String::new)
     }
 
-    fn path(&self) -> &Path {
-        &self.path
-    }
-
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         self.metadata.bls_setting.unwrap_or_default().check()?;
 

tests/ef_tests/src/cases/sanity_blocks.rs

@@ -6,7 +6,6 @@ use serde_derive::Deserialize;
 use state_processing::{
     per_block_processing, per_slot_processing, BlockInvalid, BlockProcessingError,
 };
-use std::path::PathBuf;
 use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch};
 
 #[derive(Debug, Clone, Deserialize)]
@@ -19,7 +18,6 @@ pub struct Metadata {
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
 pub struct SanityBlocks<E: EthSpec> {
-    pub path: PathBuf,
     pub metadata: Metadata,
     pub pre: BeaconState<E>,
     pub blocks: Vec<BeaconBlock<E>>,
@@ -44,7 +42,6 @@ impl<E: EthSpec> LoadCase for SanityBlocks<E> {
         };
 
         Ok(Self {
-            path: path.into(),
             metadata,
             pre,
             blocks,
@@ -61,10 +58,6 @@ impl<E: EthSpec> Case for SanityBlocks<E> {
             .unwrap_or_else(String::new)
     }
 
-    fn path(&self) -> &Path {
-        &self.path
-    }
-
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         self.metadata.bls_setting.unwrap_or_default().check()?;
 

tests/ef_tests/src/cases/sanity_slots.rs

@@ -4,7 +4,6 @@ use crate::case_result::compare_beacon_state_results_without_caches;
 use crate::decode::{ssz_decode_file, yaml_decode_file};
 use serde_derive::Deserialize;
 use state_processing::per_slot_processing;
-use std::path::PathBuf;
 use types::{BeaconState, EthSpec};
 
 #[derive(Debug, Clone, Default, Deserialize)]
@@ -16,7 +15,6 @@ pub struct Metadata {
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
 pub struct SanitySlots<E: EthSpec> {
-    pub path: PathBuf,
     pub metadata: Metadata,
     pub pre: BeaconState<E>,
     pub slots: u64,
@@ -41,7 +39,6 @@ impl<E: EthSpec> LoadCase for SanitySlots<E> {
         };
 
         Ok(Self {
-            path: path.into(),
             metadata,
             pre,
             slots,
@@ -58,10 +55,6 @@ impl<E: EthSpec> Case for SanitySlots<E> {
             .unwrap_or_else(String::new)
     }
 
-    fn path(&self) -> &Path {
-        &self.path
-    }
-
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         self.metadata.bls_setting.unwrap_or_default().check()?;
 

tests/ef_tests/src/cases/ssz_generic.rs

@@ -1,12 +1,16 @@
+#![allow(non_snake_case)]
+
 use super::*;
 use crate::cases::common::{SszStaticType, TestU128, TestU256};
 use crate::cases::ssz_static::{check_serialization, check_tree_hash};
 use crate::decode::yaml_decode_file;
 use serde_derive::Deserialize;
+use ssz_derive::{Decode, Encode};
 use std::fs;
 use std::path::{Path, PathBuf};
+use tree_hash_derive::TreeHash;
 use types::typenum::*;
-use types::{BitList, BitVector, FixedVector};
+use types::{BitList, BitVector, FixedVector, VariableList};
 
 #[derive(Debug, Clone, Deserialize)]
 struct Metadata {
@@ -54,7 +58,7 @@ macro_rules! type_dispatch {
             "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*),
             "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU128>, $($rest)*),
             "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU256>, $($rest)*),
-            _ => { println!("unsupported: {}", $value); Ok(()) },
+            _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
         }
     };
     ($function:ident,
@@ -86,7 +90,23 @@ macro_rules! type_dispatch {
             "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*),
             "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*),
             "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*),
-            _ => { println!("unsupported: {}", $value); Ok(()) },
+            _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
+        }
+    };
+    ($function:ident,
+     ($($arg:expr),*),
+     $base_ty:tt,
+     <$($param_ty:ty),*>,
+     [ $value:expr => test_container ] $($rest:tt)*) => {
+        match $value {
+            "SingleFieldTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SingleFieldTestStruct>, $($rest)*),
+            "SmallTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SmallTestStruct>, $($rest)*),
+            "FixedTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* FixedTestStruct>, $($rest)*),
+            "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*),
+            "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*),
+            // TODO: enable ComplexTestStruct
+            "ComplexTestStruct" => Err(Error::SkippedKnownFailure),
+            _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
         }
     };
     // No base type: apply type params to function
@@ -99,10 +119,6 @@ macro_rules! type_dispatch {
 }
 
 impl Case for SszGeneric {
-    fn path(&self) -> &Path {
-        &self.path
-    }
-
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         let parts = self.case_name.split('_').collect::<Vec<_>>();
 
@@ -162,7 +178,17 @@ impl Case for SszGeneric {
                     [type_name.as_str() => primitive_type]
                 )?;
             }
-            // FIXME(michael): support for the containers tests
+            "containers" => {
+                let type_name = parts[0];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    _,
+                    <>,
+                    [type_name => test_container]
+                )?;
+            }
             _ => panic!("unsupported handler: {}", self.handler_name),
         }
         Ok(())
@@ -187,7 +213,7 @@ fn ssz_generic_test<T: SszStaticType>(path: &Path) -> Result<(), Error> {
     };
 
     // Valid
-    // TODO: signing root
+    // TODO: signing root (annoying because of traits)
     if let Some(value) = value {
         check_serialization(&value, &serialized)?;
 
@@ -207,3 +233,38 @@ fn ssz_generic_test<T: SszStaticType>(path: &Path) -> Result<(), Error> {
 
     Ok(())
 }
+
+// Containers for SSZ generic tests
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct SingleFieldTestStruct {
+    A: u8,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct SmallTestStruct {
+    A: u16,
+    B: u16,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct FixedTestStruct {
+    A: u8,
+    B: u64,
+    C: u32,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct VarTestStruct {
+    A: u16,
+    B: VariableList<u16, U1024>,
+    C: u8,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct BitsStruct {
+    A: BitList<U5>,
+    B: BitVector<U2>,
+    C: BitVector<U1>,
+    D: BitList<U6>,
+    E: BitVector<U8>,
+}

tests/ef_tests/src/cases/ssz_static.rs

@@ -28,7 +28,6 @@ pub struct SszStaticSR<T> {
 }
 
 fn load_from_dir<T: SszStaticType>(path: &Path) -> Result<(SszStaticRoots, Vec<u8>, T), Error> {
-    // FIXME(michael): set description/name
     let roots = yaml_decode_file(&path.join("roots.yaml"))?;
     let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists");
     let value = yaml_decode_file(&path.join("value.yaml"))?;

tests/ef_tests/src/handler.rs

@@ -32,19 +32,21 @@ pub trait Handler {
             .join(Self::handler_name());
 
         // Iterate through test suites
-        // TODO: parallelism
-        // TODO: error handling?
         let test_cases = fs::read_dir(&handler_path)
-            .expect("open main directory")
+            .expect("handler dir exists")
             .flat_map(|entry| {
                 entry
                     .ok()
                     .filter(|e| e.file_type().map(|ty| ty.is_dir()).unwrap_or(false))
             })
-            .flat_map(|suite| fs::read_dir(suite.path()).expect("open suite dir"))
+            .flat_map(|suite| fs::read_dir(suite.path()).expect("suite dir exists"))
             .flat_map(Result::ok)
-            .map(|test_case_dir| Self::Case::load_from_dir(&test_case_dir.path()).expect("loads"))
-            .collect::<Vec<_>>();
+            .map(|test_case_dir| {
+                let path = test_case_dir.path();
+                let case = Self::Case::load_from_dir(&path).expect("test should load");
+                (path, case)
+            })
+            .collect();
 
         let results = Cases { test_cases }.test_results();
 
@@ -286,3 +288,5 @@ pub struct Boolean;
 type_name!(Boolean, "boolean");
 pub struct Uints;
 type_name!(Uints, "uints");
+pub struct Containers;
+type_name!(Containers, "containers");