Update to Spec v0.10 (#817)

* Start updating types

* WIP

* Signature hacking

* Existing EF tests passing with fake_crypto

* Updates

* Delete outdated API spec

* The refactor continues

* It compiles

* WIP test fixes

* All release tests passing bar genesis state parsing

* Update and test YamlConfig

* Update to spec v0.10 compatible BLS

* Updates to BLS EF tests

* Add EF test for AggregateVerify

And delete unused hash2curve tests for uncompressed points

* Update EF tests to v0.10.1

* Use optional block root correctly in block proc

* Use genesis fork in deposit domain. All tests pass

* Cargo fmt

* Fast aggregate verify test

* Update REST API docs

* Cargo fmt

* Fix unused import

* Bump spec tags to v0.10.1

* Add `seconds_per_eth1_block` to chainspec

* Update to timestamp based eth1 voting scheme

* Return None from `get_votes_to_consider` if block cache is empty

* Handle overflows in `is_candidate_block`

* Revert to failing tests

* Fix eth1 data sets test

* Choose default vote according to spec

* Fix collect_valid_votes tests

* Fix `get_votes_to_consider` to choose all eligible blocks

* Uncomment winning_vote tests

* Add comments; remove unused code

* Reduce seconds_per_eth1_block for simulation

* Addressed review comments

* Add test for default vote case

* Fix logs

* Remove unused functions

* Meter default eth1 votes

* Fix comments

* Address review comments; remove unused dependency

* Disable/delete two outdated tests

* Bump eth1 default vote warn to error

* Delete outdated eth1 test

Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>

tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs

@@ -1,36 +0,0 @@
-use super::*;
-use crate::case_result::compare_result;
-use crate::cases::common::BlsCase;
-use bls::{AggregatePublicKey, PublicKey};
-use serde_derive::Deserialize;
-
-#[derive(Debug, Clone, Deserialize)]
-pub struct BlsAggregatePubkeys {
-    pub input: Vec<String>,
-    pub output: String,
-}
-
-impl BlsCase for BlsAggregatePubkeys {}
-
-impl Case for BlsAggregatePubkeys {
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let mut aggregate_pubkey = AggregatePublicKey::new();
-
-        for key_str in &self.input {
-            let key = hex::decode(&key_str[2..])
-                .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-            let key = PublicKey::from_bytes(&key)
-                .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-
-            aggregate_pubkey.add(&key);
-        }
-
-        let output_bytes = Some(
-            hex::decode(&self.output[2..])
-                .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?,
-        );
-        let aggregate_pubkey = Ok(aggregate_pubkey.as_raw().as_bytes());
-
-        compare_result::<Vec<u8>, Vec<u8>>(&aggregate_pubkey, &output_bytes)
-    }
-}

tests/ef_tests/src/cases/bls_aggregate_verify.rs

@@ -0,0 +1,59 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
+use bls::{AggregatePublicKey, AggregateSignature};
+use serde_derive::Deserialize;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsAggregatePair {
+    pub pubkey: AggregatePublicKey,
+    pub message: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsAggregateVerifyInput {
+    pub pairs: Vec<BlsAggregatePair>,
+    pub signature: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsAggregateVerify {
+    pub input: BlsAggregateVerifyInput,
+    pub output: bool,
+}
+
+impl BlsCase for BlsAggregateVerify {}
+
+impl Case for BlsAggregateVerify {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let messages = self
+            .input
+            .pairs
+            .iter()
+            .map(|pair| {
+                hex::decode(&pair.message[2..])
+                    .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))
+            })
+            .collect::<Result<Vec<Vec<_>>, _>>()?;
+
+        let message_refs = messages
+            .iter()
+            .map(|x| x.as_slice())
+            .collect::<Vec<&[u8]>>();
+
+        let pubkey_refs = self
+            .input
+            .pairs
+            .iter()
+            .map(|p| &p.pubkey)
+            .collect::<Vec<_>>();
+
+        let signature_ok = hex::decode(&self.input.signature[2..])
+            .ok()
+            .and_then(|bytes: Vec<u8>| AggregateSignature::from_bytes(&bytes).ok())
+            .map(|signature| signature.verify_multiple(&message_refs, &pubkey_refs))
+            .unwrap_or(false);
+
+        compare_result::<bool, ()>(&Ok(signature_ok), &Some(self.output))
+    }
+}

tests/ef_tests/src/cases/bls_fast_aggregate_verify.rs

@@ -0,0 +1,50 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
+use bls::{AggregatePublicKey, AggregateSignature, PublicKey, PublicKeyBytes};
+use serde_derive::Deserialize;
+use std::convert::TryInto;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsFastAggregateVerifyInput {
+    pub pubkeys: Vec<PublicKeyBytes>,
+    pub message: String,
+    pub signature: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsFastAggregateVerify {
+    pub input: BlsFastAggregateVerifyInput,
+    pub output: bool,
+}
+
+impl BlsCase for BlsFastAggregateVerify {}
+
+impl Case for BlsFastAggregateVerify {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let message = hex::decode(&self.input.message[2..])
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+        let signature_ok = self
+            .input
+            .pubkeys
+            .iter()
+            .try_fold(
+                AggregatePublicKey::new(),
+                |mut agg, pkb| -> Option<AggregatePublicKey> {
+                    let pk: Result<PublicKey, ssz::DecodeError> = pkb.try_into();
+                    agg.add(&pk.ok()?);
+                    Some(agg)
+                },
+            )
+            .and_then(|aggregate_pubkey| {
+                hex::decode(&self.input.signature[2..])
+                    .ok()
+                    .and_then(|bytes: Vec<u8>| AggregateSignature::from_bytes(&bytes).ok())
+                    .map(|signature| signature.verify(&message, &aggregate_pubkey))
+            })
+            .unwrap_or(false);
+
+        compare_result::<bool, ()>(&Ok(signature_ok), &Some(self.output))
+    }
+}

tests/ef_tests/src/cases/bls_g2_compressed.rs

@@ -1,50 +0,0 @@
-use super::*;
-use crate::case_result::compare_result;
-use crate::cases::common::BlsCase;
-use bls::{compress_g2, hash_on_g2};
-use serde_derive::Deserialize;
-
-#[derive(Debug, Clone, Deserialize)]
-pub struct BlsG2CompressedInput {
-    pub message: String,
-    pub domain: String,
-}
-
-#[derive(Debug, Clone, Deserialize)]
-pub struct BlsG2Compressed {
-    pub input: BlsG2CompressedInput,
-    pub output: Vec<String>,
-}
-
-impl BlsCase for BlsG2Compressed {}
-
-impl Case for BlsG2Compressed {
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        // Convert message and domain to required types
-        let msg = hex::decode(&self.input.message[2..])
-            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-        let d = hex::decode(&self.input.domain[2..])
-            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-        let d = bytes_to_u64(&d);
-
-        // Calculate the point and convert it to compressed bytes
-        let mut point = hash_on_g2(&msg, d);
-        let point = compress_g2(&mut point);
-
-        // Convert the output to one set of bytes
-        let mut decoded = hex::decode(&self.output[0][2..])
-            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-        let mut decoded_y = hex::decode(&self.output[1][2..])
-            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-        decoded.append(&mut decoded_y);
-
-        compare_result::<Vec<u8>, Vec<u8>>(&Ok(point), &Some(decoded))
-    }
-}
-
-// Converts a vector to u64 (from little endian)
-fn bytes_to_u64(array: &[u8]) -> u64 {
-    let mut bytes = [0u8; 8];
-    bytes.copy_from_slice(array);
-    u64::from_le_bytes(bytes)
-}

tests/ef_tests/src/cases/bls_g2_uncompressed.rs

@@ -1,65 +0,0 @@
-use super::*;
-use crate::case_result::compare_result;
-use crate::cases::common::BlsCase;
-use bls::hash_on_g2;
-use serde_derive::Deserialize;
-
-#[derive(Debug, Clone, Deserialize)]
-pub struct BlsG2UncompressedInput {
-    pub message: String,
-    pub domain: String,
-}
-
-impl BlsCase for BlsG2UncompressedInput {}
-
-#[derive(Debug, Clone, Deserialize)]
-pub struct BlsG2Uncompressed {
-    pub input: BlsG2UncompressedInput,
-    pub output: Vec<Vec<String>>,
-}
-
-impl Case for BlsG2Uncompressed {
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        // Convert message and domain to required types
-        let msg = hex::decode(&self.input.message[2..])
-            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-        let d = hex::decode(&self.input.domain[2..])
-            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-        let d = bytes_to_u64(&d);
-
-        // Calculate the point and convert it to compressed bytes
-        let point = hash_on_g2(&msg, d);
-        let mut point_bytes = [0 as u8; 288];
-        point.getpx().geta().tobytearray(&mut point_bytes, 0);
-        point.getpx().getb().tobytearray(&mut point_bytes, 48);
-        point.getpy().geta().tobytearray(&mut point_bytes, 96);
-        point.getpy().getb().tobytearray(&mut point_bytes, 144);
-        point.getpz().geta().tobytearray(&mut point_bytes, 192);
-        point.getpz().getb().tobytearray(&mut point_bytes, 240);
-
-        // Convert the output to one set of bytes (x.a, x.b, y.a, y.b, z.a, z.b)
-        let mut decoded: Vec<u8> = vec![];
-        for coordinate in &self.output {
-            let mut decoded_part = hex::decode(&coordinate[0][2..])
-                .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-            decoded.append(&mut decoded_part);
-            decoded_part = hex::decode(&coordinate[1][2..])
-                .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-            decoded.append(&mut decoded_part);
-        }
-
-        compare_result::<Vec<u8>, Vec<u8>>(&Ok(point_bytes.to_vec()), &Some(decoded))
-    }
-}
-
-// Converts a vector to u64 (from big endian)
-fn bytes_to_u64(array: &[u8]) -> u64 {
-    let mut result: u64 = 0;
-    for (i, value) in array.iter().rev().enumerate() {
-        if i == 8 {
-            break;
-        }
-        result += u64::pow(2, i as u32 * 8) * u64::from(*value);
-    }
-    result
-}

tests/ef_tests/src/cases/bls_priv_to_pub.rs

@@ -1,38 +0,0 @@
-use super::*;
-use crate::case_result::compare_result;
-use crate::cases::common::BlsCase;
-use bls::{PublicKey, SecretKey};
-use serde_derive::Deserialize;
-
-#[derive(Debug, Clone, Deserialize)]
-pub struct BlsPrivToPub {
-    pub input: String,
-    pub output: String,
-}
-
-impl BlsCase for BlsPrivToPub {}
-
-impl Case for BlsPrivToPub {
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let secret = &self.input;
-
-        // Convert message and domain to required types
-        let mut sk =
-            hex::decode(&secret[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-        pad_to_48(&mut sk);
-        let sk = SecretKey::from_bytes(&sk).unwrap();
-        let pk = PublicKey::from_secret_key(&sk);
-
-        let decoded = hex::decode(&self.output[2..])
-            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-
-        compare_result::<Vec<u8>, Vec<u8>>(&Ok(pk.as_raw().as_bytes()), &Some(decoded))
-    }
-}
-
-// Increase the size of an array to 48 bytes
-fn pad_to_48(array: &mut Vec<u8>) {
-    while array.len() < 48 {
-        array.insert(0, 0);
-    }
-}

tests/ef_tests/src/cases/bls_sign_msg.rs

@@ -8,7 +8,6 @@ use serde_derive::Deserialize;
 pub struct BlsSignInput {
     pub privkey: String,
     pub message: String,
-    pub domain: String,
 }
 
 #[derive(Debug, Clone, Deserialize)]
@@ -21,18 +20,15 @@ impl BlsCase for BlsSign {}
 
 impl Case for BlsSign {
     fn result(&self, _case_index: usize) -> Result<(), Error> {
-        // Convert private_key, message and domain to required types
+        // Convert private_key and message to required types
         let mut sk = hex::decode(&self.input.privkey[2..])
             .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
         pad_to_48(&mut sk);
         let sk = SecretKey::from_bytes(&sk).unwrap();
         let msg = hex::decode(&self.input.message[2..])
             .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-        let d = hex::decode(&self.input.domain[2..])
-            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-        let d = bytes_to_u64(&d);
 
-        let signature = Signature::new(&msg, d, &sk);
+        let signature = Signature::new(&msg, &sk);
 
         // Convert the output to one set of bytes
         let decoded = hex::decode(&self.output[2..])
@@ -42,13 +38,6 @@ impl Case for BlsSign {
     }
 }
 
-// Converts a vector to u64 (from little endian)
-fn bytes_to_u64(array: &[u8]) -> u64 {
-    let mut bytes = [0u8; 8];
-    bytes.copy_from_slice(array);
-    u64::from_le_bytes(bytes)
-}
-
 // Increase the size of an array to 48 bytes
 fn pad_to_48(array: &mut Vec<u8>) {
     while array.len() < 48 {

tests/ef_tests/src/cases/bls_verify_msg.rs

@@ -0,0 +1,35 @@
+use super::*;
+use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
+use bls::{PublicKey, Signature, SignatureBytes};
+use serde_derive::Deserialize;
+use std::convert::TryInto;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsVerifyInput {
+    pub pubkey: PublicKey,
+    pub message: String,
+    pub signature: SignatureBytes,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsVerify {
+    pub input: BlsVerifyInput,
+    pub output: bool,
+}
+
+impl BlsCase for BlsVerify {}
+
+impl Case for BlsVerify {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let message = hex::decode(&self.input.message[2..])
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+        let signature_ok = (&self.input.signature)
+            .try_into()
+            .map(|signature: Signature| signature.verify(&message, &self.input.pubkey))
+            .unwrap_or(false);
+
+        compare_result::<bool, ()>(&Ok(signature_ok), &Some(self.output))
+    }
+}

tests/ef_tests/src/cases/operations.rs

@@ -14,7 +14,7 @@ use std::fmt::Debug;
 use std::path::Path;
 use types::{
     Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec,
-    ProposerSlashing, VoluntaryExit,
+    ProposerSlashing, SignedVoluntaryExit,
 };
 
 #[derive(Debug, Clone, Default, Deserialize)]
@@ -95,7 +95,7 @@ impl<E: EthSpec> Operation<E> for ProposerSlashing {
     }
 }
 
-impl<E: EthSpec> Operation<E> for VoluntaryExit {
+impl<E: EthSpec> Operation<E> for SignedVoluntaryExit {
     fn handler_name() -> String {
         "voluntary_exit".into()
     }
@@ -123,13 +123,7 @@ impl<E: EthSpec> Operation<E> for BeaconBlock<E> {
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
     ) -> Result<(), BlockProcessingError> {
-        Ok(process_block_header(
-            state,
-            self,
-            None,
-            VerifySignatures::True,
-            spec,
-        )?)
+        Ok(process_block_header(state, self, spec)?)
     }
 }
 

tests/ef_tests/src/cases/sanity_blocks.rs

@@ -6,7 +6,7 @@ use serde_derive::Deserialize;
 use state_processing::{
     per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy,
 };
-use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch};
+use types::{BeaconState, EthSpec, RelativeEpoch, SignedBeaconBlock};
 
 #[derive(Debug, Clone, Deserialize)]
 pub struct Metadata {
@@ -20,7 +20,7 @@ pub struct Metadata {
 pub struct SanityBlocks<E: EthSpec> {
     pub metadata: Metadata,
     pub pre: BeaconState<E>,
-    pub blocks: Vec<BeaconBlock<E>>,
+    pub blocks: Vec<SignedBeaconBlock<E>>,
     pub post: Option<BeaconState<E>>,
 }
 
@@ -28,7 +28,7 @@ impl<E: EthSpec> LoadCase for SanityBlocks<E> {
     fn load_from_dir(path: &Path) -> Result<Self, Error> {
         let metadata: Metadata = yaml_decode_file(&path.join("meta.yaml"))?;
         let pre = ssz_decode_file(&path.join("pre.ssz"))?;
-        let blocks: Vec<BeaconBlock<E>> = (0..metadata.blocks_count)
+        let blocks: Vec<SignedBeaconBlock<E>> = (0..metadata.blocks_count)
             .map(|i| {
                 let filename = format!("blocks_{}.ssz", i);
                 ssz_decode_file(&path.join(filename))
@@ -75,7 +75,8 @@ impl<E: EthSpec> Case for SanityBlocks<E> {
         let result = self
             .blocks
             .iter()
-            .try_for_each(|block| {
+            .try_for_each(|signed_block| {
+                let block = &signed_block.message;
                 while bulk_state.slot < block.slot {
                     per_slot_processing(&mut bulk_state, None, spec).unwrap();
                     per_slot_processing(&mut indiv_state, None, spec).unwrap();
@@ -91,7 +92,7 @@ impl<E: EthSpec> Case for SanityBlocks<E> {
 
                 per_block_processing(
                     &mut indiv_state,
-                    block,
+                    signed_block,
                     None,
                     BlockSignatureStrategy::VerifyIndividual,
                     spec,
@@ -99,7 +100,7 @@ impl<E: EthSpec> Case for SanityBlocks<E> {
 
                 per_block_processing(
                     &mut bulk_state,
-                    block,
+                    signed_block,
                     None,
                     BlockSignatureStrategy::VerifyBulk,
                     spec,

tests/ef_tests/src/cases/ssz_static.rs

@@ -6,7 +6,6 @@ use cached_tree_hash::{CacheArena, CachedTreeHash};
 use serde_derive::Deserialize;
 use std::fs;
 use std::marker::PhantomData;
-use tree_hash::SignedRoot;
 use types::Hash256;
 
 #[derive(Debug, Clone, Deserialize)]
@@ -22,13 +21,6 @@ pub struct SszStatic<T> {
     value: T,
 }
 
-#[derive(Debug, Clone)]
-pub struct SszStaticSR<T> {
-    roots: SszStaticRoots,
-    serialized: Vec<u8>,
-    value: T,
-}
-
 #[derive(Debug, Clone)]
 pub struct SszStaticTHC<T, C> {
     roots: SszStaticRoots,
@@ -55,16 +47,6 @@ impl<T: SszStaticType> LoadCase for SszStatic<T> {
     }
 }
 
-impl<T: SszStaticType + SignedRoot> LoadCase for SszStaticSR<T> {
-    fn load_from_dir(path: &Path) -> Result<Self, Error> {
-        load_from_dir(path).map(|(roots, serialized, value)| Self {
-            roots,
-            serialized,
-            value,
-        })
-    }
-}
-
 impl<T: SszStaticType + CachedTreeHash<C>, C: Debug + Sync> LoadCase for SszStaticTHC<T, C> {
     fn load_from_dir(path: &Path) -> Result<Self, Error> {
         load_from_dir(path).map(|(roots, serialized, value)| Self {
@@ -105,22 +87,6 @@ impl<T: SszStaticType> Case for SszStatic<T> {
     }
 }
 
-impl<T: SszStaticType + SignedRoot> Case for SszStaticSR<T> {
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        check_serialization(&self.value, &self.serialized)?;
-        check_tree_hash(&self.roots.root, &self.value.tree_hash_root())?;
-        check_tree_hash(
-            &self
-                .roots
-                .signing_root
-                .as_ref()
-                .expect("signed root exists"),
-            &self.value.signed_root(),
-        )?;
-        Ok(())
-    }
-}
-
 impl<T: SszStaticType + CachedTreeHash<C>, C: Debug + Sync> Case for SszStaticTHC<T, C> {
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         check_serialization(&self.value, &self.serialized)?;