Update to Spec v0.10 (#817)

* Start updating types

* WIP

* Signature hacking

* Existing EF tests passing with fake_crypto

* Updates

* Delete outdated API spec

* The refactor continues

* It compiles

* WIP test fixes

* All release tests passing bar genesis state parsing

* Update and test YamlConfig

* Update to spec v0.10 compatible BLS

* Updates to BLS EF tests

* Add EF test for AggregateVerify

And delete unused hash2curve tests for uncompressed points

* Update EF tests to v0.10.1

* Use optional block root correctly in block proc

* Use genesis fork in deposit domain. All tests pass

* Cargo fmt

* Fast aggregate verify test

* Update REST API docs

* Cargo fmt

* Fix unused import

* Bump spec tags to v0.10.1

* Add `seconds_per_eth1_block` to chainspec

* Update to timestamp based eth1 voting scheme

* Return None from `get_votes_to_consider` if block cache is empty

* Handle overflows in `is_candidate_block`

* Revert to failing tests

* Fix eth1 data sets test

* Choose default vote according to spec

* Fix collect_valid_votes tests

* Fix `get_votes_to_consider` to choose all eligible blocks

* Uncomment winning_vote tests

* Add comments; remove unused code

* Reduce seconds_per_eth1_block for simulation

* Addressed review comments

* Add test for default vote case

* Fix logs

* Remove unused functions

* Meter default eth1 votes

* Fix comments

* Address review comments; remove unused dependency

* Disable/delete two outdated tests

* Bump eth1 default vote warn to error

* Delete outdated eth1 test

Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>

eth2/utils/bls/src/aggregate_public_key.rs

@@ -1,5 +1,9 @@
-use super::PublicKey;
+use super::{PublicKey, BLS_PUBLIC_KEY_BYTE_SIZE};
 use milagro_bls::{AggregatePublicKey as RawAggregatePublicKey, G1Point};
+use serde::de::{Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
+use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
+use ssz::{Decode, DecodeError, Encode};
 
 /// A BLS aggregate public key.
 ///
@@ -13,6 +17,16 @@ impl AggregatePublicKey {
         AggregatePublicKey(RawAggregatePublicKey::new())
     }
 
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
+        let pubkey = RawAggregatePublicKey::from_bytes(&bytes).map_err(|_| {
+            DecodeError::BytesInvalid(
+                format!("Invalid AggregatePublicKey bytes: {:?}", bytes).to_string(),
+            )
+        })?;
+
+        Ok(AggregatePublicKey(pubkey))
+    }
+
     pub fn add_without_affine(&mut self, public_key: &PublicKey) {
         self.0.point.add(&public_key.as_raw().point)
     }
@@ -34,6 +48,11 @@ impl AggregatePublicKey {
         &self.0
     }
 
+    /// Returns the underlying point as compressed bytes.
+    pub fn as_bytes(&self) -> Vec<u8> {
+        self.as_raw().as_bytes()
+    }
+
     pub fn into_raw(self) -> RawAggregatePublicKey {
         self.0
     }
@@ -41,6 +60,37 @@ impl AggregatePublicKey {
     /// Return a hex string representation of this key's bytes.
     #[cfg(test)]
     pub fn as_hex_string(&self) -> String {
-        serde_hex::encode(self.as_raw().as_bytes())
+        serde_hex::encode(self.as_bytes())
+    }
+}
+
+impl_ssz!(
+    AggregatePublicKey,
+    BLS_PUBLIC_KEY_BYTE_SIZE,
+    "AggregatePublicKey"
+);
+impl_tree_hash!(AggregatePublicKey, BLS_PUBLIC_KEY_BYTE_SIZE);
+
+impl Serialize for AggregatePublicKey {
+    /// Serde serialization is compliant the Ethereum YAML test format.
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&hex_encode(self.as_bytes()))
+    }
+}
+
+impl<'de> Deserialize<'de> for AggregatePublicKey {
+    /// Serde serialization is compliant the Ethereum YAML test format.
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+        let agg_sig = AggregatePublicKey::from_ssz_bytes(&bytes)
+            .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?;
+
+        Ok(agg_sig)
     }
 }

eth2/utils/bls/src/aggregate_signature.rs

@@ -47,17 +47,12 @@ impl AggregateSignature {
     ///
     /// Only returns `true` if the set of keys in the `AggregatePublicKey` match the set of keys
     /// that signed the `AggregateSignature`.
-    pub fn verify(
-        &self,
-        msg: &[u8],
-        domain: u64,
-        aggregate_public_key: &AggregatePublicKey,
-    ) -> bool {
+    pub fn verify(&self, msg: &[u8], aggregate_public_key: &AggregatePublicKey) -> bool {
         if self.is_empty {
             return false;
         }
         self.aggregate_signature
-            .verify(msg, domain, aggregate_public_key.as_raw())
+            .verify(msg, aggregate_public_key.as_raw())
     }
 
     /// Verify this AggregateSignature against multiple AggregatePublickeys with multiple Messages.
@@ -67,7 +62,6 @@ impl AggregateSignature {
     pub fn verify_multiple(
         &self,
         messages: &[&[u8]],
-        domain: u64,
         aggregate_public_keys: &[&AggregatePublicKey],
     ) -> bool {
         if self.is_empty {
@@ -83,7 +77,7 @@ impl AggregateSignature {
         }
 
         self.aggregate_signature
-            .verify_multiple(&msgs, domain, &aggregate_public_keys[..])
+            .verify_multiple(&msgs, &aggregate_public_keys[..])
     }
 
     /// Return AggregateSignature as bytes
@@ -193,7 +187,7 @@ mod tests {
         let keypair = Keypair::random();
 
         let mut original = AggregateSignature::new();
-        original.add(&Signature::new(&[42, 42], 0, &keypair.sk));
+        original.add(&Signature::new(&[42, 42], &keypair.sk));
 
         let bytes = original.as_ssz_bytes();
         let decoded = AggregateSignature::from_ssz_bytes(&bytes).unwrap();

eth2/utils/bls/src/fake_aggregate_public_key.rs

@@ -1,5 +1,10 @@
 use super::{PublicKey, BLS_PUBLIC_KEY_BYTE_SIZE};
+use hex::encode as hex_encode;
 use milagro_bls::G1Point;
+use serde::de::{Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
+use serde_hex::PrefixedHexVisitor;
+use ssz::{ssz_encode, Decode, DecodeError, Encode};
 
 /// A BLS aggregate public key.
 ///
@@ -17,6 +22,20 @@ impl FakeAggregatePublicKey {
         Self::zero()
     }
 
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
+        if bytes.len() != BLS_PUBLIC_KEY_BYTE_SIZE {
+            Err(DecodeError::InvalidByteLength {
+                len: bytes.len(),
+                expected: BLS_PUBLIC_KEY_BYTE_SIZE,
+            })
+        } else {
+            Ok(Self {
+                bytes: bytes.to_vec(),
+                point: G1Point::new(),
+            })
+        }
+    }
+
     pub fn add_without_affine(&mut self, _public_key: &PublicKey) {
         // No nothing.
     }
@@ -53,3 +72,32 @@ impl FakeAggregatePublicKey {
         self.bytes.clone()
     }
 }
+
+impl_ssz!(
+    FakeAggregatePublicKey,
+    BLS_PUBLIC_KEY_BYTE_SIZE,
+    "FakeAggregatePublicKey"
+);
+
+impl_tree_hash!(FakeAggregatePublicKey, BLS_PUBLIC_KEY_BYTE_SIZE);
+
+impl Serialize for FakeAggregatePublicKey {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&hex_encode(ssz_encode(self)))
+    }
+}
+
+impl<'de> Deserialize<'de> for FakeAggregatePublicKey {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+        let pubkey = <_>::from_ssz_bytes(&bytes[..])
+            .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?;
+        Ok(pubkey)
+    }
+}

eth2/utils/bls/src/fake_aggregate_signature.rs

@@ -48,12 +48,7 @@ impl FakeAggregateSignature {
     }
 
     /// _Always_ returns `true`.
-    pub fn verify(
-        &self,
-        _msg: &[u8],
-        _domain: u64,
-        _aggregate_public_key: &FakeAggregatePublicKey,
-    ) -> bool {
+    pub fn verify(&self, _msg: &[u8], _aggregate_public_key: &FakeAggregatePublicKey) -> bool {
         true
     }
 
@@ -61,7 +56,6 @@ impl FakeAggregateSignature {
     pub fn verify_multiple(
         &self,
         _messages: &[&[u8]],
-        _domain: u64,
         _aggregate_public_keys: &[&FakeAggregatePublicKey],
     ) -> bool {
         true
@@ -127,7 +121,7 @@ mod tests {
         let keypair = Keypair::random();
 
         let mut original = FakeAggregateSignature::new();
-        original.add(&Signature::new(&[42, 42], 0, &keypair.sk));
+        original.add(&Signature::new(&[42, 42], &keypair.sk));
 
         let bytes = ssz_encode(&original);
         let decoded = FakeAggregateSignature::from_ssz_bytes(&bytes).unwrap();

eth2/utils/bls/src/fake_signature.rs

@@ -20,7 +20,7 @@ pub struct FakeSignature {
 
 impl FakeSignature {
     /// Creates a new all-zero's signature
-    pub fn new(_msg: &[u8], _domain: u64, _sk: &SecretKey) -> Self {
+    pub fn new(_msg: &[u8], _sk: &SecretKey) -> Self {
         FakeSignature::zero()
     }
 
@@ -39,7 +39,7 @@ impl FakeSignature {
     }
 
     /// _Always_ returns `true`.
-    pub fn verify(&self, _msg: &[u8], _domain: u64, _pk: &PublicKey) -> bool {
+    pub fn verify(&self, _msg: &[u8], _pk: &PublicKey) -> bool {
         true
     }
 
@@ -124,7 +124,7 @@ mod tests {
     pub fn test_ssz_round_trip() {
         let keypair = Keypair::random();
 
-        let original = FakeSignature::new(&[42, 42], 0, &keypair.sk);
+        let original = FakeSignature::new(&[42, 42], &keypair.sk);
 
         let bytes = ssz_encode(&original);
         let decoded = FakeSignature::from_ssz_bytes(&bytes).unwrap();

eth2/utils/bls/src/lib.rs

@@ -75,7 +75,6 @@ pub fn bls_verify_aggregate(
     pubkey: &AggregatePublicKey,
     message: &[u8],
     signature: &AggregateSignature,
-    domain: u64,
 ) -> bool {
-    signature.verify(message, domain, pubkey)
+    signature.verify(message, pubkey)
 }

eth2/utils/bls/src/signature.rs

@@ -17,39 +17,19 @@ pub struct Signature {
 
 impl Signature {
     /// Instantiate a new Signature from a message and a SecretKey.
-    pub fn new(msg: &[u8], domain: u64, sk: &SecretKey) -> Self {
+    pub fn new(msg: &[u8], sk: &SecretKey) -> Self {
         Signature {
-            signature: RawSignature::new(msg, domain, sk.as_raw()),
-            is_empty: false,
-        }
-    }
-
-    /// Instantiate a new Signature from a message and a SecretKey, where the message has already
-    /// been hashed.
-    pub fn new_hashed(x_real_hashed: &[u8], x_imaginary_hashed: &[u8], sk: &SecretKey) -> Self {
-        Signature {
-            signature: RawSignature::new_hashed(x_real_hashed, x_imaginary_hashed, sk.as_raw()),
+            signature: RawSignature::new(msg, sk.as_raw()),
             is_empty: false,
         }
     }
 
     /// Verify the Signature against a PublicKey.
-    pub fn verify(&self, msg: &[u8], domain: u64, pk: &PublicKey) -> bool {
+    pub fn verify(&self, msg: &[u8], pk: &PublicKey) -> bool {
         if self.is_empty {
             return false;
         }
-        self.signature.verify(msg, domain, pk.as_raw())
-    }
-
-    /// Verify the Signature against a PublicKey, where the message has already been hashed.
-    pub fn verify_hashed(
-        &self,
-        x_real_hashed: &[u8],
-        x_imaginary_hashed: &[u8],
-        pk: &PublicKey,
-    ) -> bool {
-        self.signature
-            .verify_hashed(x_real_hashed, x_imaginary_hashed, pk.as_raw())
+        self.signature.verify(msg, pk.as_raw())
     }
 
     /// Returns the underlying signature.
@@ -141,7 +121,7 @@ mod tests {
     pub fn test_ssz_round_trip() {
         let keypair = Keypair::random();
 
-        let original = Signature::new(&[42, 42], 0, &keypair.sk);
+        let original = Signature::new(&[42, 42], &keypair.sk);
 
         let bytes = ssz_encode(&original);
         let decoded = Signature::from_ssz_bytes(&bytes).unwrap();
@@ -153,7 +133,7 @@ mod tests {
     pub fn test_byte_size() {
         let keypair = Keypair::random();
 
-        let signature = Signature::new(&[42, 42], 0, &keypair.sk);
+        let signature = Signature::new(&[42, 42], &keypair.sk);
         let bytes = ssz_encode(&signature);
         assert_eq!(bytes.len(), BLS_SIG_BYTE_SIZE);
     }

eth2/utils/bls/src/signature_bytes.rs

@@ -16,7 +16,7 @@ mod tests {
     #[test]
     pub fn test_valid_signature() {
         let keypair = Keypair::random();
-        let original = Signature::new(&[42, 42], 0, &keypair.sk);
+        let original = Signature::new(&[42, 42], &keypair.sk);
 
         let bytes = ssz_encode(&original);
         let signature_bytes = SignatureBytes::from_bytes(&bytes).unwrap();

eth2/utils/bls/src/signature_set.rs

@@ -6,7 +6,6 @@ use std::borrow::Cow;
 use milagro_bls::AggregateSignature as RawAggregateSignature;
 
 type Message = Vec<u8>;
-type Domain = u64;
 
 #[derive(Clone, Debug)]
 pub struct SignedMessage<'a> {
@@ -27,23 +26,16 @@ impl<'a> SignedMessage<'a> {
 pub struct SignatureSet<'a> {
     pub signature: &'a G2Point,
     signed_messages: Vec<SignedMessage<'a>>,
-    domain: Domain,
 }
 
 impl<'a> SignatureSet<'a> {
-    pub fn single<S>(
-        signature: &'a S,
-        signing_key: Cow<'a, G1Point>,
-        message: Message,
-        domain: Domain,
-    ) -> Self
+    pub fn single<S>(signature: &'a S, signing_key: Cow<'a, G1Point>, message: Message) -> Self
     where
         S: G2Ref,
     {
         Self {
             signature: signature.g2_ref(),
             signed_messages: vec![SignedMessage::new(vec![signing_key], message)],
-            domain,
         }
     }
 
@@ -53,7 +45,6 @@ impl<'a> SignatureSet<'a> {
         message_0_signing_keys: Vec<Cow<'a, G1Point>>,
         message_1: Message,
         message_1_signing_keys: Vec<Cow<'a, G1Point>>,
-        domain: Domain,
     ) -> Self
     where
         T: G1Ref + Clone,
@@ -65,18 +56,16 @@ impl<'a> SignatureSet<'a> {
                 SignedMessage::new(message_0_signing_keys, message_0),
                 SignedMessage::new(message_1_signing_keys, message_1),
             ],
-            domain,
         }
     }
 
-    pub fn new<S>(signature: &'a S, signed_messages: Vec<SignedMessage<'a>>, domain: Domain) -> Self
+    pub fn new<S>(signature: &'a S, signed_messages: Vec<SignedMessage<'a>>) -> Self
     where
         S: G2Ref,
     {
         Self {
             signature: signature.g2_ref(),
             signed_messages,
-            domain,
         }
     }
 
@@ -103,7 +92,7 @@ impl<'a> SignatureSet<'a> {
         let pubkey_refs: Vec<&milagro_bls::AggregatePublicKey> =
             pubkeys.iter().map(std::borrow::Borrow::borrow).collect();
 
-        sig.verify_multiple(&messages, self.domain, &pubkey_refs)
+        sig.verify_multiple(&messages, &pubkey_refs)
     }
 }
 
@@ -118,7 +107,7 @@ pub fn verify_signature_sets<'a>(_iter: impl Iterator<Item = SignatureSet<'a>>)
     true
 }
 
-type VerifySet<'a> = (G2Point, Vec<G1Point>, Vec<Vec<u8>>, u64);
+type VerifySet<'a> = (G2Point, Vec<G1Point>, Vec<Vec<u8>>);
 
 impl<'a> Into<VerifySet<'a>> for SignatureSet<'a> {
     fn into(self) -> VerifySet<'a> {
@@ -138,7 +127,7 @@ impl<'a> Into<VerifySet<'a>> for SignatureSet<'a> {
             })
             .unzip();
 
-        (signature, pubkeys, messages, self.domain)
+        (signature, pubkeys, messages)
     }
 }