Milagro BLS update (#985)

* Start updating types

* WIP

* Signature hacking

* Existing EF tests passing with fake_crypto

* Updates

* Delete outdated API spec

* The refactor continues

* It compiles

* WIP test fixes

* All release tests passing bar genesis state parsing

* Update and test YamlConfig

* Update to spec v0.10 compatible BLS

* Updates to BLS EF tests

* Add EF test for AggregateVerify

And delete unused hash2curve tests for uncompressed points

* Update EF tests to v0.10.1

* Use optional block root correctly in block proc

* Use genesis fork in deposit domain. All tests pass

* Cargo fmt

* Fast aggregate verify test

* Update REST API docs

* Cargo fmt

* Fix unused import

* Bump spec tags to v0.10.1

* Add `seconds_per_eth1_block` to chainspec

* Update to timestamp based eth1 voting scheme

* Return None from `get_votes_to_consider` if block cache is empty

* Handle overflows in `is_candidate_block`

* Revert to failing tests

* Fix eth1 data sets test

* Choose default vote according to spec

* Fix collect_valid_votes tests

* Fix `get_votes_to_consider` to choose all eligible blocks

* Uncomment winning_vote tests

* Add comments; remove unused code

* Reduce seconds_per_eth1_block for simulation

* Addressed review comments

* Add test for default vote case

* Fix logs

* Remove unused functions

* Meter default eth1 votes

* Fix comments

* Address review comments; remove unused dependency

* Add first attempt at attestation proc. re-write

* Add version 2 of attestation processing

* Minor fixes

* Add validator pubkey cache

* Make get_indexed_attestation take a committee

* Link signature processing into new attn verification

* First working version

* Ensure pubkey cache is updated

* Add more metrics, slight optimizations

* Clone committee cache during attestation processing

* Update shuffling cache during block processing

* Remove old commented-out code

* Fix shuffling cache insert bug

* Used indexed attestation in fork choice

* Restructure attn processing, add metrics

* Add more detailed metrics

* Tidy, fix failing tests

* Fix failing tests, tidy

* Disable/delete two outdated tests

* Add new Pubkeys struct to signature_sets

* Refactor with functional approach

* Update beacon chain

* Remove decompressed member from pubkey bytes

* Add hashmap for indices lookup

* Change `get_attesting_indices` to use Vec

* Fix failing test

* Tidy

* Add pubkey cache persistence file

* Add more comments

* Integrate persistence file into builder

* Add pubkey cache tests

* Add data_dir to beacon chain builder

* Remove Option in pubkey cache persistence file

* Ensure consistency between datadir/data_dir

* Fix failing network test

* Tidy

* Fix todos

* Improve tests

* Split up block processing metrics

* Tidy

* Refactor get_pubkey_from_state

* Remove commented-out code

* Add BeaconChain::validator_pubkey

* Update milagro_bls

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Cargo fmt

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Use Option::filter

* Remove Box

* Comment out tests that fail due to hard-coded

* Fix fake crypto

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Fix Cow::Borrowed

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Cargo fmt

Signed-off-by: Kirk Baird <baird.k@outlook.com>

Co-authored-by: Michael Sproul <michael@sigmaprime.io>
Co-authored-by: Michael Sproul <micsproul@gmail.com>
Co-authored-by: pawan <pawandhananjay@gmail.com>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

eth2/utils/bls/src/aggregate_public_key.rs

@@ -1,5 +1,5 @@
 use super::{PublicKey, BLS_PUBLIC_KEY_BYTE_SIZE};
-use milagro_bls::{AggregatePublicKey as RawAggregatePublicKey, G1Point};
+use milagro_bls::AggregatePublicKey as RawAggregatePublicKey;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
@@ -37,10 +37,6 @@ impl AggregatePublicKey {
         self.0.add(public_key.as_raw())
     }
 
-    pub fn add_point(&mut self, point: &G1Point) {
-        self.0.point.add(point)
-    }
-
     /// Returns the underlying public key.
     pub fn as_raw(&self) -> &RawAggregatePublicKey {
         &self.0

eth2/utils/bls/src/aggregate_signature.rs

@@ -1,8 +1,5 @@
 use super::*;
-use milagro_bls::{
-    AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature,
-    G2Point,
-};
+use milagro_bls::{AggregateSignature as RawAggregateSignature, G2Point};
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
@@ -32,16 +29,19 @@ impl AggregateSignature {
 
     /// Add (aggregate) a signature to the `AggregateSignature`.
     pub fn add(&mut self, signature: &Signature) {
-        if self.is_empty {
-            self.aggregate_signature = RawAggregateSignature::new();
-            self.is_empty = false;
-        }
+        // Only empty if both are empty
+        self.is_empty = self.is_empty && signature.is_empty();
 
+        // Note: empty signatures will have point at infinity which is equivalent of adding 0.
         self.aggregate_signature.add(signature.as_raw())
     }
 
     /// Add (aggregate) another `AggregateSignature`.
     pub fn add_aggregate(&mut self, agg_signature: &AggregateSignature) {
+        // Only empty if both are empty
+        self.is_empty = self.is_empty && agg_signature.is_empty();
+
+        // Note: empty signatures will have point at infinity which is equivalent of adding 0.
         self.aggregate_signature
             .add_aggregate(&agg_signature.aggregate_signature)
     }
@@ -55,32 +55,32 @@ impl AggregateSignature {
             return false;
         }
         self.aggregate_signature
-            .verify(msg, aggregate_public_key.as_raw())
+            .fast_aggregate_verify_pre_aggregated(msg, aggregate_public_key.as_raw())
     }
 
-    /// Verify this AggregateSignature against multiple AggregatePublickeys with multiple Messages.
+    /// Verify the `AggregateSignature` against an `AggregatePublicKey`.
     ///
-    ///  All PublicKeys related to a Message should be aggregated into one AggregatePublicKey.
-    ///  Each AggregatePublicKey has a 1:1 ratio with a 32 byte Message.
-    pub fn verify_multiple(
-        &self,
-        messages: &[&[u8]],
-        aggregate_public_keys: &[&AggregatePublicKey],
-    ) -> bool {
+    /// Only returns `true` if the set of keys in the `AggregatePublicKey` match the set of keys
+    /// that signed the `AggregateSignature`.
+    pub fn verify_unaggregated(&self, msg: &[u8], public_keys: &[&PublicKey]) -> bool {
         if self.is_empty {
             return false;
         }
-        let aggregate_public_keys: Vec<&RawAggregatePublicKey> =
-            aggregate_public_keys.iter().map(|pk| pk.as_raw()).collect();
-
-        // Messages are concatenated into one long message.
-        let mut msgs: Vec<Vec<u8>> = vec![];
-        for message in messages {
-            msgs.push(message.to_vec());
-        }
-
+        let public_key_refs: Vec<_> = public_keys.iter().map(|pk| pk.as_raw()).collect();
         self.aggregate_signature
-            .verify_multiple(&msgs, &aggregate_public_keys[..])
+            .fast_aggregate_verify(msg, &public_key_refs)
+    }
+
+    /// Verify this AggregateSignature against multiple AggregatePublickeys and Messages.
+    ///
+    /// Each AggregatePublicKey has a 1:1 ratio with a 32 byte Message.
+    pub fn verify_multiple(&self, messages: &[&[u8]], public_keys: &[&PublicKey]) -> bool {
+        if self.is_empty {
+            return false;
+        }
+        let public_keys_refs: Vec<_> = public_keys.iter().map(|pk| pk.as_raw()).collect();
+        self.aggregate_signature
+            .aggregate_verify(&messages, &public_keys_refs)
     }
 
     /// Return AggregateSignature as bytes

eth2/utils/bls/src/fake_aggregate_public_key.rs

@@ -67,6 +67,17 @@ impl FakeAggregatePublicKey {
         // No nothing.
     }
 
+    pub fn aggregate(_pks: &[&PublicKey]) -> Self {
+        Self::new()
+    }
+
+    pub fn from_public_key(public_key: &PublicKey) -> Self {
+        Self {
+            bytes: public_key.as_bytes(),
+            point: public_key.point.clone(),
+        }
+    }
+
     pub fn as_raw(&self) -> &Self {
         &self
     }

eth2/utils/bls/src/fake_aggregate_signature.rs

@@ -1,6 +1,6 @@
 use super::{
-    fake_aggregate_public_key::FakeAggregatePublicKey, fake_signature::FakeSignature,
-    BLS_AGG_SIG_BYTE_SIZE,
+    fake_aggregate_public_key::FakeAggregatePublicKey, fake_public_key::FakePublicKey,
+    fake_signature::FakeSignature, BLS_AGG_SIG_BYTE_SIZE,
 };
 use milagro_bls::G2Point;
 use serde::de::{Deserialize, Deserializer};
@@ -47,6 +47,11 @@ impl FakeAggregateSignature {
         // Do nothing.
     }
 
+    /// Does glorious nothing.
+    pub fn aggregate(&mut self, _agg_sig: &FakeAggregateSignature) {
+        // Do nothing.
+    }
+
     /// _Always_ returns `true`.
     pub fn verify(&self, _msg: &[u8], _aggregate_public_key: &FakeAggregatePublicKey) -> bool {
         true
@@ -56,11 +61,28 @@ impl FakeAggregateSignature {
     pub fn verify_multiple(
         &self,
         _messages: &[&[u8]],
-        _aggregate_public_keys: &[&FakeAggregatePublicKey],
+        _aggregate_public_keys: &[&FakePublicKey],
     ) -> bool {
         true
     }
 
+    /// _Always_ returns `true`.
+    pub fn fast_aggregate_verify_pre_aggregated(
+        &self,
+        _messages: &[u8],
+        _aggregate_public_keys: &FakeAggregatePublicKey,
+    ) -> bool {
+        true
+    }
+
+    /// _Always_ returns `true`.
+    pub fn from_signature(signature: &FakeSignature) -> Self {
+        Self {
+            bytes: signature.as_bytes(),
+            point: signature.point.clone(),
+        }
+    }
+
     /// Convert bytes to fake BLS aggregate signature
     pub fn from_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
         if bytes.len() != BLS_AGG_SIG_BYTE_SIZE {

eth2/utils/bls/src/lib.rs

@@ -13,8 +13,8 @@ pub use crate::keypair::Keypair;
 pub use crate::public_key_bytes::PublicKeyBytes;
 pub use crate::secret_key::SecretKey;
 pub use crate::signature_bytes::SignatureBytes;
-pub use milagro_bls::{compress_g2, hash_on_g2, G1Point};
-pub use signature_set::{verify_signature_sets, G1Ref, SignatureSet, SignedMessage};
+pub use milagro_bls::{compress_g2, hash_to_curve_g2};
+pub use signature_set::{verify_signature_sets, SignatureSet};
 
 #[cfg(feature = "fake_crypto")]
 mod fake_aggregate_public_key;

eth2/utils/bls/src/signature_set.rs

@@ -1,179 +1,74 @@
-use crate::{AggregatePublicKey, AggregateSignature, PublicKey, Signature};
-use milagro_bls::{G1Point, G2Point};
+use crate::{AggregateSignature, PublicKey, Signature};
 use std::borrow::Cow;
 
 #[cfg(not(feature = "fake_crypto"))]
-use milagro_bls::AggregateSignature as RawAggregateSignature;
+use milagro_bls::{
+    AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature,
+    PublicKey as RawPublicKey,
+};
+
+#[cfg(feature = "fake_crypto")]
+use crate::fakes::{
+    AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature,
+    PublicKey as RawPublicKey,
+};
 
 type Message = Vec<u8>;
 
 #[derive(Clone, Debug)]
-pub struct SignedMessage<'a> {
-    signing_keys: Vec<Cow<'a, G1Point>>,
+pub struct SignatureSet {
+    pub signature: RawAggregateSignature,
+    signing_keys: RawAggregatePublicKey,
     message: Message,
 }
 
-impl<'a> SignedMessage<'a> {
-    pub fn new(signing_keys: Vec<Cow<'a, G1Point>>, message: Message) -> Self {
+impl SignatureSet {
+    pub fn single(signature: &Signature, signing_key: Cow<PublicKey>, message: Message) -> Self {
         Self {
-            signing_keys,
+            signature: RawAggregateSignature::from_signature(signature.as_raw()),
+            signing_keys: RawAggregatePublicKey::from_public_key(signing_key.as_raw()),
             message,
         }
     }
-}
 
-#[derive(Clone, Debug)]
-pub struct SignatureSet<'a> {
-    pub signature: &'a G2Point,
-    signed_messages: Vec<SignedMessage<'a>>,
-}
-
-impl<'a> SignatureSet<'a> {
-    pub fn single<S>(signature: &'a S, signing_key: Cow<'a, G1Point>, message: Message) -> Self
-    where
-        S: G2Ref,
-    {
-        Self {
-            signature: signature.g2_ref(),
-            signed_messages: vec![SignedMessage::new(vec![signing_key], message)],
-        }
-    }
-
-    pub fn dual<S, T>(
-        signature: &'a S,
-        message_0: Message,
-        message_0_signing_keys: Vec<Cow<'a, G1Point>>,
-        message_1: Message,
-        message_1_signing_keys: Vec<Cow<'a, G1Point>>,
+    pub fn new(
+        signature: &AggregateSignature,
+        signing_keys: Vec<Cow<PublicKey>>,
+        message: Message,
     ) -> Self
-    where
-        T: G1Ref + Clone,
-        S: G2Ref,
-    {
+where {
+        let signing_keys_refs: Vec<&RawPublicKey> =
+            signing_keys.iter().map(|pk| pk.as_raw()).collect();
         Self {
-            signature: signature.g2_ref(),
-            signed_messages: vec![
-                SignedMessage::new(message_0_signing_keys, message_0),
-                SignedMessage::new(message_1_signing_keys, message_1),
-            ],
-        }
-    }
-
-    pub fn new<S>(signature: &'a S, signed_messages: Vec<SignedMessage<'a>>) -> Self
-    where
-        S: G2Ref,
-    {
-        Self {
-            signature: signature.g2_ref(),
-            signed_messages,
+            signature: signature.as_raw().clone(),
+            signing_keys: RawAggregatePublicKey::aggregate(&signing_keys_refs),
+            message,
         }
     }
 
     pub fn is_valid(&self) -> bool {
-        let sig = milagro_bls::AggregateSignature {
-            point: self.signature.clone(),
-        };
-
-        let mut messages: Vec<Vec<u8>> = vec![];
-        let mut pubkeys = vec![];
-
-        self.signed_messages.iter().for_each(|signed_message| {
-            messages.push(signed_message.message.clone());
-
-            let point = if signed_message.signing_keys.len() == 1 {
-                signed_message.signing_keys[0].clone().into_owned()
-            } else {
-                aggregate_public_keys(&signed_message.signing_keys)
-            };
-
-            pubkeys.push(milagro_bls::AggregatePublicKey { point });
-        });
-
-        let pubkey_refs: Vec<&milagro_bls::AggregatePublicKey> =
-            pubkeys.iter().map(std::borrow::Borrow::borrow).collect();
-
-        sig.verify_multiple(&messages, &pubkey_refs)
+        self.signature
+            .fast_aggregate_verify_pre_aggregated(&self.message, &self.signing_keys)
     }
 }
 
+type VerifySet<'a> = (
+    &'a RawAggregateSignature,
+    &'a RawAggregatePublicKey,
+    &'a [u8],
+);
+
 #[cfg(not(feature = "fake_crypto"))]
-pub fn verify_signature_sets<'a>(iter: impl Iterator<Item = SignatureSet<'a>>) -> bool {
+pub fn verify_signature_sets<'a>(sets: Vec<SignatureSet>) -> bool {
     let rng = &mut rand::thread_rng();
-    RawAggregateSignature::verify_multiple_signatures(rng, iter.map(Into::into))
+    let verify_set: Vec<VerifySet> = sets
+        .iter()
+        .map(|ss| (&ss.signature, &ss.signing_keys, ss.message.as_slice()))
+        .collect();
+    RawAggregateSignature::verify_multiple_aggregate_signatures(rng, verify_set.into_iter())
 }
 
 #[cfg(feature = "fake_crypto")]
-pub fn verify_signature_sets<'a>(_iter: impl Iterator<Item = SignatureSet<'a>>) -> bool {
+pub fn verify_signature_sets<'a>(sets: Vec<SignatureSet>) -> bool {
     true
 }
-
-type VerifySet<'a> = (G2Point, Vec<G1Point>, Vec<Vec<u8>>);
-
-impl<'a> Into<VerifySet<'a>> for SignatureSet<'a> {
-    fn into(self) -> VerifySet<'a> {
-        let signature = self.signature.clone();
-
-        let (pubkeys, messages): (Vec<G1Point>, Vec<Message>) = self
-            .signed_messages
-            .into_iter()
-            .map(|signed_message| {
-                let key = if signed_message.signing_keys.len() == 1 {
-                    signed_message.signing_keys[0].clone().into_owned()
-                } else {
-                    aggregate_public_keys(&signed_message.signing_keys)
-                };
-
-                (key, signed_message.message)
-            })
-            .unzip();
-
-        (signature, pubkeys, messages)
-    }
-}
-
-/// Create an aggregate public key for a list of validators, failing if any key can't be found.
-fn aggregate_public_keys<'a>(public_keys: &'a [Cow<'a, G1Point>]) -> G1Point {
-    let mut aggregate =
-        public_keys
-            .iter()
-            .fold(AggregatePublicKey::new(), |mut aggregate, pubkey| {
-                aggregate.add_point(&pubkey);
-                aggregate
-            });
-
-    aggregate.affine();
-
-    aggregate.into_raw().point
-}
-
-pub trait G1Ref {
-    fn g1_ref(&self) -> Cow<'_, G1Point>;
-}
-
-impl G1Ref for AggregatePublicKey {
-    fn g1_ref(&self) -> Cow<'_, G1Point> {
-        Cow::Borrowed(&self.as_raw().point)
-    }
-}
-
-impl G1Ref for PublicKey {
-    fn g1_ref(&self) -> Cow<'_, G1Point> {
-        Cow::Borrowed(&self.as_raw().point)
-    }
-}
-
-pub trait G2Ref {
-    fn g2_ref(&self) -> &G2Point;
-}
-
-impl G2Ref for AggregateSignature {
-    fn g2_ref(&self) -> &G2Point {
-        &self.as_raw().point
-    }
-}
-
-impl G2Ref for Signature {
-    fn g2_ref(&self) -> &G2Point {
-        &self.as_raw().point
-    }
-}