Milagro BLS update (#985)

* Start updating types

* WIP

* Signature hacking

* Existing EF tests passing with fake_crypto

* Updates

* Delete outdated API spec

* The refactor continues

* It compiles

* WIP test fixes

* All release tests passing bar genesis state parsing

* Update and test YamlConfig

* Update to spec v0.10 compatible BLS

* Updates to BLS EF tests

* Add EF test for AggregateVerify

And delete unused hash2curve tests for uncompressed points

* Update EF tests to v0.10.1

* Use optional block root correctly in block proc

* Use genesis fork in deposit domain. All tests pass

* Cargo fmt

* Fast aggregate verify test

* Update REST API docs

* Cargo fmt

* Fix unused import

* Bump spec tags to v0.10.1

* Add `seconds_per_eth1_block` to chainspec

* Update to timestamp based eth1 voting scheme

* Return None from `get_votes_to_consider` if block cache is empty

* Handle overflows in `is_candidate_block`

* Revert to failing tests

* Fix eth1 data sets test

* Choose default vote according to spec

* Fix collect_valid_votes tests

* Fix `get_votes_to_consider` to choose all eligible blocks

* Uncomment winning_vote tests

* Add comments; remove unused code

* Reduce seconds_per_eth1_block for simulation

* Addressed review comments

* Add test for default vote case

* Fix logs

* Remove unused functions

* Meter default eth1 votes

* Fix comments

* Address review comments; remove unused dependency

* Add first attempt at attestation proc. re-write

* Add version 2 of attestation processing

* Minor fixes

* Add validator pubkey cache

* Make get_indexed_attestation take a committee

* Link signature processing into new attn verification

* First working version

* Ensure pubkey cache is updated

* Add more metrics, slight optimizations

* Clone committee cache during attestation processing

* Update shuffling cache during block processing

* Remove old commented-out code

* Fix shuffling cache insert bug

* Used indexed attestation in fork choice

* Restructure attn processing, add metrics

* Add more detailed metrics

* Tidy, fix failing tests

* Fix failing tests, tidy

* Disable/delete two outdated tests

* Add new Pubkeys struct to signature_sets

* Refactor with functional approach

* Update beacon chain

* Remove decompressed member from pubkey bytes

* Add hashmap for indices lookup

* Change `get_attesting_indices` to use Vec

* Fix failing test

* Tidy

* Add pubkey cache persistence file

* Add more comments

* Integrate persistence file into builder

* Add pubkey cache tests

* Add data_dir to beacon chain builder

* Remove Option in pubkey cache persistence file

* Ensure consistency between datadir/data_dir

* Fix failing network test

* Tidy

* Fix todos

* Improve tests

* Split up block processing metrics

* Tidy

* Refactor get_pubkey_from_state

* Remove commented-out code

* Add BeaconChain::validator_pubkey

* Update milagro_bls

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Cargo fmt

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Use Option::filter

* Remove Box

* Comment out tests that fail due to hard-coded

* Fix fake crypto

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Fix Cow::Borrowed

Signed-off-by: Kirk Baird <baird.k@outlook.com>

* Cargo fmt

Signed-off-by: Kirk Baird <baird.k@outlook.com>

Co-authored-by: Michael Sproul <michael@sigmaprime.io>
Co-authored-by: Michael Sproul <micsproul@gmail.com>
Co-authored-by: pawan <pawandhananjay@gmail.com>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

eth2/state_processing/src/per_block_processing/block_signature_verifier.rs

@@ -3,7 +3,7 @@
 use super::signature_sets::{Error as SignatureSetError, Result as SignatureSetResult, *};
 use crate::common::get_indexed_attestation;
 use crate::per_block_processing::errors::{AttestationInvalid, BlockOperationError};
-use bls::{verify_signature_sets, SignatureSet};
+use bls::{verify_signature_sets, PublicKey, SignatureSet};
 use rayon::prelude::*;
 use std::borrow::Cow;
 use types::{
@@ -11,8 +11,6 @@ use types::{
     SignedBeaconBlock,
 };
 
-pub use bls::G1Point;
-
 pub type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Debug, PartialEq)]
@@ -53,18 +51,18 @@ impl From<BlockOperationError<AttestationInvalid>> for Error {
 pub struct BlockSignatureVerifier<'a, T, F>
 where
     T: EthSpec,
-    F: Fn(usize) -> Option<Cow<'a, G1Point>> + Clone,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>> + Clone,
 {
     get_pubkey: F,
     state: &'a BeaconState<T>,
     spec: &'a ChainSpec,
-    sets: Vec<SignatureSet<'a>>,
+    sets: Vec<SignatureSet>,
 }
 
 impl<'a, T, F> BlockSignatureVerifier<'a, T, F>
 where
     T: EthSpec,
-    F: Fn(usize) -> Option<Cow<'a, G1Point>> + Clone,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>> + Clone,
 {
     /// Create a new verifier without any included signatures. See the `include...` functions to
     /// add signatures, and the `verify`
@@ -116,7 +114,7 @@ where
             .sets
             .into_par_iter()
             .chunks(num_chunks)
-            .map(|chunk| verify_signature_sets(chunk.into_iter()))
+            .map(|chunk| verify_signature_sets(chunk))
             .reduce(|| true, |current, this| current && this);
 
         if result {

eth2/state_processing/src/per_block_processing/signature_sets.rs

@@ -2,7 +2,7 @@
 //! validated individually, or alongside in others in a potentially cheaper bulk operation.
 //!
 //! This module exposes one function to extract each type of `SignatureSet` from a `BeaconBlock`.
-use bls::{G1Point, G1Ref, SignatureSet, SignedMessage};
+use bls::SignatureSet;
 use ssz::DecodeError;
 use std::borrow::Cow;
 use std::convert::TryInto;
@@ -44,7 +44,7 @@ impl From<BeaconStateError> for Error {
 pub fn get_pubkey_from_state<'a, T>(
     state: &'a BeaconState<T>,
     validator_index: usize,
-) -> Option<Cow<'a, G1Point>>
+) -> Option<Cow<'a, PublicKey>>
 where
     T: EthSpec,
 {
@@ -55,7 +55,7 @@ where
             let pk: Option<PublicKey> = (&v.pubkey).try_into().ok();
             pk
         })
-        .map(|pk| Cow::Owned(pk.into_point()))
+        .map(Cow::Owned)
 }
 
 /// A signature set that is valid if a block was signed by the expected block producer.
@@ -65,10 +65,10 @@ pub fn block_proposal_signature_set<'a, T, F>(
     signed_block: &'a SignedBeaconBlock<T>,
     block_root: Option<Hash256>,
     spec: &'a ChainSpec,
-) -> Result<SignatureSet<'a>>
+) -> Result<SignatureSet>
 where
     T: EthSpec,
-    F: Fn(usize) -> Option<Cow<'a, G1Point>>,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>>,
 {
     let block = &signed_block.message;
     let proposer_index = state.get_beacon_proposer_index(block.slot, spec)?;
@@ -103,10 +103,10 @@ pub fn randao_signature_set<'a, T, F>(
     get_pubkey: F,
     block: &'a BeaconBlock<T>,
     spec: &'a ChainSpec,
-) -> Result<SignatureSet<'a>>
+) -> Result<SignatureSet>
 where
     T: EthSpec,
-    F: Fn(usize) -> Option<Cow<'a, G1Point>>,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>>,
 {
     let proposer_index = state.get_beacon_proposer_index(block.slot, spec)?;
 
@@ -132,10 +132,10 @@ pub fn proposer_slashing_signature_set<'a, T, F>(
     get_pubkey: F,
     proposer_slashing: &'a ProposerSlashing,
     spec: &'a ChainSpec,
-) -> Result<(SignatureSet<'a>, SignatureSet<'a>)>
+) -> Result<(SignatureSet, SignatureSet)>
 where
     T: EthSpec,
-    F: Fn(usize) -> Option<Cow<'a, G1Point>>,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>>,
 {
     let proposer_index = proposer_slashing.signed_header_1.message.proposer_index as usize;
 
@@ -161,9 +161,9 @@ where
 fn block_header_signature_set<'a, T: EthSpec>(
     state: &'a BeaconState<T>,
     signed_header: &'a SignedBeaconBlockHeader,
-    pubkey: Cow<'a, G1Point>,
+    pubkey: Cow<'a, PublicKey>,
     spec: &'a ChainSpec,
-) -> Result<SignatureSet<'a>> {
+) -> Result<SignatureSet> {
     let domain = spec.get_domain(
         signed_header.message.slot.epoch(T::slots_per_epoch()),
         Domain::BeaconProposer,
@@ -191,10 +191,10 @@ pub fn indexed_attestation_signature_set<'a, 'b, T, F>(
     signature: &'a AggregateSignature,
     indexed_attestation: &'b IndexedAttestation<T>,
     spec: &'a ChainSpec,
-) -> Result<SignatureSet<'a>>
+) -> Result<SignatureSet>
 where
     T: EthSpec,
-    F: Fn(usize) -> Option<Cow<'a, G1Point>>,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>>,
 {
     let pubkeys = indexed_attestation
         .attesting_indices
@@ -213,9 +213,9 @@ where
     );
 
     let message = indexed_attestation.data.signing_root(domain);
-    let signed_message = SignedMessage::new(pubkeys, message.as_bytes().to_vec());
+    let message = message.as_bytes().to_vec();
 
-    Ok(SignatureSet::new(signature, vec![signed_message]))
+    Ok(SignatureSet::new(signature, pubkeys, message))
 }
 
 /// Returns the signature set for the given `indexed_attestation` but pubkeys are supplied directly
@@ -227,10 +227,10 @@ pub fn indexed_attestation_signature_set_from_pubkeys<'a, 'b, T, F>(
     fork: &Fork,
     genesis_validators_root: Hash256,
     spec: &'a ChainSpec,
-) -> Result<SignatureSet<'a>>
+) -> Result<SignatureSet>
 where
     T: EthSpec,
-    F: Fn(usize) -> Option<Cow<'a, G1Point>>,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>>,
 {
     let pubkeys = indexed_attestation
         .attesting_indices
@@ -249,9 +249,9 @@ where
     );
 
     let message = indexed_attestation.data.signing_root(domain);
-    let signed_message = SignedMessage::new(pubkeys, message.as_bytes().to_vec());
+    let message = message.as_bytes().to_vec();
 
-    Ok(SignatureSet::new(signature, vec![signed_message]))
+    Ok(SignatureSet::new(signature, pubkeys, message))
 }
 
 /// Returns the signature set for the given `attester_slashing` and corresponding `pubkeys`.
@@ -260,10 +260,10 @@ pub fn attester_slashing_signature_sets<'a, T, F>(
     get_pubkey: F,
     attester_slashing: &'a AttesterSlashing<T>,
     spec: &'a ChainSpec,
-) -> Result<(SignatureSet<'a>, SignatureSet<'a>)>
+) -> Result<(SignatureSet, SignatureSet)>
 where
     T: EthSpec,
-    F: Fn(usize) -> Option<Cow<'a, G1Point>> + Clone,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>> + Clone,
 {
     Ok((
         indexed_attestation_signature_set(
@@ -305,12 +305,12 @@ pub fn deposit_pubkey_signature_message(
 /// `deposit_pubkey_signature_message`.
 pub fn deposit_signature_set<'a>(
     pubkey_signature_message: &'a (PublicKey, Signature, Vec<u8>),
-) -> SignatureSet<'a> {
+) -> SignatureSet {
     let (pubkey, signature, message) = pubkey_signature_message;
 
     // Note: Deposits are valid across forks, thus the deposit domain is computed
-    // with the fork zeroed.
-    SignatureSet::single(signature, pubkey.g1_ref(), message.clone())
+    // with the fok zeroed.
+    SignatureSet::single(&signature, Cow::Borrowed(pubkey), message.clone())
 }
 
 /// Returns a signature set that is valid if the `SignedVoluntaryExit` was signed by the indicated
@@ -320,10 +320,10 @@ pub fn exit_signature_set<'a, T, F>(
     get_pubkey: F,
     signed_exit: &'a SignedVoluntaryExit,
     spec: &'a ChainSpec,
-) -> Result<SignatureSet<'a>>
+) -> Result<SignatureSet>
 where
     T: EthSpec,
-    F: Fn(usize) -> Option<Cow<'a, G1Point>>,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>>,
 {
     let exit = &signed_exit.message;
     let proposer_index = exit.validator_index as usize;