Bulk signature verification (#507)

* Add basic block processing benches

* Start reviving state processing benches

* Fix old block builders

* Add optimization for faster pubkey add

* Tidy benches, add another

* Add extra block processing bench

* Start working on faster BLS scheme

* Add partially complete sig verify optimization

* Add .gitignore to state processing

* Add progress on faster signature verification

* Fix SignatureSet for fake_crypto

* Tidy attester slashings sig set

* Tidy bulk signature verifier

* Refactor signature sets to be cleaner

* Start threading SignatureStrategy through code

* Add (empty) test dir

* Move BenchingBlockBuilder

* Add initial block signature verification tests

* Add tests for bulk signature verification

* Start threading SignatureStrategy in block proc.

* Refactor per_block_processing errors

* Use sig set tuples instead of lists of two

* Remove dead code

* Thread VerifySignatures through per_block_processing

* Add bulk signature verification

* Introduce parallel bulk signature verification

* Expand state processing benches

* Fix additional compile errors

* Fix issue where par iter chunks is 0

* Update milagro_bls dep

* Remove debugs, code fragment in beacon chain

* Tidy, add comments to block sig verifier

* Fix various PR comments

* Add block_root option to per_block_processing

* Fix comment in block signature verifier

* Fix comments from PR review

* Remove old comment

* Fix comment

tests/ef_tests/src/cases/operations_attestation.rs

@@ -2,7 +2,7 @@ use super::*;
 use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
 use serde_derive::Deserialize;
-use state_processing::per_block_processing::process_attestations;
+use state_processing::per_block_processing::{process_attestations, VerifySignatures};
 use types::{Attestation, BeaconState, EthSpec};
 
 #[derive(Debug, Clone, Deserialize)]
@@ -38,7 +38,7 @@ impl<E: EthSpec> Case for OperationsAttestation<E> {
         // Processing requires the epoch cache.
         state.build_all_caches(spec).unwrap();
 
-        let result = process_attestations(&mut state, &[attestation], spec);
+        let result = process_attestations(&mut state, &[attestation], VerifySignatures::True, spec);
 
         let mut result = result.and_then(|_| Ok(state));
 

tests/ef_tests/src/cases/operations_attester_slashing.rs

@@ -2,7 +2,7 @@ use super::*;
 use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
 use serde_derive::Deserialize;
-use state_processing::per_block_processing::process_attester_slashings;
+use state_processing::per_block_processing::{process_attester_slashings, VerifySignatures};
 use types::{AttesterSlashing, BeaconState, EthSpec};
 
 #[derive(Debug, Clone, Deserialize)]
@@ -38,8 +38,12 @@ impl<E: EthSpec> Case for OperationsAttesterSlashing<E> {
         // Processing requires the epoch cache.
         state.build_all_caches(&E::default_spec()).unwrap();
 
-        let result =
-            process_attester_slashings(&mut state, &[attester_slashing], &E::default_spec());
+        let result = process_attester_slashings(
+            &mut state,
+            &[attester_slashing],
+            VerifySignatures::True,
+            &E::default_spec(),
+        );
 
         let mut result = result.and_then(|_| Ok(state));
 

tests/ef_tests/src/cases/operations_block_header.rs

@@ -2,7 +2,7 @@ use super::*;
 use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
 use serde_derive::Deserialize;
-use state_processing::per_block_processing::process_block_header;
+use state_processing::per_block_processing::{process_block_header, VerifySignatures};
 use types::{BeaconBlock, BeaconState, EthSpec};
 
 #[derive(Debug, Clone, Deserialize)]
@@ -37,7 +37,9 @@ impl<E: EthSpec> Case for OperationsBlockHeader<E> {
         // Processing requires the epoch cache.
         state.build_all_caches(spec).unwrap();
 
-        let mut result = process_block_header(&mut state, &self.block, spec, true).map(|_| state);
+        let mut result =
+            process_block_header(&mut state, &self.block, None, VerifySignatures::True, spec)
+                .map(|_| state);
 
         compare_beacon_state_results_without_caches(&mut result, &mut expected)
     }

tests/ef_tests/src/cases/operations_exit.rs

@@ -2,7 +2,7 @@ use super::*;
 use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
 use serde_derive::Deserialize;
-use state_processing::per_block_processing::process_exits;
+use state_processing::per_block_processing::{process_exits, VerifySignatures};
 use types::{BeaconState, EthSpec, VoluntaryExit};
 
 #[derive(Debug, Clone, Deserialize)]
@@ -36,7 +36,12 @@ impl<E: EthSpec> Case for OperationsExit<E> {
         // Exit processing requires the epoch cache.
         state.build_all_caches(&E::default_spec()).unwrap();
 
-        let result = process_exits(&mut state, &[exit], &E::default_spec());
+        let result = process_exits(
+            &mut state,
+            &[exit],
+            VerifySignatures::True,
+            &E::default_spec(),
+        );
 
         let mut result = result.and_then(|_| Ok(state));
 

tests/ef_tests/src/cases/operations_proposer_slashing.rs

@@ -2,7 +2,7 @@ use super::*;
 use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
 use serde_derive::Deserialize;
-use state_processing::per_block_processing::process_proposer_slashings;
+use state_processing::per_block_processing::{process_proposer_slashings, VerifySignatures};
 use types::{BeaconState, EthSpec, ProposerSlashing};
 
 #[derive(Debug, Clone, Deserialize)]
@@ -36,8 +36,12 @@ impl<E: EthSpec> Case for OperationsProposerSlashing<E> {
         // Processing requires the epoch cache.
         state.build_all_caches(&E::default_spec()).unwrap();
 
-        let result =
-            process_proposer_slashings(&mut state, &[proposer_slashing], &E::default_spec());
+        let result = process_proposer_slashings(
+            &mut state,
+            &[proposer_slashing],
+            VerifySignatures::True,
+            &E::default_spec(),
+        );
 
         let mut result = result.and_then(|_| Ok(state));
 

tests/ef_tests/src/cases/operations_transfer.rs

@@ -2,7 +2,7 @@ use super::*;
 use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
 use serde_derive::Deserialize;
-use state_processing::per_block_processing::process_transfers;
+use state_processing::per_block_processing::{process_transfers, VerifySignatures};
 use types::{BeaconState, EthSpec, Transfer};
 
 #[derive(Debug, Clone, Deserialize)]
@@ -38,7 +38,7 @@ impl<E: EthSpec> Case for OperationsTransfer<E> {
 
         let spec = E::default_spec();
 
-        let result = process_transfers(&mut state, &[transfer], &spec);
+        let result = process_transfers(&mut state, &[transfer], VerifySignatures::True, &spec);
 
         let mut result = result.and_then(|_| Ok(state));
 

tests/ef_tests/src/cases/sanity_blocks.rs

@@ -3,7 +3,7 @@ use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
 use serde_derive::Deserialize;
 use state_processing::{
-    per_block_processing, per_slot_processing, BlockInvalid, BlockProcessingError,
+    per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy,
 };
 use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch};
 
@@ -50,14 +50,18 @@ impl<E: EthSpec> Case for SanityBlocks<E> {
                     .build_committee_cache(RelativeEpoch::Current, spec)
                     .unwrap();
 
-                per_block_processing(&mut state, block, spec)?;
+                per_block_processing(
+                    &mut state,
+                    block,
+                    None,
+                    BlockSignatureStrategy::VerifyIndividual,
+                    spec,
+                )?;
 
                 if block.state_root == state.canonical_root() {
                     Ok(())
                 } else {
-                    Err(BlockProcessingError::Invalid(
-                        BlockInvalid::StateRootMismatch,
-                    ))
+                    Err(BlockProcessingError::StateRootMismatch)
                 }
             })
             .map(|_| state);