Use global pubkey cache for block processing (#849)

* Start updating types

* WIP

* Signature hacking

* Existing EF tests passing with fake_crypto

* Updates

* Delete outdated API spec

* The refactor continues

* It compiles

* WIP test fixes

* All release tests passing bar genesis state parsing

* Update and test YamlConfig

* Update to spec v0.10 compatible BLS

* Updates to BLS EF tests

* Add EF test for AggregateVerify

And delete unused hash2curve tests for uncompressed points

* Update EF tests to v0.10.1

* Use optional block root correctly in block proc

* Use genesis fork in deposit domain. All tests pass

* Cargo fmt

* Fast aggregate verify test

* Update REST API docs

* Cargo fmt

* Fix unused import

* Bump spec tags to v0.10.1

* Add `seconds_per_eth1_block` to chainspec

* Update to timestamp based eth1 voting scheme

* Return None from `get_votes_to_consider` if block cache is empty

* Handle overflows in `is_candidate_block`

* Revert to failing tests

* Fix eth1 data sets test

* Choose default vote according to spec

* Fix collect_valid_votes tests

* Fix `get_votes_to_consider` to choose all eligible blocks

* Uncomment winning_vote tests

* Add comments; remove unused code

* Reduce seconds_per_eth1_block for simulation

* Addressed review comments

* Add test for default vote case

* Fix logs

* Remove unused functions

* Meter default eth1 votes

* Fix comments

* Address review comments; remove unused dependency

* Add first attempt at attestation proc. re-write

* Add version 2 of attestation processing

* Minor fixes

* Add validator pubkey cache

* Make get_indexed_attestation take a committee

* Link signature processing into new attn verification

* First working version

* Ensure pubkey cache is updated

* Add more metrics, slight optimizations

* Clone committee cache during attestation processing

* Update shuffling cache during block processing

* Remove old commented-out code

* Fix shuffling cache insert bug

* Used indexed attestation in fork choice

* Restructure attn processing, add metrics

* Add more detailed metrics

* Tidy, fix failing tests

* Fix failing tests, tidy

* Disable/delete two outdated tests

* Add new Pubkeys struct to signature_sets

* Refactor with functional approach

* Update beacon chain

* Remove decompressed member from pubkey bytes

* Add hashmap for indices lookup

* Change `get_attesting_indices` to use Vec

* Fix failing test

* Tidy

* Add pubkey cache persistence file

* Add more comments

* Integrate persistence file into builder

* Add pubkey cache tests

* Add data_dir to beacon chain builder

* Remove Option in pubkey cache persistence file

* Ensure consistency between datadir/data_dir

* Fix failing network test

* Tidy

* Fix todos

* Improve tests

* Split up block processing metrics

* Tidy

* Refactor get_pubkey_from_state

* Remove commented-out code

* Add BeaconChain::validator_pubkey

* Use Option::filter

* Remove Box

* Comment out tests that fail due to hard-coded

Co-authored-by: Michael Sproul <michael@sigmaprime.io>
Co-authored-by: Michael Sproul <micsproul@gmail.com>
Co-authored-by: pawan <pawandhananjay@gmail.com>

eth2/utils/bls/src/fake_aggregate_public_key.rs

@@ -22,6 +22,13 @@ impl FakeAggregatePublicKey {
         Self::zero()
     }
 
+    pub fn empty_signature() -> Self {
+        Self {
+            bytes: vec![0; BLS_PUBLIC_KEY_BYTE_SIZE],
+            point: G1Point::new(),
+        }
+    }
+
     pub fn from_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
         if bytes.len() != BLS_PUBLIC_KEY_BYTE_SIZE {
             Err(DecodeError::InvalidByteLength {

eth2/utils/bls/src/fake_public_key.rs

@@ -85,6 +85,14 @@ impl FakePublicKey {
     pub fn as_raw(&self) -> &Self {
         self
     }
+
+    pub fn as_point(&self) -> &G1Point {
+        &self.point
+    }
+
+    pub fn into_point(self) -> G1Point {
+        self.point
+    }
 }
 
 impl fmt::Display for FakePublicKey {

eth2/utils/bls/src/macros.rs

@@ -92,7 +92,6 @@ macro_rules! bytes_struct {
         #[derive(Clone)]
         pub struct $name {
             bytes: [u8; $byte_size],
-            decompressed: Option<$type>
         }
     };
     ($name: ident, $type: ty, $byte_size: expr, $small_name: expr) => {
@@ -103,14 +102,12 @@ macro_rules! bytes_struct {
             pub fn from_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
                 Ok(Self {
                     bytes: Self::get_bytes(bytes)?,
-                    decompressed: None
                 })
             }
 
             pub fn empty() -> Self {
                 Self {
                     bytes: [0; $byte_size],
-                    decompressed: None
                 }
             }
 
@@ -134,15 +131,6 @@ macro_rules! bytes_struct {
                     Ok(result)
                 }
             }
-
-            pub fn decompress(&mut self) -> Result<(), ssz::DecodeError> {
-                self.decompressed = Some(<&Self as std::convert::TryInto<$type>>::try_into(self)?);
-                Ok(())
-            }
-
-            pub fn decompressed(&self) -> &Option<$type> {
-                &self.decompressed
-            }
         }
 
         impl std::fmt::Debug for $name {
@@ -180,7 +168,44 @@ macro_rules! bytes_struct {
             }
         }
 
-        impl_ssz!($name, $byte_size, "$type");
+        impl ssz::Encode for $name {
+            fn is_ssz_fixed_len() -> bool {
+                true
+            }
+
+            fn ssz_fixed_len() -> usize {
+                $byte_size
+            }
+
+            fn ssz_bytes_len(&self) -> usize {
+                $byte_size
+            }
+
+            fn ssz_append(&self, buf: &mut Vec<u8>) {
+                buf.extend_from_slice(&self.bytes)
+            }
+        }
+
+        impl ssz::Decode for $name {
+            fn is_ssz_fixed_len() -> bool {
+                true
+            }
+
+            fn ssz_fixed_len() -> usize {
+                $byte_size
+            }
+
+            fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
+                let len = bytes.len();
+                let expected = <Self as ssz::Decode>::ssz_fixed_len();
+
+                if len != expected {
+                    Err(ssz::DecodeError::InvalidByteLength { len, expected })
+                } else {
+                    Self::from_bytes(bytes)
+                }
+            }
+        }
 
         impl tree_hash::TreeHash for $name {
             fn tree_hash_type() -> tree_hash::TreeHashType {

eth2/utils/bls/src/public_key.rs

@@ -1,5 +1,5 @@
 use super::{SecretKey, BLS_PUBLIC_KEY_BYTE_SIZE};
-use milagro_bls::PublicKey as RawPublicKey;
+use milagro_bls::{G1Point, PublicKey as RawPublicKey};
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
@@ -24,11 +24,21 @@ impl PublicKey {
         Self(raw)
     }
 
-    /// Returns the underlying signature.
+    /// Returns a reference to the underlying signature.
     pub fn as_raw(&self) -> &RawPublicKey {
         &self.0
     }
 
+    /// Consumes self and returns the underlying signature.
+    pub fn as_point(&self) -> &G1Point {
+        &self.0.point
+    }
+
+    /// Consumes self and returns the underlying signature.
+    pub fn into_point(self) -> G1Point {
+        self.0.point
+    }
+
     /// Returns the underlying point as compressed bytes.
     ///
     /// Identical to `self.as_uncompressed_bytes()`.