Update to spec v1.0.0-rc.0 and BLSv4 (#1765)

## Issue Addressed

Closes #1504 
Closes #1505
Replaces #1703
Closes #1707

## Proposed Changes

* Update BLST and Milagro to versions compatible with BLSv4 spec
* Update Lighthouse to spec v1.0.0-rc.0, and update EF test vectors
* Use the v1.0.0 constants for `MainnetEthSpec`.
* Rename `InteropEthSpec` -> `V012LegacyEthSpec`
    * Change all constants to suit the mainnet `v0.12.3` specification (i.e., Medalla).
* Deprecate the `--spec` flag for the `lighthouse` binary
    * This value is now obtained from the `config_name` field of the `YamlConfig`.
        * Built in testnet YAML files have been updated.
    * Ignore the `--spec` value, if supplied, log a warning that it will be deprecated
    * `lcli` still has the spec flag, that's fine because it's dev tooling.
* Remove the `E: EthSpec` from `YamlConfig`
    * This means we need to deser the genesis `BeaconState` on-demand, but this is fine.
* Swap the old "minimal", "mainnet" strings over to the new `EthSpecId` enum.
* Always require a `CONFIG_NAME` field in `YamlConfig` (it used to have a default).

## Additional Info

Lots of breaking changes, do not merge! ~~We will likely need a Lighthouse v0.4.0 branch, and possibly a long-term v0.3.0 branch to keep Medalla alive~~.

Co-authored-by: Kirk Baird <baird.k@outlook.com>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

testing/ef_tests/Makefile

@@ -1,4 +1,4 @@
-TESTS_TAG := v0.12.3
+TESTS_TAG := v1.0.0-rc.0
 TESTS = general minimal mainnet
 TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS))
 

testing/ef_tests/src/cases/bls_aggregate_verify.rs

@@ -1,13 +1,13 @@
 use super::*;
 use crate::case_result::compare_result;
 use crate::cases::common::BlsCase;
-use bls::{AggregateSignature, PublicKey};
+use bls::{AggregateSignature, PublicKeyBytes};
 use serde_derive::Deserialize;
 use types::Hash256;
 
 #[derive(Debug, Clone, Deserialize)]
 pub struct BlsAggregateVerifyInput {
-    pub pubkeys: Vec<PublicKey>,
+    pub pubkeys: Vec<PublicKeyBytes>,
     pub messages: Vec<String>,
     pub signature: String,
 }
@@ -33,14 +33,29 @@ impl Case for BlsAggregateVerify {
             })
             .collect::<Result<Vec<_>, _>>()?;
 
-        let pubkey_refs = self.input.pubkeys.iter().collect::<Vec<_>>();
+        let pubkeys_result = self
+            .input
+            .pubkeys
+            .iter()
+            .map(|pkb| pkb.decompress())
+            .collect::<Result<Vec<_>, _>>();
+
+        let pubkeys = match pubkeys_result {
+            Ok(pubkeys) => pubkeys,
+            Err(bls::Error::InvalidInfinityPublicKey) if !self.output => {
+                return Ok(());
+            }
+            Err(e) => return Err(Error::FailedToParseTest(format!("{:?}", e))),
+        };
+
+        let pubkey_refs = pubkeys.iter().collect::<Vec<_>>();
 
         let signature_bytes = hex::decode(&self.input.signature[2..])
             .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
 
         let signature_valid = AggregateSignature::deserialize(&signature_bytes)
             .ok()
-            .map(|signature| signature.aggregate_verify(&messages, &pubkey_refs))
+            .map(|signature| signature.aggregate_verify(&messages, &pubkey_refs[..]))
             .unwrap_or(false);
 
         compare_result::<bool, ()>(&Ok(signature_valid), &Some(self.output))

testing/ef_tests/src/cases/bls_fast_aggregate_verify.rs

@@ -1,7 +1,7 @@
 use super::*;
 use crate::case_result::compare_result;
 use crate::cases::common::BlsCase;
-use bls::{AggregateSignature, PublicKey, PublicKeyBytes};
+use bls::{AggregateSignature, PublicKeyBytes};
 use serde_derive::Deserialize;
 use std::convert::TryInto;
 use types::Hash256;
@@ -9,6 +9,7 @@ use types::Hash256;
 #[derive(Debug, Clone, Deserialize)]
 pub struct BlsFastAggregateVerifyInput {
     pub pubkeys: Vec<PublicKeyBytes>,
+    #[serde(alias = "messages")]
     pub message: String,
     pub signature: String,
 }
@@ -28,13 +29,20 @@ impl Case for BlsFastAggregateVerify {
                 .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?,
         );
 
-        let pubkeys = self
+        let pubkeys_result = self
             .input
             .pubkeys
             .iter()
             .map(|pkb| pkb.try_into())
-            .collect::<Result<Vec<PublicKey>, bls::Error>>()
-            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+            .collect::<Result<Vec<_>, _>>();
+
+        let pubkeys = match pubkeys_result {
+            Ok(pubkeys) => pubkeys,
+            Err(bls::Error::InvalidInfinityPublicKey) if !self.output => {
+                return Ok(());
+            }
+            Err(e) => return Err(Error::FailedToParseTest(format!("{:?}", e))),
+        };
 
         let pubkey_refs = pubkeys.iter().collect::<Vec<_>>();
 

testing/ef_tests/src/cases/bls_sign_msg.rs

@@ -14,7 +14,7 @@ pub struct BlsSignInput {
 #[derive(Debug, Clone, Deserialize)]
 pub struct BlsSign {
     pub input: BlsSignInput,
-    pub output: String,
+    pub output: Option<String>,
 }
 
 impl BlsCase for BlsSign {}
@@ -27,16 +27,25 @@ impl Case for BlsSign {
 
         assert_eq!(sk.len(), 32);
 
-        let sk = SecretKey::deserialize(&sk).unwrap();
+        let sk = match SecretKey::deserialize(&sk) {
+            Ok(sk) => sk,
+            Err(_) if self.output.is_none() => {
+                return Ok(());
+            }
+            Err(e) => return Err(Error::FailedToParseTest(format!("{:?}", e))),
+        };
         let msg = hex::decode(&self.input.message[2..])
             .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
 
         let signature = sk.sign(Hash256::from_slice(&msg));
 
-        // Convert the output to one set of bytes
-        let decoded = hex::decode(&self.output[2..])
+        let decoded = self
+            .output
+            .as_ref()
+            .map(|output| hex::decode(&output[2..]))
+            .transpose()
             .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
 
-        compare_result::<Vec<u8>, Vec<u8>>(&Ok(signature.serialize().to_vec()), &Some(decoded))
+        compare_result::<Vec<u8>, Vec<u8>>(&Ok(signature.serialize().to_vec()), &decoded)
     }
 }

testing/ef_tests/src/cases/bls_verify_msg.rs

@@ -1,14 +1,14 @@
 use super::*;
 use crate::case_result::compare_result;
 use crate::cases::common::BlsCase;
-use bls::{PublicKey, Signature, SignatureBytes};
+use bls::{PublicKeyBytes, Signature, SignatureBytes};
 use serde_derive::Deserialize;
 use std::convert::TryInto;
 use types::Hash256;
 
 #[derive(Debug, Clone, Deserialize)]
 pub struct BlsVerifyInput {
-    pub pubkey: PublicKey,
+    pub pubkey: PublicKeyBytes,
     pub message: String,
     pub signature: SignatureBytes,
 }
@@ -28,8 +28,9 @@ impl Case for BlsVerify {
 
         let signature_ok = (&self.input.signature)
             .try_into()
-            .map(|signature: Signature| {
-                signature.verify(&self.input.pubkey, Hash256::from_slice(&message))
+            .and_then(|signature: Signature| {
+                let pk = self.input.pubkey.decompress()?;
+                Ok(signature.verify(&pk, Hash256::from_slice(&message)))
             })
             .unwrap_or(false);
 

testing/ef_tests/src/type_name.rs

@@ -36,6 +36,7 @@ macro_rules! type_name_generic {
 type_name!(MinimalEthSpec, "minimal");
 type_name!(MainnetEthSpec, "mainnet");
 
+type_name_generic!(AggregateAndProof);
 type_name_generic!(Attestation);
 type_name!(AttestationData);
 type_name_generic!(AttesterSlashing);
@@ -46,12 +47,18 @@ type_name_generic!(BeaconState);
 type_name!(Checkpoint);
 type_name!(Deposit);
 type_name!(DepositData);
+type_name!(DepositMessage);
 type_name!(Eth1Data);
 type_name!(Fork);
+type_name!(ForkData);
 type_name_generic!(HistoricalBatch);
 type_name_generic!(IndexedAttestation);
 type_name_generic!(PendingAttestation);
 type_name!(ProposerSlashing);
+type_name_generic!(SignedAggregateAndProof);
+type_name_generic!(SignedBeaconBlock);
+type_name!(SignedBeaconBlockHeader);
 type_name!(SignedVoluntaryExit);
+type_name!(SigningData);
 type_name!(Validator);
 type_name!(VoluntaryExit);

testing/ef_tests/tests/tests.rs

@@ -172,6 +172,7 @@ mod ssz_static {
     use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler};
     use types::*;
 
+    ssz_static_test!(aggregate_and_proof, AggregateAndProof<_>);
     ssz_static_test!(attestation, Attestation<_>);
     ssz_static_test!(attestation_data, AttestationData);
     ssz_static_test!(attester_slashing, AttesterSlashing<_>);
@@ -188,12 +189,24 @@ mod ssz_static {
     ssz_static_test!(checkpoint, Checkpoint);
     ssz_static_test!(deposit, Deposit);
     ssz_static_test!(deposit_data, DepositData);
+    ssz_static_test!(deposit_message, DepositMessage);
+    // FIXME(sproul): move Eth1Block to consensus/types
+    //
+    // Tracked at: https://github.com/sigp/lighthouse/issues/1835
+    //
+    // ssz_static_test!(eth1_block, Eth1Block);
     ssz_static_test!(eth1_data, Eth1Data);
     ssz_static_test!(fork, Fork);
+    ssz_static_test!(fork_data, ForkData);
     ssz_static_test!(historical_batch, HistoricalBatch<_>);
     ssz_static_test!(indexed_attestation, IndexedAttestation<_>);
     ssz_static_test!(pending_attestation, PendingAttestation<_>);
     ssz_static_test!(proposer_slashing, ProposerSlashing);
+    ssz_static_test!(signed_aggregate_and_proof, SignedAggregateAndProof<_>);
+    ssz_static_test!(signed_beacon_block, SignedBeaconBlock<_>);
+    ssz_static_test!(signed_beacon_block_header, SignedBeaconBlockHeader);
+    ssz_static_test!(signed_voluntary_exit, SignedVoluntaryExit);
+    ssz_static_test!(signing_data, SigningData);
     ssz_static_test!(validator, Validator);
     ssz_static_test!(voluntary_exit, VoluntaryExit);
 }
@@ -217,7 +230,7 @@ fn epoch_processing_justification_and_finalization() {
 #[test]
 fn epoch_processing_rewards_and_penalties() {
     EpochProcessingHandler::<MinimalEthSpec, RewardsAndPenalties>::run();
-    // Note: there are no reward and penalty tests for mainnet yet
+    EpochProcessingHandler::<MainnetEthSpec, RewardsAndPenalties>::run();
 }
 
 #[test]