Interop chain start strategies (#479)

* Implement more flexible beacon chain genesis

* Fix compile issues from rebase on master

* Rename CLI flag

* Adds initial documentation for TOML files

* Update docs readme

* Add first version of cli_util

* Dont write cache fields in serde

* Tidy cli_util

* Add code to load genesis YAML file

* Move serde_utils out of tests in `types`

* Update logging text

* Fix serde YAML for Fork

* Make yaml hex decoding more strict

* Update deterministic key generate for interop

* Set deposit count on testing genesis state

* Make some fixes for deposit count

* Remove code fragements

* Large restructure of docs

* Tidy docs

* Fix readme link

* Add interop docs

* Tidy README

eth2/types/Cargo.toml

@@ -11,6 +11,7 @@ compare_fields = { path = "../utils/compare_fields" }
 compare_fields_derive = { path = "../utils/compare_fields_derive" }
 dirs = "1.0"
 derivative = "1.0"
+eth2_interop_keypairs = { path = "../utils/eth2_interop_keypairs" }
 ethereum-types = "0.5"
 hashing = { path = "../utils/hashing" }
 hex = "0.3"

eth2/types/src/beacon_block_body.rs

@@ -1,4 +1,5 @@
-use crate::test_utils::{graffiti_from_hex_str, TestRandom};
+use crate::test_utils::TestRandom;
+use crate::utils::graffiti_from_hex_str;
 use crate::*;
 
 use serde_derive::{Deserialize, Serialize};

eth2/types/src/beacon_state.rs

@@ -134,13 +134,13 @@ where
     pub finalized_checkpoint: Checkpoint,
 
     // Caching (not in the spec)
-    #[serde(default)]
+    #[serde(skip_serializing, skip_deserializing)]
     #[ssz(skip_serializing)]
     #[ssz(skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
     pub committee_caches: [CommitteeCache; CACHED_EPOCHS],
-    #[serde(default)]
+    #[serde(skip_serializing, skip_deserializing)]
     #[ssz(skip_serializing)]
     #[ssz(skip_deserializing)]
     #[tree_hash(skip_hashing)]

eth2/types/src/chain_spec.rs

@@ -1,7 +1,7 @@
 use crate::*;
 use int_to_bytes::int_to_bytes4;
 use serde_derive::{Deserialize, Serialize};
-use test_utils::{u8_from_hex_str, u8_to_hex_str};
+use utils::{u8_from_hex_str, u8_to_hex_str};
 
 /// Each of the BLS signature domains.
 ///

eth2/types/src/fork.rs

@@ -1,7 +1,6 @@
-use crate::{
-    test_utils::{fork_from_hex_str, TestRandom},
-    Epoch,
-};
+use crate::test_utils::TestRandom;
+use crate::utils::{fork_from_hex_str, fork_to_hex_str};
+use crate::Epoch;
 
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -25,9 +24,15 @@ use tree_hash_derive::{CachedTreeHash, TreeHash};
     TestRandom,
 )]
 pub struct Fork {
-    #[serde(deserialize_with = "fork_from_hex_str")]
+    #[serde(
+        serialize_with = "fork_to_hex_str",
+        deserialize_with = "fork_from_hex_str"
+    )]
     pub previous_version: [u8; 4],
-    #[serde(deserialize_with = "fork_from_hex_str")]
+    #[serde(
+        serialize_with = "fork_to_hex_str",
+        deserialize_with = "fork_from_hex_str"
+    )]
     pub current_version: [u8; 4],
     pub epoch: Epoch,
 }

eth2/types/src/lib.rs

@@ -30,6 +30,7 @@ pub mod indexed_attestation;
 pub mod pending_attestation;
 pub mod proposer_slashing;
 pub mod transfer;
+pub mod utils;
 pub mod voluntary_exit;
 #[macro_use]
 pub mod slot_epoch_macros;

eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs

@@ -133,6 +133,9 @@ impl<T: EthSpec> TestingBeaconStateBuilder<T> {
             spec,
         );
 
+        state.eth1_data.deposit_count = validator_count as u64;
+        state.eth1_deposit_index = validator_count as u64;
+
         let balances = vec![starting_balance; validator_count].into();
 
         debug!("Importing {} existing validators...", validator_count);

eth2/types/src/test_utils/generate_deterministic_keypairs.rs

@@ -1,14 +1,12 @@
 use crate::*;
-use int_to_bytes::int_to_bytes48;
+use eth2_interop_keypairs::be_private_key;
 use log::debug;
 use rayon::prelude::*;
 
-/// Generates `validator_count` keypairs where the secret key is the index of the
-/// validator.
+/// Generates `validator_count` keypairs where the secret key is derived solely from the index of
+/// the validator.
 ///
-/// For example, the first validator has a secret key of `int_to_bytes48(1)`, the second has
-/// `int_to_bytes48(2)` and so on. (We skip `0` as it generates a weird looking public key and is
-/// probably invalid).
+/// Uses the `eth2_interop_keypairs` crate to generate keys.
 pub fn generate_deterministic_keypairs(validator_count: usize) -> Vec<Keypair> {
     debug!(
         "Generating {} deterministic validator keypairs...",
@@ -20,6 +18,7 @@ pub fn generate_deterministic_keypairs(validator_count: usize) -> Vec<Keypair> {
         .par_iter()
         .map(|&i| generate_deterministic_keypair(i))
         .collect();
+
     keypairs
 }
 
@@ -27,8 +26,8 @@ pub fn generate_deterministic_keypairs(validator_count: usize) -> Vec<Keypair> {
 ///
 /// This is used for testing only, and not to be used in production!
 pub fn generate_deterministic_keypair(validator_index: usize) -> Keypair {
-    let secret = int_to_bytes48(validator_index as u64 + 1000);
-    let sk = SecretKey::from_bytes(&secret).unwrap();
+    let sk = SecretKey::from_bytes(&be_private_key(validator_index))
+        .expect("be_private_key always returns valid keys");
     let pk = PublicKey::from_secret_key(&sk);
     Keypair { sk, pk }
 }

eth2/types/src/test_utils/mod.rs

@@ -3,7 +3,6 @@ mod macros;
 mod builders;
 mod generate_deterministic_keypairs;
 mod keypairs_file;
-mod serde_utils;
 mod test_random;
 
 pub use builders::*;
@@ -14,5 +13,4 @@ pub use rand::{
     RngCore,
     {prng::XorShiftRng, SeedableRng},
 };
-pub use serde_utils::{fork_from_hex_str, graffiti_from_hex_str, u8_from_hex_str, u8_to_hex_str};
 pub use test_random::TestRandom;

eth2/types/src/utils.rs

@@ -0,0 +1,3 @@
+mod serde_utils;
+
+pub use serde_utils::*;

eth2/types/src/utils/serde_utils.rs

@@ -1,3 +1,4 @@
+use hex;
 use serde::de::Error;
 use serde::{Deserialize, Deserializer, Serializer};
 
@@ -32,7 +33,7 @@ where
     let mut array = [0 as u8; FORK_BYTES_LEN];
     let decoded: Vec<u8> = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?;
 
-    if decoded.len() > FORK_BYTES_LEN {
+    if decoded.len() != FORK_BYTES_LEN {
         return Err(D::Error::custom("Fork length too long"));
     }
 
@@ -45,6 +46,17 @@ where
     Ok(array)
 }
 
+// #[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `byte` to be a ref.
+pub fn fork_to_hex_str<S>(bytes: &[u8; 4], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&bytes));
+
+    serializer.serialize_str(&hex_string)
+}
+
 pub fn graffiti_from_hex_str<'de, D>(deserializer: D) -> Result<[u8; GRAFFITI_BYTES_LEN], D::Error>
 where
     D: Deserializer<'de>,

eth2/utils/eth2_interop_keypairs/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "eth2_interop_keypairs"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+num-bigint = "0.2"
+eth2_hashing = "0.1"

eth2/utils/eth2_interop_keypairs/src/lib.rs

@@ -0,0 +1,130 @@
+//! Produces the "deterministic" validator private keys used for inter-operability testing for
+//! Ethereum 2.0 clients.
+//!
+//! Each private key is the first hash in the sha2 hash-chain that is less than 2^255. As such,
+//! keys generated here are **not secret** and are **not for production use**.
+//!
+//! Note: these keys have not been tested against a reference implementation, yet.
+
+use eth2_hashing::hash;
+use num_bigint::BigUint;
+
+pub const CURVE_ORDER_BITS: usize = 255;
+pub const PRIVATE_KEY_BYTES: usize = 48;
+pub const HASH_BYTES: usize = 32;
+
+fn hash_big_int_le(uint: BigUint) -> BigUint {
+    let mut preimage = uint.to_bytes_le();
+    preimage.resize(32, 0_u8);
+    BigUint::from_bytes_le(&hash(&preimage))
+}
+
+fn private_key(validator_index: usize) -> BigUint {
+    let mut key = BigUint::from(validator_index);
+    loop {
+        key = hash_big_int_le(key);
+        if key.bits() <= CURVE_ORDER_BITS {
+            break key;
+        }
+    }
+}
+
+/// Generates an **unsafe** BLS12-381 private key for the given validator index, where that private
+/// key is represented in big-endian bytes.
+pub fn be_private_key(validator_index: usize) -> [u8; PRIVATE_KEY_BYTES] {
+    let vec = private_key(validator_index).to_bytes_be();
+
+    let mut out = [0; PRIVATE_KEY_BYTES];
+    out[PRIVATE_KEY_BYTES - vec.len()..PRIVATE_KEY_BYTES].copy_from_slice(&vec);
+    out
+}
+
+/// Generates an **unsafe** BLS12-381 private key for the given validator index, where that private
+/// key is represented in little-endian bytes.
+pub fn le_private_key(validator_index: usize) -> [u8; PRIVATE_KEY_BYTES] {
+    let vec = private_key(validator_index).to_bytes_le();
+
+    let mut out = [0; PRIVATE_KEY_BYTES];
+    out[0..vec.len()].copy_from_slice(&vec);
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn flip(vec: &[u8]) -> Vec<u8> {
+        let len = vec.len();
+        let mut out = vec![0; len];
+        for i in 0..len {
+            out[len - 1 - i] = vec[i];
+        }
+        out
+    }
+
+    fn pad_le_bls(mut vec: Vec<u8>) -> Vec<u8> {
+        vec.resize(PRIVATE_KEY_BYTES, 0_u8);
+        vec
+    }
+
+    fn pad_be_bls(mut vec: Vec<u8>) -> Vec<u8> {
+        let mut out = vec![0; PRIVATE_KEY_BYTES - vec.len()];
+        out.append(&mut vec);
+        out
+    }
+
+    fn pad_le_hash(index: usize) -> Vec<u8> {
+        let mut vec = index.to_le_bytes().to_vec();
+        vec.resize(HASH_BYTES, 0_u8);
+        vec
+    }
+
+    fn multihash(index: usize, rounds: usize) -> Vec<u8> {
+        let mut vec = pad_le_hash(index);
+        for _ in 0..rounds {
+            vec = hash(&vec);
+        }
+        vec
+    }
+
+    fn compare(validator_index: usize, preimage: &[u8]) {
+        assert_eq!(
+            &le_private_key(validator_index)[..],
+            &pad_le_bls(hash(preimage))[..]
+        );
+        assert_eq!(
+            &be_private_key(validator_index)[..],
+            &pad_be_bls(flip(&hash(preimage)))[..]
+        );
+    }
+
+    #[test]
+    fn consistency() {
+        for i in 0..256 {
+            let le = BigUint::from_bytes_le(&le_private_key(i));
+            let be = BigUint::from_bytes_be(&be_private_key(i));
+            assert_eq!(le, be);
+        }
+    }
+
+    #[test]
+    fn non_repeats() {
+        // These indices only need one hash to be in the curve order.
+        compare(0, &pad_le_hash(0));
+        compare(3, &pad_le_hash(3));
+    }
+
+    #[test]
+    fn repeats() {
+        // Index 5 needs 5x hashes to get into the curve order.
+        compare(5, &multihash(5, 5));
+    }
+
+    #[test]
+    fn doesnt_panic() {
+        for i in 0..256 {
+            be_private_key(i);
+            le_private_key(i);
+        }
+    }
+}