Interop chain start strategies (#479)

* Implement more flexible beacon chain genesis

* Fix compile issues from rebase on master

* Rename CLI flag

* Adds initial documentation for TOML files

* Update docs readme

* Add first version of cli_util

* Dont write cache fields in serde

* Tidy cli_util

* Add code to load genesis YAML file

* Move serde_utils out of tests in `types`

* Update logging text

* Fix serde YAML for Fork

* Make yaml hex decoding more strict

* Update deterministic key generate for interop

* Set deposit count on testing genesis state

* Make some fixes for deposit count

* Remove code fragements

* Large restructure of docs

* Tidy docs

* Fix readme link

* Add interop docs

* Tidy README

eth2/types/src/beacon_block_body.rs

@@ -1,4 +1,5 @@
-use crate::test_utils::{graffiti_from_hex_str, TestRandom};
+use crate::test_utils::TestRandom;
+use crate::utils::graffiti_from_hex_str;
 use crate::*;
 
 use serde_derive::{Deserialize, Serialize};

eth2/types/src/beacon_state.rs

@@ -134,13 +134,13 @@ where
     pub finalized_checkpoint: Checkpoint,
 
     // Caching (not in the spec)
-    #[serde(default)]
+    #[serde(skip_serializing, skip_deserializing)]
     #[ssz(skip_serializing)]
     #[ssz(skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
     pub committee_caches: [CommitteeCache; CACHED_EPOCHS],
-    #[serde(default)]
+    #[serde(skip_serializing, skip_deserializing)]
     #[ssz(skip_serializing)]
     #[ssz(skip_deserializing)]
     #[tree_hash(skip_hashing)]

eth2/types/src/chain_spec.rs

@@ -1,7 +1,7 @@
 use crate::*;
 use int_to_bytes::int_to_bytes4;
 use serde_derive::{Deserialize, Serialize};
-use test_utils::{u8_from_hex_str, u8_to_hex_str};
+use utils::{u8_from_hex_str, u8_to_hex_str};
 
 /// Each of the BLS signature domains.
 ///

eth2/types/src/fork.rs

@@ -1,7 +1,6 @@
-use crate::{
-    test_utils::{fork_from_hex_str, TestRandom},
-    Epoch,
-};
+use crate::test_utils::TestRandom;
+use crate::utils::{fork_from_hex_str, fork_to_hex_str};
+use crate::Epoch;
 
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -25,9 +24,15 @@ use tree_hash_derive::{CachedTreeHash, TreeHash};
     TestRandom,
 )]
 pub struct Fork {
-    #[serde(deserialize_with = "fork_from_hex_str")]
+    #[serde(
+        serialize_with = "fork_to_hex_str",
+        deserialize_with = "fork_from_hex_str"
+    )]
     pub previous_version: [u8; 4],
-    #[serde(deserialize_with = "fork_from_hex_str")]
+    #[serde(
+        serialize_with = "fork_to_hex_str",
+        deserialize_with = "fork_from_hex_str"
+    )]
     pub current_version: [u8; 4],
     pub epoch: Epoch,
 }

eth2/types/src/lib.rs

@@ -30,6 +30,7 @@ pub mod indexed_attestation;
 pub mod pending_attestation;
 pub mod proposer_slashing;
 pub mod transfer;
+pub mod utils;
 pub mod voluntary_exit;
 #[macro_use]
 pub mod slot_epoch_macros;

eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs

@@ -133,6 +133,9 @@ impl<T: EthSpec> TestingBeaconStateBuilder<T> {
             spec,
         );
 
+        state.eth1_data.deposit_count = validator_count as u64;
+        state.eth1_deposit_index = validator_count as u64;
+
         let balances = vec![starting_balance; validator_count].into();
 
         debug!("Importing {} existing validators...", validator_count);

eth2/types/src/test_utils/generate_deterministic_keypairs.rs

@@ -1,14 +1,12 @@
 use crate::*;
-use int_to_bytes::int_to_bytes48;
+use eth2_interop_keypairs::be_private_key;
 use log::debug;
 use rayon::prelude::*;
 
-/// Generates `validator_count` keypairs where the secret key is the index of the
-/// validator.
+/// Generates `validator_count` keypairs where the secret key is derived solely from the index of
+/// the validator.
 ///
-/// For example, the first validator has a secret key of `int_to_bytes48(1)`, the second has
-/// `int_to_bytes48(2)` and so on. (We skip `0` as it generates a weird looking public key and is
-/// probably invalid).
+/// Uses the `eth2_interop_keypairs` crate to generate keys.
 pub fn generate_deterministic_keypairs(validator_count: usize) -> Vec<Keypair> {
     debug!(
         "Generating {} deterministic validator keypairs...",
@@ -20,6 +18,7 @@ pub fn generate_deterministic_keypairs(validator_count: usize) -> Vec<Keypair> {
         .par_iter()
         .map(|&i| generate_deterministic_keypair(i))
         .collect();
+
     keypairs
 }
 
@@ -27,8 +26,8 @@ pub fn generate_deterministic_keypairs(validator_count: usize) -> Vec<Keypair> {
 ///
 /// This is used for testing only, and not to be used in production!
 pub fn generate_deterministic_keypair(validator_index: usize) -> Keypair {
-    let secret = int_to_bytes48(validator_index as u64 + 1000);
-    let sk = SecretKey::from_bytes(&secret).unwrap();
+    let sk = SecretKey::from_bytes(&be_private_key(validator_index))
+        .expect("be_private_key always returns valid keys");
     let pk = PublicKey::from_secret_key(&sk);
     Keypair { sk, pk }
 }

eth2/types/src/test_utils/mod.rs

@@ -3,7 +3,6 @@ mod macros;
 mod builders;
 mod generate_deterministic_keypairs;
 mod keypairs_file;
-mod serde_utils;
 mod test_random;
 
 pub use builders::*;
@@ -14,5 +13,4 @@ pub use rand::{
     RngCore,
     {prng::XorShiftRng, SeedableRng},
 };
-pub use serde_utils::{fork_from_hex_str, graffiti_from_hex_str, u8_from_hex_str, u8_to_hex_str};
 pub use test_random::TestRandom;

eth2/types/src/utils.rs

@@ -0,0 +1,3 @@
+mod serde_utils;
+
+pub use serde_utils::*;

eth2/types/src/utils/serde_utils.rs

@@ -1,3 +1,4 @@
+use hex;
 use serde::de::Error;
 use serde::{Deserialize, Deserializer, Serializer};
 
@@ -32,7 +33,7 @@ where
     let mut array = [0 as u8; FORK_BYTES_LEN];
     let decoded: Vec<u8> = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?;
 
-    if decoded.len() > FORK_BYTES_LEN {
+    if decoded.len() != FORK_BYTES_LEN {
         return Err(D::Error::custom("Fork length too long"));
     }
 
@@ -45,6 +46,17 @@ where
     Ok(array)
 }
 
+// #[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `byte` to be a ref.
+pub fn fork_to_hex_str<S>(bytes: &[u8; 4], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&bytes));
+
+    serializer.serialize_str(&hex_string)
+}
+
 pub fn graffiti_from_hex_str<'de, D>(deserializer: D) -> Result<[u8; GRAFFITI_BYTES_LEN], D::Error>
 where
     D: Deserializer<'de>,