Improve DX for loading validator keys from file

eth2/types/src/beacon_state/tests.rs

@@ -13,7 +13,7 @@ pub fn get_attestation_participants_consistency() {
     let mut rng = XorShiftRng::from_seed([42; 16]);
 
     let spec = ChainSpec::few_validators();
-    let builder = TestingBeaconStateBuilder::new(8, None, &spec);
+    let builder = TestingBeaconStateBuilder::from_deterministic_keypairs(8, &spec);
     let (mut state, _keypairs) = builder.build();
 
     state

eth2/types/src/test_utils/keypairs_file.rs

@@ -1,4 +1,5 @@
 use crate::*;
+use rayon::prelude::*;
 use std::fs::File;
 use std::io::{Error, ErrorKind, Read, Write};
 use std::path::Path;
@@ -45,19 +46,27 @@ impl KeypairsFile for Vec<Keypair> {
             let mut buf = vec![0; batch.len() * KEYPAIR_BYTES_LEN];
             keypairs_file.read_exact(&mut buf)?;
 
-            for (i, _) in batch.iter().enumerate() {
-                let sk_start = i * KEYPAIR_BYTES_LEN;
-                let sk_end = sk_start + SECRET_KEY_BYTES_LEN;
-                let sk = SecretKey::from_bytes(&buf[sk_start..sk_end])
-                    .map_err(|_| Error::new(ErrorKind::Other, "Invalid SecretKey bytes"))?;
+            let mut keypair_batch = batch
+                .par_iter()
+                .enumerate()
+                .map(|(i, _)| {
+                    let sk_start = i * KEYPAIR_BYTES_LEN;
+                    let sk_end = sk_start + SECRET_KEY_BYTES_LEN;
+                    let sk = SecretKey::from_bytes(&buf[sk_start..sk_end])
+                        .map_err(|_| Error::new(ErrorKind::Other, "Invalid SecretKey bytes"))
+                        .unwrap();
 
-                let pk_start = sk_end;
-                let pk_end = pk_start + PUBLIC_KEY_BYTES_LEN;
-                let pk = PublicKey::from_bytes(&buf[pk_start..pk_end])
-                    .map_err(|_| Error::new(ErrorKind::Other, "Invalid PublicKey bytes"))?;
+                    let pk_start = sk_end;
+                    let pk_end = pk_start + PUBLIC_KEY_BYTES_LEN;
+                    let pk = PublicKey::from_bytes(&buf[pk_start..pk_end])
+                        .map_err(|_| Error::new(ErrorKind::Other, "Invalid PublicKey bytes"))
+                        .unwrap();
 
-                keypairs.push(Keypair { sk, pk });
-            }
+                    Keypair { sk, pk }
+                })
+                .collect();
+
+            keypairs.append(&mut keypair_batch);
         }
 
         Ok(keypairs)
@@ -68,7 +77,6 @@ impl KeypairsFile for Vec<Keypair> {
 mod tests {
     use super::*;
     use rand::{distributions::Alphanumeric, thread_rng, Rng};
-    use rayon::prelude::*;
     use std::fs::remove_file;
 
     fn random_keypairs(n: usize) -> Vec<Keypair> {

eth2/types/src/test_utils/mod.rs

@@ -14,7 +14,7 @@ pub use rand::{prng::XorShiftRng, SeedableRng};
 pub use test_random::TestRandom;
 pub use testing_attestation_builder::TestingAttestationBuilder;
 pub use testing_beacon_block_builder::TestingBeaconBlockBuilder;
-pub use testing_beacon_state_builder::TestingBeaconStateBuilder;
+pub use testing_beacon_state_builder::{keypairs_path, TestingBeaconStateBuilder};
 pub use testing_deposit_builder::TestingDepositBuilder;
 pub use testing_transfer_builder::TestingTransferBuilder;
 pub use testing_voluntary_exit_builder::TestingVoluntaryExitBuilder;

eth2/types/src/test_utils/testing_beacon_state_builder.rs

@@ -2,8 +2,22 @@ use super::{generate_deterministic_keypairs, KeypairsFile};
 use crate::beacon_state::BeaconStateBuilder;
 use crate::*;
 use bls::get_withdrawal_credentials;
+use dirs;
 use rayon::prelude::*;
-use std::path::Path;
+use std::path::{Path, PathBuf};
+
+pub const KEYPAIRS_FILE: &str = "keypairs.raw_keypairs";
+
+/// Returns the directory where the generated keypairs should be stored.
+///
+/// It is either `$HOME/.lighthouse/keypairs.raw_keypairs` or, if `$HOME` is not available,
+/// `./keypairs.raw_keypairs`.
+pub fn keypairs_path() -> PathBuf {
+    let dir = dirs::home_dir()
+        .and_then(|home| Some(home.join(".lighthouse")))
+        .unwrap_or_else(|| PathBuf::from(""));
+    dir.join(KEYPAIRS_FILE)
+}
 
 pub struct TestingBeaconStateBuilder {
     state: BeaconState,
@@ -11,11 +25,52 @@ pub struct TestingBeaconStateBuilder {
 }
 
 impl TestingBeaconStateBuilder {
-    pub fn new(validator_count: usize, keypairs_path: Option<&Path>, spec: &ChainSpec) -> Self {
-        let keypairs = match keypairs_path {
-            None => generate_deterministic_keypairs(validator_count),
-            Some(path) => Vec::from_raw_file(path, validator_count).unwrap(),
-        };
+    /// Attempts to load validators from a file in the `CARGO_MANIFEST_DIR`. If the file is
+    /// unavailable, it generates the keys at runtime.
+    ///
+    /// If the `CARGO_MANIFEST_DIR` environment variable is not set, the local directory is used.
+    ///
+    /// See the `Self::from_keypairs_file` method for more info.
+    ///
+    /// # Panics
+    ///
+    /// If the file does not contain enough keypairs or is invalid.
+    pub fn from_default_keypairs_file_if_exists(validator_count: usize, spec: &ChainSpec) -> Self {
+        let dir = dirs::home_dir()
+            .and_then(|home| Some(home.join(".lighthouse")))
+            .unwrap_or_else(|| PathBuf::from(""));
+        let file = dir.join(KEYPAIRS_FILE);
+
+        if file.exists() {
+            TestingBeaconStateBuilder::from_keypairs_file(validator_count, &file, spec)
+        } else {
+            TestingBeaconStateBuilder::from_deterministic_keypairs(validator_count, spec)
+        }
+    }
+
+    /// Loads the initial validator keypairs from a file on disk.
+    ///
+    /// Loading keypairs from file is ~10x faster than generating them. Use the `gen_keys` command
+    /// on the  `test_harness` binary to generate the keys. In the `test_harness` dir, run `cargo
+    /// run -- gen_keys -h` for help.
+    ///
+    /// # Panics
+    ///
+    /// If the file does not exist, is invalid or does not contain enough keypairs.
+    pub fn from_keypairs_file(validator_count: usize, path: &Path, spec: &ChainSpec) -> Self {
+        let keypairs = Vec::from_raw_file(path, validator_count).unwrap();
+        TestingBeaconStateBuilder::from_keypairs(keypairs, spec)
+    }
+
+    /// Generates the validator keypairs deterministically.
+    pub fn from_deterministic_keypairs(validator_count: usize, spec: &ChainSpec) -> Self {
+        let keypairs = generate_deterministic_keypairs(validator_count);
+        TestingBeaconStateBuilder::from_keypairs(keypairs, spec)
+    }
+
+    /// Creates the builder from an existing set of keypairs.
+    pub fn from_keypairs(keypairs: Vec<Keypair>, spec: &ChainSpec) -> Self {
+        let validator_count = keypairs.len();
 
         let validators = keypairs
             .par_iter()
@@ -61,10 +116,15 @@ impl TestingBeaconStateBuilder {
         }
     }
 
+    /// Consume the builder and return the `BeaconState` and the keypairs for each validator.
     pub fn build(self) -> (BeaconState, Vec<Keypair>) {
         (self.state, self.keypairs)
     }
 
+    /// Ensures that the state returned from `Self::build(..)` has all caches pre-built.
+    ///
+    /// Note: this performs the build when called. Ensure that no changes are made that would
+    /// invalidate this cache.
     pub fn build_caches(&mut self, spec: &ChainSpec) -> Result<(), BeaconStateError> {
         let state = &mut self.state;
 
@@ -147,6 +207,9 @@ impl TestingBeaconStateBuilder {
     }
 }
 
+/// Maps a committee to a `PendingAttestation`.
+///
+/// The committee will be signed by all validators in the committee.
 fn committee_to_pending_attestation(
     state: &BeaconState,
     committee: &[usize],