Directory Restructure (#1163)

* Move tests -> testing

* Directory restructure

* Update Cargo.toml during restructure

* Update Makefile during restructure

* Fix arbitrary path

testing/state_transition_vectors/.gitignore

@@ -0,0 +1 @@
+/vectors/

testing/state_transition_vectors/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "state_transition_vectors"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+state_processing = { path = "../../consensus/state_processing" }
+types = { path = "../../consensus/types" }
+eth2_ssz = "0.1.2"

testing/state_transition_vectors/Makefile

@@ -0,0 +1,8 @@
+produce-vectors:
+	cargo run --release
+
+test:
+	cargo test --release
+
+clean:
+	rm -r vectors/

testing/state_transition_vectors/README.md

@@ -0,0 +1,72 @@
+# state_transition_vectors
+
+This crate contains test vectors for Lighthouse state transition functions.
+
+This crate serves two purposes:
+
+- Outputting the test vectors to disk via `make`.
+- Running the vectors against our code via `make test`.
+
+
+## Outputting vectors to disk
+
+Whilst we don't actually need to write the vectors to disk to test them, we
+provide this functionality so we can generate corpra for the fuzzer and also so
+they can be of use to other clients.
+
+To create the files in `./vectors` (directory relative to this crate), run:
+
+```bash
+make
+```
+
+This will produce a directory structure that looks roughly like this:
+
+```
+vectors
+└── exit
+    ├── invalid_bad_signature
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_duplicate
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_exit_already_initiated
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_future_exit_epoch
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_not_active_after_exit_epoch
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_not_active_before_activation_epoch
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_too_young_by_a_lot
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_too_young_by_one_epoch
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── invalid_validator_unknown
+    │   ├── block.ssz
+    │   ├── error.txt
+    │   └── pre.ssz
+    ├── valid_genesis_epoch
+    │   ├── block.ssz
+    │   ├── post.ssz
+    │   └── pre.ssz
+    └── valid_previous_epoch
+        ├── block.ssz
+        ├── post.ssz
+        └── pre.ssz
+```

testing/state_transition_vectors/src/exit.rs

@@ -0,0 +1,346 @@
+use super::*;
+use state_processing::{
+    per_block_processing, per_block_processing::errors::ExitInvalid,
+    test_utils::BlockProcessingBuilder, BlockProcessingError, BlockSignatureStrategy,
+};
+use types::{BeaconBlock, BeaconState, Epoch, EthSpec, SignedBeaconBlock};
+
+// Default validator index to exit.
+pub const VALIDATOR_INDEX: u64 = 0;
+// Epoch that the state will be transitioned to by default, equal to PERSISTENT_COMMITTEE_PERIOD.
+pub const STATE_EPOCH: Epoch = Epoch::new(2048);
+
+struct ExitTest {
+    validator_index: u64,
+    exit_epoch: Epoch,
+    state_epoch: Epoch,
+    block_modifier: Box<dyn FnOnce(&mut BeaconBlock<E>)>,
+    builder_modifier: Box<dyn FnOnce(BlockProcessingBuilder<E>) -> BlockProcessingBuilder<E>>,
+    #[allow(dead_code)]
+    expected: Result<(), BlockProcessingError>,
+}
+
+impl Default for ExitTest {
+    fn default() -> Self {
+        Self {
+            validator_index: VALIDATOR_INDEX,
+            exit_epoch: STATE_EPOCH,
+            state_epoch: STATE_EPOCH,
+            block_modifier: Box::new(|_| ()),
+            builder_modifier: Box::new(|x| x),
+            expected: Ok(()),
+        }
+    }
+}
+
+impl ExitTest {
+    fn block_and_pre_state(self) -> (SignedBeaconBlock<E>, BeaconState<E>) {
+        let spec = &E::default_spec();
+
+        (self.builder_modifier)(
+            get_builder(spec, self.state_epoch.as_u64(), VALIDATOR_COUNT)
+                .insert_exit(self.validator_index, self.exit_epoch)
+                .modify(self.block_modifier),
+        )
+        .build(None, None)
+    }
+
+    fn process(
+        block: &SignedBeaconBlock<E>,
+        state: &mut BeaconState<E>,
+    ) -> Result<(), BlockProcessingError> {
+        per_block_processing(
+            state,
+            block,
+            None,
+            BlockSignatureStrategy::VerifyIndividual,
+            &E::default_spec(),
+        )
+    }
+
+    #[cfg(test)]
+    fn run(self) -> BeaconState<E> {
+        let spec = &E::default_spec();
+        let expected = self.expected.clone();
+        assert_eq!(STATE_EPOCH, spec.persistent_committee_period);
+
+        let (block, mut state) = self.block_and_pre_state();
+
+        let result = Self::process(&block, &mut state);
+
+        assert_eq!(result, expected);
+
+        state
+    }
+
+    fn test_vector(self, title: String) -> TestVector {
+        let (block, pre_state) = self.block_and_pre_state();
+        let mut post_state = pre_state.clone();
+        let (post_state, error) = match Self::process(&block, &mut post_state) {
+            Ok(_) => (Some(post_state), None),
+            Err(e) => (None, Some(format!("{:?}", e))),
+        };
+
+        TestVector {
+            title,
+            block,
+            pre_state,
+            post_state,
+            error,
+        }
+    }
+}
+
+vectors_and_tests!(
+    // Ensures we can process a valid exit,
+    valid_single_exit,
+    ExitTest::default(),
+    // Tests three exists in the same block.
+    valid_three_exits,
+    ExitTest {
+        builder_modifier: Box::new(|builder| {
+            builder
+                .insert_exit(1, STATE_EPOCH)
+                .insert_exit(2, STATE_EPOCH)
+        }),
+        ..ExitTest::default()
+    },
+    // Ensures that a validator cannot be exited twice in the same block.
+    invalid_duplicate,
+    ExitTest {
+        block_modifier: Box::new(|block| {
+            // Duplicate the exit
+            let exit = block.body.voluntary_exits[0].clone();
+            block.body.voluntary_exits.push(exit).unwrap();
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 1,
+            reason: ExitInvalid::AlreadyExited(0),
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // validator = state.validators[voluntary_exit.validator_index]
+    // ```
+    invalid_validator_unknown,
+    ExitTest {
+        block_modifier: Box::new(|block| {
+            block.body.voluntary_exits[0].message.validator_index = VALIDATOR_COUNT as u64;
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::ValidatorUnknown(VALIDATOR_COUNT as u64),
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify exit has not been initiated
+    // assert validator.exit_epoch == FAR_FUTURE_EPOCH
+    // ```
+    invalid_exit_already_initiated,
+    ExitTest {
+        builder_modifier: Box::new(|mut builder| {
+            builder.state.validators[0].exit_epoch = STATE_EPOCH + 1;
+            builder
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::AlreadyExited(0),
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify the validator is active
+    // assert is_active_validator(validator, get_current_epoch(state))
+    // ```
+    invalid_not_active_before_activation_epoch,
+    ExitTest {
+        builder_modifier: Box::new(|mut builder| {
+            builder.state.validators[0].activation_epoch = builder.spec.far_future_epoch;
+            builder
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::NotActive(0),
+        }),
+        ..ExitTest::default()
+    },
+    // Also tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify the validator is active
+    // assert is_active_validator(validator, get_current_epoch(state))
+    // ```
+    invalid_not_active_after_exit_epoch,
+    ExitTest {
+        builder_modifier: Box::new(|mut builder| {
+            builder.state.validators[0].exit_epoch = STATE_EPOCH;
+            builder
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::NotActive(0),
+        }),
+        ..ExitTest::default()
+    },
+    // Ensures we can process an exit from genesis.
+    valid_genesis_epoch,
+    ExitTest {
+        exit_epoch: Epoch::new(0),
+        ..ExitTest::default()
+    },
+    // Ensures we can process an exit from the previous epoch.
+    valid_previous_epoch,
+    ExitTest {
+        exit_epoch: STATE_EPOCH - 1,
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Exits must specify an epoch when they become valid; they are not
+    // # valid before then
+    // assert get_current_epoch(state) >= voluntary_exit.epoch
+    // ```
+    invalid_future_exit_epoch,
+    ExitTest {
+        exit_epoch: STATE_EPOCH + 1,
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::FutureEpoch {
+                state: STATE_EPOCH,
+                exit: STATE_EPOCH + 1,
+            },
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify the validator has been active long enough
+    // assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD
+    // ```
+    invalid_too_young_by_one_epoch,
+    ExitTest {
+        state_epoch: STATE_EPOCH - 1,
+        exit_epoch: STATE_EPOCH - 1,
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::TooYoungToExit {
+                current_epoch: STATE_EPOCH - 1,
+                earliest_exit_epoch: STATE_EPOCH,
+            },
+        }),
+        ..ExitTest::default()
+    },
+    // Also tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify the validator has been active long enough
+    // assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD
+    // ```
+    invalid_too_young_by_a_lot,
+    ExitTest {
+        state_epoch: Epoch::new(0),
+        exit_epoch: Epoch::new(0),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::TooYoungToExit {
+                current_epoch: Epoch::new(0),
+                earliest_exit_epoch: STATE_EPOCH,
+            },
+        }),
+        ..ExitTest::default()
+    },
+    // Tests the following line of the spec:
+    //
+    // v0.11.2
+    //
+    // ```ignore
+    // # Verify signature
+    // domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT,
+    // voluntary_exit.epoch)
+    // signing_root = compute_signing_root(voluntary_exit, domain)
+    // assert bls.Verify(validator.pubkey, signing_root,
+    // signed_voluntary_exit.signature)
+    // ```
+    invalid_bad_signature,
+    ExitTest {
+        block_modifier: Box::new(|block| {
+            // Shift the validator index by 1 so that it's mismatched from the key that was
+            // used to sign.
+            block.body.voluntary_exits[0].message.validator_index = VALIDATOR_INDEX + 1;
+        }),
+        expected: Err(BlockProcessingError::ExitInvalid {
+            index: 0,
+            reason: ExitInvalid::BadSignature,
+        }),
+        ..ExitTest::default()
+    }
+);
+
+#[cfg(test)]
+mod custom_tests {
+    use super::*;
+
+    fn assert_exited(state: &BeaconState<E>, validator_index: usize) {
+        let spec = E::default_spec();
+
+        let validator = &state.validators[validator_index];
+        assert_eq!(
+            validator.exit_epoch,
+            // This is correct until we exceed the churn limit. If that happens, we
+            // need to introduce more complex logic.
+            state.current_epoch() + 1 + spec.max_seed_lookahead,
+            "exit epoch"
+        );
+        assert_eq!(
+            validator.withdrawable_epoch,
+            validator.exit_epoch + E::default_spec().min_validator_withdrawability_delay,
+            "withdrawable epoch"
+        );
+    }
+
+    #[test]
+    fn valid() {
+        let state = ExitTest::default().run();
+        assert_exited(&state, VALIDATOR_INDEX as usize);
+    }
+
+    #[test]
+    fn valid_three() {
+        let state = ExitTest {
+            builder_modifier: Box::new(|builder| {
+                builder
+                    .insert_exit(1, STATE_EPOCH)
+                    .insert_exit(2, STATE_EPOCH)
+            }),
+            ..ExitTest::default()
+        }
+        .run();
+
+        for i in &[VALIDATOR_INDEX, 1, 2] {
+            assert_exited(&state, *i as usize);
+        }
+    }
+}

testing/state_transition_vectors/src/macros.rs

@@ -0,0 +1,28 @@
+/// Provides:
+///
+/// - `fn vectors()`: allows for getting a `Vec<TestVector>` of all vectors for exporting.
+/// - `mod tests`: runs all the test vectors locally.
+macro_rules! vectors_and_tests {
+    ($($name: ident, $test: expr),*) => {
+        pub fn vectors() -> Vec<TestVector> {
+            let mut vec = vec![];
+
+            $(
+                vec.push($test.test_vector(stringify!($name).into()));
+            )*
+
+            vec
+        }
+
+        #[cfg(test)]
+        mod tests {
+            use super::*;
+            $(
+                #[test]
+                fn $name() {
+                    $test.run();
+                }
+            )*
+        }
+    };
+}

testing/state_transition_vectors/src/main.rs

@@ -0,0 +1,107 @@
+#[macro_use]
+mod macros;
+mod exit;
+
+use ssz::Encode;
+use state_processing::test_utils::BlockProcessingBuilder;
+use std::env;
+use std::fs::{self, File};
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::exit;
+use types::MainnetEthSpec;
+use types::{BeaconState, ChainSpec, EthSpec, SignedBeaconBlock};
+
+type E = MainnetEthSpec;
+
+pub const VALIDATOR_COUNT: usize = 64;
+
+/// The base output directory for test vectors.
+pub const BASE_VECTOR_DIR: &str = "vectors";
+
+/// Writes all known test vectors to `CARGO_MANIFEST_DIR/vectors`.
+fn main() {
+    match write_all_vectors() {
+        Ok(()) => exit(0),
+        Err(e) => {
+            eprintln!("Error: {}", e);
+            exit(1)
+        }
+    }
+}
+
+/// An abstract definition of a test vector that can be run as a test or exported to disk.
+pub struct TestVector {
+    pub title: String,
+    pub pre_state: BeaconState<E>,
+    pub block: SignedBeaconBlock<E>,
+    pub post_state: Option<BeaconState<E>>,
+    pub error: Option<String>,
+}
+
+/// Gets a `BlockProcessingBuilder` to be used in testing.
+fn get_builder(
+    spec: &ChainSpec,
+    epoch_offset: u64,
+    num_validators: usize,
+) -> BlockProcessingBuilder<MainnetEthSpec> {
+    // Set the state and block to be in the last slot of the `epoch_offset`th epoch.
+    let last_slot_of_epoch = (MainnetEthSpec::genesis_epoch() + epoch_offset)
+        .end_slot(MainnetEthSpec::slots_per_epoch());
+    BlockProcessingBuilder::new(num_validators, last_slot_of_epoch, &spec).build_caches()
+}
+
+/// Writes all vectors to file.
+fn write_all_vectors() -> Result<(), String> {
+    write_vectors_to_file("exit", &exit::vectors())
+}
+
+/// Writes a list of `vectors` to the `title` dir.
+fn write_vectors_to_file(title: &str, vectors: &[TestVector]) -> Result<(), String> {
+    let dir = env::var("CARGO_MANIFEST_DIR")
+        .map_err(|e| format!("Unable to find manifest dir: {:?}", e))?
+        .parse::<PathBuf>()
+        .map_err(|e| format!("Unable to parse manifest dir: {:?}", e))?
+        .join(BASE_VECTOR_DIR)
+        .join(title);
+
+    if dir.exists() {
+        fs::remove_dir_all(&dir).map_err(|e| format!("Unable to remove {:?}: {:?}", dir, e))?;
+    }
+    fs::create_dir_all(&dir).map_err(|e| format!("Unable to create {:?}: {:?}", dir, e))?;
+
+    for vector in vectors {
+        let dir = dir.clone().join(&vector.title);
+        if dir.exists() {
+            fs::remove_dir_all(&dir).map_err(|e| format!("Unable to remove {:?}: {:?}", dir, e))?;
+        }
+        fs::create_dir_all(&dir).map_err(|e| format!("Unable to create {:?}: {:?}", dir, e))?;
+
+        write_to_ssz_file(&dir.clone().join("pre.ssz"), &vector.pre_state)?;
+        write_to_ssz_file(&dir.clone().join("block.ssz"), &vector.block)?;
+        if let Some(post_state) = vector.post_state.as_ref() {
+            write_to_ssz_file(&dir.clone().join("post.ssz"), post_state)?;
+        }
+        if let Some(error) = vector.error.as_ref() {
+            write_to_file(&dir.clone().join("error.txt"), error.as_bytes())?;
+        }
+    }
+
+    Ok(())
+}
+
+/// Write some SSZ object to file.
+fn write_to_ssz_file<T: Encode>(path: &PathBuf, item: &T) -> Result<(), String> {
+    write_to_file(path, &item.as_ssz_bytes())
+}
+
+/// Write some bytes to file.
+fn write_to_file(path: &PathBuf, item: &[u8]) -> Result<(), String> {
+    File::create(path)
+        .map_err(|e| format!("Unable to create {:?}: {:?}", path, e))
+        .and_then(|mut file| {
+            file.write_all(item)
+                .map(|_| ())
+                .map_err(|e| format!("Unable to write to {:?}: {:?}", path, e))
+        })
+}