From 6484d4197d7b54b6b893b639dac141dcd8afd246 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 22 Oct 2019 12:35:35 +0900 Subject: [PATCH] Run EF tests on GitLab (#562) --- .gitignore | 1 + .gitlab-ci.yml | 19 +++++++++++++--- tests/ef_tests/Cargo.toml | 2 ++ tests/ef_tests/Makefile | 24 ++++++++++++++------ tests/ef_tests/README.md | 42 ++++++++++++++++++++++------------- tests/ef_tests/src/handler.rs | 5 ----- tests/ef_tests/tests/tests.rs | 2 ++ 7 files changed, 65 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 6b8d4ab214..01aafef05b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ Cargo.lock *.raw_keypairs flamegraph.svg perf.data* +*.tar.gz diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4bc09d1a2c..1636d51724 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,10 @@ #Adapted from https://users.rust-lang.org/t/my-gitlab-config-docs-tests/16396 -image: 'sigp/lighthouse:latest' +default: + image: 'sigp/lighthouse:latest' + cache: + paths: + - tests/ef_tests/*-v0.8.3.tar.gz stages: - test @@ -29,12 +33,21 @@ test-release: script: - cargo test --verbose --all --release -test-fake-crypto: +test-ef: stage: test variables: GIT_SUBMODULE_STRATEGY: normal script: - - cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features fake_crypto + - make make-ef-tests + - cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features ef_tests + +test-ef-fake-crypto: + stage: test + variables: + GIT_SUBMODULE_STRATEGY: normal + script: + - make make-ef-tests + - cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features ef_tests fake_crypto documentation: stage: document diff --git a/tests/ef_tests/Cargo.toml b/tests/ef_tests/Cargo.toml index 2f1dea11d1..24fe75fb3e 100644 --- a/tests/ef_tests/Cargo.toml +++ b/tests/ef_tests/Cargo.toml @@ -5,6 +5,8 @@ authors = ["Paul Hauner "] edition = "2018" [features] +# `ef_tests` feature must be enabled to actually run the tests +ef_tests = [] fake_crypto = ["bls/fake_crypto"] [dependencies] diff --git a/tests/ef_tests/Makefile b/tests/ef_tests/Makefile index ff7e9e8156..a24c21b529 100644 --- a/tests/ef_tests/Makefile +++ b/tests/ef_tests/Makefile @@ -1,18 +1,28 @@ +# Bump the test tag here and in .gitlab-ci.yml and CI will take care of updating the cached tarballs TESTS_TAG := v0.8.3 TESTS = general minimal mainnet +TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) REPO_NAME := eth2.0-spec-tests OUTPUT_DIR := ./$(REPO_NAME) BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(TESTS_TAG) -eth2.0-spec-tests: +$(OUTPUT_DIR): $(TARBALLS) mkdir $(OUTPUT_DIR) - for test in $(TESTS); do \ - wget $(BASE_URL)/$$test.tar.gz; \ - tar -xzvf $$test.tar.gz -C $(OUTPUT_DIR);\ - rm $$test.tar.gz;\ + for test_tarball in $^; do \ + tar -xzf $$test_tarball -C $(OUTPUT_DIR);\ done -clean: - rm -r $(OUTPUT_DIR) +%-$(TESTS_TAG).tar.gz: + wget $(BASE_URL)/$*.tar.gz -O $@ + +clean-test-files: + rm -rf $(OUTPUT_DIR) + +clean-archives: + rm -f $(TARBALLS) + +clean: clean-test-files clean-archives + +.PHONY: clean clean-archives clean-test-files diff --git a/tests/ef_tests/README.md b/tests/ef_tests/README.md index 944891d755..50af57e1fc 100644 --- a/tests/ef_tests/README.md +++ b/tests/ef_tests/README.md @@ -2,22 +2,12 @@ This crate parses and executes the test vectors at [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests). -Functionality is achieved only via the `$ cargo test` command. +Functionality is achieved only via the `$ cargo test --features ef_tests` command. -## Tests +## Running the Tests -Because the test vectors are very large, we do not download the -tests vectors or require that the tests pass by default. Specifically; - -- If the `tests/ef_tests/eth2.0-spec-tests` directory is not present, all tests - indicate a `pass` when they did not actually run. -- If that directory _is_ present, the tests are executed faithfully, failing if - a discrepancy is found. - -## Downloading Test Vectors - -The `eth2.0-spec-tests` directory is not present by default. To -obtain it, use the Makefile: +Because the test vectors are very large, we do not download or run them by default. +To download them, run (in this directory): ``` $ make @@ -27,4 +17,26 @@ _Note: this may download hundreds of MB of compressed archives from the [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/), which may expand into several GB of files._ -Remove the tests to save space or update to a new version with `$ make clean`. +If successful, you should now have the extracted tests in `./eth2.0-spec-tests`. + +Run them with: + +``` +$ cargo test --features ef_tests +``` + +The tests won't run without the `ef_tests` feature enabled (this is to ensure that a top-level +`cargo test --all` won't fail on missing files). + +## Saving Space + +When you download the tests, the downloaded archives will be kept in addition to the extracted +files. You have several options for saving space: + +1. Delete the archives (`make clean-archives`), and keep the extracted files. Suitable for everyday + use, just don't re-run `make` or it will redownload the archives. +2. Delete the extracted files (`make clean-test-files`), and keep the archives. Suitable for CI, or + temporarily saving space. If you re-run `make` it will extract the archives rather than + redownloading them. +3. Delete everything (`make clean`). Good for updating to a new version, or if you no longer wish to + run the EF tests. diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index e8c83e1f84..e5d175e115 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -31,11 +31,6 @@ pub trait Handler { .join(Self::runner_name()) .join(Self::handler_name()); - // If the directory containing the tests does not exist, just let all tests pass. - if !handler_path.exists() { - return; - } - // Iterate through test suites let test_cases = fs::read_dir(&handler_path) .expect("handler dir exists") diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 337c54b461..43cb79a8dd 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "ef_tests")] + use ef_tests::*; use types::*;