diff --git a/Makefile b/Makefile index fe5dfbe551..75b6811b74 100644 --- a/Makefile +++ b/Makefile @@ -218,6 +218,9 @@ run-state-transition-tests: # Downloads and runs the EF test vectors. test-ef: make-ef-tests run-ef-tests +# Downloads and runs the nightly EF test vectors. +test-ef-nightly: make-ef-tests-nightly run-ef-tests + # Downloads and runs the EF test vectors with nextest. nextest-ef: make-ef-tests nextest-run-ef-tests @@ -278,6 +281,10 @@ lint-full: make-ef-tests: make -C $(EF_TESTS) +# Download/extract the nightly EF test vectors. +make-ef-tests-nightly: + CONSENSUS_SPECS_TEST_VERSION=nightly make -C $(EF_TESTS) + # Verifies that crates compile with fuzzing features enabled arbitrary-fuzz: cargo check -p state_processing --features arbitrary-fuzz,$(TEST_FEATURES) diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index 3f72863e3a..6b780b1f92 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,44 +1,33 @@ -TESTS_TAG := v1.6.0-alpha.0 -TESTS = general minimal mainnet -TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) - +# To download/extract nightly tests, run: +# CONSENSUS_SPECS_TEST_VERSION=nightly make +CONSENSUS_SPECS_TEST_VERSION ?= v1.6.0-alpha.0 REPO_NAME := consensus-spec-tests OUTPUT_DIR := ./$(REPO_NAME) -BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(TESTS_TAG) BLS_TEST_REPO_NAME := bls12-381-tests -BLS_TEST_TAG := v0.1.1 +BLS_TEST_VERSION := v0.1.1 BLS_TEST = bls_tests_yaml -BLS_TARBALL = $(patsubst %,%-$(BLS_TEST_TAG).tar.gz,$(BLS_TEST)) BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME) -BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_TAG) +BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_VERSION) -CURL := $(if $(LIGHTHOUSE_GITHUB_TOKEN),curl -L --header "Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",curl -L) +.PHONY: all clean -all: - make $(OUTPUT_DIR) - make $(BLS_OUTPUT_DIR) +all: clean $(OUTPUT_DIR) $(BLS_OUTPUT_DIR) -$(OUTPUT_DIR): $(TARBALLS) - mkdir $(OUTPUT_DIR) - for test_tarball in $^; do \ - tar -xzf $$test_tarball -C $(OUTPUT_DIR);\ +clean: + rm -rf *.tar.gz $(OUTPUT_DIR) $(BLS_OUTPUT_DIR) + +$(OUTPUT_DIR): + mkdir -p $(OUTPUT_DIR) + ./download_test_vectors.sh $(CONSENSUS_SPECS_TEST_VERSION) + for test_tarball in *.tar.gz; do \ + tar -xzf $$test_tarball -C $(OUTPUT_DIR); \ + rm -f $$test_tarball; \ done $(BLS_OUTPUT_DIR): - mkdir $(BLS_OUTPUT_DIR) - $(CURL) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -o $(BLS_TARBALL) - tar -xzf $(BLS_TARBALL) -C $(BLS_OUTPUT_DIR) - -%-$(TESTS_TAG).tar.gz: - $(CURL) $(BASE_URL)/$*.tar.gz -o $@ - -clean-test-files: - rm -rf $(OUTPUT_DIR) $(BLS_OUTPUT_DIR) - -clean-archives: - rm -f $(TARBALLS) $(BLS_TARBALL) - -clean: clean-test-files clean-archives - -.PHONY: clean clean-archives clean-test-files + mkdir -p $(BLS_OUTPUT_DIR) + curl --progress-bar --location --remote-name --show-error --retry 3 --retry-all-errors --fail \ + $(BLS_BASE_URL)/$(BLS_TEST).tar.gz + tar -xzf *.tar.gz -C $(BLS_OUTPUT_DIR) + rm -f *.tar.gz diff --git a/testing/ef_tests/download_test_vectors.sh b/testing/ef_tests/download_test_vectors.sh new file mode 100755 index 0000000000..7297f7eeb8 --- /dev/null +++ b/testing/ef_tests/download_test_vectors.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +TESTS=("general" "minimal" "mainnet") + +version=${1} +if [[ "$version" == "nightly" ]]; then + if [[ -z "${GITHUB_TOKEN:-}" ]]; then + echo "Error GITHUB_TOKEN is not set" + exit 1 + fi + + for cmd in unzip jq; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + echo "Error ${cmd} is not installed" + exit 1 + fi + done + + repo="ethereum/consensus-specs" + api="https://api.github.com" + auth_header="Authorization: token ${GITHUB_TOKEN}" + + run_id=$(curl -s -H "${auth_header}" \ + "${api}/repos/${repo}/actions/workflows/generate_vectors.yml/runs?branch=dev&status=success&per_page=1" | + jq -r '.workflow_runs[0].id') + + if [[ "${run_id}" == "null" || -z "${run_id}" ]]; then + echo "No successful nightly workflow run found" + exit 1 + fi + + echo "Downloading nightly test vectors for run: ${run_id}" + curl -s -H "${auth_header}" "${api}/repos/${repo}/actions/runs/${run_id}/artifacts" | + jq -c '.artifacts[] | {name, url: .archive_download_url}' | + while read -r artifact; do + name=$(echo "${artifact}" | jq -r .name) + url=$(echo "${artifact}" | jq -r .url) + + if [[ "$name" == "consensustestgen.log" ]]; then + continue + fi + + echo "Downloading artifact: ${name}" + curl --progress-bar --location --show-error --retry 3 --retry-all-errors --fail \ + -H "${auth_header}" -H "Accept: application/vnd.github+json" \ + --output "${name}.zip" "${url}" || { + echo "Failed to download ${name}" + exit 1 + } + + unzip -qo "${name}.zip" + rm -f "${name}.zip" + done +else + for test in "${TESTS[@]}"; do + if [[ ! -e "${test}.tar.gz" ]]; then + echo "Downloading: ${version}/${test}.tar.gz" + curl --progress-bar --location --remote-name --show-error --retry 3 --retry-all-errors --fail \ + "https://github.com/ethereum/consensus-spec-tests/releases/download/${version}/${test}.tar.gz" \ + || { + echo "Curl failed. Aborting" + rm -f "${test}.tar.gz" + exit 1 + } + fi + done +fi