Add support for nightly tests (#7538)

This PR adds the ability to download [nightly reference tests from the consensus-specs repo](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml). This will be used by spec maintainers to ensure that there are no unexpected test failures prior to new releases. Also, we will keep track of test compliance with [this website](https://jtraglia.github.io/nyx/); eventually this will be integrated into Hive.


  * A new script (`download_test_vectors.sh`) is added to handle downloads.
* The logic for downloading GitHub artifacts is a bit complex.
* Rename the variables which store test versions:
* `TESTS_TAG` to `CONSENSUS_SPECS_TEST_VERSION`.
* `BLS_TEST_TAG` to `BLS_TEST_VERSION`, for consistency.
* Delete tarballs after extracting them.
* I see no need to keep these; they just use extra disk.
* Consolidate `clean` rules into a single rule.
* Do `clean` prior to downloading/extracting tests.
* Remove `CURL` variable with GitHub token; don't need it for downloading releases.
* Do `mkdir -p` when creating directories.
* Probably more small stuff...

Makefile

@@ -218,6 +218,9 @@ run-state-transition-tests:
 # Downloads and runs the EF test vectors.
 test-ef: make-ef-tests run-ef-tests
 
+# Downloads and runs the nightly EF test vectors.
+test-ef-nightly: make-ef-tests-nightly run-ef-tests
+
 # Downloads and runs the EF test vectors with nextest.
 nextest-ef: make-ef-tests nextest-run-ef-tests
 
@@ -278,6 +281,10 @@ lint-full:
 make-ef-tests:
 	make -C $(EF_TESTS)
 
+# Download/extract the nightly EF test vectors.
+make-ef-tests-nightly:
+	CONSENSUS_SPECS_TEST_VERSION=nightly make -C $(EF_TESTS)
+
 # Verifies that crates compile with fuzzing features enabled
 arbitrary-fuzz:
 	cargo check -p state_processing --features arbitrary-fuzz,$(TEST_FEATURES)

testing/ef_tests/Makefile

@@ -1,44 +1,33 @@
-TESTS_TAG := v1.6.0-alpha.0
+# To download/extract nightly tests, run:
-TESTS = general minimal mainnet
+# CONSENSUS_SPECS_TEST_VERSION=nightly make
-TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS))
+CONSENSUS_SPECS_TEST_VERSION ?= v1.6.0-alpha.0
-
 REPO_NAME := consensus-spec-tests
 OUTPUT_DIR := ./$(REPO_NAME)
-BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(TESTS_TAG)
 
 BLS_TEST_REPO_NAME := bls12-381-tests
-BLS_TEST_TAG := v0.1.1
+BLS_TEST_VERSION := v0.1.1
 BLS_TEST = bls_tests_yaml
-BLS_TARBALL = $(patsubst %,%-$(BLS_TEST_TAG).tar.gz,$(BLS_TEST))
 BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME)
-BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_TAG)
+BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_VERSION)
 
-CURL := $(if $(LIGHTHOUSE_GITHUB_TOKEN),curl -L --header "Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",curl -L)
+.PHONY: all clean
 
-all:
+all: clean $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
-	make $(OUTPUT_DIR)
-	make $(BLS_OUTPUT_DIR)
 
-$(OUTPUT_DIR): $(TARBALLS)
+clean:
-	mkdir $(OUTPUT_DIR)
+	rm -rf *.tar.gz $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
-	for test_tarball in $^; do \
+
-		tar -xzf $$test_tarball -C $(OUTPUT_DIR);\
+$(OUTPUT_DIR):
+	mkdir -p $(OUTPUT_DIR)
+	./download_test_vectors.sh $(CONSENSUS_SPECS_TEST_VERSION)
+	for test_tarball in *.tar.gz; do \
+		tar -xzf $$test_tarball -C $(OUTPUT_DIR); \
+		rm -f $$test_tarball; \
 	done
 
 $(BLS_OUTPUT_DIR):
-	mkdir $(BLS_OUTPUT_DIR)
+	mkdir -p $(BLS_OUTPUT_DIR)
-	$(CURL) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -o $(BLS_TARBALL)
+	curl --progress-bar --location --remote-name --show-error --retry 3 --retry-all-errors --fail \
-	tar -xzf $(BLS_TARBALL) -C $(BLS_OUTPUT_DIR)
+		$(BLS_BASE_URL)/$(BLS_TEST).tar.gz
-
+	tar -xzf *.tar.gz -C $(BLS_OUTPUT_DIR)
-%-$(TESTS_TAG).tar.gz:
+	rm -f *.tar.gz
-	$(CURL) $(BASE_URL)/$*.tar.gz -o $@
-
-clean-test-files:
-	rm -rf $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
-
-clean-archives:
-	rm -f $(TARBALLS) $(BLS_TARBALL)
-
-clean: clean-test-files clean-archives
-
-.PHONY: clean clean-archives clean-test-files

testing/ef_tests/download_test_vectors.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+TESTS=("general" "minimal" "mainnet")
+
+version=${1}
+if [[ "$version" == "nightly" ]]; then
+	if [[ -z "${GITHUB_TOKEN:-}" ]]; then
+		echo "Error GITHUB_TOKEN is not set"
+		exit 1
+	fi
+
+	for cmd in unzip jq; do
+		if ! command -v "${cmd}" >/dev/null 2>&1; then
+			echo "Error ${cmd} is not installed"
+			exit 1
+		fi
+	done
+
+	repo="ethereum/consensus-specs"
+	api="https://api.github.com"
+	auth_header="Authorization: token ${GITHUB_TOKEN}"
+
+	run_id=$(curl -s -H "${auth_header}" \
+		"${api}/repos/${repo}/actions/workflows/generate_vectors.yml/runs?branch=dev&status=success&per_page=1" |
+		jq -r '.workflow_runs[0].id')
+
+	if [[ "${run_id}" == "null" || -z "${run_id}" ]]; then
+		echo "No successful nightly workflow run found"
+		exit 1
+	fi
+
+	echo "Downloading nightly test vectors for run: ${run_id}"
+	curl -s -H "${auth_header}" "${api}/repos/${repo}/actions/runs/${run_id}/artifacts" |
+		jq -c '.artifacts[] | {name, url: .archive_download_url}' |
+		while read -r artifact; do
+			name=$(echo "${artifact}" | jq -r .name)
+			url=$(echo "${artifact}" | jq -r .url)
+
+			if [[ "$name" == "consensustestgen.log" ]]; then
+				continue
+			fi
+
+			echo "Downloading artifact: ${name}"
+			curl --progress-bar --location --show-error --retry 3 --retry-all-errors --fail \
+				-H "${auth_header}" -H "Accept: application/vnd.github+json" \
+				--output "${name}.zip" "${url}" || {
+				echo "Failed to download ${name}"
+				exit 1
+			}
+
+			unzip -qo "${name}.zip"
+			rm -f "${name}.zip"
+		done
+else
+	for test in "${TESTS[@]}"; do
+		if [[ ! -e "${test}.tar.gz" ]]; then
+			echo "Downloading: ${version}/${test}.tar.gz"
+			curl --progress-bar --location --remote-name --show-error --retry 3 --retry-all-errors --fail \
+				"https://github.com/ethereum/consensus-spec-tests/releases/download/${version}/${test}.tar.gz" \
+				|| {
+					echo "Curl failed. Aborting"
+					rm -f "${test}.tar.gz"
+					exit 1
+				}
+		fi
+	done
+fi