mirror of
https://github.com/sigp/lighthouse.git
synced 2026-05-30 04:37:13 +00:00
Use makefile rather than script
This commit is contained in:
Michael Sproul
@@ -1,296 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# scripts/compliance-fc-report.sh
|
|
||||||
#
|
|
||||||
# Run the consensus-specs fork-choice compliance suites against this branch and
|
|
||||||
# print an aggregated pass/fail summary. Test-only — no production code changes.
|
|
||||||
# See --help for data-source options.
|
|
||||||
|
|
||||||
set -uo pipefail
|
|
||||||
|
|
||||||
# ---- defaults (overridable via flags or env) ---------------------------------
|
|
||||||
|
|
||||||
PRESET="${COMPLIANCE_FC_PRESET:-minimal}"
|
|
||||||
DIR="${COMPLIANCE_FC_DIR:-}"
|
|
||||||
TARBALL="${COMPLIANCE_FC_TARBALL:-}"
|
|
||||||
URL="${COMPLIANCE_FC_URL:-}"
|
|
||||||
RUN_ID="${COMPLIANCE_FC_RUN_ID:-}"
|
|
||||||
CACHE_ROOT="${COMPLIANCE_FC_CACHE_DIR:-/var/tmp/compliance_fc_cache}"
|
|
||||||
SUITE_FILTER="${COMPLIANCE_FC_SUITE:-}"
|
|
||||||
|
|
||||||
ALL_SUITES=(
|
|
||||||
attester_slashing_test
|
|
||||||
block_cover_test
|
|
||||||
block_tree_test
|
|
||||||
block_weight_test
|
|
||||||
invalid_message_test
|
|
||||||
shuffling_test
|
|
||||||
)
|
|
||||||
ALL_FORKS=(fulu gloas)
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
cat <<'EOF'
|
|
||||||
Run the fork-choice compliance suites and print a pass/fail report.
|
|
||||||
|
|
||||||
USAGE
|
|
||||||
scripts/compliance-fc-report.sh [options] [-- cargo_args...]
|
|
||||||
|
|
||||||
DATA SOURCE (first non-empty wins)
|
|
||||||
--dir PATH Use a pre-extracted tree at PATH (must contain tests/).
|
|
||||||
--tarball PATH Use a local .tar.gz; extracted to cache on first use.
|
|
||||||
--url URL Download tarball via curl; cached + extracted.
|
|
||||||
--run-id ID Pin to a consensus-specs run id; download via gh.
|
|
||||||
(default) Resolve the latest successful run of the consensus-specs
|
|
||||||
"Compliance Tests" workflow on master via gh.
|
|
||||||
|
|
||||||
OTHER OPTIONS
|
|
||||||
--preset NAME Preset: minimal or mainnet (default: minimal).
|
|
||||||
Only minimal currently ships compliance data.
|
|
||||||
--suite NAME Run only one suite (e.g. block_tree_test). Repeatable
|
|
||||||
via comma-separated list.
|
|
||||||
--cache-dir PATH Cache root (default: /var/tmp/compliance_fc_cache).
|
|
||||||
-h, --help Print this help and exit.
|
|
||||||
|
|
||||||
ENVIRONMENT (each flag has a matching env var; the flag wins if both are set)
|
|
||||||
COMPLIANCE_FC_PRESET, COMPLIANCE_FC_DIR, COMPLIANCE_FC_TARBALL,
|
|
||||||
COMPLIANCE_FC_URL, COMPLIANCE_FC_RUN_ID, COMPLIANCE_FC_CACHE_DIR,
|
|
||||||
COMPLIANCE_FC_SUITE
|
|
||||||
GITHUB_TOKEN Required only for --run-id and the default auto-fetch path
|
|
||||||
(GitHub Actions artifact downloads return 403 to anonymous
|
|
||||||
requests, even on public repos). Use --tarball or --url to
|
|
||||||
avoid the token entirely.
|
|
||||||
|
|
||||||
Anything after '--' is forwarded verbatim to 'cargo test'.
|
|
||||||
|
|
||||||
EXAMPLES
|
|
||||||
# Auto-fetch latest (needs token)
|
|
||||||
GITHUB_TOKEN=... scripts/compliance-fc-report.sh
|
|
||||||
|
|
||||||
# Use a manually-downloaded tarball — no token, no gh
|
|
||||||
scripts/compliance-fc-report.sh --tarball ~/Downloads/small.tar.gz
|
|
||||||
|
|
||||||
# Pull the artifact through a public mirror via curl — no token
|
|
||||||
scripts/compliance-fc-report.sh --url https://example.org/small.tar.gz
|
|
||||||
|
|
||||||
# Re-use an already-extracted tree
|
|
||||||
scripts/compliance-fc-report.sh --dir /var/tmp/compliance_fc_root
|
|
||||||
|
|
||||||
# Run only one suite
|
|
||||||
scripts/compliance-fc-report.sh --tarball ./small.tar.gz --suite block_tree_test
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---- argument parsing --------------------------------------------------------
|
|
||||||
|
|
||||||
while (( $# > 0 )); do
|
|
||||||
case "$1" in
|
|
||||||
--preset) PRESET="$2"; shift 2 ;;
|
|
||||||
--dir) DIR="$2"; shift 2 ;;
|
|
||||||
--tarball) TARBALL="$2"; shift 2 ;;
|
|
||||||
--url) URL="$2"; shift 2 ;;
|
|
||||||
--run-id) RUN_ID="$2"; shift 2 ;;
|
|
||||||
--suite) SUITE_FILTER="$2"; shift 2 ;;
|
|
||||||
--cache-dir) CACHE_ROOT="$2"; shift 2 ;;
|
|
||||||
-h|--help) usage; exit 0 ;;
|
|
||||||
--) shift; break ;;
|
|
||||||
-*) echo "error: unknown option: $1" >&2; usage >&2; exit 1 ;;
|
|
||||||
*) echo "error: unexpected positional arg: $1 (use -- to forward to cargo)" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
CARGO_EXTRA_ARGS=("$@")
|
|
||||||
|
|
||||||
case "$PRESET" in
|
|
||||||
minimal|mainnet) ;;
|
|
||||||
*) echo "error: --preset must be minimal or mainnet (got: $PRESET)" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
if [[ "$PRESET" == "mainnet" ]]; then
|
|
||||||
echo "error: the consensus-specs Compliance Tests workflow ships only minimal preset" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Resolve repo root (script may be invoked from any cwd).
|
|
||||||
REPO_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." &> /dev/null && pwd)
|
|
||||||
TESTS_DST="${REPO_ROOT}/testing/ef_tests/consensus-spec-tests"
|
|
||||||
|
|
||||||
# ---- data source resolution --------------------------------------------------
|
|
||||||
|
|
||||||
extract_into() {
|
|
||||||
# extract_into <tarball> <cache-key>; sets DIR to the cache target.
|
|
||||||
local tarball="$1" key="$2" target="$CACHE_ROOT/$key"
|
|
||||||
if [[ -d "$target/tests" ]]; then
|
|
||||||
echo "Reusing cached compliance data at $target"
|
|
||||||
else
|
|
||||||
mkdir -p "$target"
|
|
||||||
echo "Extracting $tarball -> $target..."
|
|
||||||
tar -xzf "$tarball" -C "$target"
|
|
||||||
fi
|
|
||||||
DIR="$target"
|
|
||||||
}
|
|
||||||
|
|
||||||
require_token() {
|
|
||||||
: "${GITHUB_TOKEN:?required for gh artifact download (use --tarball or --url to avoid)}"
|
|
||||||
command -v gh >/dev/null || {
|
|
||||||
echo "error: gh CLI required for auto-fetch (brew install gh, or use --tarball/--url/--dir)" >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if [[ -n "$DIR" ]]; then
|
|
||||||
: # use as-is
|
|
||||||
elif [[ -n "$TARBALL" ]]; then
|
|
||||||
[[ -f "$TARBALL" ]] || { echo "error: tarball not found: $TARBALL" >&2; exit 1; }
|
|
||||||
key="tarball-$(shasum -a 256 "$TARBALL" | awk '{print $1}')"
|
|
||||||
extract_into "$TARBALL" "$key"
|
|
||||||
elif [[ -n "$URL" ]]; then
|
|
||||||
command -v curl >/dev/null || { echo "error: curl required for --url" >&2; exit 1; }
|
|
||||||
key="url-$(printf '%s' "$URL" | shasum -a 256 | awk '{print $1}')"
|
|
||||||
target="$CACHE_ROOT/$key"
|
|
||||||
if [[ -d "$target/tests" ]]; then
|
|
||||||
echo "Reusing cached compliance data at $target"
|
|
||||||
DIR="$target"
|
|
||||||
else
|
|
||||||
tmpfile="$(mktemp)"
|
|
||||||
trap 'rm -f "$tmpfile"' EXIT
|
|
||||||
echo "Downloading $URL..."
|
|
||||||
curl -fL --output "$tmpfile" "$URL"
|
|
||||||
extract_into "$tmpfile" "$key"
|
|
||||||
rm -f "$tmpfile"
|
|
||||||
trap - EXIT
|
|
||||||
fi
|
|
||||||
elif [[ -n "$RUN_ID" || -n "${GITHUB_TOKEN:-}" ]]; then
|
|
||||||
require_token
|
|
||||||
if [[ -z "$RUN_ID" ]]; then
|
|
||||||
# 261432977 = "Compliance Tests" workflow on ethereum/consensus-specs
|
|
||||||
echo "Resolving latest successful Compliance Tests run on master..."
|
|
||||||
RUN_ID=$(gh api \
|
|
||||||
'repos/ethereum/consensus-specs/actions/workflows/261432977/runs?branch=master&status=success&per_page=1' \
|
|
||||||
--jq '.workflow_runs[0].id // empty')
|
|
||||||
[[ -n "$RUN_ID" ]] || { echo "error: no successful runs found" >&2; exit 1; }
|
|
||||||
echo "Latest run: $RUN_ID"
|
|
||||||
fi
|
|
||||||
target="$CACHE_ROOT/run-$RUN_ID"
|
|
||||||
if [[ -d "$target/tests" ]]; then
|
|
||||||
echo "Reusing cached compliance data at $target"
|
|
||||||
DIR="$target"
|
|
||||||
else
|
|
||||||
tmpdir="$(mktemp -d)"
|
|
||||||
trap 'rm -rf "$tmpdir"' EXIT
|
|
||||||
echo "Downloading artifact from run $RUN_ID..."
|
|
||||||
gh run download "$RUN_ID" --repo ethereum/consensus-specs --name small.tar.gz --dir "$tmpdir"
|
|
||||||
[[ -f "$tmpdir/small.tar.gz" ]] || { echo "error: small.tar.gz not present in artifact" >&2; exit 1; }
|
|
||||||
extract_into "$tmpdir/small.tar.gz" "run-$RUN_ID"
|
|
||||||
rm -rf "$tmpdir"
|
|
||||||
trap - EXIT
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "error: no data source given." >&2
|
|
||||||
echo " Pass --dir / --tarball / --url / --run-id, or set GITHUB_TOKEN" >&2
|
|
||||||
echo " to auto-fetch the latest run. See --help for details." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! -d "$DIR/tests/$PRESET" ]]; then
|
|
||||||
echo "error: no $PRESET-preset compliance data at $DIR/tests/$PRESET" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ---- stage data into the ef_tests crate --------------------------------------
|
|
||||||
#
|
|
||||||
# `testing/ef_tests` resolves test paths from `env!("CARGO_MANIFEST_DIR")` at
|
|
||||||
# compile time, so the corpus must live under the crate. We copy only the
|
|
||||||
# fork_choice_compliance subtree to avoid clobbering any existing test corpus.
|
|
||||||
|
|
||||||
echo "Staging compliance data under ${TESTS_DST}/tests/${PRESET}/..."
|
|
||||||
mkdir -p "${TESTS_DST}/tests/${PRESET}"
|
|
||||||
for fork in "${ALL_FORKS[@]}"; do
|
|
||||||
src="${DIR}/tests/${PRESET}/${fork}/fork_choice_compliance"
|
|
||||||
if [[ ! -d "$src" ]]; then
|
|
||||||
echo " skip ${fork}: no fork_choice_compliance directory in source"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
dst="${TESTS_DST}/tests/${PRESET}/${fork}"
|
|
||||||
mkdir -p "$dst"
|
|
||||||
rm -rf "${dst}/fork_choice_compliance"
|
|
||||||
cp -R "$src" "${dst}/fork_choice_compliance"
|
|
||||||
done
|
|
||||||
|
|
||||||
# ---- run ---------------------------------------------------------------------
|
|
||||||
|
|
||||||
# Resolve which suites to run.
|
|
||||||
SUITES=()
|
|
||||||
if [[ -n "$SUITE_FILTER" ]]; then
|
|
||||||
IFS=',' read -ra SUITES <<< "$SUITE_FILTER"
|
|
||||||
else
|
|
||||||
SUITES=("${ALL_SUITES[@]}")
|
|
||||||
fi
|
|
||||||
|
|
||||||
LOGS_DIR="${COMPLIANCE_FC_LOGS_DIR:-/tmp/compliance_fc_logs}"
|
|
||||||
rm -rf "$LOGS_DIR"
|
|
||||||
mkdir -p "$LOGS_DIR"
|
|
||||||
|
|
||||||
declare -a results=() # tab-separated rows: handler\tfork\ttotal\tpass\tfail\tskipped
|
|
||||||
|
|
||||||
run_one() {
|
|
||||||
local handler="$1" fork="$2"
|
|
||||||
local fn="fork_choice_compliance_${handler}_${fork}"
|
|
||||||
local log="${LOGS_DIR}/${handler}_${fork}.log"
|
|
||||||
echo "==> ${fn}"
|
|
||||||
|
|
||||||
RUST_MIN_STACK=8388608 \
|
|
||||||
cargo test --release --features "ef_tests,fake_crypto" \
|
|
||||||
-p ef_tests --test tests "$fn" \
|
|
||||||
${CARGO_EXTRA_ARGS[@]+"${CARGO_EXTRA_ARGS[@]}"} \
|
|
||||||
-- --nocapture --include-ignored \
|
|
||||||
> "$log" 2>&1 || true
|
|
||||||
|
|
||||||
# Parse the harness summary. Two cases:
|
|
||||||
# 1. "N tests, F failed, K skipped (known failure), B skipped (bls), P passed."
|
|
||||||
# 2. "Passed N tests in <path>" (when nothing failed at all).
|
|
||||||
local total=0 pass=0 fail=0 skip=0
|
|
||||||
local summary
|
|
||||||
summary=$(grep -E "^[0-9]+ tests, " "$log" | head -1)
|
|
||||||
if [[ -n "$summary" ]]; then
|
|
||||||
# shellcheck disable=SC2001
|
|
||||||
read -r total fail kfail bls pass <<< \
|
|
||||||
"$(sed -E 's/^([0-9]+) tests, ([0-9]+) failed, ([0-9]+) skipped \(known failure\), ([0-9]+) skipped \(bls\), ([0-9]+) passed.*/\1 \2 \3 \4 \5/' <<< "$summary")"
|
|
||||||
skip=$((kfail + bls))
|
|
||||||
else
|
|
||||||
summary=$(grep -E "^Passed [0-9]+ tests in " "$log" | head -1)
|
|
||||||
if [[ -n "$summary" ]]; then
|
|
||||||
total=$(awk '{print $2}' <<< "$summary")
|
|
||||||
pass="$total"
|
|
||||||
else
|
|
||||||
total=0; pass=0; fail=0; skip=0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
results+=("$(printf '%s\t%s\t%d\t%d\t%d\t%d' "$handler" "$fork" "$total" "$pass" "$fail" "$skip")")
|
|
||||||
}
|
|
||||||
|
|
||||||
for handler in "${SUITES[@]}"; do
|
|
||||||
for fork in "${ALL_FORKS[@]}"; do
|
|
||||||
run_one "$handler" "$fork"
|
|
||||||
done
|
|
||||||
done
|
|
||||||
|
|
||||||
# ---- aggregate report --------------------------------------------------------
|
|
||||||
|
|
||||||
echo
|
|
||||||
echo "=== Fork-choice compliance report ($PRESET) ==="
|
|
||||||
printf '%-32s %-8s %8s %8s %8s %8s\n' "Suite" "Fork" "Total" "Pass" "Fail" "Skip"
|
|
||||||
printf '%-32s %-8s %8s %8s %8s %8s\n' "-----" "----" "-----" "----" "----" "----"
|
|
||||||
tt=0; tp=0; tf=0; ts=0
|
|
||||||
for row in "${results[@]}"; do
|
|
||||||
IFS=$'\t' read -r handler fork total pass fail skip <<< "$row"
|
|
||||||
printf '%-32s %-8s %8d %8d %8d %8d\n' "$handler" "$fork" "$total" "$pass" "$fail" "$skip"
|
|
||||||
tt=$((tt+total)); tp=$((tp+pass)); tf=$((tf+fail)); ts=$((ts+skip))
|
|
||||||
done
|
|
||||||
printf '%-32s %-8s %8s %8s %8s %8s\n' "-----" "----" "-----" "----" "----" "----"
|
|
||||||
printf '%-32s %-8s %8d %8d %8d %8d\n' "TOTAL" "" "$tt" "$tp" "$tf" "$ts"
|
|
||||||
|
|
||||||
echo
|
|
||||||
echo "Per-suite logs (look here for failure details):"
|
|
||||||
echo " ${LOGS_DIR}/<suite>_<fork>.log"
|
|
||||||
|
|
||||||
# Exit non-zero if any case failed.
|
|
||||||
(( tf == 0 ))
|
|
||||||
@@ -10,9 +10,15 @@ BLS_TEST = bls_tests_yaml
|
|||||||
BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME)
|
BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME)
|
||||||
BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_VERSION)
|
BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_VERSION)
|
||||||
|
|
||||||
|
# Fork-choice compliance tests from consensus-specs CI.
|
||||||
|
# Pin to a specific workflow run ID for reproducibility. Update to pull newer vectors.
|
||||||
|
COMPLIANCE_RUN_ID ?= 25410712895
|
||||||
|
COMPLIANCE_PRESET ?= minimal
|
||||||
|
COMPLIANCE_OUTPUT_DIR := $(OUTPUT_DIR)/tests/$(COMPLIANCE_PRESET)/fulu/fork_choice_compliance
|
||||||
|
|
||||||
.PHONY: all clean
|
.PHONY: all clean
|
||||||
|
|
||||||
all: clean $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
|
all: clean $(OUTPUT_DIR) $(BLS_OUTPUT_DIR) $(COMPLIANCE_OUTPUT_DIR)
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf *.tar.gz $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
|
rm -rf *.tar.gz $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
|
||||||
@@ -31,3 +37,12 @@ $(BLS_OUTPUT_DIR):
|
|||||||
$(BLS_BASE_URL)/$(BLS_TEST).tar.gz
|
$(BLS_BASE_URL)/$(BLS_TEST).tar.gz
|
||||||
tar -xzf *.tar.gz -C $(BLS_OUTPUT_DIR)
|
tar -xzf *.tar.gz -C $(BLS_OUTPUT_DIR)
|
||||||
rm -f *.tar.gz
|
rm -f *.tar.gz
|
||||||
|
|
||||||
|
$(COMPLIANCE_OUTPUT_DIR):
|
||||||
|
@echo "Fetching fork-choice compliance tests (run $(COMPLIANCE_RUN_ID))..."
|
||||||
|
@curl -L -f -H "Authorization: token $(GITHUB_TOKEN)" \
|
||||||
|
"https://api.github.com/repos/ethereum/consensus-specs/actions/runs/$(COMPLIANCE_RUN_ID)/artifacts" \
|
||||||
|
| python3 -c "import sys,json; arts=json.load(sys.stdin)['artifacts']; url=next(a['archive_download_url'] for a in arts if a['name']=='small.tar.gz'); print(url)" \
|
||||||
|
| xargs curl -L -f --output compliance.tar.gz -H "Authorization: token $(GITHUB_TOKEN)"
|
||||||
|
tar -xzf compliance.tar.gz -C $(OUTPUT_DIR)
|
||||||
|
rm -f compliance.tar.gz
|
||||||
|
|||||||
Reference in New Issue
Block a user