Replace local testnet script with Kurtosis (#5865)

* Kurtosis local testnet.

* Remove unused `lcli` subcommands

* Migrate doppelganger_protection test to kurtosis and further cleanup.

* Fix lint

* Add missing download image step and remove unused `lcli` dependencies.

* doppelganger success case working

* Run tests on hosted runner and improve error handling.

* Start the dp vc only after epoch 1

* Add more logging to test results.

* Fix exit code and speed up docker build.

* Fix incorrect exit codes and split doppelganger tests on CI.

* Missing the escape for double quotes 😫

* Remove unnecessary vc params in kurtosis config.
This commit is contained in:
Jimmy Chen
2024-06-04 13:03:26 +10:00
committed by GitHub
parent 1b7c4a4523
commit 5fc01454dc
37 changed files with 333 additions and 3909 deletions

1
scripts/local_testnet/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
logs

View File

@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
source ./vars.env
exec anvil \
--balance 1000000000 \
--gas-limit 1000000000 \
--accounts 10 \
--mnemonic "$ETH1_NETWORK_MNEMONIC" \
--block-time $SECONDS_PER_ETH1_BLOCK \
--port 8545 \
--chain-id "$CHAIN_ID"

View File

@@ -1,70 +0,0 @@
#!/usr/bin/env bash
#
# Starts a beacon node based upon a genesis state created by `./setup.sh`.
#
set -Eeuo pipefail
source ./vars.env
SUBSCRIBE_ALL_SUBNETS=
DEBUG_LEVEL=${DEBUG_LEVEL:-info}
# Get options
while getopts "d:sh" flag; do
case "${flag}" in
d) DEBUG_LEVEL=${OPTARG};;
s) SUBSCRIBE_ALL_SUBNETS="--subscribe-all-subnets";;
h)
echo "Start a beacon node"
echo
echo "usage: $0 <Options> <DATADIR> <NETWORK-PORT> <HTTP-PORT>"
echo
echo "Options:"
echo " -s: pass --subscribe-all-subnets to 'lighthouse bn ...', default is not passed"
echo " -d: DEBUG_LEVEL, default info"
echo " -h: this help"
echo
echo "Positional arguments:"
echo " DATADIR Value for --datadir parameter"
echo " NETWORK-PORT Value for --enr-udp-port, --enr-tcp-port and --port"
echo " HTTP-PORT Value for --http-port"
echo " EXECUTION-ENDPOINT Value for --execution-endpoint"
echo " EXECUTION-JWT Value for --execution-jwt"
exit
;;
esac
done
# Get positional arguments
data_dir=${@:$OPTIND+0:1}
tcp_port=${@:$OPTIND+1:1}
quic_port=${@:$OPTIND+2:1}
http_port=${@:$OPTIND+3:1}
execution_endpoint=${@:$OPTIND+4:1}
execution_jwt=${@:$OPTIND+5:1}
lighthouse_binary=lighthouse
exec $lighthouse_binary \
--debug-level $DEBUG_LEVEL \
bn \
$SUBSCRIBE_ALL_SUBNETS \
--datadir $data_dir \
--testnet-dir $TESTNET_DIR \
--enable-private-discovery \
--disable-peer-scoring \
--staking \
--enr-address 127.0.0.1 \
--enr-udp-port $tcp_port \
--enr-tcp-port $tcp_port \
--enr-quic-port $quic_port \
--port $tcp_port \
--quic-port $quic_port \
--http-port $http_port \
--disable-packet-filter \
--target-peers $((BN_COUNT - 1)) \
--execution-endpoint $execution_endpoint \
--execution-jwt $execution_jwt \
$BN_ARGS

View File

@@ -1,36 +0,0 @@
#!/usr/bin/env bash
#
# Generates a bootnode enr and saves it in $TESTNET/boot_enr.yaml
# Starts a bootnode from the generated enr.
#
set -Eeuo pipefail
source ./vars.env
echo "Generating bootnode enr"
lcli \
generate-bootnode-enr \
--ip 127.0.0.1 \
--udp-port $BOOTNODE_PORT \
--tcp-port $BOOTNODE_PORT \
--genesis-fork-version $GENESIS_FORK_VERSION \
--output-dir $DATADIR/bootnode
bootnode_enr=`cat $DATADIR/bootnode/enr.dat`
echo "- $bootnode_enr" > $TESTNET_DIR/boot_enr.yaml
echo "Generated bootnode enr and written to $TESTNET_DIR/boot_enr.yaml"
DEBUG_LEVEL=${1:-info}
echo "Starting bootnode"
exec lighthouse boot_node \
--testnet-dir $TESTNET_DIR \
--port $BOOTNODE_PORT \
--listen-address 127.0.0.1 \
--disable-packet-filter \
--network-dir $DATADIR/bootnode \

View File

@@ -1,13 +0,0 @@
#!/usr/bin/env bash
#
# Deletes all files associated with the local testnet.
#
set -Eeuo pipefail
source ./vars.env
if [ -d $DATADIR ]; then
rm -rf $DATADIR
fi

View File

@@ -1,17 +0,0 @@
#!/usr/bin/env bash
# Print all the logs output from local testnet
set -Eeuo pipefail
source ./vars.env
for f in "$TESTNET_DIR"/*.log
do
[[ -e "$f" ]] || break # handle the case of no *.log files
echo "============================================================================="
echo "$f"
echo "============================================================================="
cat "$f"
echo ""
done

View File

@@ -1,3 +0,0 @@
priv_key="02fd74636e96a8ffac8e7b01b0de8dea94d6bcf4989513b38cf59eb32163ff91"
source ./vars.env
exec $EL_BOOTNODE_BINARY --nodekeyhex $priv_key

File diff suppressed because one or more lines are too long

View File

@@ -1,53 +0,0 @@
set -Eeuo pipefail
source ./vars.env
# Get options
while getopts "d:sh" flag; do
case "${flag}" in
d) DEBUG_LEVEL=${OPTARG};;
s) SUBSCRIBE_ALL_SUBNETS="--subscribe-all-subnets";;
h)
echo "Start a geth node"
echo
echo "usage: $0 <Options> <DATADIR> <NETWORK-PORT> <HTTP-PORT>"
echo
echo "Options:"
echo " -h: this help"
echo
echo "Positional arguments:"
echo " DATADIR Value for --datadir parameter"
echo " NETWORK-PORT Value for --port"
echo " HTTP-PORT Value for --http.port"
echo " AUTH-PORT Value for --authrpc.port"
echo " GENESIS_FILE Value for geth init"
exit
;;
esac
done
# Get positional arguments
data_dir=${@:$OPTIND+0:1}
network_port=${@:$OPTIND+1:1}
http_port=${@:$OPTIND+2:1}
auth_port=${@:$OPTIND+3:1}
genesis_file=${@:$OPTIND+4:1}
# Init
$GETH_BINARY init \
--datadir $data_dir \
$genesis_file
echo "Completed init"
exec $GETH_BINARY \
--datadir $data_dir \
--ipcdisable \
--http \
--http.api="engine,eth,web3,net,debug" \
--networkid=$CHAIN_ID \
--syncmode=full \
--bootnodes $EL_BOOTNODE_ENODE \
--port $network_port \
--http.port $http_port \
--authrpc.port $auth_port

View File

@@ -1,19 +0,0 @@
#!/usr/bin/env bash
# Kill processes
set -Euo pipefail
# First parameter is the file with
# one pid per line.
if [ -f "$1" ]; then
while read pid
do
# handle the case of blank lines
[[ -n "$pid" ]] || continue
echo killing $pid
kill $pid || true
done < $1
fi

View File

@@ -0,0 +1,14 @@
# Full configuration reference [here](https://github.com/kurtosis-tech/ethereum-package?tab=readme-ov-file#configuration).
participants:
- el_type: geth
el_image: ethereum/client-go:latest
cl_type: lighthouse
cl_image: lighthouse:local
cl_extra_params:
- --target-peers=3
count: 4
network_params:
deneb_fork_epoch: 0
seconds_per_slot: 3
global_log_level: debug
snooper_enabled: false

View File

@@ -1,18 +0,0 @@
#!/bin/bash
#
# Resets the beacon state genesis time to now.
#
set -Eeuo pipefail
source ./vars.env
NOW=$(date +%s)
lcli \
change-genesis-time \
$TESTNET_DIR/genesis.ssz \
$(date +%s)
echo "Reset genesis time to now ($NOW)"

View File

@@ -1,53 +0,0 @@
#!/usr/bin/env bash
#
# Produces a testnet specification and a genesis state where the genesis time
# is now + $GENESIS_DELAY.
#
# Generates datadirs for multiple validator keys according to the
# $VALIDATOR_COUNT and $BN_COUNT variables.
#
set -o nounset -o errexit -o pipefail
source ./vars.env
NOW=`date +%s`
GENESIS_TIME=`expr $NOW + $GENESIS_DELAY`
lcli \
new-testnet \
--spec $SPEC_PRESET \
--deposit-contract-address $DEPOSIT_CONTRACT_ADDRESS \
--testnet-dir $TESTNET_DIR \
--min-genesis-active-validator-count $GENESIS_VALIDATOR_COUNT \
--min-genesis-time $GENESIS_TIME \
--genesis-delay $GENESIS_DELAY \
--genesis-fork-version $GENESIS_FORK_VERSION \
--altair-fork-epoch $ALTAIR_FORK_EPOCH \
--bellatrix-fork-epoch $BELLATRIX_FORK_EPOCH \
--capella-fork-epoch $CAPELLA_FORK_EPOCH \
--deneb-fork-epoch $DENEB_FORK_EPOCH \
--electra-fork-epoch $ELECTRA_FORK_EPOCH \
--ttd $TTD \
--eth1-block-hash $ETH1_BLOCK_HASH \
--eth1-id $CHAIN_ID \
--eth1-follow-distance 128 \
--seconds-per-slot $SECONDS_PER_SLOT \
--seconds-per-eth1-block $SECONDS_PER_ETH1_BLOCK \
--proposer-score-boost "$PROPOSER_SCORE_BOOST" \
--validator-count $GENESIS_VALIDATOR_COUNT \
--interop-genesis-state \
--force
echo Specification and genesis.ssz generated at $TESTNET_DIR.
echo "Generating $VALIDATOR_COUNT validators concurrently... (this may take a while)"
lcli \
insecure-validators \
--count $VALIDATOR_COUNT \
--base-dir $DATADIR \
--node-count $VC_COUNT
echo Validators generated with keystore passwords at $DATADIR.

View File

@@ -1,35 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
source ./vars.env
# Function to output SLOT_PER_EPOCH for mainnet or minimal
get_spec_preset_value() {
case "$SPEC_PRESET" in
mainnet) echo 32 ;;
minimal) echo 8 ;;
gnosis) echo 16 ;;
*) echo "Unsupported preset: $SPEC_PRESET" >&2; exit 1 ;;
esac
}
SLOT_PER_EPOCH=$(get_spec_preset_value $SPEC_PRESET)
echo "slot_per_epoch=$SLOT_PER_EPOCH"
genesis_file=$1
# Update future hardforks time in the EL genesis file based on the CL genesis time
GENESIS_TIME=$(lcli pretty-ssz --spec $SPEC_PRESET --testnet-dir $TESTNET_DIR BeaconState $TESTNET_DIR/genesis.ssz | jq | grep -Po 'genesis_time": "\K.*\d')
echo $GENESIS_TIME
CAPELLA_TIME=$((GENESIS_TIME + (CAPELLA_FORK_EPOCH * $SLOT_PER_EPOCH * SECONDS_PER_SLOT)))
echo $CAPELLA_TIME
sed -i 's/"shanghaiTime".*$/"shanghaiTime": '"$CAPELLA_TIME"',/g' $genesis_file
CANCUN_TIME=$((GENESIS_TIME + (DENEB_FORK_EPOCH * $SLOT_PER_EPOCH * SECONDS_PER_SLOT)))
echo $CANCUN_TIME
sed -i 's/"cancunTime".*$/"cancunTime": '"$CANCUN_TIME"',/g' $genesis_file
PRAGUE_TIME=$((GENESIS_TIME + (ELECTRA_FORK_EPOCH * $SLOT_PER_EPOCH * SECONDS_PER_SLOT)))
echo $PRAGUE_TIME
sed -i 's/"pragueTime".*$/"pragueTime": '"$PRAGUE_TIME"',/g' $genesis_file
cat $genesis_file

View File

@@ -1,147 +1,83 @@
#!/usr/bin/env bash
# Start all processes necessary to create a local testnet
# Requires `docker`, `kurtosis`, `yq`
set -Eeuo pipefail
source ./vars.env
SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
ENCLAVE_NAME=local-testnet
NETWORK_PARAMS_FILE=$SCRIPT_DIR/network_params.yaml
# Set a higher ulimit in case we want to import 1000s of validators.
ulimit -n 65536
# VC_COUNT is defaulted in vars.env
DEBUG_LEVEL=${DEBUG_LEVEL:-info}
BUILDER_PROPOSALS=
BUILD_IMAGE=true
BUILDER_PROPOSALS=false
CI=false
# Get options
while getopts "v:d:ph" flag; do
while getopts "e:b:n:phc" flag; do
case "${flag}" in
v) VC_COUNT=${OPTARG};;
d) DEBUG_LEVEL=${OPTARG};;
p) BUILDER_PROPOSALS="-p";;
e) ENCLAVE_NAME=${OPTARG};;
b) BUILD_IMAGE=${OPTARG};;
n) NETWORK_PARAMS_FILE=${OPTARG};;
p) BUILDER_PROPOSALS=true;;
c) CI=true;;
h)
validators=$(( $VALIDATOR_COUNT / $BN_COUNT ))
echo "Start local testnet, defaults: 1 eth1 node, $BN_COUNT beacon nodes,"
echo "and $VC_COUNT validator clients with each vc having $validators validators."
echo "Start a local testnet with kurtosis."
echo
echo "usage: $0 <Options>"
echo
echo "Options:"
echo " -v: VC_COUNT default: $VC_COUNT"
echo " -d: DEBUG_LEVEL default: info"
echo " -p: enable builder proposals"
echo " -h: this help"
echo " -e: enclave name default: $ENCLAVE_NAME"
echo " -b: whether to build Lighthouse docker image default: $BUILD_IMAGE"
echo " -n: kurtosis network params file path default: $NETWORK_PARAMS_FILE"
echo " -p: enable builder proposals"
echo " -c: CI mode, run without other additional services like Grafana and Dora explorer"
echo " -h: this help"
exit
;;
esac
done
if (( $VC_COUNT > $BN_COUNT )); then
echo "Error $VC_COUNT is too large, must be <= BN_COUNT=$BN_COUNT"
LH_IMAGE_NAME=$(yq eval ".participants[0].cl_image" $NETWORK_PARAMS_FILE)
if ! command -v docker &> /dev/null; then
echo "Docker is not installed. Please install Docker and try again."
exit 1
fi
if ! command -v kurtosis &> /dev/null; then
echo "kurtosis command not found. Please install kurtosis and try again."
exit
fi
genesis_file=${@:$OPTIND+0:1}
if ! command -v yq &> /dev/null; then
echo "yq not found. Please install yq and try again."
fi
# Init some constants
PID_FILE=$TESTNET_DIR/PIDS.pid
LOG_DIR=$TESTNET_DIR
if [ "$BUILDER_PROPOSALS" = true ]; then
yq eval '.participants[0].vc_extra_params = ["--builder-proposals"]' -i $NETWORK_PARAMS_FILE
echo "--builder-proposals VC flag added to network_params.yaml"
fi
# Stop local testnet and remove $PID_FILE
./stop_local_testnet.sh
if [ "$CI" = true ]; then
# TODO: run assertoor tests
yq eval '.additional_services = []' -i $NETWORK_PARAMS_FILE
echo "Running without additional services (CI mode)."
else
yq eval '.additional_services = ["dora", "prometheus_grafana"]' -i $NETWORK_PARAMS_FILE
echo "Additional services dora and prometheus_grafana added to network_params.yaml"
fi
# Clean $DATADIR and create empty log files so the
# user can "tail -f" right after starting this script
# even before its done.
./clean.sh
mkdir -p $LOG_DIR
for (( bn=1; bn<=$BN_COUNT; bn++ )); do
touch $LOG_DIR/beacon_node_$bn.log
done
for (( el=1; el<=$BN_COUNT; el++ )); do
touch $LOG_DIR/geth_$el.log
done
for (( vc=1; vc<=$VC_COUNT; vc++ )); do
touch $LOG_DIR/validator_node_$vc.log
done
if [ "$BUILD_IMAGE" = true ]; then
echo "Building Lighthouse Docker image."
ROOT_DIR="$SCRIPT_DIR/../.."
docker build --build-arg FEATURES=portable -f $ROOT_DIR/Dockerfile -t $LH_IMAGE_NAME $ROOT_DIR
else
echo "Not rebuilding Lighthouse Docker image."
fi
# Sleep with a message
sleeping() {
echo sleeping $1
sleep $1
}
# Stop local testnet
kurtosis enclave rm -f $ENCLAVE_NAME 2>/dev/null || true
# Execute the command with logs saved to a file.
#
# First parameter is log file name
# Second parameter is executable name
# Remaining parameters are passed to executable
execute_command() {
LOG_NAME=$1
EX_NAME=$2
shift
shift
CMD="$EX_NAME $@ >> $LOG_DIR/$LOG_NAME 2>&1"
echo "executing: $CMD"
echo "$CMD" > "$LOG_DIR/$LOG_NAME"
eval "$CMD &"
}
# Execute the command with logs saved to a file
# and is PID is saved to $PID_FILE.
#
# First parameter is log file name
# Second parameter is executable name
# Remaining parameters are passed to executable
execute_command_add_PID() {
execute_command $@
echo "$!" >> $PID_FILE
}
# Setup data
echo "executing: ./setup.sh >> $LOG_DIR/setup.log"
./setup.sh >> $LOG_DIR/setup.log 2>&1
# Call setup_time.sh to update future hardforks time in the EL genesis file based on the CL genesis time
./setup_time.sh $genesis_file
# Delay to let boot_enr.yaml to be created
execute_command_add_PID bootnode.log ./bootnode.sh
sleeping 3
execute_command_add_PID el_bootnode.log ./el_bootnode.sh
sleeping 3
# Start beacon nodes
BN_udp_tcp_base=9000
BN_http_port_base=8000
EL_base_network=7000
EL_base_http=6000
EL_base_auth_http=5000
(( $VC_COUNT < $BN_COUNT )) && SAS=-s || SAS=
for (( el=1; el<=$BN_COUNT; el++ )); do
execute_command_add_PID geth_$el.log ./geth.sh $DATADIR/geth_datadir$el $((EL_base_network + $el)) $((EL_base_http + $el)) $((EL_base_auth_http + $el)) $genesis_file
done
sleeping 20
# Reset the `genesis.json` config file fork times.
sed -i 's/"shanghaiTime".*$/"shanghaiTime": 0,/g' $genesis_file
sed -i 's/"cancunTime".*$/"cancunTime": 0,/g' $genesis_file
sed -i 's/"pragueTime".*$/"pragueTime": 0,/g' $genesis_file
for (( bn=1; bn<=$BN_COUNT; bn++ )); do
secret=$DATADIR/geth_datadir$bn/geth/jwtsecret
echo $secret
execute_command_add_PID beacon_node_$bn.log ./beacon_node.sh $SAS -d $DEBUG_LEVEL $DATADIR/node_$bn $((BN_udp_tcp_base + $bn)) $((BN_udp_tcp_base + $bn + 100)) $((BN_http_port_base + $bn)) http://localhost:$((EL_base_auth_http + $bn)) $secret
done
# Start requested number of validator clients
for (( vc=1; vc<=$VC_COUNT; vc++ )); do
execute_command_add_PID validator_node_$vc.log ./validator_client.sh $BUILDER_PROPOSALS -d $DEBUG_LEVEL $DATADIR/node_$vc http://localhost:$((BN_http_port_base + $vc))
done
kurtosis run --enclave $ENCLAVE_NAME github.com/kurtosis-tech/ethereum-package --args-file $NETWORK_PARAMS_FILE
echo "Started!"

View File

@@ -1,10 +1,15 @@
#!/usr/bin/env bash
# Stop all processes that were started with start_local_testnet.sh
set -Eeuo pipefail
source ./vars.env
SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
ENCLAVE_NAME=${1:-local-testnet}
LOGS_PATH=$SCRIPT_DIR/logs
LOGS_SUBDIR=$LOGS_PATH/$ENCLAVE_NAME
PID_FILE=$TESTNET_DIR/PIDS.pid
./kill_processes.sh $PID_FILE
rm -f $PID_FILE
# Delete existing logs directory and make sure parent directory exists.
rm -rf $LOGS_SUBDIR && mkdir -p $LOGS_PATH
kurtosis enclave dump $ENCLAVE_NAME $LOGS_SUBDIR
echo "Local testnet logs stored to $LOGS_SUBDIR."
kurtosis enclave rm -f $ENCLAVE_NAME
echo "Local testnet stopped."

View File

@@ -1,34 +0,0 @@
#!/usr/bin/env bash
#
# Starts a validator client based upon a genesis state created by
# `./setup.sh`.
#
# Usage: ./validator_client.sh <DATADIR> <BEACON-NODE-HTTP> <OPTIONAL-DEBUG-LEVEL>
set -Eeuo pipefail
source ./vars.env
DEBUG_LEVEL=info
BUILDER_PROPOSALS=
# Get options
while getopts "pd:" flag; do
case "${flag}" in
p) BUILDER_PROPOSALS="--builder-proposals";;
d) DEBUG_LEVEL=${OPTARG};;
esac
done
exec lighthouse \
--debug-level $DEBUG_LEVEL \
vc \
$BUILDER_PROPOSALS \
--datadir ${@:$OPTIND:1} \
--testnet-dir $TESTNET_DIR \
--init-slashing-protection \
--beacon-nodes ${@:$OPTIND+1:1} \
--suggested-fee-recipient 0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990 \
$VC_ARGS

View File

@@ -1,69 +0,0 @@
# Path to the geth binary
GETH_BINARY=geth
EL_BOOTNODE_BINARY=bootnode
# Base directories for the validator keys and secrets
DATADIR=~/.lighthouse/local-testnet
# Directory for the eth2 config
TESTNET_DIR=$DATADIR/testnet
# Mnemonic for generating validator keys
MNEMONIC_PHRASE="vast thought differ pull jewel broom cook wrist tribe word before omit"
EL_BOOTNODE_ENODE="enode://51ea9bb34d31efc3491a842ed13b8cab70e753af108526b57916d716978b380ed713f4336a80cdb85ec2a115d5a8c0ae9f3247bed3c84d3cb025c6bab311062c@127.0.0.1:0?discport=30301"
# Hardcoded deposit contract
DEPOSIT_CONTRACT_ADDRESS=4242424242424242424242424242424242424242
GENESIS_FORK_VERSION=0x42424242
# Block hash generated from genesis.json in directory
ETH1_BLOCK_HASH=4b0e17cf5c04616d64526d292b80a1f2720cf2195d990006e4ea6950c5bbcb9f
VALIDATOR_COUNT=80
GENESIS_VALIDATOR_COUNT=80
# Number of beacon_node instances that you intend to run
BN_COUNT=4
# Number of validator clients
VC_COUNT=$BN_COUNT
# Number of seconds to delay to start genesis block.
# If started by a script this can be 0, if starting by hand
# use something like 180.
GENESIS_DELAY=0
# Port for P2P communication with bootnode
BOOTNODE_PORT=4242
# Network ID and Chain ID of local eth1 test network
CHAIN_ID=4242
# Hard fork configuration
ALTAIR_FORK_EPOCH=0
BELLATRIX_FORK_EPOCH=0
CAPELLA_FORK_EPOCH=0
DENEB_FORK_EPOCH=1
ELECTRA_FORK_EPOCH=9999999
TTD=0
# Spec version (mainnet or minimal)
SPEC_PRESET=mainnet
# Seconds per Eth2 slot
SECONDS_PER_SLOT=3
# Seconds per Eth1 block
SECONDS_PER_ETH1_BLOCK=3
# Proposer score boost percentage
PROPOSER_SCORE_BOOST=40
# Command line arguments for beacon node client
BN_ARGS=""
# Command line arguments for validator client
VC_ARGS=""