lighthouse/Makefile

.PHONY: tests

EF_TESTS = "testing/ef_tests"
STATE_TRANSITION_VECTORS = "testing/state_transition_vectors"
EXECUTION_ENGINE_INTEGRATION = "testing/execution_engine_integration"
GIT_TAG := $(shell git describe --tags --candidates 1)
BIN_DIR = "bin"

X86_64_TAG = "x86_64-unknown-linux-gnu"
BUILD_PATH_X86_64 = "target/$(X86_64_TAG)/release"
AARCH64_TAG = "aarch64-unknown-linux-gnu"
BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
RISCV64_TAG = "riscv64gc-unknown-linux-gnu"
BUILD_PATH_RISCV64 = "target/$(RISCV64_TAG)/release"

PINNED_NIGHTLY ?= nightly

# List of features to use when cross-compiling. Can be overridden via the environment.
CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,beacon-node-leveldb,beacon-node-redb

# Cargo profile for Cross builds. Default is for local builds, CI uses an override.
CROSS_PROFILE ?= release

# List of features to use when running EF tests.
EF_TEST_FEATURES ?=

# List of features to use when running CI tests.
TEST_FEATURES ?=

# Cargo profile for regular builds.
PROFILE ?= release

# List of all recent hard forks. This list is used to set env variables for http_api tests
RECENT_FORKS=electra fulu gloas

# Extra flags for Cargo
CARGO_INSTALL_EXTRA_FLAGS?=

# Builds the Lighthouse binary in release (optimized).
#
# Binaries will most likely be found in `./target/release`
install:
	cargo install --path lighthouse --force --locked \
		--features "$(FEATURES)" \
		--profile "$(PROFILE)" \
		$(CARGO_INSTALL_EXTRA_FLAGS)

# Builds the lcli binary in release (optimized).
install-lcli:
	cargo install --path lcli --force --locked \
		--features "$(FEATURES)" \
		--profile "$(PROFILE)" \
		$(CARGO_INSTALL_EXTRA_FLAGS)

# The following commands use `cross` to build a cross-compile.
#
# These commands require that:
#
# - `cross` is installed (`cargo install cross`).
# - Docker is running.
# - The current user is in the `docker` group.
#
# The resulting binaries will be created in the `target/` directory.
build-x86_64:
	cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked
build-aarch64:
	# JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB
	# pages, which are commonly used by aarch64 systems.
	# See: https://github.com/sigp/lighthouse/issues/5244
	JEMALLOC_SYS_WITH_LG_PAGE=16 cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked
build-riscv64:
	cross build --bin lighthouse --target riscv64gc-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked

build-lcli-x86_64:
	cross build --bin lcli --target x86_64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked
build-lcli-aarch64:
	# JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB
	# pages, which are commonly used by aarch64 systems.
	# See: https://github.com/sigp/lighthouse/issues/5244
	JEMALLOC_SYS_WITH_LG_PAGE=16 cross build --bin lcli --target aarch64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked
build-lcli-riscv64:
	cross build --bin lcli --target riscv64gc-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked

# Environment variables for reproducible builds
# Initialize RUSTFLAGS
RUST_BUILD_FLAGS =
# Remove build ID from the binary to ensure reproducibility across builds
RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none
# Remove metadata hash from symbol names to ensure reproducible builds
RUST_BUILD_FLAGS += -C metadata=''

# Set timestamp from last git commit for reproducible builds
SOURCE_DATE ?= $(shell git log -1 --pretty=%ct)

# Disable incremental compilation to avoid non-deterministic artifacts
CARGO_INCREMENTAL_VAL = 0
# Set C locale for consistent string handling and sorting
LOCALE_VAL = C
# Set UTC timezone for consistent time handling across builds
TZ_VAL = UTC

# Features for reproducible builds
FEATURES_REPRODUCIBLE = $(CROSS_FEATURES),jemalloc-unprefixed

# Derive the architecture-specific library path from RUST_TARGET
JEMALLOC_LIB_ARCH = $(word 1,$(subst -, ,$(RUST_TARGET)))
JEMALLOC_OVERRIDE = /usr/lib/$(JEMALLOC_LIB_ARCH)-linux-gnu/libjemalloc.a

# Default target architecture
RUST_TARGET ?= x86_64-unknown-linux-gnu

# Default images for different architectures
RUST_IMAGE_AMD64 ?= rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816
RUST_IMAGE_ARM64 ?= rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94

.PHONY: build-reproducible
build-reproducible: ## Build the lighthouse binary into `target` directory with reproducible builds
	SOURCE_DATE_EPOCH=$(SOURCE_DATE) \
	RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \
	CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \
	LC_ALL=${LOCALE_VAL} \
	TZ=${TZ_VAL} \
	JEMALLOC_OVERRIDE=${JEMALLOC_OVERRIDE} \
	cargo build --bin lighthouse --features "$(FEATURES_REPRODUCIBLE)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET)

.PHONY: build-reproducible-x86_64
build-reproducible-x86_64: ## Build reproducible x86_64 Docker image
	DOCKER_BUILDKIT=1 docker build \
		--build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
		--build-arg RUST_IMAGE=$(RUST_IMAGE_AMD64) \
		-f Dockerfile.reproducible \
		-t lighthouse:reproducible-amd64 .

.PHONY: build-reproducible-aarch64
build-reproducible-aarch64: ## Build reproducible aarch64 Docker image
	DOCKER_BUILDKIT=1 docker build \
		--platform linux/arm64 \
		--build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
		--build-arg RUST_IMAGE=$(RUST_IMAGE_ARM64) \
		-f Dockerfile.reproducible \
		-t lighthouse:reproducible-arm64 .

.PHONY: build-reproducible-all
build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64 ## Build both x86_64 and aarch64 reproducible Docker images

# Create a `.tar.gz` containing a binary for a specific target.
define tarball_release_binary
	cp $(1)/lighthouse $(BIN_DIR)/lighthouse
	cd $(BIN_DIR) && \
		tar -czf lighthouse-$(GIT_TAG)-$(2)$(3).tar.gz lighthouse && \
		rm lighthouse
endef

# Create a series of `.tar.gz` files in the BIN_DIR directory, each containing
# a `lighthouse` binary for a different target.
#
# The current git tag will be used as the version in the output file names. You
# will likely need to use `git tag` and create a semver tag (e.g., `v0.2.3`).
build-release-tarballs:
	[ -d $(BIN_DIR) ] || mkdir -p $(BIN_DIR)
	$(MAKE) build-x86_64
	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"")
	$(MAKE) build-aarch64
	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"")
	$(MAKE) build-riscv64
	$(call tarball_release_binary,$(BUILD_PATH_RISCV64),$(RISCV64_TAG),"")



# Runs the full workspace tests in **release**, without downloading any additional
# test vectors.
test-release:
	cargo nextest run --workspace --release --features "$(TEST_FEATURES)" \
		--exclude ef_tests --exclude beacon_chain --exclude slasher --exclude network \
		--exclude http_api


# Runs the full workspace tests in **debug**, without downloading any additional test
# vectors.
test-debug:
	cargo nextest run --workspace --features "$(TEST_FEATURES)" \
		--exclude ef_tests --exclude beacon_chain --exclude network --exclude http_api

# Runs cargo-fmt (linter).
cargo-fmt:
	cargo fmt --all -- --check

# Typechecks benchmark code
check-benches:
	cargo check --workspace --benches --features "$(TEST_FEATURES)"


# Runs EF test vectors
run-ef-tests:
	rm -rf $(EF_TESTS)/.accessed_file_log.txt
	cargo nextest run --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES)"
	cargo nextest run --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES),fake_crypto"
	./$(EF_TESTS)/check_all_files_accessed.py $(EF_TESTS)/.accessed_file_log.txt $(EF_TESTS)/consensus-spec-tests

# Run the tests in the `beacon_chain` crate for recent forks.
test-beacon-chain: $(patsubst %,test-beacon-chain-%,$(RECENT_FORKS))

test-beacon-chain-%:
	env FORK_NAME=$* cargo nextest run --release --features "fork_from_env,slasher/lmdb,$(TEST_FEATURES)" -p beacon_chain

# Run the tests in the `http_api` crate for recent forks.
test-http-api: $(patsubst %,test-http-api-%,$(RECENT_FORKS))

test-http-api-%:
	env FORK_NAME=$* cargo nextest run --release --features "beacon_chain/fork_from_env" -p http_api


# Run the tests in the `operation_pool` crate for all known forks.
test-op-pool: $(patsubst %,test-op-pool-%,$(RECENT_FORKS))

test-op-pool-%:
	env FORK_NAME=$* cargo nextest run --release \
		--features "beacon_chain/fork_from_env,$(TEST_FEATURES)"\
		-p operation_pool

# Run the tests in the `network` crate for recent forks.
test-network: $(patsubst %,test-network-%,$(RECENT_FORKS))

test-network-%:
	env FORK_NAME=$* cargo nextest run --release \
		--features "fork_from_env,$(TEST_FEATURES)" \
		-p network

# Run the tests in the `slasher` crate for all supported database backends.
test-slasher:
	cargo nextest run --release -p slasher --features "lmdb,$(TEST_FEATURES)"
	cargo nextest run --release -p slasher --no-default-features --features "redb,$(TEST_FEATURES)"
	cargo nextest run --release -p slasher --no-default-features --features "mdbx,$(TEST_FEATURES)"
	cargo nextest run --release -p slasher --features "lmdb,mdbx,redb,$(TEST_FEATURES)" # all backends enabled

# Runs only the tests/state_transition_vectors tests.
run-state-transition-tests:
	make -C $(STATE_TRANSITION_VECTORS) test

# Downloads and runs the EF test vectors.
test-ef: make-ef-tests run-ef-tests

# Downloads and runs the nightly EF test vectors.
test-ef-nightly: make-ef-tests-nightly run-ef-tests

# Runs tests checking interop between Lighthouse and execution clients.
test-exec-engine:
	make -C $(EXECUTION_ENGINE_INTEGRATION) test

# Runs the full workspace tests in release, without downloading any additional
# test vectors.
test: test-release

# Updates the CLI help text pages in the Lighthouse book, building with Docker (primarily for Windows users).
cli:
	docker run --rm --user=root \
	-v ${PWD}:/home/runner/actions-runner/lighthouse sigmaprime/github-runner \
	bash -c 'cd lighthouse && make && ./scripts/cli.sh'

# Updates the CLI help text pages in the Lighthouse book, building using local
# `cargo`.
cli-local:
	make && ./scripts/cli.sh

# Check for markdown files
mdlint:
	./scripts/mdlint.sh

# Runs the entire test suite, downloading test vectors if required.
test-full: cargo-fmt test-release test-debug test-ef test-exec-engine

# Lints the code for bad style and potentially unsafe arithmetic using Clippy.
# Clippy lints are opt-in per-crate for now. By default, everything is allowed except for performance and correctness lints.
lint:
	cargo clippy --workspace --benches --tests $(EXTRA_CLIPPY_OPTS) --features "$(TEST_FEATURES)" -- \
		-D clippy::fn_to_numeric_cast_any \
		-D clippy::manual_let_else \
		-D clippy::large_stack_frames \
		-D clippy::disallowed_methods \
		-D warnings \
		-A clippy::derive_partial_eq_without_eq \
		-A clippy::upper-case-acronyms \
		-A clippy::vec-init-then-push \
		-A clippy::question-mark \
		-A clippy::uninlined-format-args \
		-A clippy::enum_variant_names

# Lints the code using Clippy and automatically fix some simple compiler warnings.
lint-fix:
	EXTRA_CLIPPY_OPTS="--fix --allow-staged --allow-dirty" $(MAKE) lint-full

# Also run the lints on the optimized-only tests
lint-full:
	TEST_FEATURES="beacon-node-leveldb,beacon-node-redb,${TEST_FEATURES}"  RUSTFLAGS="-C debug-assertions=no $(RUSTFLAGS)" $(MAKE) lint

# Runs the makefile in the `ef_tests` repo.
#
# May download and extract an archive of test vectors from the ethereum
# repositories. At the time of writing, this was several hundred MB of
# downloads which extracts into several GB of test vectors.
make-ef-tests:
	make -C $(EF_TESTS)

# Download/extract the nightly EF test vectors.
make-ef-tests-nightly:
	CONSENSUS_SPECS_TEST_VERSION=nightly make -C $(EF_TESTS)

# Verifies that crates compile with fuzzing features enabled
arbitrary-fuzz:
	cargo check -p state_processing --features arbitrary-fuzz,$(TEST_FEATURES)
	cargo check -p slashing_protection --features arbitrary-fuzz,$(TEST_FEATURES)

# Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database)
audit: install-audit audit-CI

install-audit:
	cargo install --force cargo-audit

audit-CI:
	cargo audit

# Runs `cargo vendor` to make sure dependencies can be vendored for packaging, reproducibility and archival purpose.
vendor:
	cargo vendor

# Runs `cargo udeps` to check for unused dependencies
udeps:
	cargo +$(PINNED_NIGHTLY) udeps --tests --all-targets --release --features "$(TEST_FEATURES)"

# Performs a `cargo` clean and cleans the `ef_tests` directory.
clean:
	cargo clean
	make -C $(EF_TESTS) clean
	make -C $(STATE_TRANSITION_VECTORS) clean