feat: Add reproducible builds release workflows and push images to DockerHub (#7614)

This pull request introduces workflows and updates to ensure reproducible builds for the Lighthouse project. It adds two GitHub Actions workflows for building and testing reproducible Docker images and binaries, updates the `Makefile` to streamline reproducible build configurations, and modifies the `Dockerfile.reproducible` to align with the new build process. Additionally, it removes the `reproducible` profile from `Cargo.toml`.


  ### New GitHub Actions Workflows:

* [`.github/workflows/docker-reproducible.yml`](diffhunk://#diff-222af23bee616920b04f5b92a83eb5106fce08abd885cd3a3b15b8beb5e789c3R1-R145): Adds a workflow to build and push reproducible multi-architecture Docker images for releases, including support for dry runs without pushing an image.

### Build Configuration Updates:

* [`Makefile`](diffhunk://#diff-76ed074a9305c04054cdebb9e9aad2d818052b07091de1f20cad0bbac34ffb52L85-R143): Refactors reproducible build targets, centralizes environment variables for reproducibility, and updates Docker build arguments for `x86_64` and `aarch64` architectures.
* [`Dockerfile.reproducible`](diffhunk://#diff-587298ff141278ce3be7c54a559f9f31472cc5b384e285e2105b3dee319ba31dL1-R24): Updates the base Rust image to version 1.86, removes hardcoded reproducibility settings, and delegates build logic to the `Makefile`.
* Switch to using jemalloc-sys from Debian repos instead of building it from source. A Debian version is [reproducible](https://tests.reproducible-builds.org/debian/rb-pkg/trixie/amd64/jemalloc.html) which is [hard to achieve](https://github.com/NixOS/nixpkgs/issues/380852) if you build it from source.

### Profile Removal:

* [`Cargo.toml`](diffhunk://#diff-2e9d962a08321605940b5a657135052fbcef87b5e360662bb527c96d9a615542L289-L295): Removes the `reproducible` profile, simplifying build configurations and relying on external tooling for reproducibility.


Co-Authored-By: Moe Mahhouk <mohammed-mahhouk@hotmail.com>

Co-Authored-By: chonghe <44791194+chong-he@users.noreply.github.com>

Co-Authored-By: Michael Sproul <michaelsproul@users.noreply.github.com>

.github/workflows/docker-reproducible.yml

@@ -0,0 +1,176 @@
+name: docker-reproducible
+
+on:
+  push:
+    branches:
+      - unstable
+      - stable
+    tags:
+      - v*
+  workflow_dispatch: # allows manual triggering for testing purposes and skips publishing an image
+
+env:
+  DOCKER_REPRODUCIBLE_IMAGE_NAME: >-
+    ${{ github.repository_owner }}/lighthouse-reproducible
+  DOCKER_PASSWORD: ${{ secrets.DH_KEY }}
+  DOCKER_USERNAME: ${{ secrets.DH_ORG }}
+
+jobs:
+  extract-version:
+    name: extract version
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Extract version
+        run: |
+          if [[ "${{ github.ref }}" == refs/tags/* ]]; then
+            # It's a tag (e.g., v1.2.3)
+            VERSION="${GITHUB_REF#refs/tags/}"
+          elif [[ "${{ github.ref }}" == refs/heads/stable ]]; then
+            # stable branch -> latest
+            VERSION="latest"
+          elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then
+            # unstable branch -> latest-unstable
+            VERSION="latest-unstable"
+          else
+            # For manual triggers from other branches and will not publish any image
+            VERSION="test-build"
+          fi
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+        id: extract_version
+    outputs:
+      VERSION: ${{ steps.extract_version.outputs.VERSION }}
+
+  verify-and-build:
+    name: verify reproducibility and build
+    needs: extract-version
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+        include:
+          - arch: amd64
+            rust_target: x86_64-unknown-linux-gnu
+            rust_image: >-
+              rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816
+            platform: linux/amd64
+            runner: ubuntu-22.04
+          - arch: arm64
+            rust_target: aarch64-unknown-linux-gnu
+            rust_image: >-
+              rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94
+            platform: linux/arm64
+            runner: ubuntu-22.04-arm
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+
+      - name: Verify reproducible builds (${{ matrix.arch }})
+        run: |
+          # Build first image
+          docker build -f Dockerfile.reproducible \
+            --platform ${{ matrix.platform }} \
+            --build-arg RUST_TARGET="${{ matrix.rust_target }}" \
+            --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \
+            -t lighthouse-verify-1-${{ matrix.arch }} .
+
+          # Extract binary from first build
+          docker create --name extract-1-${{ matrix.arch }} lighthouse-verify-1-${{ matrix.arch }}
+          docker cp extract-1-${{ matrix.arch }}:/lighthouse ./lighthouse-1-${{ matrix.arch }}
+          docker rm extract-1-${{ matrix.arch }}
+
+          # Clean state for second build
+          docker buildx prune -f
+          docker system prune -f
+
+          # Build second image
+          docker build -f Dockerfile.reproducible \
+            --platform ${{ matrix.platform }} \
+            --build-arg RUST_TARGET="${{ matrix.rust_target }}" \
+            --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \
+            -t lighthouse-verify-2-${{ matrix.arch }} .
+
+          # Extract binary from second build
+          docker create --name extract-2-${{ matrix.arch }} lighthouse-verify-2-${{ matrix.arch }}
+          docker cp extract-2-${{ matrix.arch }}:/lighthouse ./lighthouse-2-${{ matrix.arch }}
+          docker rm extract-2-${{ matrix.arch }}
+
+          # Compare binaries
+          echo "=== Comparing binaries ==="
+          echo "Build 1 SHA256: $(sha256sum lighthouse-1-${{ matrix.arch }})"
+          echo "Build 2 SHA256: $(sha256sum lighthouse-2-${{ matrix.arch }})"
+
+          if cmp lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }}; then
+            echo "Reproducible build verified for ${{ matrix.arch }}"
+          else
+            echo "Reproducible build FAILED for ${{ matrix.arch }}"
+            echo "BLOCKING RELEASE: Builds are not reproducible!"
+            echo "First 10 differences:"
+            cmp -l lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }} | head -10
+            exit 1
+          fi
+
+          # Clean up verification artifacts but keep one image for publishing
+          rm -f lighthouse-*-${{ matrix.arch }}
+          docker rmi lighthouse-verify-1-${{ matrix.arch }} || true
+
+          # Re-tag the second image for publishing (we verified it's identical to first)
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+          FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
+          docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG"
+
+      - name: Log in to Docker Hub
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
+
+      - name: Push verified image (${{ matrix.arch }})
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        run: |
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+          IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
+          docker push "$IMAGE_TAG"
+
+      - name: Clean up local images
+        run: |
+          docker rmi lighthouse-verify-2-${{ matrix.arch }} || true
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+          docker rmi "${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" || true
+
+      - name: Upload verification artifacts (on failure)
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: verification-failure-${{ matrix.arch }}
+          path: |
+            lighthouse-*-${{ matrix.arch }}
+
+  create-manifest:
+    name: create multi-arch manifest
+    runs-on: ubuntu-22.04
+    needs: [extract-version, verify-and-build]
+    if: ${{ github.event_name != 'workflow_dispatch' }}
+    steps:
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
+
+      - name: Create and push multi-arch manifest
+        run: |
+          IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+
+          # Create manifest for the version tag
+          docker manifest create \
+            ${IMAGE_NAME}:${VERSION} \
+            ${IMAGE_NAME}:${VERSION}-amd64 \
+            ${IMAGE_NAME}:${VERSION}-arm64
+
+          docker manifest push ${IMAGE_NAME}:${VERSION}

Cargo.toml

@@ -279,13 +279,6 @@ lto = "fat"
 codegen-units = 1
 incremental = false
 
-[profile.reproducible]
-inherits = "release"
-debug = false
-panic = "abort"
-codegen-units = 1
-overflow-checks = true
-
 [profile.release-debug]
 inherits = "release"
 debug = true

Dockerfile.reproducible

@@ -3,42 +3,22 @@ ARG RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9
 FROM ${RUST_IMAGE} AS builder
 
 # Install specific version of the build dependencies
-RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1
+RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1 libjemalloc-dev=5.2.1-3
 
-# Add target architecture argument with default value
 ARG RUST_TARGET="x86_64-unknown-linux-gnu"
 
 # Copy the project to the container
-COPY . /app
+COPY ./ /app
 WORKDIR /app
 
-# Get the latest commit timestamp and set SOURCE_DATE_EPOCH (default it to 0 if not passed)
-ARG SOURCE_DATE=0
-
-# Set environment variables for reproducibility
-ARG RUSTFLAGS="-C link-arg=-Wl,--build-id=none -C metadata='' --remap-path-prefix $(pwd)=."
-ENV SOURCE_DATE_EPOCH=$SOURCE_DATE \
-    CARGO_INCREMENTAL=0 \
-    LC_ALL=C \
-    TZ=UTC \
-    RUSTFLAGS="${RUSTFLAGS}"
-
-# Set the default features if not provided
-ARG FEATURES="gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc"
-
-# Set the default profile if not provided
-ARG PROFILE="reproducible"
-
 # Build the project with the reproducible settings
-RUN cargo build --bin lighthouse \
+RUN make build-reproducible
-    --features "${FEATURES}" \
-    --profile "${PROFILE}" \
-    --locked \
-    --target "${RUST_TARGET}"
 
-RUN mv /app/target/${RUST_TARGET}/${PROFILE}/lighthouse /lighthouse
+# Move the binary to a standard location
+RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse
 
 # Create a minimal final image with just the binary
 FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a
 COPY --from=builder /lighthouse /lighthouse
+
 ENTRYPOINT [ "/lighthouse" ]

Makefile

@@ -81,36 +81,67 @@ build-lcli-aarch64:
 build-lcli-riscv64:
 	cross build --bin lcli --target riscv64gc-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked
 
-# extracts the current source date for reproducible builds
+# Environment variables for reproducible builds
-SOURCE_DATE := $(shell git log -1 --pretty=%ct)
+# Initialize RUSTFLAGS
+RUST_BUILD_FLAGS =
+# Remove build ID from the binary to ensure reproducibility across builds
+RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none
+# Remove metadata hash from symbol names to ensure reproducible builds
+RUST_BUILD_FLAGS += -C metadata=''
 
-# Default image for x86_64
+# Set timestamp from last git commit for reproducible builds
+SOURCE_DATE ?= $(shell git log -1 --pretty=%ct)
+
+# Disable incremental compilation to avoid non-deterministic artifacts
+CARGO_INCREMENTAL_VAL = 0
+# Set C locale for consistent string handling and sorting
+LOCALE_VAL = C
+# Set UTC timezone for consistent time handling across builds
+TZ_VAL = UTC
+
+# Features for reproducible builds
+FEATURES_REPRODUCIBLE = $(CROSS_FEATURES),jemalloc-unprefixed
+
+# Derive the architecture-specific library path from RUST_TARGET
+JEMALLOC_LIB_ARCH = $(word 1,$(subst -, ,$(RUST_TARGET)))
+JEMALLOC_OVERRIDE = /usr/lib/$(JEMALLOC_LIB_ARCH)-linux-gnu/libjemalloc.a
+
+# Default target architecture
+RUST_TARGET ?= x86_64-unknown-linux-gnu
+
+# Default images for different architectures
 RUST_IMAGE_AMD64 ?= rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816
+RUST_IMAGE_ARM64 ?= rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94
 
-# Reproducible build for x86_64
+.PHONY: build-reproducible
-build-reproducible-x86_64:
+build-reproducible: ## Build the lighthouse binary into `target` directory with reproducible builds
+	SOURCE_DATE_EPOCH=$(SOURCE_DATE) \
+	RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \
+	CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \
+	LC_ALL=${LOCALE_VAL} \
+	TZ=${TZ_VAL} \
+	JEMALLOC_OVERRIDE=${JEMALLOC_OVERRIDE} \
+	cargo build --bin lighthouse --features "$(FEATURES_REPRODUCIBLE)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET)
+
+.PHONY: build-reproducible-x86_64
+build-reproducible-x86_64: ## Build reproducible x86_64 Docker image
 	DOCKER_BUILDKIT=1 docker build \
 		--build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
 		--build-arg RUST_IMAGE=$(RUST_IMAGE_AMD64) \
-		--build-arg SOURCE_DATE=$(SOURCE_DATE) \
 		-f Dockerfile.reproducible \
 		-t lighthouse:reproducible-amd64 .
 
-# Default image for arm64
+.PHONY: build-reproducible-aarch64
-RUST_IMAGE_ARM64 ?= rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94
+build-reproducible-aarch64: ## Build reproducible aarch64 Docker image
-
-# Reproducible build for aarch64
-build-reproducible-aarch64:
 	DOCKER_BUILDKIT=1 docker build \
 		--platform linux/arm64 \
 		--build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
 		--build-arg RUST_IMAGE=$(RUST_IMAGE_ARM64) \
-		--build-arg SOURCE_DATE=$(SOURCE_DATE) \
 		-f Dockerfile.reproducible \
 		-t lighthouse:reproducible-arm64 .
 
-# Build both architectures
+.PHONY: build-reproducible-all
-build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64
+build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64 ## Build both x86_64 and aarch64 reproducible Docker images
 
 # Create a `.tar.gz` containing a binary for a specific target.
 define tarball_release_binary

common/malloc_utils/Cargo.toml

@@ -21,6 +21,8 @@ jemalloc-profiling = ["tikv-jemallocator/profiling"]
 # Force the use of system malloc (or glibc) rather than jemalloc.
 # This is a no-op on Windows where jemalloc is always disabled.
 sysmalloc = []
+# Enable jemalloc with unprefixed malloc (recommended for reproducible builds)
+jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator/unprefixed_malloc_on_supported_platforms"]
 
 [dependencies]
 libc = "0.2.79"

testing/state_transition_vectors/Makefile

@@ -5,4 +5,4 @@ test:
 	cargo test --release --features "$(TEST_FEATURES)"
 
 clean:
-	rm -r vectors/
+	rm -rf vectors/