Resolve rebase_on issues

* Fix clippy

* Fix beacon chain tests

* Comment out dodgy iterator test

* Fix imports for beta compiler

.cargo/config.toml

@@ -0,0 +1,4 @@
+[env]
+# Set the number of arenas to 16 when using jemalloc.
+JEMALLOC_SYS_WITH_MALLOC_CONF = "abort_conf:true,narenas:16"
+

.config/nextest.toml

@@ -0,0 +1,113 @@
+# This is the default config used by nextest. It is embedded in the binary at
+# build time. It may be used as a template for .config/nextest.toml.
+
+[store]
+# The directory under the workspace root at which nextest-related files are
+# written. Profile-specific storage is currently written to dir/<profile-name>.
+dir = "target/nextest"
+
+# This section defines the default nextest profile. Custom profiles are layered
+# on top of the default profile.
+[profile.default]
+# "retries" defines the number of times a test should be retried. If set to a
+# non-zero value, tests that succeed on a subsequent attempt will be marked as
+# non-flaky. Can be overridden through the `--retries` option.
+# Examples
+# * retries = 3
+# * retries = { backoff = "fixed", count = 2, delay = "1s" }
+# * retries = { backoff = "exponential", count = 10, delay = "1s", jitter = true, max-delay = "10s" }
+retries = 0
+
+# The number of threads to run tests with. Supported values are either an integer or
+# the string "num-cpus". Can be overridden through the `--test-threads` option.
+test-threads = 8
+
+# The number of threads required for each test. This is generally used in overrides to
+# mark certain tests as heavier than others. However, it can also be set as a global parameter.
+threads-required = 1
+
+# Show these test statuses in the output.
+#
+# The possible values this can take are:
+# * none: no output
+# * fail: show failed (including exec-failed) tests
+# * retry: show flaky and retried tests
+# * slow: show slow tests
+# * pass: show passed tests
+# * skip: show skipped tests (most useful for CI)
+# * all: all of the above
+#
+# Each value includes all the values above it; for example, "slow" includes
+# failed and retried tests.
+#
+# Can be overridden through the `--status-level` flag.
+status-level = "pass"
+
+# Similar to status-level, show these test statuses at the end of the run.
+final-status-level = "flaky"
+
+# "failure-output" defines when standard output and standard error for failing tests are produced.
+# Accepted values are
+# * "immediate": output failures as soon as they happen
+# * "final": output failures at the end of the test run
+# * "immediate-final": output failures as soon as they happen and at the end of
+#   the test run; combination of "immediate" and "final"
+# * "never": don't output failures at all
+#
+# For large test suites and CI it is generally useful to use "immediate-final".
+#
+# Can be overridden through the `--failure-output` option.
+failure-output = "immediate"
+
+# "success-output" controls production of standard output and standard error on success. This should
+# generally be set to "never".
+success-output = "never"
+
+# Cancel the test run on the first failure. For CI runs, consider setting this
+# to false.
+fail-fast = true
+
+# Treat a test that takes longer than the configured 'period' as slow, and print a message.
+# See <https://nexte.st/book/slow-tests> for more information.
+#
+# Optional: specify the parameter 'terminate-after' with a non-zero integer,
+# which will cause slow tests to be terminated after the specified number of
+# periods have passed.
+# Example: slow-timeout = { period = "60s", terminate-after = 2 }
+slow-timeout = { period = "120s" }
+
+# Treat a test as leaky if after the process is shut down, standard output and standard error
+# aren't closed within this duration.
+#
+# This usually happens in case of a test that creates a child process and lets it inherit those
+# handles, but doesn't clean the child process up (especially when it fails).
+#
+# See <https://nexte.st/book/leaky-tests> for more information.
+leak-timeout = "100ms"
+
+[profile.default.junit]
+# Output a JUnit report into the given file inside 'store.dir/<profile-name>'.
+# If unspecified, JUnit is not written out.
+
+# path = "junit.xml"
+
+# The name of the top-level "report" element in JUnit report. If aggregating
+# reports across different test runs, it may be useful to provide separate names
+# for each report.
+report-name = "lighthouse-run"
+
+# Whether standard output and standard error for passing tests should be stored in the JUnit report.
+# Output is stored in the <system-out> and <system-err> elements of the <testcase> element.
+store-success-output = false
+
+# Whether standard output and standard error for failing tests should be stored in the JUnit report.
+# Output is stored in the <system-out> and <system-err> elements of the <testcase> element.
+#
+# Note that if a description can be extracted from the output, it is always stored in the
+# <description> element.
+store-failure-output = true
+
+# This profile is activated if MIRI_SYSROOT is set.
+[profile.default-miri]
+# Miri tests take up a lot of memory, so only run 1 test at a time by default.
+test-threads = 4

.dockerignore

@@ -1,4 +1,5 @@
 testing/ef_tests/consensus-spec-tests
+testing/execution_engine_integration/execution_clients
 target/
 *.data
 *.tar.gz

.editorconfig

@@ -6,4 +6,4 @@ end_of_line=lf
 charset=utf-8
 trim_trailing_whitespace=true
 max_line_length=100
-insert_final_newline=false
+insert_final_newline=true

.github/custom/clippy.toml

@@ -0,0 +1,23 @@
+disallowed-from-async-methods = [
+    "tokio::runtime::Handle::block_on",
+    "tokio::runtime::Runtime::block_on",
+    "tokio::task::LocalSet::block_on",
+    "tokio::sync::Mutex::blocking_lock",
+    "tokio::sync::RwLock::blocking_read",
+    "tokio::sync::mpsc::Receiver::blocking_recv",
+    "tokio::sync::mpsc::UnboundedReceiver::blocking_recv",
+    "tokio::sync::oneshot::Receiver::blocking_recv",
+    "tokio::sync::mpsc::Sender::blocking_send",
+    "tokio::sync::RwLock::blocking_write",
+]
+async-wrapper-methods = [
+    "tokio::runtime::Handle::spawn_blocking",
+    "task_executor::TaskExecutor::spawn_blocking",
+    "task_executor::TaskExecutor::spawn_blocking_handle",
+    "warp_utils::task::blocking_task",
+    "warp_utils::task::blocking_json_task",
+    "beacon_chain::beacon_chain::BeaconChain::spawn_blocking_handle",
+    "validator_client::http_api::blocking_signed_json_task",
+    "execution_layer::test_utils::MockServer::new",
+    "execution_layer::test_utils::MockServer::new_with_config",
+]

.github/mergify.yml

@@ -0,0 +1,19 @@
+queue_rules:
+  - name: default
+    batch_size: 8
+    batch_max_wait_time: 60 s
+    checks_timeout: 10800 s
+    merge_method: squash
+    commit_message_template: |
+      {{ title }} (#{{ number }})
+      
+      {% for commit in commits %}
+      * {{ commit.commit_message }}
+      {% endfor %}
+    queue_conditions:
+      - "#approved-reviews-by >= 1"
+      - "check-success=license/cla"
+      - "check-success=target-branch-check"
+    merge_conditions:
+      - "check-success=test-suite-success"
+      - "check-success=local-testnet-success"

.github/workflows/book.yml

@@ -5,11 +5,15 @@ on:
     branches:
       - unstable
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build-and-upload-to-s3:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
 
       - name: Setup mdBook
         uses: peaceiris/actions-mdbook@v1

.github/workflows/cancel-previous-runs.yml

@@ -1,14 +0,0 @@
-name: cancel previous runs
-on: [push]
-jobs:
-  cancel:
-    name: 'Cancel Previous Runs'
-    runs-on: ubuntu-latest
-    timeout-minutes: 3
-    steps:
-      # https://github.com/styfle/cancel-workflow-action/releases
-      - uses: styfle/cancel-workflow-action@514c783324374c6940d1b92bfb962d0763d22de3 # 0.7.0
-        with:
-          # https://api.github.com/repos/sigp/lighthouse/actions/workflows
-          workflow_id: 697364,2434944,4462424,308241,2883401,316
-          access_token: ${{ github.token }}

.github/workflows/docker-antithesis.yml

@@ -1,31 +0,0 @@
-name: docker antithesis
-
-on:
-    push:
-        branches:
-            - unstable
-
-env:
-    ANTITHESIS_PASSWORD: ${{ secrets.ANTITHESIS_PASSWORD }}
-    ANTITHESIS_USERNAME: ${{ secrets.ANTITHESIS_USERNAME }}
-    ANTITHESIS_SERVER: ${{ secrets.ANTITHESIS_SERVER }}
-    REPOSITORY: ${{ secrets.ANTITHESIS_REPOSITORY }}
-    IMAGE_NAME: lighthouse
-    TAG: libvoidstar
-
-jobs:
-    build-docker:
-        runs-on: ubuntu-18.04
-        steps:
-            - uses: actions/checkout@v2
-            - name: Update Rust
-              run: rustup update stable
-            - name: Dockerhub login
-              run: |
-                  echo "${ANTITHESIS_PASSWORD}" | docker login --username ${ANTITHESIS_USERNAME} https://${ANTITHESIS_SERVER} --password-stdin
-            - name: Build AMD64 dockerfile (with push)
-              run: |
-                  docker build \
-                      --tag ${ANTITHESIS_SERVER}/${REPOSITORY}/${IMAGE_NAME}:${TAG} \
-                      --file ./testing/antithesis/Dockerfile.libvoidstar .
-                  docker push ${ANTITHESIS_SERVER}/${REPOSITORY}/${IMAGE_NAME}:${TAG}

.github/workflows/docker.yml

@@ -8,11 +8,17 @@ on:
         tags:
             - v*
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
     DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
     IMAGE_NAME: ${{ github.repository_owner}}/lighthouse
     LCLI_IMAGE_NAME: ${{ github.repository_owner }}/lcli
+    # Enable self-hosted runners for the sigp repo only.
+    SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }}
 
 jobs:
     # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX
@@ -22,7 +28,7 @@ jobs:
     # `unstable`, but for now we keep the two parts of the version separate for backwards
     # compatibility.
     extract-version:
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-22.04
         steps:
             - name: Extract version (if stable)
               if: github.event.ref == 'refs/heads/stable'
@@ -43,23 +49,31 @@ jobs:
             VERSION: ${{ env.VERSION }}
             VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }}
     build-docker-single-arch:
-        name: build-docker-${{ matrix.binary }}
-        runs-on: ubuntu-18.04
+        name: build-docker-${{ matrix.binary }}${{ matrix.features.version_suffix }}
+        # Use self-hosted runners only on the sigp repo.
+        runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04'  }}
         strategy:
             matrix:
                 binary: [aarch64,
                          aarch64-portable,
                          x86_64,
                          x86_64-portable]
+                features: [
+                    {version_suffix: "", env: "gnosis,slasher-lmdb,slasher-mdbx,jemalloc"},
+                    {version_suffix: "-dev", env: "jemalloc,spec-minimal"}
+                ]
+                include:
+                    - profile: maxperf
+
         needs: [extract-version]
         env:
-            # We need to enable experimental docker features in order to use `docker buildx`
-            DOCKER_CLI_EXPERIMENTAL: enabled
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
+            FEATURE_SUFFIX: ${{ matrix.features.version_suffix }}
         steps:
-            - uses: actions/checkout@v2
+            - uses: actions/checkout@v4
             - name: Update Rust
+              if: env.SELF_HOSTED_RUNNERS == 'false'
               run: rustup update stable
             - name: Dockerhub login
               run: |
@@ -67,17 +81,15 @@ jobs:
             - name: Cross build Lighthouse binary
               run: |
                   cargo install cross
-                  make build-${{ matrix.binary }}
+                  env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ matrix.features.env }} make build-${{ matrix.binary }}
+            - name: Make bin dir
+              run: mkdir ./bin
             - name: Move cross-built binary into Docker scope (if ARM)
               if: startsWith(matrix.binary, 'aarch64')
-              run: |
-                  mkdir ./bin;
-                  mv ./target/aarch64-unknown-linux-gnu/release/lighthouse ./bin;
+              run: mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin
             - name: Move cross-built binary into Docker scope (if x86_64)
               if: startsWith(matrix.binary, 'x86_64')
-              run: |
-                  mkdir ./bin;
-                  mv ./target/x86_64-unknown-linux-gnu/release/lighthouse ./bin;
+              run: mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin
             - name: Map aarch64 to arm64 short arch
               if: startsWith(matrix.binary, 'aarch64')
               run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV
@@ -87,53 +99,65 @@ jobs:
             - name: Set modernity suffix
               if: endsWith(matrix.binary, '-portable') != true
               run: echo "MODERNITY_SUFFIX=-modern" >> $GITHUB_ENV;
-            # Install dependencies for emulation. Have to create a new builder to pick up emulation support.
-            - name: Build Dockerfile and push
-              run: |
-                  docker run --privileged --rm tonistiigi/binfmt --install ${SHORT_ARCH}
-                  docker buildx create --use --name cross-builder
-                  docker buildx build \
-                      --platform=linux/${SHORT_ARCH} \
-                      --file ./Dockerfile.cross . \
-                      --tag ${IMAGE_NAME}:${VERSION}-${SHORT_ARCH}${VERSION_SUFFIX}${MODERNITY_SUFFIX} \
-                      --push
+
+            - name: Install QEMU
+              if: env.SELF_HOSTED_RUNNERS == 'false'
+              run: sudo apt-get update && sudo apt-get install -y qemu-user-static
+
+            - name: Set up Docker Buildx
+              if: env.SELF_HOSTED_RUNNERS == 'false'
+              uses: docker/setup-buildx-action@v3
+
+            - name: Build and push
+              uses: docker/build-push-action@v5
+              with:
+                file: ./Dockerfile.cross
+                context: .
+                platforms: linux/${{ env.SHORT_ARCH }}
+                push: true
+                tags: ${{ env.IMAGE_NAME }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}${{ env.MODERNITY_SUFFIX }}${{ env.FEATURE_SUFFIX }}
+
     build-docker-multiarch:
         name: build-docker-multiarch${{ matrix.modernity }}
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-22.04
         needs: [build-docker-single-arch, extract-version]
         strategy:
             matrix:
                 modernity: ["", "-modern"]
         env:
-            # We need to enable experimental docker features in order to use `docker manifest`
-            DOCKER_CLI_EXPERIMENTAL: enabled
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
         steps:
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+
             - name: Dockerhub login
               run: |
                   echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+
             - name: Create and push multiarch manifest
               run: |
-                  docker manifest create ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \
-                      --amend ${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \
-                      --amend ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }};
-                  docker manifest push ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }}
+                  docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \
+                      ${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \
+                      ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }};
+
     build-docker-lcli:
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-22.04
         needs: [extract-version]
         env:
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
         steps:
-            - uses: actions/checkout@v2
+            - uses: actions/checkout@v4
             - name: Dockerhub login
               run: |
                   echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
-            - name: Build lcli dockerfile (with push)
-              run: |
-                  docker build \
-                      --build-arg PORTABLE=true \
-                      --tag ${LCLI_IMAGE_NAME}:${VERSION}${VERSION_SUFFIX} \
-                      --file ./lcli/Dockerfile .
-                  docker push ${LCLI_IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}
+            - name: Build lcli and push
+              uses: docker/build-push-action@v5
+              with:
+                  build-args: |
+                      FEATURES=portable
+                  context: .
+                  push: true
+                  file: ./lcli/Dockerfile
+                  tags: ${{ env.LCLI_IMAGE_NAME }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }}

.github/workflows/linkcheck.yml

@@ -7,6 +7,11 @@ on:
   pull_request:
     paths:
       - 'book/**'
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   linkcheck:
@@ -15,16 +20,17 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Create docker network
-        run: docker network create book
+        uses: actions/checkout@v4
 
       - name: Run mdbook server
-        run: docker run -v ${{ github.workspace }}/book:/book --network book --name book -p 3000:3000 -d peaceiris/mdbook:latest serve --hostname 0.0.0.0
+        run: |
+          docker run -v ${{ github.workspace }}/book:/book --name book -p 3000:3000 -d peaceiris/mdbook:latest serve --hostname 0.0.0.0
+          sleep 5
 
       - name: Print logs
         run: docker logs book
 
       - name: Run linkcheck
-        run: docker run  --network book tennox/linkcheck:latest book:3000
+        run: |
+          curl -sL https://github.com/filiph/linkcheck/releases/download/3.0.0/linkcheck-3.0.0-linux-x64.tar.gz | tar xvzf - linkcheck/linkcheck --strip 1
+          ./linkcheck localhost:3000 -d

.github/workflows/local-testnet.yml

@@ -6,23 +6,47 @@ on:
     branches:
       - unstable
   pull_request:
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   run-local-testnet:
     strategy:
       matrix:
         os:
-          - ubuntu-18.04
-          - macos-latest
+          - ubuntu-22.04
+          - macos-12
     runs-on: ${{ matrix.os }}
+    env:
+      # Enable portable to prevent issues with caching `blst` for the wrong CPU type
+      FEATURES: portable,jemalloc
     steps:
-      - uses: actions/checkout@v1
-
-      - name: Install ganache
-        run: npm install ganache-cli@latest --global
+      - uses: actions/checkout@v4
 
+      - name: Get latest version of stable Rust
+        run: rustup update stable
+      - name: Install geth (ubuntu)
+        if: matrix.os == 'ubuntu-22.04'
+        run: |
+          sudo add-apt-repository -y ppa:ethereum/ethereum
+          sudo apt-get update
+          sudo apt-get install ethereum
+      - name: Install geth (mac)
+        if: matrix.os == 'macos-12'
+        run: |
+          brew tap ethereum/ethereum
+          brew install ethereum
+      - name: Install GNU sed & GNU grep
+        if: matrix.os == 'macos-12'
+        run: |
+          brew install gnu-sed grep
+          echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
+          echo "$(brew --prefix)/opt/grep/libexec/gnubin" >> $GITHUB_PATH
       # https://github.com/actions/cache/blob/main/examples.md#rust---cargo
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         id: cache-cargo
         with:
           path: |
@@ -34,17 +58,43 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Install lighthouse
-        if: steps.cache-cargo.outputs.cache-hit != 'true'
         run: make && make install-lcli
 
       - name: Start local testnet
-        run: ./start_local_testnet.sh
+        run: ./start_local_testnet.sh genesis.json && sleep 60
         working-directory: scripts/local_testnet
 
       - name: Print logs
-        run: ./print_logs.sh
+        run: ./dump_logs.sh
         working-directory: scripts/local_testnet
 
       - name: Stop local testnet
         run: ./stop_local_testnet.sh
         working-directory: scripts/local_testnet
+
+      - name: Clean-up testnet
+        run: ./clean.sh
+        working-directory: scripts/local_testnet
+
+      - name: Start local testnet with blinded block production
+        run: ./start_local_testnet.sh -p genesis.json && sleep 60
+        working-directory: scripts/local_testnet
+
+      - name: Print logs for blinded block testnet
+        run: ./dump_logs.sh
+        working-directory: scripts/local_testnet
+
+      - name: Stop local testnet with blinded block production
+        run: ./stop_local_testnet.sh
+        working-directory: scripts/local_testnet
+
+  # This job succeeds ONLY IF all others succeed. It is used by the merge queue to determine whether
+  # a PR is safe to merge. New jobs should be added here.
+  local-testnet-success:
+    name: local-testnet-success
+    runs-on: ubuntu-latest
+    needs: ["run-local-testnet"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check that success job is dependent on all others
+        run: ./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success

.github/workflows/publish-crate.yml

@@ -1,66 +0,0 @@
-name:   Publish Crate
-
-on:
-    push:
-        tags:
-            - tree-hash-v*
-            - tree-hash-derive-v*
-            - eth2-ssz-v*
-            - eth2-ssz-derive-v*
-            - eth2-ssz-types-v*
-            - eth2-serde-util-v*
-            - eth2-hashing-v*
-
-env:
-    CARGO_API_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
-
-jobs:
-    extract-tag:
-        runs-on: ubuntu-latest
-        steps:
-            - name: Extract tag
-              run: echo "::set-output name=TAG::$(echo ${GITHUB_REF#refs/tags/})"
-              id: extract_tag
-        outputs:
-            TAG: ${{ steps.extract_tag.outputs.TAG }}
-
-    publish-crate:
-        runs-on: ubuntu-latest
-        needs: [extract-tag]
-        env:
-            TAG: ${{ needs.extract-tag.outputs.TAG }}
-        steps:
-            - uses: actions/checkout@v2
-            - name: Update Rust
-              run: rustup update stable
-            - name: Cargo login
-              run: |
-                  echo "${CARGO_API_TOKEN}" | cargo login
-            - name: publish eth2 ssz derive
-              if: startsWith(env.TAG, 'eth2-ssz-derive-v')
-              run: |
-                ./scripts/ci/publish.sh consensus/ssz_derive eth2_ssz_derive "$TAG"
-            - name: publish eth2 ssz
-              if: startsWith(env.TAG, 'eth2-ssz-v')
-              run: |
-                ./scripts/ci/publish.sh consensus/ssz eth2_ssz "$TAG"
-            - name: publish eth2 hashing
-              if: startsWith(env.TAG, 'eth2-hashing-v')
-              run: |
-                ./scripts/ci/publish.sh crypto/eth2_hashing eth2_hashing "$TAG"
-            - name: publish tree hash derive
-              if: startsWith(env.TAG, 'tree-hash-derive-v')
-              run: |
-                ./scripts/ci/publish.sh consensus/tree_hash_derive tree_hash_derive "$TAG"
-            - name: publish tree hash
-              if: startsWith(env.TAG, 'tree-hash-v')
-              run: |
-                ./scripts/ci/publish.sh consensus/tree_hash tree_hash "$TAG"
-            - name: publish ssz types
-              if: startsWith(env.TAG, 'eth2-ssz-types-v')
-              run: |
-                ./scripts/ci/publish.sh consensus/ssz_types eth2_ssz_types "$TAG"
-            - name: publish serde util
-              if: startsWith(env.TAG, 'eth2-serde-util-v')
-              run: |
-                ./scripts/ci/publish.sh consensus/serde_utils eth2_serde_utils "$TAG"

.github/workflows/release.yml

@@ -5,18 +5,24 @@ on:
         tags:
             - v*
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
     DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-    REPO_NAME: sigp/lighthouse
-    IMAGE_NAME: sigp/lighthouse
+    REPO_NAME: ${{ github.repository_owner }}/lighthouse
+    IMAGE_NAME: ${{ github.repository_owner }}/lighthouse
+    # Enable self-hosted runners for the sigp repo only.
+    SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }}
 
 jobs:
     extract-version:
         runs-on: ubuntu-latest
         steps:
             - name: Extract version
-              run: echo "::set-output name=VERSION::$(echo ${GITHUB_REF#refs/tags/})"
+              run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
               id: extract_version
         outputs:
             VERSION: ${{ steps.extract_version.outputs.VERSION }}
@@ -34,42 +40,47 @@ jobs:
                        x86_64-windows-portable]
                 include:
                     -   arch: aarch64-unknown-linux-gnu
-                        platform: ubuntu-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest'  }}
+                        profile: maxperf
                     -   arch: aarch64-unknown-linux-gnu-portable
-                        platform: ubuntu-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest'  }}
+                        profile: maxperf
                     -   arch: x86_64-unknown-linux-gnu
-                        platform: ubuntu-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest'  }}
+                        profile: maxperf
                     -   arch: x86_64-unknown-linux-gnu-portable
-                        platform: ubuntu-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest'  }}
+                        profile: maxperf
                     -   arch: x86_64-apple-darwin
-                        platform: macos-latest
+                        runner: macos-latest
+                        profile: maxperf
                     -   arch: x86_64-apple-darwin-portable
-                        platform: macos-latest
+                        runner: macos-latest
+                        profile: maxperf
                     -   arch: x86_64-windows
-                        platform: windows-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019'  }}
+                        profile: maxperf
                     -   arch: x86_64-windows-portable
-                        platform: windows-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019'  }}
+                        profile: maxperf
 
-        runs-on:    ${{ matrix.platform }}
+        runs-on:    ${{ matrix.runner }}
         needs: extract-version
         steps:
             - name: Checkout sources
-              uses: actions/checkout@v2
-            - name: Build toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  toolchain: stable
-                  profile:  minimal
-                  override: true
+              uses: actions/checkout@v4
+            - name: Get latest version of stable Rust
+              if: env.SELF_HOSTED_RUNNERS == 'false'
+              run: rustup update stable
 
             # ==============================
             # Windows dependencies
             # ==============================
 
             - uses: KyleMayes/install-llvm-action@v1
-              if: startsWith(matrix.arch, 'x86_64-windows')
+              if: env.SELF_HOSTED_RUNNERS == 'false' && startsWith(matrix.arch, 'x86_64-windows')
               with:
-                version: "13.0"
+                version: "16.0"
                 directory: ${{ runner.temp }}/llvm
             - name: Set LIBCLANG_PATH
               if: startsWith(matrix.arch, 'x86_64-windows')
@@ -83,49 +94,49 @@ jobs:
               if:   matrix.arch == 'aarch64-unknown-linux-gnu-portable'
               run:  |
                 cargo install cross
-                make build-aarch64-portable
+                env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64-portable
 
             - name: Build Lighthouse for aarch64-unknown-linux-gnu
               if:   matrix.arch == 'aarch64-unknown-linux-gnu'
               run:  |
                 cargo install cross
-                make build-aarch64
+                env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64
 
             - name: Build Lighthouse for x86_64-unknown-linux-gnu-portable
               if:   matrix.arch == 'x86_64-unknown-linux-gnu-portable'
               run:  |
                 cargo install cross
-                make build-x86_64-portable
+                env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64-portable
 
             - name: Build Lighthouse for x86_64-unknown-linux-gnu
               if:   matrix.arch == 'x86_64-unknown-linux-gnu'
               run:  |
                 cargo install cross
-                make build-x86_64
+                env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64
 
             - name: Move cross-compiled binary
               if:  startsWith(matrix.arch, 'aarch64')
-              run: mv target/aarch64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse
+              run: mv target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse
 
             - name: Move cross-compiled binary
               if:  startsWith(matrix.arch, 'x86_64-unknown-linux-gnu')
-              run: mv target/x86_64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse
+              run: mv target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse
 
             - name: Build Lighthouse for x86_64-apple-darwin portable
               if:   matrix.arch == 'x86_64-apple-darwin-portable'
-              run:  cargo install --path lighthouse --force --locked --features portable,gnosis
+              run:  cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }}
 
             - name: Build Lighthouse for x86_64-apple-darwin modern
               if:   matrix.arch == 'x86_64-apple-darwin'
-              run:  cargo install --path lighthouse --force --locked --features modern,gnosis
+              run:  cargo install --path lighthouse --force --locked --features modern,gnosis --profile ${{ matrix.profile }}
 
             - name: Build Lighthouse for Windows portable
               if:   matrix.arch == 'x86_64-windows-portable'
-              run:  cargo install --path lighthouse --force --locked --features portable,gnosis
+              run:  cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }}
 
             - name: Build Lighthouse for Windows modern
               if:   matrix.arch == 'x86_64-windows'
-              run:  cargo install --path lighthouse --force --locked --features modern,gnosis
+              run:  cargo install --path lighthouse --force --locked --features modern,gnosis --profile ${{ matrix.profile }}
 
             - name: Configure GPG and create artifacts
               if: startsWith(matrix.arch, 'x86_64-windows') != true
@@ -161,17 +172,19 @@ jobs:
             # This is required to share artifacts between different jobs
             # =======================================================================
 
-            - name:  Upload artifact
-              uses:  actions/upload-artifact@v2
+            - name: Upload artifact
+              uses: actions/upload-artifact@v4
               with:
                   name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
                   path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
+                  compression-level: 0
 
             - name: Upload signature
-              uses: actions/upload-artifact@v2
+              uses: actions/upload-artifact@v4
               with:
                   name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
                   path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
+                  compression-level: 0
 
     draft-release:
         name:   Draft Release
@@ -182,7 +195,7 @@ jobs:
         steps:
             # This is necessary for generating the changelog. It has to come before "Download Artifacts" or else it deletes the artifacts.
             - name: Checkout sources
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
               with:
                   fetch-depth: 0
 
@@ -191,7 +204,7 @@ jobs:
             # ==============================
 
             - name: Download artifacts
-              uses: actions/download-artifact@v2
+              uses: actions/download-artifact@v4
 
             # ==============================
             #       Create release draft
@@ -199,11 +212,14 @@ jobs:
 
             - name: Generate Full Changelog
               id: changelog
-              run: echo "::set-output name=CHANGELOG::$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})"
+              run: |
+                echo "CHANGELOG<<EOF" >> $GITHUB_OUTPUT
+                echo "$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT
+                echo "EOF" >> $GITHUB_OUTPUT
 
             - name: Create Release Draft
               env:
-                  GITHUB_USER: sigp
+                  GITHUB_USER: ${{ github.repository_owner }}
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
               # The formatting here is borrowed from OpenEthereum: https://github.com/openethereum/openethereum/blob/main/.github/workflows/build.yml
@@ -212,7 +228,7 @@ jobs:
                   <Rick and Morty character>
 
                   ## Testing Checklist (DELETE ME)
-                  
+
                   - [ ] Run on synced Prater Sigma Prime nodes.
                   - [ ] Run on synced Canary (mainnet) Sigma Prime nodes.
                   - [ ] Resync a Prater node.
@@ -268,9 +284,6 @@ jobs:
                   | <img src="https://simpleicons.org/icons/docker.svg" style="width: 32px;"/> | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) |
                   ENDBODY
                   )
-                  assets=()
-                  for asset in ./lighthouse-*.tar.gz*; do
-                      assets+=("-a" "$asset/$asset")
-                  done
+                  assets=(./lighthouse-*.tar.gz*/lighthouse-*.tar.gz*)
                   tag_name="${{ env.VERSION }}"
-                  echo "$body" | hub release create --draft "${assets[@]}" -F "-" "$tag_name"
+                  echo "$body" | gh release create --draft -F "-" "$tag_name" "${assets[@]}"

.github/workflows/test-suite.yml

@@ -8,120 +8,219 @@ on:
       - trying
       - 'pr/*'
   pull_request:
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   # Deny warnings in CI
-  RUSTFLAGS: "-D warnings"
-  # The Nightly version used for cargo-udeps, might need updating from time to time.
-  PINNED_NIGHTLY: nightly-2021-12-01
+  # Disable debug info (see https://github.com/sigp/lighthouse/issues/4005)
+  RUSTFLAGS: "-D warnings -C debuginfo=0"
+  # Prevent Github API rate limiting.
+  LIGHTHOUSE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  # Enable self-hosted runners for the sigp repo only.
+  SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }}
+  # Self-hosted runners need to reference a different host for `./watch` tests.
+  WATCH_HOST: ${{ github.repository == 'sigp/lighthouse' && 'host.docker.internal' || 'localhost' }}
+  # Disable incremental compilation
+  CARGO_INCREMENTAL: 0
+  # Enable portable to prevent issues with caching `blst` for the wrong CPU type
+  TEST_FEATURES: portable
 jobs:
   target-branch-check:
     name: target-branch-check
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
+    if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
     steps:
-        - name: Check that pull request is targeting unstable branch
-          run: test ${{ github.base_ref }} = "unstable"
-  cargo-fmt:
-    name: cargo-fmt
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v1
-    - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Check formatting with cargo fmt
-      run: make cargo-fmt
+        - name: Check that the pull request is not targeting the stable branch
+          run: test ${{ github.base_ref }} != "stable"
   release-tests-ubuntu:
     name: release-tests-ubuntu
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
+    # Use self-hosted runners only on the sigp repo.
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest'  }}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+          bins: cargo-nextest
+      env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install Foundry (anvil)
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: foundry-rs/foundry-toolchain@v1
+      with:
+        version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d
     - name: Run tests in release
-      run: make test-release
+      run: make nextest-release
+    - name: Show cache stats
+      if: env.SELF_HOSTED_RUNNERS == 'true'
+      run: sccache --show-stats
   release-tests-windows:
     name: release-tests-windows
-    runs-on: windows-latest
-    needs: cargo-fmt
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "CI"]') || 'windows-2019'  }}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Install ganache-cli
-      run: npm install -g ganache-cli
-    - name: Install make
-      run: choco install -y make
-    - uses: KyleMayes/install-llvm-action@v1
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: moonrepo/setup-rust@v1
       with:
-        version: "13.0"
-        directory: ${{ runner.temp }}/llvm
+          channel: stable
+          cache-target: release
+          bins: cargo-nextest
+      env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install Foundry (anvil)
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: foundry-rs/foundry-toolchain@v1
+      with:
+        version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d
+    - name: Install make
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      run: choco install -y make
+#    - uses: KyleMayes/install-llvm-action@v1
+#         if: env.SELF_HOSTED_RUNNERS == 'false'
+#      with:
+#        version: "16.0"
+#        directory: ${{ runner.temp }}/llvm
     - name: Set LIBCLANG_PATH
       run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV
     - name: Run tests in release
-      run: make test-release
+      run: make nextest-release
+    - name: Show cache stats
+      if: env.SELF_HOSTED_RUNNERS == 'true'
+      run: sccache --show-stats
   beacon-chain-tests:
     name: beacon-chain-tests
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
+    # Use self-hosted runners only on the sigp repo.
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest'  }}
+    env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+          bins: cargo-nextest
     - name: Run beacon_chain tests for all known forks
       run: make test-beacon-chain
+    - name: Show cache stats
+      if: env.SELF_HOSTED_RUNNERS == 'true'
+      run: sccache --show-stats
   op-pool-tests:
     name: op-pool-tests
     runs-on: ubuntu-latest
-    needs: cargo-fmt
+    env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+          bins: cargo-nextest
     - name: Run operation_pool tests for all known forks
       run: make test-op-pool
+  network-tests:
+    name: network-tests
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get latest version of stable Rust
+      uses: moonrepo/setup-rust@v1
+      with:
+        channel: stable
+        cache-target: release
+        bins: cargo-nextest
+    - name: Run network tests for all known forks
+      run: make test-network
+  slasher-tests:
+    name: slasher-tests
+    runs-on: ubuntu-latest
+    env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get latest version of stable Rust
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+          bins: cargo-nextest
+    - name: Run slasher tests for all supported backends
+      run: make test-slasher
   debug-tests-ubuntu:
     name: debug-tests-ubuntu
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
+    # Use self-hosted runners only on the sigp repo.
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest'  }}
+    env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          bins: cargo-nextest
+    - name: Install Foundry (anvil)
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: foundry-rs/foundry-toolchain@v1
+      with:
+        version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d
     - name: Run tests in debug
-      run: make test-debug
+      run: make nextest-debug
+    - name: Show cache stats
+      if: env.SELF_HOSTED_RUNNERS == 'true'
+      run: sccache --show-stats
   state-transition-vectors-ubuntu:
     name: state-transition-vectors-ubuntu
     runs-on: ubuntu-latest
-    needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
     - name: Run state_transition_vectors in release.
       run: make run-state-transition-tests
   ef-tests-ubuntu:
     name: ef-tests-ubuntu
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
+    # Use self-hosted runners only on the sigp repo.
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest'  }}
+    env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Run consensus-spec-tests with blst, milagro and fake_crypto
-      run: make test-ef
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+          bins: cargo-nextest
+    - name: Run consensus-spec-tests with blst and fake_crypto
+      run: make nextest-ef
+    - name: Show cache stats
+      if: env.SELF_HOSTED_RUNNERS == 'true'
+      run: sccache --show-stats
   dockerfile-ubuntu:
     name: dockerfile-ubuntu
     runs-on: ubuntu-latest
-    needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
-    - name: Get latest version of stable Rust
-      run: rustup update stable
+    - uses: actions/checkout@v4
     - name: Build the root Dockerfile
       run: docker build --build-arg FEATURES=portable -t lighthouse:local .
     - name: Test the built image
@@ -129,131 +228,171 @@ jobs:
   eth1-simulator-ubuntu:
     name: eth1-simulator-ubuntu
     runs-on: ubuntu-latest
-    needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+    - name: Install Foundry (anvil)
+      uses: foundry-rs/foundry-toolchain@v1
+      with:
+        version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d
     - name: Run the beacon chain sim that starts from an eth1 contract
       run: cargo run --release --bin simulator eth1-sim
+  merge-transition-ubuntu:
+    name: merge-transition-ubuntu
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get latest version of stable Rust
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+    - name: Install Foundry (anvil)
+      uses: foundry-rs/foundry-toolchain@v1
+      with:
+        version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d
+    - name: Run the beacon chain sim and go through the merge transition
+      run: cargo run --release --bin simulator eth1-sim --post-merge
   no-eth1-simulator-ubuntu:
     name: no-eth1-simulator-ubuntu
     runs-on: ubuntu-latest
-    needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
     - name: Run the beacon chain sim without an eth1 connection
       run: cargo run --release --bin simulator no-eth1-sim
   syncing-simulator-ubuntu:
     name: syncing-simulator-ubuntu
     runs-on: ubuntu-latest
-    needs: cargo-fmt
     steps:
-      - uses: actions/checkout@v1
-      - name: Get latest version of stable Rust
-        run: rustup update stable
-      - name: Install ganache-cli
-        run: sudo npm install -g ganache-cli
-      - name: Run the syncing simulator
-        run: cargo run --release --bin simulator syncing-sim
+    - uses: actions/checkout@v4
+    - name: Get latest version of stable Rust
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+    - name: Install Foundry (anvil)
+      uses: foundry-rs/foundry-toolchain@v1
+      with:
+        version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d
+    - name: Run the syncing simulator
+      run: cargo run --release --bin simulator syncing-sim
   doppelganger-protection-test:
-      name: doppelganger-protection-test
-      runs-on: ubuntu-latest
-      needs: cargo-fmt
-      steps:
-          - uses: actions/checkout@v1
-          - name: Get latest version of stable Rust
-            run: rustup update stable
-          - name: Install ganache-cli
-            run: sudo npm install -g ganache-cli
-          - name: Install lighthouse and lcli
-            run: |
-                make
-                make install-lcli
-          - name: Run the doppelganger protection success test script
-            run: |
-                cd scripts/tests
-                ./doppelganger_protection.sh success
-          - name: Run the doppelganger protection failure test script
-            run: |
-                cd scripts/tests
-                ./doppelganger_protection.sh failure
-  check-benchmarks:
-    name: check-benchmarks
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
+    name: doppelganger-protection-test
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest'  }}
+    env:
+      # Enable portable to prevent issues with caching `blst` for the wrong CPU type
+      FEATURES: jemalloc,portable
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Typecheck benchmark code without running it
-      run: make check-benches
-  check-consensus:
-    name: check-consensus
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+    - name: Install geth
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      run: |
+          sudo add-apt-repository -y ppa:ethereum/ethereum
+          sudo apt-get update
+          sudo apt-get install ethereum
+    - name: Install lighthouse
+      run: |
+          make
+    - name: Install lcli
+      # TODO: uncomment after the version of lcli in https://github.com/sigp/lighthouse/pull/5137
+      # is installed on the runners
+      # if: env.SELF_HOSTED_RUNNERS == 'false'
+      run: make install-lcli
+    - name: Run the doppelganger protection failure test script
+      run: |
+          cd scripts/tests
+          ./doppelganger_protection.sh failure genesis.json
+    - name: Run the doppelganger protection success test script
+      run: |
+          cd scripts/tests
+          ./doppelganger_protection.sh success genesis.json
+  execution-engine-integration-ubuntu:
+    name: execution-engine-integration-ubuntu
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest'  }}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Typecheck consensus code in strict mode
-      run: make check-consensus
-  clippy:
-    name: clippy
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+          cache: false
+      env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Add go compiler to $PATH
+      if: env.SELF_HOSTED_RUNNERS == 'true'
+      run: echo "/usr/local/go/bin" >> $GITHUB_PATH
+    - name: Run exec engine integration tests in release
+      run: make test-exec-engine
+  check-code:
+    name: check-code
     runs-on: ubuntu-latest
-    needs: cargo-fmt
+    env:
+      CARGO_INCREMENTAL: 1
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Get latest version of stable Rust
-      run: rustup update stable
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+          components: rustfmt,clippy
+          bins: cargo-audit
+    - name: Check formatting with cargo fmt
+      run: make cargo-fmt
     - name: Lint code for quality and style with Clippy
       run: make lint
     - name: Certify Cargo.lock freshness
       run: git diff --exit-code Cargo.lock
-  arbitrary-check:
-    name: arbitrary-check
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
-    steps:
-    - uses: actions/checkout@v1
-    - name: Get latest version of stable Rust
-      run: rustup update stable
+    - name: Typecheck benchmark code without running it
+      run: make check-benches
     - name: Validate state_processing feature arbitrary-fuzz
       run:  make arbitrary-fuzz
-  cargo-audit:
-    name: cargo-audit
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
-    steps:
-    - uses: actions/checkout@v1
-    - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Run cargo audit to identify known security vulnerabilities reported to the RustSec Advisory Database
-      run:  make audit
-  cargo-vendor:
-    name: cargo-vendor
-    runs-on: ubuntu-latest
-    needs: cargo-fmt
-    steps:
-    - uses: actions/checkout@v1
+    - name: Run cargo audit
+      run: make audit-CI
     - name: Run cargo vendor to make sure dependencies can be vendored for packaging, reproducibility and archival purpose
       run:  CARGO_HOME=$(readlink -f $HOME) make vendor
+  check-msrv:
+    name: check-msrv
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Rust at Minimum Supported Rust Version (MSRV)
+      run: |
+        metadata=$(cargo metadata --no-deps --format-version 1)
+        msrv=$(echo $metadata | jq -r '.packages | map(select(.name == "lighthouse")) | .[0].rust_version')
+        rustup override set $msrv
+    - name: Run cargo check
+      run: cargo check --workspace
   cargo-udeps:
     name: cargo-udeps
     runs-on: ubuntu-latest
-    needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
-    - name: Install Rust (${{ env.PINNED_NIGHTLY }})
-      run: rustup toolchain install $PINNED_NIGHTLY
-    - name: Install cargo-udeps
-      run: cargo install cargo-udeps --locked
+    - uses: actions/checkout@v4
+    - name: Get latest version of nightly Rust
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: nightly
+          bins: cargo-udeps
+          cache: false
+      env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Create Cargo config dir
       run: mkdir -p .cargo
     - name: Install custom Cargo config
@@ -263,3 +402,59 @@ jobs:
     env:
       # Allow warnings on Nightly
       RUSTFLAGS: ""
+  compile-with-beta-compiler:
+    name: compile-with-beta-compiler
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install dependencies
+      run: sudo apt update && sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang
+    - name: Use Rust beta
+      run: rustup override set beta
+    - name: Run make
+      run: make
+  cli-check:
+    name: cli-check
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get latest version of stable Rust
+      uses: moonrepo/setup-rust@v1
+      with:
+        channel: stable
+        cache-target: release
+    - name: Run Makefile to trigger the bash script
+      run: make cli
+  # This job succeeds ONLY IF all others succeed. It is used by the merge queue to determine whether
+  # a PR is safe to merge. New jobs should be added here.
+  test-suite-success:
+    name: test-suite-success
+    runs-on: ubuntu-latest
+    needs: [
+      'target-branch-check',
+      'release-tests-ubuntu',
+      'release-tests-windows',
+      'beacon-chain-tests',
+      'op-pool-tests',
+      'network-tests',
+      'slasher-tests',
+      'debug-tests-ubuntu',
+      'state-transition-vectors-ubuntu',
+      'ef-tests-ubuntu',
+      'dockerfile-ubuntu',
+      'eth1-simulator-ubuntu',
+      'merge-transition-ubuntu',
+      'no-eth1-simulator-ubuntu',
+      'syncing-simulator-ubuntu',
+      'doppelganger-protection-test',
+      'execution-engine-integration-ubuntu',
+      'check-code',
+      'check-msrv',
+      'cargo-udeps',
+      'compile-with-beta-compiler',
+      'cli-check',
+    ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check that success job is dependent on all others
+        run: ./scripts/ci/check-success-job.sh ./.github/workflows/test-suite.yml test-suite-success

.gitignore

@@ -1,4 +1,5 @@
 target/
+vendor/
 **/*.rs.bk
 *.pk
 *.sk
@@ -8,3 +9,12 @@ perf.data*
 *.tar.gz
 /bin
 genesis.ssz
+/clippy.toml
+/.cargo
+
+# IntelliJ
+/*.iml
+.idea
+
+# VSCode
+/.vscode

CONTRIBUTING.md

@@ -1,4 +1,5 @@
 # Contributors Guide
+[![GitPOAP badge](https://public-api.gitpoap.io/v1/repo/sigp/lighthouse/badge)](https://www.gitpoap.io/gh/sigp/lighthouse)
 
 Lighthouse is an open-source Ethereum 2.0 client. We're community driven and
 welcome all contribution. We aim to provide a constructive, respectful and fun
@@ -44,8 +45,8 @@ questions.
 2. **Work in a feature branch** of your personal fork
    (github.com/YOUR_NAME/lighthouse) of the main repository
    (github.com/sigp/lighthouse).
-3. Once you feel you have addressed the issue, **create a pull-request** to merge
-   your changes in to the main repository.
+3. Once you feel you have addressed the issue, **create a pull-request** with
+   `unstable` as the base branch to merge your changes into the main repository.
 4. Wait for the repository maintainers to **review your changes** to ensure the
    issue is addressed satisfactorily. Optionally, mention your PR on
 [discord](https://discord.gg/cyAszAh).

Cargo.toml

@@ -4,9 +4,12 @@ members = [
 
     "beacon_node",
     "beacon_node/beacon_chain",
+    "beacon_node/beacon_processor",
+    "beacon_node/builder_client",
     "beacon_node/client",
     "beacon_node/eth1",
     "beacon_node/lighthouse_network",
+    "beacon_node/lighthouse_network/gossipsub",
     "beacon_node/execution_layer",
     "beacon_node/http_api",
     "beacon_node/http_metrics",
@@ -14,7 +17,7 @@ members = [
     "beacon_node/store",
     "beacon_node/timer",
 
-	"boot_node",
+    "boot_node",
 
     "common/account_utils",
     "common/clap_utils",
@@ -27,39 +30,38 @@ members = [
     "common/eth2_interop_keypairs",
     "common/eth2_network_config",
     "common/eth2_wallet_manager",
-    "common/hashset_delay",
     "common/lighthouse_metrics",
     "common/lighthouse_version",
     "common/lockfile",
     "common/logging",
     "common/lru_cache",
     "common/malloc_utils",
+    "common/oneshot_broadcast",
+    "common/pretty_reqwest_error",
+    "common/promise_cache",
     "common/sensitive_url",
     "common/slot_clock",
+    "common/system_health",
     "common/task_executor",
     "common/target_check",
     "common/test_random_derive",
+    "common/unused_port",
     "common/validator_dir",
     "common/warp_utils",
-    "common/fallback",
     "common/monitoring_api",
 
+    "database_manager",
+
     "consensus/cached_tree_hash",
     "consensus/int_to_bytes",
     "consensus/fork_choice",
     "consensus/proto_array",
     "consensus/safe_arith",
-    "consensus/ssz",
-    "consensus/ssz_derive",
-    "consensus/ssz_types",
-    "consensus/serde_utils",
     "consensus/state_processing",
     "consensus/swap_or_not_shuffle",
-    "consensus/tree_hash",
-    "consensus/tree_hash_derive",
 
     "crypto/bls",
-    "crypto/eth2_hashing",
+    "crypto/kzg",
     "crypto/eth2_key_derivation",
     "crypto/eth2_keystore",
     "crypto/eth2_wallet",
@@ -74,6 +76,7 @@ members = [
 
     "testing/ef_tests",
     "testing/eth1_test_rig",
+    "testing/execution_engine_integration",
     "testing/node_test_rig",
     "testing/simulator",
     "testing/test-test_logger",
@@ -82,13 +85,164 @@ members = [
 
     "validator_client",
     "validator_client/slashing_protection",
-]
 
-[patch]
+    "validator_manager",
+
+    "watch",
+]
+resolver = "2"
+
+[workspace.package]
+edition = "2021"
+
+[workspace.dependencies]
+anyhow = "1"
+arbitrary = { version = "1", features = ["derive"] }
+async-channel = "1.9.0"
+bincode = "1"
+bitvec = "1"
+byteorder = "1"
+bytes = "1"
+clap = "2"
+compare_fields_derive = { path = "common/compare_fields_derive" }
+criterion = "0.3"
+crossbeam-channel = "0.5.8"
+delay_map = "0.3"
+derivative = "2"
+dirs = "3"
+either = "1.9"
+discv5 = { version = "0.4.1", features = ["libp2p"] }
+env_logger = "0.9"
+error-chain = "0.12"
+ethereum-types = "0.14"
+ethereum_hashing = "1.0.0-beta.2"
+ethereum_serde_utils = "0.5.2"
+ethereum_ssz = "0.5"
+ethereum_ssz_derive = "0.5"
+ethers-core = "1"
+ethers-providers = { version = "1", default-features = false }
+exit-future = "0.2"
+fnv = "1"
+fs2 = "0.4"
+futures = "0.3"
+hex = "0.4"
+hyper = "1"
+itertools = "0.10"
+lazy_static = "1"
+libsecp256k1 = "0.7"
+log = "0.4"
+lru = "0.12"
+maplit = "1"
+milhouse = { git = "https://github.com/sigp/milhouse", branch = "main" }
+num_cpus = "1"
+parking_lot = "0.12"
+paste = "1"
+quickcheck = "1"
+quickcheck_macros = "1"
+quote = "1"
+r2d2 = "0.8"
+rand = "0.8"
+rayon = "1.7"
+regex = "1"
+reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "stream", "rustls-tls", "native-tls-vendored"] }
+ring = "0.16"
+rusqlite = { version = "0.28", features = ["bundled"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+serde_repr = "0.1"
+serde_yaml = "0.9"
+sha2 = "0.9"
+slog = { version = "2", features = ["max_level_trace", "release_max_level_trace", "nested-values"] }
+slog-async = "2"
+slog-term = "2"
+sloggers = { version = "2", features = ["json"] }
+smallvec = "1.11.2"
+snap = "1"
+ssz_types = "0.5"
+strum = { version = "0.24", features = ["derive"] }
+superstruct = "0.7"
+syn = "1"
+sysinfo = "0.26"
+tempfile = "3"
+tokio = { version = "1", features = ["rt-multi-thread", "sync", "signal"] }
+tokio-stream = { version = "0.1", features = ["sync"] }
+tokio-util = { version = "0.6", features = ["codec", "compat", "time"] }
+tracing = "0.1.40"
+tracing-appender = "0.2"
+tracing-core = "0.1"
+tracing-log = "0.2"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tree_hash = "0.5"
+tree_hash_derive = "0.5"
+url = "2"
+uuid = { version = "0.8", features = ["serde", "v4"] }
+warp = { version = "0.3.6", default-features = false, features = ["tls"] }
+xdelta3 = { git = "http://github.com/michaelsproul/xdelta3-rs", rev="ae9a1d2585ef998f4656acdc35cf263ee88e6dfa" }
+zeroize = { version = "1", features = ["zeroize_derive"] }
+zip = "0.6"
+zstd = "0.11.2"
+
+# Local crates.
+account_utils = { path = "common/account_utils" }
+beacon_chain = { path = "beacon_node/beacon_chain" }
+beacon_node = { path = "beacon_node" }
+beacon_processor =  { path = "beacon_node/beacon_processor" }
+bls = { path = "crypto/bls" }
+cached_tree_hash = { path = "consensus/cached_tree_hash" }
+clap_utils = { path = "common/clap_utils" }
+compare_fields = { path = "common/compare_fields" }
+deposit_contract = { path = "common/deposit_contract" }
+directory = { path = "common/directory" }
+environment = { path = "lighthouse/environment" }
+eth1 = { path = "beacon_node/eth1" }
+eth1_test_rig = { path = "testing/eth1_test_rig" }
+eth2 = { path = "common/eth2" }
+eth2_config = { path = "common/eth2_config" }
+eth2_key_derivation = { path = "crypto/eth2_key_derivation" }
+eth2_keystore = { path = "crypto/eth2_keystore" }
+eth2_network_config = { path = "common/eth2_network_config" }
+eth2_wallet = { path = "crypto/eth2_wallet" }
+execution_layer = { path = "beacon_node/execution_layer" }
+filesystem = { path = "common/filesystem" }
+fork_choice = { path = "consensus/fork_choice" }
+genesis = { path = "beacon_node/genesis" }
+gossipsub = { path = "beacon_node/lighthouse_network/gossipsub/" }
+http_api = { path = "beacon_node/http_api" }
+int_to_bytes = { path = "consensus/int_to_bytes" }
+kzg = { path = "crypto/kzg" }
+lighthouse_metrics = { path = "common/lighthouse_metrics" }
+lighthouse_network = { path = "beacon_node/lighthouse_network" }
+lighthouse_version = { path = "common/lighthouse_version" }
+lockfile = { path = "common/lockfile" }
+logging = { path = "common/logging" }
+lru_cache = { path = "common/lru_cache" }
+malloc_utils = { path = "common/malloc_utils" }
+merkle_proof = { path = "consensus/merkle_proof" }
+monitoring_api = { path = "common/monitoring_api" }
+network = { path = "beacon_node/network" }
+operation_pool = { path = "beacon_node/operation_pool" }
+pretty_reqwest_error = { path = "common/pretty_reqwest_error" }
+proto_array = { path = "consensus/proto_array" }
+safe_arith = {path = "consensus/safe_arith"}
+sensitive_url = { path = "common/sensitive_url" }
+slasher = { path = "slasher" }
+slashing_protection = { path = "validator_client/slashing_protection" }
+slot_clock = { path = "common/slot_clock" }
+state_processing = { path = "consensus/state_processing" }
+store = { path = "beacon_node/store" }
+swap_or_not_shuffle = { path = "consensus/swap_or_not_shuffle" }
+task_executor = { path = "common/task_executor" }
+types = { path = "consensus/types" }
+unused_port = { path = "common/unused_port" }
+validator_client = { path = "validator_client" }
+validator_dir = { path = "common/validator_dir" }
+warp_utils = { path = "common/warp_utils" }
+
 [patch.crates-io]
-fixed-hash = { git = "https://github.com/paritytech/parity-common", rev="df638ab0885293d21d656dc300d39236b69ce57d" }
-warp = { git = "https://github.com/macladson/warp", rev ="7e75acc" }
-eth2_ssz = { path = "consensus/ssz" }
-eth2_ssz_types = { path = "consensus/ssz_types" }
-tree_hash = { path = "consensus/tree_hash" }
-eth2_serde_utils = { path = "consensus/serde_utils" }
+yamux = { git = "https://github.com/sigp/rust-yamux.git" }
+
+[profile.maxperf]
+inherits = "release"
+lto = "fat"
+codegen-units = 1
+incremental = false

Cross.toml

@@ -1,15 +1,5 @@
-[build.env]
-passthrough = [
-    "RUSTFLAGS",
-]
-
-# These custom images are required to work around the lack of Clang in the default `cross` images.
-# We need Clang to run `bindgen` for MDBX, and the `BINDGEN_EXTRA_CLANG_ARGS` flags must also be set
-# while cross-compiling for ARM to prevent bindgen from attempting to include headers from the host.
-#
-# For more information see https://github.com/rust-embedded/cross/pull/608
 [target.x86_64-unknown-linux-gnu]
-image = "michaelsproul/cross-clang:x86_64-latest"
+pre-build = ["apt-get install -y cmake clang-5.0"]
 
 [target.aarch64-unknown-linux-gnu]
-image = "michaelsproul/cross-clang:aarch64-latest"
+pre-build = ["apt-get install -y cmake clang-5.0"]

Dockerfile

@@ -1,11 +1,15 @@
-FROM rust:1.58.1-bullseye AS builder
+FROM rust:1.75.0-bullseye AS builder
 RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev
 COPY . lighthouse
 ARG FEATURES
+ARG PROFILE=release
+ARG CARGO_USE_GIT_CLI=true
 ENV FEATURES $FEATURES
+ENV PROFILE $PROFILE
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_USE_GIT_CLI
 RUN cd lighthouse && make
 
-FROM ubuntu:latest
+FROM ubuntu:22.04
 RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
   libssl-dev \
   ca-certificates \

Dockerfile.cross

@@ -1,7 +1,7 @@
 # This image is meant to enable cross-architecture builds.
 # It assumes the lighthouse binary has already been
 # compiled for `$TARGETPLATFORM` and moved to `./bin`.
-FROM --platform=$TARGETPLATFORM ubuntu:latest
+FROM --platform=$TARGETPLATFORM ubuntu:22.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   libssl-dev \
   ca-certificates \

Makefile

@@ -2,6 +2,7 @@
 
 EF_TESTS = "testing/ef_tests"
 STATE_TRANSITION_VECTORS = "testing/state_transition_vectors"
+EXECUTION_ENGINE_INTEGRATION = "testing/execution_engine_integration"
 GIT_TAG := $(shell git describe --tags --candidates 1)
 BIN_DIR = "bin"
 
@@ -11,20 +12,53 @@ AARCH64_TAG = "aarch64-unknown-linux-gnu"
 BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
 
 PINNED_NIGHTLY ?= nightly
+CLIPPY_PINNED_NIGHTLY=nightly-2022-05-19
+
+# List of features to use when building natively. Can be overridden via the environment.
+# No jemalloc on Windows
+ifeq ($(OS),Windows_NT)
+    FEATURES?=
+else
+    FEATURES?=jemalloc
+endif
+
+# List of features to use when cross-compiling. Can be overridden via the environment.
+CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,jemalloc
+
+# Cargo profile for Cross builds. Default is for local builds, CI uses an override.
+CROSS_PROFILE ?= release
+
+# List of features to use when running EF tests.
+EF_TEST_FEATURES ?=
+
+# List of features to use when running CI tests.
+TEST_FEATURES ?=
+
+# Cargo profile for regular builds.
+PROFILE ?= release
 
 # List of all hard forks. This list is used to set env variables for several tests so that
 # they run for different forks.
-FORKS=phase0 altair
+FORKS=phase0 altair merge capella deneb electra
+
+# Extra flags for Cargo
+CARGO_INSTALL_EXTRA_FLAGS?=
 
 # Builds the Lighthouse binary in release (optimized).
 #
 # Binaries will most likely be found in `./target/release`
 install:
-	cargo install --path lighthouse --force --locked --features "$(FEATURES)"
+	cargo install --path lighthouse --force --locked \
+		--features "$(FEATURES)" \
+		--profile "$(PROFILE)" \
+		$(CARGO_INSTALL_EXTRA_FLAGS)
 
 # Builds the lcli binary in release (optimized).
 install-lcli:
-	cargo install --path lcli --force --locked --features "$(FEATURES)"
+	cargo install --path lcli --force --locked \
+		--features "$(FEATURES)" \
+		--profile "$(PROFILE)" \
+		$(CARGO_INSTALL_EXTRA_FLAGS)
 
 # The following commands use `cross` to build a cross-compile.
 #
@@ -40,13 +74,13 @@ install-lcli:
 # optimized CPU functions that may not be available on some systems. This
 # results in a more portable binary with ~20% slower BLS verification.
 build-x86_64:
-	cross build --release --bin lighthouse --target x86_64-unknown-linux-gnu --features modern,gnosis
+	cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "modern,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked
 build-x86_64-portable:
-	cross build --release --bin lighthouse --target x86_64-unknown-linux-gnu --features portable,gnosis
+	cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked
 build-aarch64:
-	cross build --release --bin lighthouse --target aarch64-unknown-linux-gnu --features gnosis
+	cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked
 build-aarch64-portable:
-	cross build --release --bin lighthouse --target aarch64-unknown-linux-gnu --features portable,gnosis
+	cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked
 
 # Create a `.tar.gz` containing a binary for a specific target.
 define tarball_release_binary
@@ -75,12 +109,26 @@ build-release-tarballs:
 # Runs the full workspace tests in **release**, without downloading any additional
 # test vectors.
 test-release:
-	cargo test --workspace --release --exclude ef_tests --exclude beacon_chain
+	cargo test --workspace --release --features "$(TEST_FEATURES)" \
+ 		--exclude ef_tests --exclude beacon_chain --exclude slasher --exclude network
+
+# Runs the full workspace tests in **release**, without downloading any additional
+# test vectors, using nextest.
+nextest-release:
+	cargo nextest run --workspace --release --features "$(TEST_FEATURES)" \
+		--exclude ef_tests --exclude beacon_chain --exclude slasher --exclude network
 
 # Runs the full workspace tests in **debug**, without downloading any additional test
 # vectors.
 test-debug:
-	cargo test --workspace --exclude ef_tests --exclude beacon_chain
+	cargo test --workspace --features "$(TEST_FEATURES)" \
+		--exclude ef_tests --exclude beacon_chain --exclude network
+
+# Runs the full workspace tests in **debug**, without downloading any additional test
+# vectors, using nextest.
+nextest-debug:
+	cargo nextest run --workspace --features "$(TEST_FEATURES)" \
+		--exclude ef_tests --exclude beacon_chain --exclude network
 
 # Runs cargo-fmt (linter).
 cargo-fmt:
@@ -88,34 +136,50 @@ cargo-fmt:
 
 # Typechecks benchmark code
 check-benches:
-	cargo check --workspace --benches
-
-# Typechecks consensus code *without* allowing deprecated legacy arithmetic or metrics.
-check-consensus:
-	cargo check -p state_processing --no-default-features
+	cargo check --workspace --benches --features "$(TEST_FEATURES)"
 
 # Runs only the ef-test vectors.
 run-ef-tests:
 	rm -rf $(EF_TESTS)/.accessed_file_log.txt
-	cargo test --release -p ef_tests --features "ef_tests"
-	cargo test --release -p ef_tests --features "ef_tests,fake_crypto"
-	cargo test --release -p ef_tests --features "ef_tests,milagro"
+	cargo test --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES)"
+	cargo test --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES),fake_crypto"
+	./$(EF_TESTS)/check_all_files_accessed.py $(EF_TESTS)/.accessed_file_log.txt $(EF_TESTS)/consensus-spec-tests
+
+# Runs EF test vectors with nextest
+nextest-run-ef-tests:
+	rm -rf $(EF_TESTS)/.accessed_file_log.txt
+	cargo nextest run --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES)"
+	cargo nextest run --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES),fake_crypto"
 	./$(EF_TESTS)/check_all_files_accessed.py $(EF_TESTS)/.accessed_file_log.txt $(EF_TESTS)/consensus-spec-tests
 
 # Run the tests in the `beacon_chain` crate for all known forks.
 test-beacon-chain: $(patsubst %,test-beacon-chain-%,$(FORKS))
 
 test-beacon-chain-%:
-	env FORK_NAME=$* cargo test --release --features fork_from_env -p beacon_chain
+	env FORK_NAME=$* cargo nextest run --release --features "fork_from_env,slasher/lmdb,$(TEST_FEATURES)" -p beacon_chain
 
 # Run the tests in the `operation_pool` crate for all known forks.
 test-op-pool: $(patsubst %,test-op-pool-%,$(FORKS))
 
 test-op-pool-%:
-	env FORK_NAME=$* cargo test --release \
-		--features 'beacon_chain/fork_from_env'\
+	env FORK_NAME=$* cargo nextest run --release \
+		--features "beacon_chain/fork_from_env,$(TEST_FEATURES)"\
 		-p operation_pool
 
+# Run the tests in the `network` crate for all known forks.
+test-network: $(patsubst %,test-network-%,$(FORKS))
+
+test-network-%:
+	env FORK_NAME=$* cargo nextest run --release \
+		--features "fork_from_env,$(TEST_FEATURES)" \
+		-p network
+
+# Run the tests in the `slasher` crate for all supported database backends.
+test-slasher:
+	cargo nextest run --release -p slasher --features "lmdb,$(TEST_FEATURES)"
+	cargo nextest run --release -p slasher --no-default-features --features "mdbx,$(TEST_FEATURES)"
+	cargo nextest run --release -p slasher --features "lmdb,mdbx,$(TEST_FEATURES)" # both backends enabled
+
 # Runs only the tests/state_transition_vectors tests.
 run-state-transition-tests:
 	make -C $(STATE_TRANSITION_VECTORS) test
@@ -123,22 +187,56 @@ run-state-transition-tests:
 # Downloads and runs the EF test vectors.
 test-ef: make-ef-tests run-ef-tests
 
+# Downloads and runs the EF test vectors with nextest.
+nextest-ef: make-ef-tests nextest-run-ef-tests
+
+# Runs tests checking interop between Lighthouse and execution clients.
+test-exec-engine:
+	make -C $(EXECUTION_ENGINE_INTEGRATION) test
+
 # Runs the full workspace tests in release, without downloading any additional
 # test vectors.
 test: test-release
 
+# Updates the CLI help text pages in the Lighthouse book, building with Docker.
+cli:
+	docker run --rm --user=root \
+	-v ${PWD}:/home/runner/actions-runner/lighthouse sigmaprime/github-runner \
+	bash -c 'cd lighthouse && make && ./scripts/cli.sh'
+
+# Updates the CLI help text pages in the Lighthouse book, building using local
+# `cargo`.
+cli-local:
+	make && ./scripts/cli.sh
+
 # Runs the entire test suite, downloading test vectors if required.
-test-full: cargo-fmt test-release test-debug test-ef
+test-full: cargo-fmt test-release test-debug test-ef test-exec-engine
 
 # Lints the code for bad style and potentially unsafe arithmetic using Clippy.
 # Clippy lints are opt-in per-crate for now. By default, everything is allowed except for performance and correctness lints.
 lint:
-	cargo clippy --workspace --tests -- \
+	cargo clippy --workspace --tests $(EXTRA_CLIPPY_OPTS) --features "$(TEST_FEATURES)" -- \
 		-D clippy::fn_to_numeric_cast_any \
+		-D clippy::manual_let_else \
 		-D warnings \
+		-A clippy::derive_partial_eq_without_eq \
 		-A clippy::from-over-into \
 		-A clippy::upper-case-acronyms \
-		-A clippy::vec-init-then-push
+		-A clippy::vec-init-then-push \
+		-A clippy::question-mark \
+		-A clippy::uninlined-format-args \
+		-A clippy::enum_variant_names
+
+# Lints the code using Clippy and automatically fix some simple compiler warnings.
+lint-fix:
+	EXTRA_CLIPPY_OPTS="--fix --allow-staged --allow-dirty" $(MAKE) lint
+
+nightly-lint:
+	cp .github/custom/clippy.toml .
+	cargo +$(CLIPPY_PINNED_NIGHTLY) clippy --workspace --tests --release -- \
+		-A clippy::all \
+		-D clippy::disallowed_from_async
+	rm clippy.toml
 
 # Runs the makefile in the `ef_tests` repo.
 #
@@ -150,13 +248,17 @@ make-ef-tests:
 
 # Verifies that crates compile with fuzzing features enabled
 arbitrary-fuzz:
-	cargo check -p state_processing --features arbitrary-fuzz
-	cargo check -p slashing_protection --features arbitrary-fuzz
+	cargo check -p state_processing --features arbitrary-fuzz,$(TEST_FEATURES)
+	cargo check -p slashing_protection --features arbitrary-fuzz,$(TEST_FEATURES)
 
 # Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database)
-audit:
+audit: install-audit audit-CI
+
+install-audit:
 	cargo install --force cargo-audit
-	cargo audit --ignore RUSTSEC-2020-0071 --ignore RUSTSEC-2020-0159 --ignore RUSTSEC-2022-0009
+
+audit-CI:
+	cargo audit
 
 # Runs `cargo vendor` to make sure dependencies can be vendored for packaging, reproducibility and archival purpose.
 vendor:
@@ -164,7 +266,7 @@ vendor:
 
 # Runs `cargo udeps` to check for unused dependencies
 udeps:
-	cargo +$(PINNED_NIGHTLY) udeps --tests --all-targets --release
+	cargo +$(PINNED_NIGHTLY) udeps --tests --all-targets --release --features "$(TEST_FEATURES)"
 
 # Performs a `cargo` clean and cleans the `ef_tests` directory.
 clean:

README.md

@@ -1,18 +1,16 @@
-# Lighthouse: Ethereum 2.0
+# Lighthouse: Ethereum consensus client
 
-An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prime.
+An open-source Ethereum consensus client, written in Rust and maintained by Sigma Prime.
 
-[![Build Status]][Build Link] [![Book Status]][Book Link] [![Chat Badge]][Chat Link]
+[![Book Status]][Book Link] [![Chat Badge]][Chat Link]
 
-[Build Status]: https://github.com/sigp/lighthouse/workflows/test-suite/badge.svg?branch=stable
-[Build Link]: https://github.com/sigp/lighthouse/actions
 [Chat Badge]: https://img.shields.io/badge/chat-discord-%237289da
 [Chat Link]: https://discord.gg/cyAszAh
 [Book Status]:https://img.shields.io/badge/user--docs-unstable-informational
 [Book Link]: https://lighthouse-book.sigmaprime.io
 [stable]: https://github.com/sigp/lighthouse/tree/stable
 [unstable]: https://github.com/sigp/lighthouse/tree/unstable
-[blog]: https://lighthouse.sigmaprime.io
+[blog]: https://lighthouse-blog.sigmaprime.io
 
 [Documentation](https://lighthouse-book.sigmaprime.io)
 
@@ -22,7 +20,7 @@ An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prim
 
 Lighthouse is:
 
-- Ready for use on Eth2 mainnet.
+- Ready for use on Ethereum consensus mainnet.
 - Fully open-source, licensed under Apache 2.0.
 - Security-focused. Fuzzing techniques have been continuously applied and several external security reviews have been performed.
 - Built in [Rust](https://www.rust-lang.org), a modern language providing unique safety guarantees and
@@ -30,20 +28,20 @@ Lighthouse is:
 - Funded by various organisations, including Sigma Prime, the
 	Ethereum Foundation, ConsenSys, the Decentralization Foundation and private individuals.
 - Actively involved in the specification and security analysis of the
-	Ethereum 2.0 specification.
+	Ethereum proof-of-stake consensus specification.
 
-## Eth2 Deposit Contract
+## Staking Deposit Contract
 
 The Lighthouse team acknowledges
 [`0x00000000219ab540356cBB839Cbe05303d7705Fa`](https://etherscan.io/address/0x00000000219ab540356cbb839cbe05303d7705fa)
-as the canonical Eth2 deposit contract address.
+as the canonical staking deposit contract address.
 
 ## Documentation
 
 The [Lighthouse Book](https://lighthouse-book.sigmaprime.io) contains information for users and
 developers.
 
-The Lighthouse team maintains a blog at [lighthouse.sigmaprime.io][blog] which contains periodical
+The Lighthouse team maintains a blog at [lighthouse-blog.sigmaprime.io][blog] which contains periodic
 progress updates, roadmap insights and interesting findings.
 
 ## Branches
@@ -66,9 +64,9 @@ of the Lighthouse book.
 ## Contact
 
 The best place for discussion is the [Lighthouse Discord
-server](https://discord.gg/cyAszAh). 
+server](https://discord.gg/cyAszAh).
 
-Sign up to the [Lighthouse Development Updates](http://eepurl.com/dh9Lvb) mailing list for email
+Sign up to the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb) mailing list for email
 notifications about releases, network status and other important information.
 
 Encrypt sensitive messages using our [PGP

account_manager/Cargo.toml

@@ -1,29 +1,35 @@
 [package]
 name = "account_manager"
 version = "0.3.5"
-authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
-edition = "2018"
+authors = [
+    "Paul Hauner <paul@paulhauner.com>",
+    "Luke Anderson <luke@sigmaprime.io>",
+]
+edition = { workspace = true }
 
 [dependencies]
-bls = { path = "../crypto/bls" }
-clap = "2.33.3"
-types = { path = "../consensus/types" }
-environment = { path = "../lighthouse/environment" }
-eth2_network_config = { path = "../common/eth2_network_config" }
-clap_utils = { path = "../common/clap_utils" }
-directory = { path = "../common/directory" }
-eth2_wallet = { path = "../crypto/eth2_wallet" }
+bls = { workspace = true }
+clap = { workspace = true }
+types = { workspace = true }
+environment = { workspace = true }
+eth2_network_config = { workspace = true }
+clap_utils = { workspace = true }
+directory = { workspace = true }
+eth2_wallet = { workspace = true }
 eth2_wallet_manager = { path = "../common/eth2_wallet_manager" }
-validator_dir = { path = "../common/validator_dir" }
-tokio = { version = "1.14.0", features = ["full"] }
-eth2_keystore = { path = "../crypto/eth2_keystore" }
-account_utils = { path = "../common/account_utils" }
-slashing_protection = { path = "../validator_client/slashing_protection" }
-eth2 = {path = "../common/eth2"}
-safe_arith = {path = "../consensus/safe_arith"}
-slot_clock = { path = "../common/slot_clock" }
-filesystem = { path = "../common/filesystem" }
-sensitive_url = { path = "../common/sensitive_url" }
+validator_dir = { workspace = true }
+tokio = { workspace = true }
+eth2_keystore = { workspace = true }
+account_utils = { workspace = true }
+slashing_protection = { workspace = true }
+eth2 = { workspace = true }
+safe_arith = { workspace = true }
+slot_clock = { workspace = true }
+filesystem = { workspace = true }
+sensitive_url = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+slog = { workspace = true }
 
 [dev-dependencies]
-tempfile = "3.1.0"
+tempfile = { workspace = true }

account_manager/README.md

@@ -29,6 +29,6 @@ Simply run `./account_manager generate` to generate a new random private key,
 which will be automatically saved to the correct directory.
 
 If you prefer to use our "deterministic" keys for testing purposes, simply
-run `./accounts_manager generate_deterministic -i <index>`, where `index` is
+run `./account_manager generate_deterministic -i <index>`, where `index` is
 the validator index for the key. This will reliably produce the same key each time
-and save it to the directory.
+and save it to the directory.

account_manager/src/common.rs

@@ -1,55 +1,7 @@
-use account_utils::PlainText;
-use account_utils::{read_input_from_user, strip_off_newlines};
-use eth2_wallet::bip39::{Language, Mnemonic};
-use std::fs;
-use std::path::PathBuf;
-use std::str::from_utf8;
-use std::thread::sleep;
-use std::time::Duration;
+use account_utils::read_input_from_user;
 
-pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
 pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:";
 
-pub fn read_mnemonic_from_cli(
-    mnemonic_path: Option<PathBuf>,
-    stdin_inputs: bool,
-) -> Result<Mnemonic, String> {
-    let mnemonic = match mnemonic_path {
-        Some(path) => fs::read(&path)
-            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
-            .and_then(|bytes| {
-                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
-                let phrase = from_utf8(bytes_no_newlines.as_ref())
-                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
-                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
-                    format!(
-                        "Unable to derive mnemonic from string {:?}: {:?}",
-                        phrase, e
-                    )
-                })
-            })?,
-        None => loop {
-            eprintln!();
-            eprintln!("{}", MNEMONIC_PROMPT);
-
-            let mnemonic = read_input_from_user(stdin_inputs)?;
-
-            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
-                Ok(mnemonic_m) => {
-                    eprintln!("Valid mnemonic provided.");
-                    eprintln!();
-                    sleep(Duration::from_secs(1));
-                    break mnemonic_m;
-                }
-                Err(_) => {
-                    eprintln!("Invalid mnemonic");
-                }
-            }
-        },
-    };
-    Ok(mnemonic)
-}
-
 /// Reads in a wallet name from the user. If the `--wallet-name` flag is provided, use it. Otherwise
 /// read from an interactive prompt using tty unless the `--stdin-inputs` flag is provided.
 pub fn read_wallet_name_from_cli(

account_manager/src/lib.rs

@@ -10,6 +10,7 @@ use types::EthSpec;
 pub const CMD: &str = "account_manager";
 pub const SECRETS_DIR_FLAG: &str = "secrets-dir";
 pub const VALIDATOR_DIR_FLAG: &str = "validator-dir";
+pub const VALIDATOR_DIR_FLAG_ALIAS: &str = "validators-dir";
 pub const WALLETS_DIR_FLAG: &str = "wallets-dir";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
@@ -21,7 +22,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 }
 
 /// Run the account manager, returning an error if the operation did not succeed.
-pub fn run<T: EthSpec>(matches: &ArgMatches<'_>, env: Environment<T>) -> Result<(), String> {
+pub fn run<E: EthSpec>(matches: &ArgMatches<'_>, env: Environment<E>) -> Result<(), String> {
     match matches.subcommand() {
         (wallet::CMD, Some(matches)) => wallet::cli_run(matches)?,
         (validator::CMD, Some(matches)) => validator::cli_run(matches, env)?,

account_manager/src/validator/create.rs

@@ -112,9 +112,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
 }
 
-pub fn cli_run<T: EthSpec>(
+pub fn cli_run<E: EthSpec>(
     matches: &ArgMatches,
-    mut env: Environment<T>,
+    env: Environment<E>,
     validator_dir: PathBuf,
 ) -> Result<(), String> {
     let spec = env.core_context().eth2_config.spec;

account_manager/src/validator/exit.rs

@@ -14,7 +14,7 @@ use slot_clock::{SlotClock, SystemTimeSlotClock};
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 use tokio::time::sleep;
-use types::{ChainSpec, Epoch, EthSpec, Fork, VoluntaryExit};
+use types::{ChainSpec, Epoch, EthSpec, VoluntaryExit};
 
 pub const CMD: &str = "exit";
 pub const KEYSTORE_FLAG: &str = "keystore";
@@ -27,7 +27,6 @@ pub const PASSWORD_PROMPT: &str = "Enter the keystore password";
 pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/";
 pub const CONFIRMATION_PHRASE: &str = "Exit my validator";
 pub const WEBSITE_URL: &str = "https://lighthouse-book.sigmaprime.io/voluntary-exit.html";
-pub const PROMPT: &str = "WARNING: WITHDRAWING STAKED ETH IS NOT CURRENTLY POSSIBLE";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new("exit")
@@ -124,10 +123,8 @@ async fn publish_voluntary_exit<E: EthSpec>(
 ) -> Result<(), String> {
     let genesis_data = get_geneisis_data(client).await?;
     let testnet_genesis_root = eth2_network_config
-        .beacon_state::<E>()
-        .as_ref()
-        .expect("network should have valid genesis state")
-        .genesis_validators_root();
+        .genesis_validators_root::<E>()?
+        .ok_or("Genesis state is unknown")?;
 
     // Verify that the beacon node and validator being exited are on the same network.
     if genesis_data.genesis_validators_root != testnet_genesis_root {
@@ -149,7 +146,6 @@ async fn publish_voluntary_exit<E: EthSpec>(
         .ok_or("Failed to get current epoch. Please check your system time")?;
     let validator_index = get_validator_index_for_exit(client, &keypair.pk, epoch, spec).await?;
 
-    let fork = get_beacon_state_fork(client).await?;
     let voluntary_exit = VoluntaryExit {
         epoch,
         validator_index,
@@ -161,7 +157,6 @@ async fn publish_voluntary_exit<E: EthSpec>(
     );
     if !no_confirmation {
         eprintln!("WARNING: THIS IS AN IRREVERSIBLE OPERATION\n");
-        eprintln!("{}\n", PROMPT);
         eprintln!(
             "PLEASE VISIT {} TO MAKE SURE YOU UNDERSTAND THE IMPLICATIONS OF A VOLUNTARY EXIT.",
             WEBSITE_URL
@@ -177,12 +172,8 @@ async fn publish_voluntary_exit<E: EthSpec>(
 
     if confirmation == CONFIRMATION_PHRASE {
         // Sign and publish the voluntary exit to network
-        let signed_voluntary_exit = voluntary_exit.sign(
-            &keypair.sk,
-            &fork,
-            genesis_data.genesis_validators_root,
-            spec,
-        );
+        let signed_voluntary_exit =
+            voluntary_exit.sign(&keypair.sk, genesis_data.genesis_validators_root, spec);
         client
             .post_beacon_pool_voluntary_exits(&signed_voluntary_exit)
             .await
@@ -212,8 +203,8 @@ async fn publish_voluntary_exit<E: EthSpec>(
         let validator_data = get_validator_data(client, &keypair.pk).await?;
         match validator_data.status {
             ValidatorStatus::ActiveExiting => {
-                let exit_epoch = validator_data.validator.exit_epoch;
-                let withdrawal_epoch = validator_data.validator.withdrawable_epoch;
+                let exit_epoch = validator_data.validator.exit_epoch();
+                let withdrawal_epoch = validator_data.validator.withdrawable_epoch();
                 let current_epoch = get_current_epoch::<E>(genesis_data.genesis_time, spec)
                     .ok_or("Failed to get current epoch. Please check your system time")?;
                 eprintln!("Voluntary exit has been accepted into the beacon chain, but not yet finalized. \
@@ -233,7 +224,7 @@ async fn publish_voluntary_exit<E: EthSpec>(
             ValidatorStatus::ExitedSlashed | ValidatorStatus::ExitedUnslashed => {
                 eprintln!(
                     "Validator has exited on epoch: {}",
-                    validator_data.validator.exit_epoch
+                    validator_data.validator.exit_epoch()
                 );
                 break;
             }
@@ -259,7 +250,7 @@ async fn get_validator_index_for_exit(
         ValidatorStatus::ActiveOngoing => {
             let eligible_epoch = validator_data
                 .validator
-                .activation_epoch
+                .activation_epoch()
                 .safe_add(spec.shard_committee_period)
                 .map_err(|e| format!("Failed to calculate eligible epoch, validator activation epoch too high: {:?}", e))?;
 
@@ -320,16 +311,6 @@ async fn is_syncing(client: &BeaconNodeHttpClient) -> Result<bool, String> {
         .is_syncing)
 }
 
-/// Get fork object for the current state by querying the beacon node client.
-async fn get_beacon_state_fork(client: &BeaconNodeHttpClient) -> Result<Fork, String> {
-    Ok(client
-        .get_beacon_states_fork(StateId::Head)
-        .await
-        .map_err(|e| format!("Failed to get get fork: {:?}", e))?
-        .ok_or("Failed to get fork, state not found")?
-        .data)
-}
-
 /// Calculates the current epoch from the genesis time and current time.
 fn get_current_epoch<E: EthSpec>(genesis_time: u64, spec: &ChainSpec) -> Option<Epoch> {
     let slot_clock = SystemTimeSlotClock::new(
@@ -349,7 +330,7 @@ fn load_voting_keypair(
     password_file_path: Option<&PathBuf>,
     stdin_inputs: bool,
 ) -> Result<Keypair, String> {
-    let keystore = Keystore::from_json_file(&voting_keystore_path).map_err(|e| {
+    let keystore = Keystore::from_json_file(voting_keystore_path).map_err(|e| {
         format!(
             "Unable to read keystore JSON {:?}: {:?}",
             voting_keystore_path, e

account_manager/src/validator/import.rs

@@ -4,8 +4,8 @@ use account_utils::{
     eth2_keystore::Keystore,
     read_password_from_user,
     validator_definitions::{
-        recursively_find_voting_keystores, ValidatorDefinition, ValidatorDefinitions,
-        CONFIG_FILENAME,
+        recursively_find_voting_keystores, PasswordStorage, ValidatorDefinition,
+        ValidatorDefinitions, CONFIG_FILENAME,
     },
     ZeroizeString,
 };
@@ -176,7 +176,7 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
 
             let password = match keystore_password_path.as_ref() {
                 Some(path) => {
-                    let password_from_file: ZeroizeString = fs::read_to_string(&path)
+                    let password_from_file: ZeroizeString = fs::read_to_string(path)
                         .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))?
                         .into();
                     password_from_file.without_newlines()
@@ -256,7 +256,7 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
             .ok_or_else(|| format!("Badly formatted file name: {:?}", src_keystore))?;
 
         // Copy the keystore to the new location.
-        fs::copy(&src_keystore, &dest_keystore)
+        fs::copy(src_keystore, &dest_keystore)
             .map_err(|e| format!("Unable to copy keystore: {:?}", e))?;
 
         // Register with slashing protection.
@@ -277,9 +277,15 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
         let suggested_fee_recipient = None;
         let validator_def = ValidatorDefinition::new_keystore_with_password(
             &dest_keystore,
-            password_opt,
+            password_opt
+                .map(PasswordStorage::ValidatorDefinitions)
+                .unwrap_or(PasswordStorage::None),
             graffiti,
             suggested_fee_recipient,
+            None,
+            None,
+            None,
+            None,
         )
         .map_err(|e| format!("Unable to create new validator definition: {:?}", e))?;
 

account_manager/src/validator/mod.rs

@@ -6,7 +6,7 @@ pub mod modify;
 pub mod recover;
 pub mod slashing_protection;
 
-use crate::VALIDATOR_DIR_FLAG;
+use crate::{VALIDATOR_DIR_FLAG, VALIDATOR_DIR_FLAG_ALIAS};
 use clap::{App, Arg, ArgMatches};
 use directory::{parse_path_or_default_with_flag, DEFAULT_VALIDATOR_DIR};
 use environment::Environment;
@@ -21,6 +21,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         .arg(
             Arg::with_name(VALIDATOR_DIR_FLAG)
                 .long(VALIDATOR_DIR_FLAG)
+                .alias(VALIDATOR_DIR_FLAG_ALIAS)
                 .value_name("VALIDATOR_DIRECTORY")
                 .help(
                     "The path to search for validator directories. \
@@ -38,7 +39,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         .subcommand(exit::cli_app())
 }
 
-pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
+pub fn cli_run<E: EthSpec>(matches: &ArgMatches, env: Environment<E>) -> Result<(), String> {
     let validator_base_dir = if matches.value_of("datadir").is_some() {
         let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
         path.join(DEFAULT_VALIDATOR_DIR)
@@ -48,7 +49,7 @@ pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<
     eprintln!("validator-dir path: {:?}", validator_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, validator_base_dir),
+        (create::CMD, Some(matches)) => create::cli_run::<E>(matches, env, validator_base_dir),
         (modify::CMD, Some(matches)) => modify::cli_run(matches, validator_base_dir),
         (import::CMD, Some(matches)) => import::cli_run(matches, validator_base_dir),
         (list::CMD, Some(_)) => list::cli_run(validator_base_dir),

account_manager/src/validator/recover.rs

@@ -1,10 +1,9 @@
 use super::create::STORE_WITHDRAW_FLAG;
-use crate::common::read_mnemonic_from_cli;
 use crate::validator::create::COUNT_FLAG;
 use crate::wallet::create::STDIN_INPUTS_FLAG;
 use crate::SECRETS_DIR_FLAG;
 use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
-use account_utils::random_password;
+use account_utils::{random_password, read_mnemonic_from_cli};
 use clap::{App, Arg, ArgMatches};
 use directory::ensure_dir_exists;
 use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR};

account_manager/src/validator/slashing_protection.rs

@@ -7,7 +7,7 @@ use slashing_protection::{
 use std::fs::File;
 use std::path::PathBuf;
 use std::str::FromStr;
-use types::{BeaconState, Epoch, EthSpec, PublicKeyBytes, Slot};
+use types::{Epoch, EthSpec, PublicKeyBytes, Slot};
 
 pub const CMD: &str = "slashing-protection";
 pub const IMPORT_CMD: &str = "import";
@@ -16,7 +16,6 @@ pub const EXPORT_CMD: &str = "export";
 pub const IMPORT_FILE_ARG: &str = "IMPORT-FILE";
 pub const EXPORT_FILE_ARG: &str = "EXPORT-FILE";
 
-pub const MINIFY_FLAG: &str = "minify";
 pub const PUBKEYS_FLAG: &str = "pubkeys";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
@@ -31,16 +30,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                         .value_name("FILE")
                         .help("The slashing protection interchange file to import (.json)"),
                 )
-                .arg(
-                    Arg::with_name(MINIFY_FLAG)
-                        .long(MINIFY_FLAG)
-                        .takes_value(true)
-                        .possible_values(&["false", "true"])
-                        .help(
-                            "Deprecated: Lighthouse no longer requires minification on import \
-                             because it always minifies",
-                        ),
-                ),
         )
         .subcommand(
             App::new(EXPORT_CMD)
@@ -61,45 +50,26 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                              comma-separated. All known keys will be exported if omitted",
                         ),
                 )
-                .arg(
-                    Arg::with_name(MINIFY_FLAG)
-                        .long(MINIFY_FLAG)
-                        .takes_value(true)
-                        .default_value("false")
-                        .possible_values(&["false", "true"])
-                        .help(
-                            "Minify the output file. This will make it smaller and faster to \
-                             import, but not faster to generate.",
-                        ),
-                ),
         )
 }
 
-pub fn cli_run<T: EthSpec>(
+pub fn cli_run<E: EthSpec>(
     matches: &ArgMatches<'_>,
-    env: Environment<T>,
+    env: Environment<E>,
     validator_base_dir: PathBuf,
 ) -> Result<(), String> {
     let slashing_protection_db_path = validator_base_dir.join(SLASHING_PROTECTION_FILENAME);
-
     let eth2_network_config = env
         .eth2_network_config
         .ok_or("Unable to get testnet configuration from the environment")?;
 
     let genesis_validators_root = eth2_network_config
-        .beacon_state::<T>()
-        .map(|state: BeaconState<T>| state.genesis_validators_root())
-        .map_err(|e| {
-            format!(
-                "Unable to get genesis state, has genesis occurred? Detail: {:?}",
-                e
-            )
-        })?;
+        .genesis_validators_root::<E>()?
+        .ok_or_else(|| "Unable to get genesis state, has genesis occurred?".to_string())?;
 
     match matches.subcommand() {
         (IMPORT_CMD, Some(matches)) => {
             let import_filename: PathBuf = clap_utils::parse_required(matches, IMPORT_FILE_ARG)?;
-            let minify: Option<bool> = clap_utils::parse_optional(matches, MINIFY_FLAG)?;
             let import_file = File::open(&import_filename).map_err(|e| {
                 format!(
                     "Unable to open import file at {}: {:?}",
@@ -109,23 +79,10 @@ pub fn cli_run<T: EthSpec>(
             })?;
 
             eprint!("Loading JSON file into memory & deserializing");
-            let mut interchange = Interchange::from_json_reader(&import_file)
+            let interchange = Interchange::from_json_reader(&import_file)
                 .map_err(|e| format!("Error parsing file for import: {:?}", e))?;
             eprintln!(" [done].");
 
-            if let Some(minify) = minify {
-                eprintln!(
-                    "WARNING: --minify flag is deprecated and will be removed in a future release"
-                );
-                if minify {
-                    eprint!("Minifying input file for faster loading");
-                    interchange = interchange
-                        .minify()
-                        .map_err(|e| format!("Minification failed: {:?}", e))?;
-                    eprintln!(" [done].");
-                }
-            }
-
             let slashing_protection_database =
                 SlashingDatabase::open_or_create(&slashing_protection_db_path).map_err(|e| {
                     format!(
@@ -158,7 +115,7 @@ pub fn cli_run<T: EthSpec>(
                             InterchangeImportOutcome::Success { pubkey, summary } => {
                                 eprintln!("- {:?}", pubkey);
                                 eprintln!(
-                                    "    - latest block: {}",
+                                    "    - latest proposed block: {}",
                                     display_slot(summary.max_block_slot)
                                 );
                                 eprintln!(
@@ -213,7 +170,6 @@ pub fn cli_run<T: EthSpec>(
         }
         (EXPORT_CMD, Some(matches)) => {
             let export_filename: PathBuf = clap_utils::parse_required(matches, EXPORT_FILE_ARG)?;
-            let minify: bool = clap_utils::parse_required(matches, MINIFY_FLAG)?;
 
             let selected_pubkeys = if let Some(pubkeys) =
                 clap_utils::parse_optional::<String>(matches, PUBKEYS_FLAG)?
@@ -244,17 +200,10 @@ pub fn cli_run<T: EthSpec>(
                     )
                 })?;
 
-            let mut interchange = slashing_protection_database
+            let interchange = slashing_protection_database
                 .export_interchange_info(genesis_validators_root, selected_pubkeys.as_deref())
                 .map_err(|e| format!("Error during export: {:?}", e))?;
 
-            if minify {
-                eprintln!("Minifying output file");
-                interchange = interchange
-                    .minify()
-                    .map_err(|e| format!("Unable to minify output: {:?}", e))?;
-            }
-
             let output_file = File::create(export_filename)
                 .map_err(|e| format!("Error creating output file: {:?}", e))?;
 

account_manager/src/wallet/create.rs

@@ -159,7 +159,7 @@ pub fn create_wallet_from_mnemonic(
         unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
     };
 
-    let mgr = WalletManager::open(&wallet_base_dir)
+    let mgr = WalletManager::open(wallet_base_dir)
         .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     let wallet_password: PlainText = match wallet_password_path {

account_manager/src/wallet/list.rs

@@ -10,7 +10,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 }
 
 pub fn cli_run(wallet_base_dir: PathBuf) -> Result<(), String> {
-    let mgr = WalletManager::open(&wallet_base_dir)
+    let mgr = WalletManager::open(wallet_base_dir)
         .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     for (name, _uuid) in mgr

account_manager/src/wallet/recover.rs

@@ -1,6 +1,6 @@
-use crate::common::read_mnemonic_from_cli;
 use crate::wallet::create::{create_wallet_from_mnemonic, STDIN_INPUTS_FLAG};
 use crate::wallet::create::{HD_TYPE, NAME_FLAG, PASSWORD_FLAG, TYPE_FLAG};
+use account_utils::read_mnemonic_from_cli;
 use clap::{App, Arg, ArgMatches};
 use std::path::PathBuf;
 

beacon_node/Cargo.toml

@@ -1,8 +1,11 @@
 [package]
 name = "beacon_node"
-version = "2.1.3"
-authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
-edition = "2018"
+version = "5.1.222-exp"
+authors = [
+    "Paul Hauner <paul@paulhauner.com>",
+    "Age Manning <Age@AgeManning.com",
+]
+edition = { workspace = true }
 
 [lib]
 name = "beacon_node"
@@ -12,30 +15,36 @@ path = "src/lib.rs"
 node_test_rig = { path = "../testing/node_test_rig" }
 
 [features]
-write_ssz_files = ["beacon_chain/write_ssz_files"]  # Writes debugging .ssz files to /tmp during block processing.
+write_ssz_files = [
+    "beacon_chain/write_ssz_files",
+] # Writes debugging .ssz files to /tmp during block processing.
 
 [dependencies]
-eth2_config = { path = "../common/eth2_config" }
-beacon_chain = { path = "beacon_chain" }
-types = { path = "../consensus/types" }
-store = { path = "./store" }
+eth2_config = { workspace = true }
+beacon_chain = { workspace = true }
+types = { workspace = true }
+store = { workspace = true }
 client = { path = "client" }
-clap = "2.33.3"
-slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_trace"] }
-dirs = "3.0.1"
-directory = {path = "../common/directory"}
-futures = "0.3.7"
-environment = { path = "../lighthouse/environment" }
-task_executor = { path = "../common/task_executor" }
-genesis = { path = "genesis" }
-eth2_network_config = { path = "../common/eth2_network_config" }
-lighthouse_network = { path = "./lighthouse_network" }
-serde = "1.0.116"
-clap_utils = { path = "../common/clap_utils" }
-hyper = "0.14.4"
-lighthouse_version = { path = "../common/lighthouse_version" }
-hex = "0.4.2"
-slasher = { path = "../slasher" }
-monitoring_api = { path = "../common/monitoring_api" }
-sensitive_url = { path = "../common/sensitive_url" }
-http_api = { path = "http_api" }
+clap = { workspace = true }
+slog = { workspace = true }
+dirs = { workspace = true }
+directory = { workspace = true }
+futures = { workspace = true }
+environment = { workspace = true }
+task_executor = { workspace = true }
+genesis = { workspace = true }
+eth2_network_config = { workspace = true }
+execution_layer = { workspace = true }
+lighthouse_network = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+clap_utils = { workspace = true }
+hyper = { workspace = true }
+lighthouse_version = { workspace = true }
+hex = { workspace = true }
+slasher = { workspace = true }
+monitoring_api = { workspace = true }
+sensitive_url = { workspace = true }
+http_api = { workspace = true }
+unused_port = { workspace = true }
+strum = { workspace = true }

beacon_node/beacon_chain/Cargo.toml

@@ -2,7 +2,7 @@
 name = "beacon_chain"
 version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com>"]
-edition = "2018"
+edition = { workspace = true }
 autotests = false # using a single test binary compiles faster
 
 [features]
@@ -10,55 +10,69 @@ default = ["participation_metrics"]
 write_ssz_files = []  # Writes debugging .ssz files to /tmp during block processing.
 participation_metrics = []  # Exposes validator participation metrics to Prometheus.
 fork_from_env = [] # Initialise the harness chain spec from the FORK_NAME env variable
+portable = ["bls/supranational-portable"]
+test_backfill = []
 
 [dev-dependencies]
-maplit = "1.0.2"
-environment = { path = "../../lighthouse/environment" }
+maplit = { workspace = true }
+environment = { workspace = true }
+serde_json = { workspace = true }
 
 [dependencies]
-merkle_proof = { path = "../../consensus/merkle_proof" }
-store = { path = "../store" }
-parking_lot = "0.11.0"
-lazy_static = "1.4.0"
-smallvec = "1.6.1"
-lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-operation_pool = { path = "../operation_pool" }
-rayon = "1.4.1"
-serde = "1.0.116"
-serde_derive = "1.0.116"
-slog = { version = "2.5.2", features = ["max_level_trace"] }
-sloggers = { version = "2.1.1", features = ["json"] }
-slot_clock = { path = "../../common/slot_clock" }
-eth2_hashing = "0.2.0"
-eth2_ssz = "0.4.1"
-eth2_ssz_types = "0.2.2"
-eth2_ssz_derive = "0.3.0"
-state_processing = { path = "../../consensus/state_processing" }
-tree_hash = "0.4.1"
-types = { path = "../../consensus/types" }
-tokio = "1.14.0"
-eth1 = { path = "../eth1" }
-futures = "0.3.7"
-genesis = { path = "../genesis" }
-int_to_bytes = { path = "../../consensus/int_to_bytes" }
-rand = "0.7.3"
-proto_array = { path = "../../consensus/proto_array" }
-lru = "0.7.1"
-tempfile = "3.1.0"
-bitvec = "0.19.3"
-bls = { path = "../../crypto/bls" }
-safe_arith = { path = "../../consensus/safe_arith" }
-fork_choice = { path = "../../consensus/fork_choice" }
-task_executor = { path = "../../common/task_executor" }
-derivative = "2.1.1"
-itertools = "0.10.0"
-slasher = { path = "../../slasher" }
-eth2 = { path = "../../common/eth2" }
-strum = { version = "0.21.0", features = ["derive"] }
-logging = { path = "../../common/logging" }
-execution_layer = { path = "../execution_layer" }
-sensitive_url = { path = "../../common/sensitive_url" }
-superstruct = "0.4.0"
+bitvec = { workspace = true }
+bls = { workspace = true }
+crossbeam-channel = { workspace = true }
+derivative = { workspace = true }
+eth1 = { workspace = true }
+eth2 = { workspace = true }
+eth2_network_config = { workspace = true }
+ethereum_hashing = { workspace = true }
+ethereum_serde_utils = { workspace = true }
+ethereum_ssz = { workspace = true }
+ethereum_ssz_derive = { workspace = true }
+execution_layer = { workspace = true }
+fork_choice = { workspace = true }
+futures = { workspace = true }
+genesis = { workspace = true }
+hex = { workspace = true }
+int_to_bytes = { workspace = true }
+itertools = { workspace = true }
+kzg = { workspace = true }
+lazy_static = { workspace = true }
+lighthouse_metrics = { workspace = true }
+logging = { workspace = true }
+lru = { workspace = true }
+merkle_proof = { workspace = true }
+oneshot_broadcast = { path = "../../common/oneshot_broadcast/" }
+operation_pool = { workspace = true }
+parking_lot = { workspace = true }
+promise_cache = { path = "../../common/promise_cache" }
+proto_array = { workspace = true }
+rand = { workspace = true }
+rayon = { workspace = true }
+safe_arith = { workspace = true }
+sensitive_url = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+slasher = { workspace = true }
+slog = { workspace = true }
+slog-async = { workspace = true }
+slog-term = { workspace = true }
+sloggers = { workspace = true }
+slot_clock = { workspace = true }
+smallvec = { workspace = true }
+ssz_types = { workspace = true }
+state_processing = { workspace = true }
+store = { workspace = true }
+strum = { workspace = true }
+superstruct = { workspace = true }
+task_executor = { workspace = true }
+tempfile = { workspace = true }
+tokio = { workspace = true }
+tokio-stream = { workspace = true }
+tree_hash = { workspace = true }
+tree_hash_derive = { workspace = true }
+types = { workspace = true }
 
 [[test]]
 name = "beacon_chain_tests"

beacon_node/beacon_chain/src/attestation_rewards.rs

@@ -0,0 +1,452 @@
+use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use eth2::lighthouse::attestation_rewards::{IdealAttestationRewards, TotalAttestationRewards};
+use eth2::lighthouse::StandardAttestationRewards;
+use eth2::types::ValidatorId;
+use safe_arith::SafeArith;
+use serde_utils::quoted_u64::Quoted;
+use slog::debug;
+use state_processing::common::base::{self, SqrtTotalActiveBalance};
+use state_processing::per_epoch_processing::altair::{
+    process_inactivity_updates_slow, process_justification_and_finalization,
+};
+use state_processing::per_epoch_processing::base::rewards_and_penalties::{
+    get_attestation_component_delta, get_attestation_deltas_all, get_attestation_deltas_subset,
+    get_inactivity_penalty_delta, get_inclusion_delay_delta,
+};
+use state_processing::per_epoch_processing::base::validator_statuses::InclusionInfo;
+use state_processing::per_epoch_processing::base::{
+    process_justification_and_finalization as process_justification_and_finalization_base,
+    TotalBalances, ValidatorStatus, ValidatorStatuses,
+};
+use state_processing::{
+    common::altair::BaseRewardPerIncrement,
+    common::update_progressive_balances_cache::initialize_progressive_balances_cache,
+    epoch_cache::initialize_epoch_cache,
+    per_epoch_processing::altair::rewards_and_penalties::get_flag_weight,
+};
+use std::collections::HashMap;
+use store::consts::altair::{
+    PARTICIPATION_FLAG_WEIGHTS, TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX,
+    TIMELY_TARGET_FLAG_INDEX,
+};
+use types::consts::altair::WEIGHT_DENOMINATOR;
+use types::{BeaconState, Epoch, EthSpec, RelativeEpoch};
+
+impl<T: BeaconChainTypes> BeaconChain<T> {
+    pub fn compute_attestation_rewards(
+        &self,
+        epoch: Epoch,
+        validators: Vec<ValidatorId>,
+    ) -> Result<StandardAttestationRewards, BeaconChainError> {
+        debug!(self.log, "computing attestation rewards"; "epoch" => epoch, "validator_count" => validators.len());
+
+        // Get state
+        let state_slot = (epoch + 1).end_slot(T::EthSpec::slots_per_epoch());
+
+        let state_root = self
+            .state_root_at_slot(state_slot)?
+            .ok_or(BeaconChainError::NoStateForSlot(state_slot))?;
+
+        let state = self
+            .get_state(&state_root, Some(state_slot))?
+            .ok_or(BeaconChainError::MissingBeaconState(state_root))?;
+
+        match state {
+            BeaconState::Base(_) => self.compute_attestation_rewards_base(state, validators),
+            BeaconState::Altair(_)
+            | BeaconState::Merge(_)
+            | BeaconState::Capella(_)
+            | BeaconState::Deneb(_)
+            | BeaconState::Electra(_) => self.compute_attestation_rewards_altair(state, validators),
+        }
+    }
+
+    fn compute_attestation_rewards_base(
+        &self,
+        mut state: BeaconState<T::EthSpec>,
+        validators: Vec<ValidatorId>,
+    ) -> Result<StandardAttestationRewards, BeaconChainError> {
+        let spec = &self.spec;
+        let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
+        validator_statuses.process_attestations(&state)?;
+
+        process_justification_and_finalization_base(
+            &state,
+            &validator_statuses.total_balances,
+            spec,
+        )?
+        .apply_changes_to_state(&mut state);
+
+        let ideal_rewards =
+            self.compute_ideal_rewards_base(&state, &validator_statuses.total_balances)?;
+
+        let indices_to_attestation_delta = if validators.is_empty() {
+            get_attestation_deltas_all(&state, &validator_statuses, spec)?
+                .into_iter()
+                .enumerate()
+                .collect()
+        } else {
+            let validator_indices = Self::validators_ids_to_indices(&mut state, validators)?;
+            get_attestation_deltas_subset(&state, &validator_statuses, &validator_indices, spec)?
+        };
+
+        let mut total_rewards = vec![];
+
+        for (index, delta) in indices_to_attestation_delta.into_iter() {
+            let head_delta = delta.head_delta;
+            let head = (head_delta.rewards as i64).safe_sub(head_delta.penalties as i64)?;
+
+            let target_delta = delta.target_delta;
+            let target = (target_delta.rewards as i64).safe_sub(target_delta.penalties as i64)?;
+
+            let source_delta = delta.source_delta;
+            let source = (source_delta.rewards as i64).safe_sub(source_delta.penalties as i64)?;
+
+            // No penalties associated with inclusion delay
+            let inclusion_delay = delta.inclusion_delay_delta.rewards;
+            let inactivity = delta.inactivity_penalty_delta.penalties.wrapping_neg() as i64;
+
+            let rewards = TotalAttestationRewards {
+                validator_index: index as u64,
+                head,
+                target,
+                source,
+                inclusion_delay: Some(Quoted {
+                    value: inclusion_delay,
+                }),
+                inactivity,
+            };
+
+            total_rewards.push(rewards);
+        }
+
+        Ok(StandardAttestationRewards {
+            ideal_rewards,
+            total_rewards,
+        })
+    }
+
+    fn compute_attestation_rewards_altair(
+        &self,
+        mut state: BeaconState<T::EthSpec>,
+        validators: Vec<ValidatorId>,
+    ) -> Result<StandardAttestationRewards, BeaconChainError> {
+        let spec = &self.spec;
+
+        // Build required caches.
+        initialize_epoch_cache(&mut state, spec)?;
+        initialize_progressive_balances_cache(&mut state, spec)?;
+        state.build_exit_cache(spec)?;
+        state.build_committee_cache(RelativeEpoch::Previous, spec)?;
+        state.build_committee_cache(RelativeEpoch::Current, spec)?;
+
+        // Calculate ideal_rewards
+        process_justification_and_finalization(&state)?.apply_changes_to_state(&mut state);
+        process_inactivity_updates_slow(&mut state, spec)?;
+
+        let previous_epoch = state.previous_epoch();
+
+        let mut ideal_rewards_hashmap = HashMap::new();
+
+        for flag_index in 0..PARTICIPATION_FLAG_WEIGHTS.len() {
+            let weight = get_flag_weight(flag_index)
+                .map_err(|_| BeaconChainError::AttestationRewardsError)?;
+
+            let unslashed_participating_balance = state
+                .progressive_balances_cache()
+                .previous_epoch_flag_attesting_balance(flag_index)?;
+
+            let unslashed_participating_increments =
+                unslashed_participating_balance.safe_div(spec.effective_balance_increment)?;
+
+            let total_active_balance = state.get_total_active_balance()?;
+
+            let active_increments =
+                total_active_balance.safe_div(spec.effective_balance_increment)?;
+
+            let base_reward_per_increment =
+                BaseRewardPerIncrement::new(total_active_balance, spec)?;
+
+            for effective_balance_eth in 1..=self.max_effective_balance_increment_steps()? {
+                let effective_balance =
+                    effective_balance_eth.safe_mul(spec.effective_balance_increment)?;
+                let base_reward =
+                    effective_balance_eth.safe_mul(base_reward_per_increment.as_u64())?;
+
+                let penalty = -(base_reward.safe_mul(weight)?.safe_div(WEIGHT_DENOMINATOR)? as i64);
+
+                let reward_numerator = base_reward
+                    .safe_mul(weight)?
+                    .safe_mul(unslashed_participating_increments)?;
+
+                let ideal_reward = reward_numerator
+                    .safe_div(active_increments)?
+                    .safe_div(WEIGHT_DENOMINATOR)?;
+                if !state.is_in_inactivity_leak(previous_epoch, spec)? {
+                    ideal_rewards_hashmap
+                        .insert((flag_index, effective_balance), (ideal_reward, penalty));
+                } else {
+                    ideal_rewards_hashmap.insert((flag_index, effective_balance), (0, penalty));
+                }
+            }
+        }
+
+        // Calculate total_rewards
+        let mut total_rewards: Vec<TotalAttestationRewards> = Vec::new();
+
+        let validators = if validators.is_empty() {
+            Self::all_eligible_validator_indices(&state, previous_epoch)?
+        } else {
+            Self::validators_ids_to_indices(&mut state, validators)?
+        };
+
+        for &validator_index in &validators {
+            // Return 0s for unknown/inactive validator indices.
+            let Ok(validator) = state.get_validator(validator_index) else {
+                debug!(
+                    self.log,
+                    "No rewards for inactive/unknown validator";
+                    "index" => validator_index,
+                    "epoch" => previous_epoch
+                );
+                total_rewards.push(TotalAttestationRewards {
+                    validator_index: validator_index as u64,
+                    head: 0,
+                    target: 0,
+                    source: 0,
+                    inclusion_delay: None,
+                    inactivity: 0,
+                });
+                continue;
+            };
+            let previous_epoch_participation_flags = state
+                .previous_epoch_participation()?
+                .get(validator_index)
+                .ok_or(BeaconChainError::AttestationRewardsError)?;
+            let eligible = state.is_eligible_validator(previous_epoch, validator)?;
+            let mut head_reward = 0i64;
+            let mut target_reward = 0i64;
+            let mut source_reward = 0i64;
+            let mut inactivity_penalty = 0i64;
+
+            if eligible {
+                let effective_balance = validator.effective_balance();
+
+                for flag_index in 0..PARTICIPATION_FLAG_WEIGHTS.len() {
+                    let (ideal_reward, penalty) = ideal_rewards_hashmap
+                        .get(&(flag_index, effective_balance))
+                        .ok_or(BeaconChainError::AttestationRewardsError)?;
+                    let voted_correctly = !validator.slashed()
+                        && previous_epoch_participation_flags.has_flag(flag_index)?;
+                    if voted_correctly {
+                        if flag_index == TIMELY_HEAD_FLAG_INDEX {
+                            head_reward += *ideal_reward as i64;
+                        } else if flag_index == TIMELY_TARGET_FLAG_INDEX {
+                            target_reward += *ideal_reward as i64;
+                        } else if flag_index == TIMELY_SOURCE_FLAG_INDEX {
+                            source_reward += *ideal_reward as i64;
+                        }
+                    } else if flag_index == TIMELY_HEAD_FLAG_INDEX {
+                        head_reward = 0;
+                    } else if flag_index == TIMELY_TARGET_FLAG_INDEX {
+                        target_reward = *penalty;
+
+                        let penalty_numerator = effective_balance
+                            .safe_mul(state.get_inactivity_score(validator_index)?)?;
+                        let penalty_denominator = spec.inactivity_score_bias.safe_mul(
+                            spec.inactivity_penalty_quotient_for_fork(state.fork_name_unchecked()),
+                        )?;
+                        inactivity_penalty =
+                            -(penalty_numerator.safe_div(penalty_denominator)? as i64);
+                    } else if flag_index == TIMELY_SOURCE_FLAG_INDEX {
+                        source_reward = *penalty;
+                    }
+                }
+            }
+            total_rewards.push(TotalAttestationRewards {
+                validator_index: validator_index as u64,
+                head: head_reward,
+                target: target_reward,
+                source: source_reward,
+                inclusion_delay: None,
+                inactivity: inactivity_penalty,
+            });
+        }
+
+        // Convert hashmap to vector
+        let mut ideal_rewards: Vec<IdealAttestationRewards> = ideal_rewards_hashmap
+            .iter()
+            .map(
+                |((flag_index, effective_balance), (ideal_reward, _penalty))| {
+                    (flag_index, effective_balance, ideal_reward)
+                },
+            )
+            .fold(
+                HashMap::new(),
+                |mut acc, (flag_index, &effective_balance, ideal_reward)| {
+                    let entry = acc
+                        .entry(effective_balance)
+                        .or_insert(IdealAttestationRewards {
+                            effective_balance,
+                            head: 0,
+                            target: 0,
+                            source: 0,
+                            inclusion_delay: None,
+                            inactivity: 0,
+                        });
+                    match *flag_index {
+                        TIMELY_SOURCE_FLAG_INDEX => entry.source += ideal_reward,
+                        TIMELY_TARGET_FLAG_INDEX => entry.target += ideal_reward,
+                        TIMELY_HEAD_FLAG_INDEX => entry.head += ideal_reward,
+                        _ => {}
+                    }
+                    acc
+                },
+            )
+            .into_values()
+            .collect::<Vec<IdealAttestationRewards>>();
+        ideal_rewards.sort_by(|a, b| a.effective_balance.cmp(&b.effective_balance));
+
+        Ok(StandardAttestationRewards {
+            ideal_rewards,
+            total_rewards,
+        })
+    }
+
+    fn max_effective_balance_increment_steps(&self) -> Result<u64, BeaconChainError> {
+        let spec = &self.spec;
+        let max_steps = spec
+            .max_effective_balance
+            .safe_div(spec.effective_balance_increment)?;
+        Ok(max_steps)
+    }
+
+    fn all_eligible_validator_indices(
+        state: &BeaconState<T::EthSpec>,
+        previous_epoch: Epoch,
+    ) -> Result<Vec<usize>, BeaconChainError> {
+        state
+            .validators()
+            .iter()
+            .enumerate()
+            .filter_map(|(i, validator)| {
+                state
+                    .is_eligible_validator(previous_epoch, validator)
+                    .map(|eligible| eligible.then_some(i))
+                    .map_err(BeaconChainError::BeaconStateError)
+                    .transpose()
+            })
+            .collect()
+    }
+
+    fn validators_ids_to_indices(
+        state: &mut BeaconState<T::EthSpec>,
+        validators: Vec<ValidatorId>,
+    ) -> Result<Vec<usize>, BeaconChainError> {
+        let indices = validators
+            .into_iter()
+            .map(|validator| match validator {
+                ValidatorId::Index(i) => Ok(i as usize),
+                ValidatorId::PublicKey(pubkey) => state
+                    .get_validator_index(&pubkey)?
+                    .ok_or(BeaconChainError::ValidatorPubkeyUnknown(pubkey)),
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        Ok(indices)
+    }
+
+    fn compute_ideal_rewards_base(
+        &self,
+        state: &BeaconState<T::EthSpec>,
+        total_balances: &TotalBalances,
+    ) -> Result<Vec<IdealAttestationRewards>, BeaconChainError> {
+        let spec = &self.spec;
+        let previous_epoch = state.previous_epoch();
+        let finality_delay = previous_epoch
+            .safe_sub(state.finalized_checkpoint().epoch)?
+            .as_u64();
+
+        let ideal_validator_status = ValidatorStatus {
+            is_previous_epoch_attester: true,
+            is_slashed: false,
+            inclusion_info: Some(InclusionInfo {
+                delay: 1,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let mut ideal_attestation_rewards_list = Vec::new();
+        let sqrt_total_active_balance = SqrtTotalActiveBalance::new(total_balances.current_epoch());
+        for effective_balance_step in 1..=self.max_effective_balance_increment_steps()? {
+            let effective_balance =
+                effective_balance_step.safe_mul(spec.effective_balance_increment)?;
+            let base_reward =
+                base::get_base_reward(effective_balance, sqrt_total_active_balance, spec)?;
+
+            // compute ideal head rewards
+            let head = get_attestation_component_delta(
+                true,
+                total_balances.previous_epoch_head_attesters(),
+                total_balances,
+                base_reward,
+                finality_delay,
+                spec,
+            )?
+            .rewards;
+
+            // compute ideal target rewards
+            let target = get_attestation_component_delta(
+                true,
+                total_balances.previous_epoch_target_attesters(),
+                total_balances,
+                base_reward,
+                finality_delay,
+                spec,
+            )?
+            .rewards;
+
+            // compute ideal source rewards
+            let source = get_attestation_component_delta(
+                true,
+                total_balances.previous_epoch_attesters(),
+                total_balances,
+                base_reward,
+                finality_delay,
+                spec,
+            )?
+            .rewards;
+
+            // compute ideal inclusion delay rewards
+            let inclusion_delay =
+                get_inclusion_delay_delta(&ideal_validator_status, base_reward, spec)?
+                    .0
+                    .rewards;
+
+            // compute inactivity penalty
+            let inactivity = get_inactivity_penalty_delta(
+                &ideal_validator_status,
+                base_reward,
+                finality_delay,
+                spec,
+            )?
+            .penalties
+            .wrapping_neg() as i64;
+
+            let ideal_attestation_rewards = IdealAttestationRewards {
+                effective_balance,
+                head,
+                target,
+                source,
+                inclusion_delay: Some(Quoted {
+                    value: inclusion_delay,
+                }),
+                inactivity,
+            };
+
+            ideal_attestation_rewards_list.push(ideal_attestation_rewards);
+        }
+
+        Ok(ideal_attestation_rewards_list)
+    }
+}

beacon_node/beacon_chain/src/attestation_simulator.rs

@@ -0,0 +1,107 @@
+use crate::{BeaconChain, BeaconChainTypes};
+use slog::{debug, error};
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use task_executor::TaskExecutor;
+use tokio::time::sleep;
+use types::{EthSpec, Slot};
+
+/// Don't run the attestation simulator if the head slot is this many epochs
+/// behind the wall-clock slot.
+const SYNCING_TOLERANCE_EPOCHS: u64 = 2;
+
+/// Spawns a routine which produces an unaggregated attestation at every slot.
+///
+/// This routine will run once per slot
+pub fn start_attestation_simulator_service<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    chain: Arc<BeaconChain<T>>,
+) {
+    executor.clone().spawn(
+        async move { attestation_simulator_service(executor, chain).await },
+        "attestation_simulator_service",
+    );
+}
+
+/// Loop indefinitely, calling `BeaconChain::produce_unaggregated_attestation` every 4s into each slot.
+async fn attestation_simulator_service<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    chain: Arc<BeaconChain<T>>,
+) {
+    let slot_duration = chain.slot_clock.slot_duration();
+    let additional_delay = slot_duration / 3;
+
+    loop {
+        match chain.slot_clock.duration_to_next_slot() {
+            Some(duration) => {
+                sleep(duration + additional_delay).await;
+
+                debug!(
+                    chain.log,
+                    "Simulating unagg. attestation production";
+                );
+
+                // Run the task in the executor
+                let inner_chain = chain.clone();
+                executor.spawn(
+                    async move {
+                        if let Ok(current_slot) = inner_chain.slot() {
+                            produce_unaggregated_attestation(inner_chain, current_slot);
+                        }
+                    },
+                    "attestation_simulator_service",
+                );
+            }
+            None => {
+                error!(chain.log, "Failed to read slot clock");
+                // If we can't read the slot clock, just wait another slot.
+                sleep(slot_duration).await;
+            }
+        };
+    }
+}
+
+pub fn produce_unaggregated_attestation<T: BeaconChainTypes>(
+    chain: Arc<BeaconChain<T>>,
+    current_slot: Slot,
+) {
+    // Don't run the attestation simulator when the head slot is far behind the
+    // wall-clock slot.
+    //
+    // This helps prevent the simulator from becoming a burden by computing
+    // committees from old states.
+    let syncing_tolerance_slots = SYNCING_TOLERANCE_EPOCHS * T::EthSpec::slots_per_epoch();
+    if chain.best_slot() + syncing_tolerance_slots < current_slot {
+        return;
+    }
+
+    // Since attestations for different committees are practically identical (apart from the committee index field)
+    // Committee 0 is guaranteed to exist. That means there's no need to load the committee.
+    let beacon_committee_index = 0;
+
+    // Store the unaggregated attestation in the validator monitor for later processing
+    match chain.produce_unaggregated_attestation(current_slot, beacon_committee_index) {
+        Ok(unaggregated_attestation) => {
+            let data = &unaggregated_attestation.data;
+
+            debug!(
+                chain.log,
+                "Produce unagg. attestation";
+                "attestation_source" => data.source.root.to_string(),
+                "attestation_target" => data.target.root.to_string(),
+            );
+
+            chain
+                .validator_monitor
+                .write()
+                .set_unaggregated_attestation(unaggregated_attestation);
+        }
+        Err(e) => {
+            debug!(
+                chain.log,
+                "Failed to simulate attestation";
+                "error" => ?e
+            );
+        }
+    }
+}

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -27,13 +27,16 @@
 //!                                ▼
 //!                  impl VerifiedAttestation
 //! ```
+
+// Ignore this lint for `AttestationSlashInfo` which is of comparable size to the non-error types it
+// is returned alongside.
+#![allow(clippy::result_large_err)]
+
 mod batch;
 
 use crate::{
-    beacon_chain::{MAXIMUM_GOSSIP_CLOCK_DISPARITY, VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT},
-    metrics,
-    observed_aggregates::ObserveOutcome,
-    observed_attesters::Error as ObservedAttestersError,
+    beacon_chain::VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT, metrics,
+    observed_aggregates::ObserveOutcome, observed_attesters::Error as ObservedAttestersError,
     BeaconChain, BeaconChainError, BeaconChainTypes,
 };
 use bls::verify_signature_sets;
@@ -52,8 +55,8 @@ use std::borrow::Cow;
 use strum::AsRefStr;
 use tree_hash::TreeHash;
 use types::{
-    Attestation, BeaconCommittee, CommitteeIndex, Epoch, EthSpec, Hash256, IndexedAttestation,
-    SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
+    Attestation, BeaconCommittee, ChainSpec, CommitteeIndex, Epoch, EthSpec, ForkName, Hash256,
+    IndexedAttestation, SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
 };
 
 pub use batch::{batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations};
@@ -112,14 +115,14 @@ pub enum Error {
     ///
     /// The peer has sent an invalid message.
     AggregatorPubkeyUnknown(u64),
-    /// The attestation has been seen before; either in a block, on the gossip network or from a
-    /// local validator.
+    /// The attestation or a superset of this attestation's aggregations bits for the same data
+    /// has been seen before; either in a block, on the gossip network or from a local validator.
     ///
     /// ## Peer scoring
     ///
     /// It's unclear if this attestation is valid, however we have already observed it and do not
     /// need to observe it again.
-    AttestationAlreadyKnown(Hash256),
+    AttestationSupersetKnown(Hash256),
     /// There has already been an aggregation observed for this validator, we refuse to process a
     /// second.
     ///
@@ -263,7 +266,7 @@ enum CheckAttestationSignature {
 struct IndexedAggregatedAttestation<'a, T: BeaconChainTypes> {
     signed_aggregate: &'a SignedAggregateAndProof<T::EthSpec>,
     indexed_attestation: IndexedAttestation<T::EthSpec>,
-    attestation_root: Hash256,
+    attestation_data_root: Hash256,
 }
 
 /// Wraps a `Attestation` that has been verified up until the point that an `IndexedAttestation` can
@@ -318,10 +321,17 @@ impl<'a, T: BeaconChainTypes> Clone for IndexedUnaggregatedAttestation<'a, T> {
 
 /// A helper trait implemented on wrapper types that can be progressed to a state where they can be
 /// verified for application to fork choice.
-pub trait VerifiedAttestation<T: BeaconChainTypes> {
+pub trait VerifiedAttestation<T: BeaconChainTypes>: Sized {
     fn attestation(&self) -> &Attestation<T::EthSpec>;
 
     fn indexed_attestation(&self) -> &IndexedAttestation<T::EthSpec>;
+
+    // Inefficient default implementation. This is overridden for gossip verified attestations.
+    fn into_attestation_and_indices(self) -> (Attestation<T::EthSpec>, Vec<u64>) {
+        let attestation = self.attestation().clone();
+        let attesting_indices = self.indexed_attestation().attesting_indices.clone().into();
+        (attestation, attesting_indices)
+    }
 }
 
 impl<'a, T: BeaconChainTypes> VerifiedAttestation<T> for VerifiedAggregatedAttestation<'a, T> {
@@ -442,7 +452,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> {
         // MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance).
         //
         // We do not queue future attestations for later processing.
-        verify_propagation_slot_range(&chain.slot_clock, attestation)?;
+        verify_propagation_slot_range(&chain.slot_clock, attestation, &chain.spec)?;
 
         // Check the attestation's epoch matches its target.
         if attestation.data.slot.epoch(T::EthSpec::slots_per_epoch())
@@ -455,14 +465,17 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> {
         }
 
         // Ensure the valid aggregated attestation has not already been seen locally.
-        let attestation_root = attestation.tree_hash_root();
+        let attestation_data = &attestation.data;
+        let attestation_data_root = attestation_data.tree_hash_root();
+
         if chain
             .observed_attestations
             .write()
-            .is_known(attestation, attestation_root)
+            .is_known_subset(attestation, attestation_data_root)
             .map_err(|e| Error::BeaconChainError(e.into()))?
         {
-            return Err(Error::AttestationAlreadyKnown(attestation_root));
+            metrics::inc_counter(&metrics::AGGREGATED_ATTESTATION_SUBSETS);
+            return Err(Error::AttestationSupersetKnown(attestation_data_root));
         }
 
         let aggregator_index = signed_aggregate.message.aggregator_index;
@@ -508,7 +521,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> {
         if attestation.aggregation_bits.is_zero() {
             Err(Error::EmptyAggregationBitfield)
         } else {
-            Ok(attestation_root)
+            Ok(attestation_data_root)
         }
     }
 
@@ -521,13 +534,13 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> {
 
         let attestation = &signed_aggregate.message.aggregate;
         let aggregator_index = signed_aggregate.message.aggregator_index;
-        let attestation_root = match Self::verify_early_checks(signed_aggregate, chain) {
+        let attestation_data_root = match Self::verify_early_checks(signed_aggregate, chain) {
             Ok(root) => root,
             Err(e) => return Err(SignatureNotChecked(&signed_aggregate.message.aggregate, e)),
         };
 
-        let indexed_attestation =
-            match map_attestation_committee(chain, attestation, |(committee, _)| {
+        let get_indexed_attestation_with_committee =
+            |(committee, _): (BeaconCommittee, CommitteesPerSlot)| {
                 // Note: this clones the signature which is known to be a relatively slow operation.
                 //
                 // Future optimizations should remove this clone.
@@ -548,15 +561,21 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> {
 
                 get_indexed_attestation(committee.committee, attestation)
                     .map_err(|e| BeaconChainError::from(e).into())
-            }) {
-                Ok(indexed_attestation) => indexed_attestation,
-                Err(e) => return Err(SignatureNotChecked(&signed_aggregate.message.aggregate, e)),
             };
 
+        let indexed_attestation = match map_attestation_committee(
+            chain,
+            attestation,
+            get_indexed_attestation_with_committee,
+        ) {
+            Ok(indexed_attestation) => indexed_attestation,
+            Err(e) => return Err(SignatureNotChecked(&signed_aggregate.message.aggregate, e)),
+        };
+
         Ok(IndexedAggregatedAttestation {
             signed_aggregate,
             indexed_attestation,
-            attestation_root,
+            attestation_data_root,
         })
     }
 }
@@ -565,7 +584,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> {
     /// Run the checks that happen after the indexed attestation and signature have been checked.
     fn verify_late_checks(
         signed_aggregate: &SignedAggregateAndProof<T::EthSpec>,
-        attestation_root: Hash256,
+        attestation_data_root: Hash256,
         chain: &BeaconChain<T>,
     ) -> Result<(), Error> {
         let attestation = &signed_aggregate.message.aggregate;
@@ -575,13 +594,14 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> {
         //
         // It's important to double check that the attestation is not already known, otherwise two
         // attestations processed at the same time could be published.
-        if let ObserveOutcome::AlreadyKnown = chain
+        if let ObserveOutcome::Subset = chain
             .observed_attestations
             .write()
-            .observe_item(attestation, Some(attestation_root))
+            .observe_item(attestation, Some(attestation_data_root))
             .map_err(|e| Error::BeaconChainError(e.into()))?
         {
-            return Err(Error::AttestationAlreadyKnown(attestation_root));
+            metrics::inc_counter(&metrics::AGGREGATED_ATTESTATION_SUBSETS);
+            return Err(Error::AttestationSupersetKnown(attestation_data_root));
         }
 
         // Observe the aggregator so we don't process another aggregate from them.
@@ -641,7 +661,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> {
         let IndexedAggregatedAttestation {
             signed_aggregate,
             indexed_attestation,
-            attestation_root,
+            attestation_data_root,
         } = signed_aggregate;
 
         match check_signature {
@@ -665,7 +685,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> {
             CheckAttestationSignature::No => (),
         };
 
-        if let Err(e) = Self::verify_late_checks(signed_aggregate, attestation_root, chain) {
+        if let Err(e) = Self::verify_late_checks(signed_aggregate, attestation_data_root, chain) {
             return Err(SignatureValid(indexed_attestation, e));
         }
 
@@ -706,7 +726,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> {
         // MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance).
         //
         // We do not queue future attestations for later processing.
-        verify_propagation_slot_range(&chain.slot_clock, attestation)?;
+        verify_propagation_slot_range(&chain.slot_clock, attestation, &chain.spec)?;
 
         // Check to ensure that the attestation is "unaggregated". I.e., it has exactly one
         // aggregation bit set.
@@ -961,7 +981,6 @@ impl<'a, T: BeaconChainTypes> VerifiedUnaggregatedAttestation<'a, T> {
 }
 
 /// Returns `Ok(())` if the `attestation.data.beacon_block_root` is known to this chain.
-/// You can use this `shuffling_id` to read from the shuffling cache.
 ///
 /// The block root may not be known for two reasons:
 ///
@@ -977,8 +996,8 @@ fn verify_head_block_is_known<T: BeaconChainTypes>(
     max_skip_slots: Option<u64>,
 ) -> Result<ProtoBlock, Error> {
     let block_opt = chain
-        .fork_choice
-        .read()
+        .canonical_head
+        .fork_choice_read_lock()
         .get_block(&attestation.data.beacon_block_root)
         .or_else(|| {
             chain
@@ -1022,11 +1041,11 @@ fn verify_head_block_is_known<T: BeaconChainTypes>(
 pub fn verify_propagation_slot_range<S: SlotClock, E: EthSpec>(
     slot_clock: &S,
     attestation: &Attestation<E>,
+    spec: &ChainSpec,
 ) -> Result<(), Error> {
     let attestation_slot = attestation.data.slot;
-
     let latest_permissible_slot = slot_clock
-        .now_with_future_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY)
+        .now_with_future_tolerance(spec.maximum_gossip_clock_disparity())
         .ok_or(BeaconChainError::UnableToReadSlot)?;
     if attestation_slot > latest_permissible_slot {
         return Err(Error::FutureSlot {
@@ -1036,10 +1055,21 @@ pub fn verify_propagation_slot_range<S: SlotClock, E: EthSpec>(
     }
 
     // Taking advantage of saturating subtraction on `Slot`.
-    let earliest_permissible_slot = slot_clock
-        .now_with_past_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY)
+    let one_epoch_prior = slot_clock
+        .now_with_past_tolerance(spec.maximum_gossip_clock_disparity())
         .ok_or(BeaconChainError::UnableToReadSlot)?
         - E::slots_per_epoch();
+
+    let current_fork =
+        spec.fork_name_at_slot::<E>(slot_clock.now().ok_or(BeaconChainError::UnableToReadSlot)?);
+    let earliest_permissible_slot = match current_fork {
+        ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => one_epoch_prior,
+        // EIP-7045
+        ForkName::Deneb | ForkName::Electra => one_epoch_prior
+            .epoch(E::slots_per_epoch())
+            .start_slot(E::slots_per_epoch()),
+    };
+
     if attestation_slot < earliest_permissible_slot {
         return Err(Error::PastSlot {
             attestation_slot,
@@ -1091,13 +1121,13 @@ pub fn verify_attestation_signature<T: BeaconChainTypes>(
 
 /// Verifies that the `attestation.data.target.root` is indeed the target root of the block at
 /// `attestation.data.beacon_block_root`.
-pub fn verify_attestation_target_root<T: EthSpec>(
+pub fn verify_attestation_target_root<E: EthSpec>(
     head_block: &ProtoBlock,
-    attestation: &Attestation<T>,
+    attestation: &Attestation<E>,
 ) -> Result<(), Error> {
     // Check the attestation target root.
-    let head_block_epoch = head_block.slot.epoch(T::slots_per_epoch());
-    let attestation_epoch = attestation.data.slot.epoch(T::slots_per_epoch());
+    let head_block_epoch = head_block.slot.epoch(E::slots_per_epoch());
+    let attestation_epoch = attestation.data.slot.epoch(E::slots_per_epoch());
     if head_block_epoch > attestation_epoch {
         // The epoch references an invalid head block from a future epoch.
         //
@@ -1246,7 +1276,10 @@ where
     // processing an attestation that does not include our latest finalized block in its chain.
     //
     // We do not delay consideration for later, we simply drop the attestation.
-    if !chain.fork_choice.read().contains_block(&target.root)
+    if !chain
+        .canonical_head
+        .fork_choice_read_lock()
+        .contains_block(&target.root)
         && !chain.early_attester_cache.contains_block(target.root)
     {
         return Err(Error::UnknownTargetRoot(target.root));

beacon_node/beacon_chain/src/attestation_verification/batch.rs

@@ -65,14 +65,15 @@ where
             .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
             .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
 
-        let fork = chain.with_head(|head| Ok::<_, BeaconChainError>(head.beacon_state.fork()))?;
-
         let mut signature_sets = Vec::with_capacity(num_indexed * 3);
 
         // Iterate, flattening to get only the `Ok` values.
         for indexed in indexing_results.iter().flatten() {
             let signed_aggregate = &indexed.signed_aggregate;
             let indexed_attestation = &indexed.indexed_attestation;
+            let fork = chain
+                .spec
+                .fork_at_epoch(indexed_attestation.data.target.epoch);
 
             signature_sets.push(
                 signed_aggregate_selection_proof_signature_set(
@@ -174,13 +175,14 @@ where
             .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
             .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
 
-        let fork = chain.with_head(|head| Ok::<_, BeaconChainError>(head.beacon_state.fork()))?;
-
         let mut signature_sets = Vec::with_capacity(num_partially_verified);
 
         // Iterate, flattening to get only the `Ok` values.
         for partially_verified in partial_results.iter().flatten() {
             let indexed_attestation = &partially_verified.indexed_attestation;
+            let fork = chain
+                .spec
+                .fork_at_epoch(indexed_attestation.data.target.epoch);
 
             let signature_set = indexed_attestation_signature_set_from_pubkeys(
                 |validator_index| pubkey_cache.get(validator_index).map(Cow::Borrowed),

beacon_node/beacon_chain/src/attester_cache.rs

@@ -84,7 +84,7 @@ pub struct CommitteeLengths {
 
 impl CommitteeLengths {
     /// Instantiate `Self` using `state.current_epoch()`.
-    pub fn new<T: EthSpec>(state: &BeaconState<T>, spec: &ChainSpec) -> Result<Self, Error> {
+    pub fn new<E: EthSpec>(state: &BeaconState<E>, spec: &ChainSpec) -> Result<Self, Error> {
         let active_validator_indices_len = if let Ok(committee_cache) =
             state.committee_cache(RelativeEpoch::Current)
         {
@@ -102,21 +102,21 @@ impl CommitteeLengths {
     }
 
     /// Get the count of committees per each slot of `self.epoch`.
-    pub fn get_committee_count_per_slot<T: EthSpec>(
+    pub fn get_committee_count_per_slot<E: EthSpec>(
         &self,
         spec: &ChainSpec,
     ) -> Result<usize, Error> {
-        T::get_committee_count_per_slot(self.active_validator_indices_len, spec).map_err(Into::into)
+        E::get_committee_count_per_slot(self.active_validator_indices_len, spec).map_err(Into::into)
     }
 
     /// Get the length of the committee at the given `slot` and `committee_index`.
-    pub fn get_committee_length<T: EthSpec>(
+    pub fn get_committee_length<E: EthSpec>(
         &self,
         slot: Slot,
         committee_index: CommitteeIndex,
         spec: &ChainSpec,
     ) -> Result<CommitteeLength, Error> {
-        let slots_per_epoch = T::slots_per_epoch();
+        let slots_per_epoch = E::slots_per_epoch();
         let request_epoch = slot.epoch(slots_per_epoch);
 
         // Sanity check.
@@ -128,7 +128,7 @@ impl CommitteeLengths {
         }
 
         let slots_per_epoch = slots_per_epoch as usize;
-        let committees_per_slot = self.get_committee_count_per_slot::<T>(spec)?;
+        let committees_per_slot = self.get_committee_count_per_slot::<E>(spec)?;
         let index_in_epoch = compute_committee_index_in_epoch(
             slot,
             slots_per_epoch,
@@ -162,7 +162,7 @@ pub struct AttesterCacheValue {
 
 impl AttesterCacheValue {
     /// Instantiate `Self` using `state.current_epoch()`.
-    pub fn new<T: EthSpec>(state: &BeaconState<T>, spec: &ChainSpec) -> Result<Self, Error> {
+    pub fn new<E: EthSpec>(state: &BeaconState<E>, spec: &ChainSpec) -> Result<Self, Error> {
         let current_justified_checkpoint = state.current_justified_checkpoint();
         let committee_lengths = CommitteeLengths::new(state, spec)?;
         Ok(Self {
@@ -172,14 +172,14 @@ impl AttesterCacheValue {
     }
 
     /// Get the justified checkpoint and committee length for some `slot` and `committee_index`.
-    fn get<T: EthSpec>(
+    fn get<E: EthSpec>(
         &self,
         slot: Slot,
         committee_index: CommitteeIndex,
         spec: &ChainSpec,
     ) -> Result<(JustifiedCheckpoint, CommitteeLength), Error> {
         self.committee_lengths
-            .get_committee_length::<T>(slot, committee_index, spec)
+            .get_committee_length::<E>(slot, committee_index, spec)
             .map(|committee_length| (self.current_justified_checkpoint, committee_length))
     }
 }
@@ -216,12 +216,12 @@ impl AttesterCacheKey {
     /// ## Errors
     ///
     /// May error if `epoch` is out of the range of `state.block_roots`.
-    pub fn new<T: EthSpec>(
+    pub fn new<E: EthSpec>(
         epoch: Epoch,
-        state: &BeaconState<T>,
+        state: &BeaconState<E>,
         latest_block_root: Hash256,
     ) -> Result<Self, Error> {
-        let slots_per_epoch = T::slots_per_epoch();
+        let slots_per_epoch = E::slots_per_epoch();
         let decision_slot = epoch.start_slot(slots_per_epoch).saturating_sub(1_u64);
 
         let decision_root = if decision_slot.epoch(slots_per_epoch) == epoch {
@@ -255,7 +255,7 @@ pub struct AttesterCache {
 impl AttesterCache {
     /// Get the justified checkpoint and committee length for the `slot` and `committee_index` in
     /// the state identified by the cache `key`.
-    pub fn get<T: EthSpec>(
+    pub fn get<E: EthSpec>(
         &self,
         key: &AttesterCacheKey,
         slot: Slot,
@@ -265,14 +265,14 @@ impl AttesterCache {
         self.cache
             .read()
             .get(key)
-            .map(|cache_item| cache_item.get::<T>(slot, committee_index, spec))
+            .map(|cache_item| cache_item.get::<E>(slot, committee_index, spec))
             .transpose()
     }
 
     /// Cache the `state.current_epoch()` values if they are not already present in the state.
-    pub fn maybe_cache_state<T: EthSpec>(
+    pub fn maybe_cache_state<E: EthSpec>(
         &self,
-        state: &BeaconState<T>,
+        state: &BeaconState<E>,
         latest_block_root: Hash256,
         spec: &ChainSpec,
     ) -> Result<(), Error> {

beacon_node/beacon_chain/src/beacon_block_reward.rs

@@ -0,0 +1,246 @@
+use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use eth2::lighthouse::StandardBlockReward;
+use operation_pool::RewardCache;
+use safe_arith::SafeArith;
+use slog::error;
+use state_processing::{
+    common::{get_attestation_participation_flag_indices, get_attesting_indices_from_state},
+    epoch_cache::initialize_epoch_cache,
+    per_block_processing::{
+        altair::sync_committee::compute_sync_aggregate_rewards, get_slashable_indices,
+    },
+};
+use store::{
+    consts::altair::{PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, WEIGHT_DENOMINATOR},
+    RelativeEpoch,
+};
+use types::{AbstractExecPayload, BeaconBlockRef, BeaconState, BeaconStateError, Hash256};
+
+type BeaconBlockSubRewardValue = u64;
+
+impl<T: BeaconChainTypes> BeaconChain<T> {
+    pub fn compute_beacon_block_reward<Payload: AbstractExecPayload<T::EthSpec>>(
+        &self,
+        block: BeaconBlockRef<'_, T::EthSpec, Payload>,
+        block_root: Hash256,
+        state: &mut BeaconState<T::EthSpec>,
+    ) -> Result<StandardBlockReward, BeaconChainError> {
+        if block.slot() != state.slot() {
+            return Err(BeaconChainError::BlockRewardSlotError);
+        }
+
+        state.build_committee_cache(RelativeEpoch::Previous, &self.spec)?;
+        state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
+        initialize_epoch_cache(state, &self.spec)?;
+
+        self.compute_beacon_block_reward_with_cache(block, block_root, state)
+    }
+
+    // This should only be called after a committee cache has been built
+    // for both the previous and current epoch
+    fn compute_beacon_block_reward_with_cache<Payload: AbstractExecPayload<T::EthSpec>>(
+        &self,
+        block: BeaconBlockRef<'_, T::EthSpec, Payload>,
+        block_root: Hash256,
+        state: &BeaconState<T::EthSpec>,
+    ) -> Result<StandardBlockReward, BeaconChainError> {
+        let proposer_index = block.proposer_index();
+
+        let sync_aggregate_reward =
+            self.compute_beacon_block_sync_aggregate_reward(block, state)?;
+
+        let proposer_slashing_reward = self
+            .compute_beacon_block_proposer_slashing_reward(block, state)
+            .map_err(|e| {
+                error!(
+                self.log,
+                "Error calculating proposer slashing reward";
+                "error" => ?e
+                );
+                BeaconChainError::BlockRewardError
+            })?;
+
+        let attester_slashing_reward = self
+            .compute_beacon_block_attester_slashing_reward(block, state)
+            .map_err(|e| {
+                error!(
+                self.log,
+                "Error calculating attester slashing reward";
+                "error" => ?e
+                );
+                BeaconChainError::BlockRewardError
+            })?;
+
+        let block_attestation_reward = if let BeaconState::Base(_) = state {
+            self.compute_beacon_block_attestation_reward_base(block, block_root, state)
+                .map_err(|e| {
+                    error!(
+                        self.log,
+                        "Error calculating base block attestation reward";
+                        "error" => ?e
+                    );
+                    BeaconChainError::BlockRewardAttestationError
+                })?
+        } else {
+            self.compute_beacon_block_attestation_reward_altair_deneb(block, state)
+                .map_err(|e| {
+                    error!(
+                        self.log,
+                        "Error calculating altair block attestation reward";
+                        "error" => ?e
+                    );
+                    BeaconChainError::BlockRewardAttestationError
+                })?
+        };
+
+        let total_reward = sync_aggregate_reward
+            .safe_add(proposer_slashing_reward)?
+            .safe_add(attester_slashing_reward)?
+            .safe_add(block_attestation_reward)?;
+
+        Ok(StandardBlockReward {
+            proposer_index,
+            total: total_reward,
+            attestations: block_attestation_reward,
+            sync_aggregate: sync_aggregate_reward,
+            proposer_slashings: proposer_slashing_reward,
+            attester_slashings: attester_slashing_reward,
+        })
+    }
+
+    fn compute_beacon_block_sync_aggregate_reward<Payload: AbstractExecPayload<T::EthSpec>>(
+        &self,
+        block: BeaconBlockRef<'_, T::EthSpec, Payload>,
+        state: &BeaconState<T::EthSpec>,
+    ) -> Result<BeaconBlockSubRewardValue, BeaconChainError> {
+        if let Ok(sync_aggregate) = block.body().sync_aggregate() {
+            let (_, proposer_reward_per_bit) = compute_sync_aggregate_rewards(state, &self.spec)
+                .map_err(|_| BeaconChainError::BlockRewardSyncError)?;
+            Ok(sync_aggregate.sync_committee_bits.num_set_bits() as u64 * proposer_reward_per_bit)
+        } else {
+            Ok(0)
+        }
+    }
+
+    fn compute_beacon_block_proposer_slashing_reward<Payload: AbstractExecPayload<T::EthSpec>>(
+        &self,
+        block: BeaconBlockRef<'_, T::EthSpec, Payload>,
+        state: &BeaconState<T::EthSpec>,
+    ) -> Result<BeaconBlockSubRewardValue, BeaconChainError> {
+        let mut proposer_slashing_reward = 0;
+
+        let proposer_slashings = block.body().proposer_slashings();
+
+        for proposer_slashing in proposer_slashings {
+            proposer_slashing_reward.safe_add_assign(
+                state
+                    .get_validator(proposer_slashing.proposer_index() as usize)?
+                    .effective_balance()
+                    .safe_div(self.spec.whistleblower_reward_quotient)?,
+            )?;
+        }
+
+        Ok(proposer_slashing_reward)
+    }
+
+    fn compute_beacon_block_attester_slashing_reward<Payload: AbstractExecPayload<T::EthSpec>>(
+        &self,
+        block: BeaconBlockRef<'_, T::EthSpec, Payload>,
+        state: &BeaconState<T::EthSpec>,
+    ) -> Result<BeaconBlockSubRewardValue, BeaconChainError> {
+        let mut attester_slashing_reward = 0;
+
+        let attester_slashings = block.body().attester_slashings();
+
+        for attester_slashing in attester_slashings {
+            for attester_index in get_slashable_indices(state, attester_slashing)? {
+                attester_slashing_reward.safe_add_assign(
+                    state
+                        .get_validator(attester_index as usize)?
+                        .effective_balance()
+                        .safe_div(self.spec.whistleblower_reward_quotient)?,
+                )?;
+            }
+        }
+
+        Ok(attester_slashing_reward)
+    }
+
+    fn compute_beacon_block_attestation_reward_base<Payload: AbstractExecPayload<T::EthSpec>>(
+        &self,
+        block: BeaconBlockRef<'_, T::EthSpec, Payload>,
+        block_root: Hash256,
+        state: &BeaconState<T::EthSpec>,
+    ) -> Result<BeaconBlockSubRewardValue, BeaconChainError> {
+        // Call compute_block_reward in the base case
+        // Since base does not have sync aggregate, we only grab attesation portion of the returned
+        // value
+        let mut reward_cache = RewardCache::default();
+        let block_attestation_reward = self
+            .compute_block_reward(block, block_root, state, &mut reward_cache, true)?
+            .attestation_rewards
+            .total;
+
+        Ok(block_attestation_reward)
+    }
+
+    fn compute_beacon_block_attestation_reward_altair_deneb<
+        Payload: AbstractExecPayload<T::EthSpec>,
+    >(
+        &self,
+        block: BeaconBlockRef<'_, T::EthSpec, Payload>,
+        state: &BeaconState<T::EthSpec>,
+    ) -> Result<BeaconBlockSubRewardValue, BeaconChainError> {
+        let mut total_proposer_reward = 0;
+
+        let proposer_reward_denominator = WEIGHT_DENOMINATOR
+            .safe_sub(PROPOSER_WEIGHT)?
+            .safe_mul(WEIGHT_DENOMINATOR)?
+            .safe_div(PROPOSER_WEIGHT)?;
+
+        let mut current_epoch_participation = state.current_epoch_participation()?.clone();
+        let mut previous_epoch_participation = state.previous_epoch_participation()?.clone();
+
+        for attestation in block.body().attestations() {
+            let data = &attestation.data;
+            let inclusion_delay = state.slot().safe_sub(data.slot)?.as_u64();
+            // [Modified in Deneb:EIP7045]
+            let participation_flag_indices = get_attestation_participation_flag_indices(
+                state,
+                data,
+                inclusion_delay,
+                &self.spec,
+            )?;
+
+            let attesting_indices = get_attesting_indices_from_state(state, attestation)?;
+            let mut proposer_reward_numerator = 0;
+            for index in attesting_indices {
+                let index = index as usize;
+                for (flag_index, &weight) in PARTICIPATION_FLAG_WEIGHTS.iter().enumerate() {
+                    let epoch_participation = if data.target.epoch == state.current_epoch() {
+                        &mut current_epoch_participation
+                    } else {
+                        &mut previous_epoch_participation
+                    };
+
+                    let validator_participation = epoch_participation
+                        .get_mut(index)
+                        .ok_or(BeaconStateError::ParticipationOutOfBounds(index))?;
+
+                    if participation_flag_indices.contains(&flag_index)
+                        && !validator_participation.has_flag(flag_index)?
+                    {
+                        validator_participation.add_flag(flag_index)?;
+                        proposer_reward_numerator
+                            .safe_add_assign(state.get_base_reward(index)?.safe_mul(weight)?)?;
+                    }
+                }
+            }
+            total_proposer_reward.safe_add_assign(
+                proposer_reward_numerator.safe_div(proposer_reward_denominator)?,
+            )?;
+        }
+
+        Ok(total_proposer_reward)
+    }
+}

beacon_node/beacon_chain/src/beacon_block_streamer.rs

@@ -0,0 +1,981 @@
+use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use execution_layer::{ExecutionLayer, ExecutionPayloadBodyV1};
+use slog::{crit, debug, Logger};
+use std::collections::HashMap;
+use std::sync::Arc;
+use store::{DatabaseBlock, ExecutionPayloadDeneb};
+use task_executor::TaskExecutor;
+use tokio::sync::{
+    mpsc::{self, UnboundedSender},
+    RwLock,
+};
+use tokio_stream::{wrappers::UnboundedReceiverStream, Stream};
+use types::{
+    ChainSpec, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, Hash256, SignedBeaconBlock,
+    SignedBlindedBeaconBlock, Slot,
+};
+use types::{
+    ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadElectra, ExecutionPayloadHeader,
+    ExecutionPayloadMerge,
+};
+
+#[derive(PartialEq)]
+pub enum CheckCaches {
+    Yes,
+    No,
+}
+
+#[derive(Debug)]
+pub enum Error {
+    PayloadReconstruction(String),
+    BlocksByRangeFailure(Box<execution_layer::Error>),
+    RequestNotFound,
+    BlockResultNotFound,
+}
+
+const BLOCKS_PER_RANGE_REQUEST: u64 = 32;
+
+// This is the same as a DatabaseBlock but the Arc allows us to avoid an unnecessary clone.
+enum LoadedBeaconBlock<E: EthSpec> {
+    Full(Arc<SignedBeaconBlock<E>>),
+    Blinded(Box<SignedBlindedBeaconBlock<E>>),
+}
+type LoadResult<E> = Result<Option<LoadedBeaconBlock<E>>, BeaconChainError>;
+type BlockResult<E> = Result<Option<Arc<SignedBeaconBlock<E>>>, BeaconChainError>;
+
+enum RequestState<E: EthSpec> {
+    UnSent(Vec<BlockParts<E>>),
+    Sent(HashMap<Hash256, Arc<BlockResult<E>>>),
+}
+
+struct BodiesByRange<E: EthSpec> {
+    start: u64,
+    count: u64,
+    state: RequestState<E>,
+}
+
+// stores the components of a block for future re-construction in a small form
+struct BlockParts<E: EthSpec> {
+    blinded_block: Box<SignedBlindedBeaconBlock<E>>,
+    header: Box<ExecutionPayloadHeader<E>>,
+    body: Option<Box<ExecutionPayloadBodyV1<E>>>,
+}
+
+impl<E: EthSpec> BlockParts<E> {
+    pub fn new(
+        blinded: Box<SignedBlindedBeaconBlock<E>>,
+        header: ExecutionPayloadHeader<E>,
+    ) -> Self {
+        Self {
+            blinded_block: blinded,
+            header: Box::new(header),
+            body: None,
+        }
+    }
+
+    pub fn root(&self) -> Hash256 {
+        self.blinded_block.canonical_root()
+    }
+
+    pub fn slot(&self) -> Slot {
+        self.blinded_block.message().slot()
+    }
+
+    pub fn block_hash(&self) -> ExecutionBlockHash {
+        self.header.block_hash()
+    }
+}
+
+fn reconstruct_default_header_block<E: EthSpec>(
+    blinded_block: Box<SignedBlindedBeaconBlock<E>>,
+    header_from_block: ExecutionPayloadHeader<E>,
+    spec: &ChainSpec,
+) -> BlockResult<E> {
+    let fork = blinded_block
+        .fork_name(spec)
+        .map_err(BeaconChainError::InconsistentFork)?;
+
+    let payload: ExecutionPayload<E> = match fork {
+        ForkName::Merge => ExecutionPayloadMerge::default().into(),
+        ForkName::Capella => ExecutionPayloadCapella::default().into(),
+        ForkName::Deneb => ExecutionPayloadDeneb::default().into(),
+        ForkName::Electra => ExecutionPayloadElectra::default().into(),
+        ForkName::Base | ForkName::Altair => {
+            return Err(Error::PayloadReconstruction(format!(
+                "Block with fork variant {} has execution payload",
+                fork
+            ))
+            .into())
+        }
+    };
+
+    let header_from_payload = ExecutionPayloadHeader::from(payload.to_ref());
+    if header_from_payload == header_from_block {
+        blinded_block
+            .try_into_full_block(Some(payload))
+            .ok_or(BeaconChainError::AddPayloadLogicError)
+            .map(Arc::new)
+            .map(Some)
+    } else {
+        Err(BeaconChainError::InconsistentPayloadReconstructed {
+            slot: blinded_block.slot(),
+            exec_block_hash: header_from_block.block_hash(),
+            canonical_transactions_root: header_from_block.transactions_root(),
+            reconstructed_transactions_root: header_from_payload.transactions_root(),
+        })
+    }
+}
+
+fn reconstruct_blocks<E: EthSpec>(
+    block_map: &mut HashMap<Hash256, Arc<BlockResult<E>>>,
+    block_parts_with_bodies: HashMap<Hash256, BlockParts<E>>,
+    log: &Logger,
+) {
+    for (root, block_parts) in block_parts_with_bodies {
+        if let Some(payload_body) = block_parts.body {
+            match payload_body.to_payload(block_parts.header.as_ref().clone()) {
+                Ok(payload) => {
+                    let header_from_payload = ExecutionPayloadHeader::from(payload.to_ref());
+                    if header_from_payload == *block_parts.header {
+                        block_map.insert(
+                            root,
+                            Arc::new(
+                                block_parts
+                                    .blinded_block
+                                    .try_into_full_block(Some(payload))
+                                    .ok_or(BeaconChainError::AddPayloadLogicError)
+                                    .map(Arc::new)
+                                    .map(Some),
+                            ),
+                        );
+                    } else {
+                        let error = BeaconChainError::InconsistentPayloadReconstructed {
+                            slot: block_parts.blinded_block.slot(),
+                            exec_block_hash: block_parts.header.block_hash(),
+                            canonical_transactions_root: block_parts.header.transactions_root(),
+                            reconstructed_transactions_root: header_from_payload
+                                .transactions_root(),
+                        };
+                        debug!(log, "Failed to reconstruct block"; "root" => ?root, "error" => ?error);
+                        block_map.insert(root, Arc::new(Err(error)));
+                    }
+                }
+                Err(string) => {
+                    block_map.insert(
+                        root,
+                        Arc::new(Err(Error::PayloadReconstruction(string).into())),
+                    );
+                }
+            }
+        } else {
+            block_map.insert(
+                root,
+                Arc::new(Err(BeaconChainError::BlockHashMissingFromExecutionLayer(
+                    block_parts.block_hash(),
+                ))),
+            );
+        }
+    }
+}
+
+impl<E: EthSpec> BodiesByRange<E> {
+    pub fn new(maybe_block_parts: Option<BlockParts<E>>) -> Self {
+        if let Some(block_parts) = maybe_block_parts {
+            Self {
+                start: block_parts.header.block_number(),
+                count: 1,
+                state: RequestState::UnSent(vec![block_parts]),
+            }
+        } else {
+            Self {
+                start: 0,
+                count: 0,
+                state: RequestState::UnSent(vec![]),
+            }
+        }
+    }
+
+    pub fn is_unsent(&self) -> bool {
+        matches!(self.state, RequestState::UnSent(_))
+    }
+
+    pub fn push_block_parts(&mut self, block_parts: BlockParts<E>) -> Result<(), BlockParts<E>> {
+        if self.count == BLOCKS_PER_RANGE_REQUEST {
+            return Err(block_parts);
+        }
+
+        match &mut self.state {
+            RequestState::Sent(_) => Err(block_parts),
+            RequestState::UnSent(blocks_parts_vec) => {
+                let block_number = block_parts.header.block_number();
+                if self.count == 0 {
+                    self.start = block_number;
+                    self.count = 1;
+                    blocks_parts_vec.push(block_parts);
+                    Ok(())
+                } else {
+                    // need to figure out if this block fits in the request
+                    if block_number < self.start
+                        || self.start + BLOCKS_PER_RANGE_REQUEST <= block_number
+                    {
+                        return Err(block_parts);
+                    }
+
+                    blocks_parts_vec.push(block_parts);
+                    if self.start + self.count <= block_number {
+                        self.count = block_number - self.start + 1;
+                    }
+
+                    Ok(())
+                }
+            }
+        }
+    }
+
+    async fn execute(&mut self, execution_layer: &ExecutionLayer<E>, log: &Logger) {
+        if let RequestState::UnSent(blocks_parts_ref) = &mut self.state {
+            let block_parts_vec = std::mem::take(blocks_parts_ref);
+
+            let mut block_map = HashMap::new();
+            match execution_layer
+                .get_payload_bodies_by_range(self.start, self.count)
+                .await
+            {
+                Ok(bodies) => {
+                    let mut range_map = (self.start..(self.start + self.count))
+                        .zip(bodies.into_iter().chain(std::iter::repeat(None)))
+                        .collect::<HashMap<_, _>>();
+
+                    let mut with_bodies = HashMap::new();
+                    for mut block_parts in block_parts_vec {
+                        with_bodies
+                            // it's possible the same block is requested twice, using
+                            // or_insert_with() skips duplicates
+                            .entry(block_parts.root())
+                            .or_insert_with(|| {
+                                let block_number = block_parts.header.block_number();
+                                block_parts.body =
+                                    range_map.remove(&block_number).flatten().map(Box::new);
+
+                                block_parts
+                            });
+                    }
+
+                    reconstruct_blocks(&mut block_map, with_bodies, log);
+                }
+                Err(e) => {
+                    let block_result =
+                        Arc::new(Err(Error::BlocksByRangeFailure(Box::new(e)).into()));
+                    debug!(log, "Payload bodies by range failure"; "error" => ?block_result);
+                    for block_parts in block_parts_vec {
+                        block_map.insert(block_parts.root(), block_result.clone());
+                    }
+                }
+            }
+            self.state = RequestState::Sent(block_map);
+        }
+    }
+
+    pub async fn get_block_result(
+        &mut self,
+        root: &Hash256,
+        execution_layer: &ExecutionLayer<E>,
+        log: &Logger,
+    ) -> Option<Arc<BlockResult<E>>> {
+        self.execute(execution_layer, log).await;
+        if let RequestState::Sent(map) = &self.state {
+            return map.get(root).cloned();
+        }
+        // Shouldn't reach this point
+        None
+    }
+}
+
+#[derive(Clone)]
+enum EngineRequest<E: EthSpec> {
+    ByRange(Arc<RwLock<BodiesByRange<E>>>),
+    // When we already have the data or there's an error
+    NoRequest(Arc<RwLock<HashMap<Hash256, Arc<BlockResult<E>>>>>),
+}
+
+impl<E: EthSpec> EngineRequest<E> {
+    pub fn new_by_range() -> Self {
+        Self::ByRange(Arc::new(RwLock::new(BodiesByRange::new(None))))
+    }
+    pub fn new_no_request() -> Self {
+        Self::NoRequest(Arc::new(RwLock::new(HashMap::new())))
+    }
+
+    pub async fn is_unsent(&self) -> bool {
+        match self {
+            Self::ByRange(bodies_by_range) => bodies_by_range.read().await.is_unsent(),
+            Self::NoRequest(_) => false,
+        }
+    }
+
+    pub async fn push_block_parts(&mut self, block_parts: BlockParts<E>, log: &Logger) {
+        match self {
+            Self::ByRange(bodies_by_range) => {
+                let mut request = bodies_by_range.write().await;
+
+                if let Err(block_parts) = request.push_block_parts(block_parts) {
+                    drop(request);
+                    let new_by_range = BodiesByRange::new(Some(block_parts));
+                    *self = Self::ByRange(Arc::new(RwLock::new(new_by_range)));
+                }
+            }
+            Self::NoRequest(_) => {
+                // this should _never_ happen
+                crit!(
+                    log,
+                    "Please notify the devs";
+                    "beacon_block_streamer" => "push_block_parts called on NoRequest Variant",
+                );
+            }
+        }
+    }
+
+    pub async fn push_block_result(
+        &mut self,
+        root: Hash256,
+        block_result: BlockResult<E>,
+        log: &Logger,
+    ) {
+        // this function will only fail if something is seriously wrong
+        match self {
+            Self::ByRange(_) => {
+                // this should _never_ happen
+                crit!(
+                    log,
+                    "Please notify the devs";
+                    "beacon_block_streamer" => "push_block_result called on ByRange",
+                );
+            }
+            Self::NoRequest(results) => {
+                results.write().await.insert(root, Arc::new(block_result));
+            }
+        }
+    }
+
+    pub async fn get_block_result(
+        &self,
+        root: &Hash256,
+        execution_layer: &ExecutionLayer<E>,
+        log: &Logger,
+    ) -> Arc<BlockResult<E>> {
+        match self {
+            Self::ByRange(by_range) => {
+                by_range
+                    .write()
+                    .await
+                    .get_block_result(root, execution_layer, log)
+                    .await
+            }
+            Self::NoRequest(map) => map.read().await.get(root).cloned(),
+        }
+        .unwrap_or_else(|| {
+            crit!(
+                log,
+                "Please notify the devs";
+                "beacon_block_streamer" => "block_result not found in request",
+                "root" => ?root,
+            );
+            Arc::new(Err(Error::BlockResultNotFound.into()))
+        })
+    }
+}
+
+pub struct BeaconBlockStreamer<T: BeaconChainTypes> {
+    execution_layer: ExecutionLayer<T::EthSpec>,
+    check_caches: CheckCaches,
+    beacon_chain: Arc<BeaconChain<T>>,
+}
+
+impl<T: BeaconChainTypes> BeaconBlockStreamer<T> {
+    pub fn new(
+        beacon_chain: &Arc<BeaconChain<T>>,
+        check_caches: CheckCaches,
+    ) -> Result<Self, BeaconChainError> {
+        let execution_layer = beacon_chain
+            .execution_layer
+            .as_ref()
+            .ok_or(BeaconChainError::ExecutionLayerMissing)?
+            .clone();
+
+        Ok(Self {
+            execution_layer,
+            check_caches,
+            beacon_chain: beacon_chain.clone(),
+        })
+    }
+
+    fn check_caches(&self, root: Hash256) -> Option<Arc<SignedBeaconBlock<T::EthSpec>>> {
+        if self.check_caches == CheckCaches::Yes {
+            self.beacon_chain
+                .data_availability_checker
+                .get_block(&root)
+                .or(self.beacon_chain.early_attester_cache.get_block(root))
+        } else {
+            None
+        }
+    }
+
+    fn load_payloads(&self, block_roots: Vec<Hash256>) -> Vec<(Hash256, LoadResult<T::EthSpec>)> {
+        let mut db_blocks = Vec::new();
+
+        for root in block_roots {
+            if let Some(cached_block) = self.check_caches(root).map(LoadedBeaconBlock::Full) {
+                db_blocks.push((root, Ok(Some(cached_block))));
+                continue;
+            }
+
+            match self.beacon_chain.store.try_get_full_block(&root, None) {
+                Err(e) => db_blocks.push((root, Err(e.into()))),
+                Ok(opt_block) => db_blocks.push((
+                    root,
+                    Ok(opt_block.map(|db_block| match db_block {
+                        DatabaseBlock::Full(block) => LoadedBeaconBlock::Full(Arc::new(block)),
+                        DatabaseBlock::Blinded(block) => {
+                            LoadedBeaconBlock::Blinded(Box::new(block))
+                        }
+                    })),
+                )),
+            }
+        }
+
+        db_blocks
+    }
+
+    /// Pre-process the loaded blocks into execution engine requests.
+    ///
+    /// The purpose of this function is to separate the blocks into 2 categories:
+    /// 1) no_request - when we already have the full block or there's an error
+    /// 2) blocks_by_range - used for blinded blocks
+    ///
+    /// The function returns a vector of block roots in the same order as requested
+    /// along with the engine request that each root corresponds to.
+    async fn get_requests(
+        &self,
+        payloads: Vec<(Hash256, LoadResult<T::EthSpec>)>,
+    ) -> Vec<(Hash256, EngineRequest<T::EthSpec>)> {
+        let mut ordered_block_roots = Vec::new();
+        let mut requests = HashMap::new();
+
+        // we sort the by range blocks by slot before adding them to the
+        // request as it should *better* optimize the number of blocks that
+        // can fit in the same request
+        let mut by_range_blocks: Vec<BlockParts<T::EthSpec>> = vec![];
+        let mut no_request = EngineRequest::new_no_request();
+
+        for (root, load_result) in payloads {
+            // preserve the order of the requested blocks
+            ordered_block_roots.push(root);
+
+            let block_result = match load_result {
+                Err(e) => Err(e),
+                Ok(None) => Ok(None),
+                Ok(Some(LoadedBeaconBlock::Full(full_block))) => Ok(Some(full_block)),
+                Ok(Some(LoadedBeaconBlock::Blinded(blinded_block))) => {
+                    match blinded_block
+                        .message()
+                        .execution_payload()
+                        .map(|payload| payload.to_execution_payload_header())
+                    {
+                        Ok(header) => {
+                            if header.block_hash() == ExecutionBlockHash::zero() {
+                                reconstruct_default_header_block(
+                                    blinded_block,
+                                    header,
+                                    &self.beacon_chain.spec,
+                                )
+                            } else {
+                                // Add the block to the set requiring a by-range request.
+                                let block_parts = BlockParts::new(blinded_block, header);
+                                by_range_blocks.push(block_parts);
+                                continue;
+                            }
+                        }
+                        Err(e) => Err(BeaconChainError::BeaconStateError(e)),
+                    }
+                }
+            };
+
+            no_request
+                .push_block_result(root, block_result, &self.beacon_chain.log)
+                .await;
+            requests.insert(root, no_request.clone());
+        }
+
+        // Now deal with the by_range requests. Sort them in order of increasing slot
+        let mut by_range = EngineRequest::<T::EthSpec>::new_by_range();
+        by_range_blocks.sort_by_key(|block_parts| block_parts.slot());
+        for block_parts in by_range_blocks {
+            let root = block_parts.root();
+            by_range
+                .push_block_parts(block_parts, &self.beacon_chain.log)
+                .await;
+            requests.insert(root, by_range.clone());
+        }
+
+        let mut result = vec![];
+        for root in ordered_block_roots {
+            if let Some(request) = requests.get(&root) {
+                result.push((root, request.clone()))
+            } else {
+                crit!(
+                    self.beacon_chain.log,
+                    "Please notify the devs";
+                    "beacon_block_streamer" => "request not found",
+                    "root" => ?root,
+                );
+                no_request
+                    .push_block_result(
+                        root,
+                        Err(Error::RequestNotFound.into()),
+                        &self.beacon_chain.log,
+                    )
+                    .await;
+                result.push((root, no_request.clone()));
+            }
+        }
+
+        result
+    }
+
+    // used when the execution engine doesn't support the payload bodies methods
+    async fn stream_blocks_fallback(
+        &self,
+        block_roots: Vec<Hash256>,
+        sender: UnboundedSender<(Hash256, Arc<BlockResult<T::EthSpec>>)>,
+    ) {
+        debug!(
+            self.beacon_chain.log,
+            "Using slower fallback method of eth_getBlockByHash()"
+        );
+        for root in block_roots {
+            let cached_block = self.check_caches(root);
+            let block_result = if cached_block.is_some() {
+                Ok(cached_block)
+            } else {
+                self.beacon_chain
+                    .get_block(&root)
+                    .await
+                    .map(|opt_block| opt_block.map(Arc::new))
+            };
+
+            if sender.send((root, Arc::new(block_result))).is_err() {
+                break;
+            }
+        }
+    }
+
+    async fn stream_blocks(
+        &self,
+        block_roots: Vec<Hash256>,
+        sender: UnboundedSender<(Hash256, Arc<BlockResult<T::EthSpec>>)>,
+    ) {
+        let n_roots = block_roots.len();
+        let mut n_success = 0usize;
+        let mut n_sent = 0usize;
+        let mut engine_requests = 0usize;
+
+        let payloads = self.load_payloads(block_roots);
+        let requests = self.get_requests(payloads).await;
+
+        for (root, request) in requests {
+            if request.is_unsent().await {
+                engine_requests += 1;
+            }
+
+            let result = request
+                .get_block_result(&root, &self.execution_layer, &self.beacon_chain.log)
+                .await;
+
+            let successful = result
+                .as_ref()
+                .as_ref()
+                .map(|opt| opt.is_some())
+                .unwrap_or(false);
+
+            if sender.send((root, result)).is_err() {
+                break;
+            } else {
+                n_sent += 1;
+                if successful {
+                    n_success += 1;
+                }
+            }
+        }
+
+        debug!(
+            self.beacon_chain.log,
+            "BeaconBlockStreamer finished";
+            "requested blocks" => n_roots,
+            "sent" => n_sent,
+            "succeeded" => n_success,
+            "failed" => (n_sent - n_success),
+            "engine requests" => engine_requests,
+        );
+    }
+
+    pub async fn stream(
+        self,
+        block_roots: Vec<Hash256>,
+        sender: UnboundedSender<(Hash256, Arc<BlockResult<T::EthSpec>>)>,
+    ) {
+        match self
+            .execution_layer
+            .get_engine_capabilities(None)
+            .await
+            .map_err(Box::new)
+            .map_err(BeaconChainError::EngineGetCapabilititesFailed)
+        {
+            Ok(engine_capabilities) => {
+                if engine_capabilities.get_payload_bodies_by_range_v1 {
+                    self.stream_blocks(block_roots, sender).await;
+                } else {
+                    // use the fallback method
+                    self.stream_blocks_fallback(block_roots, sender).await;
+                }
+            }
+            Err(e) => {
+                send_errors(block_roots, sender, e).await;
+            }
+        }
+    }
+
+    pub fn launch_stream(
+        self,
+        block_roots: Vec<Hash256>,
+        executor: &TaskExecutor,
+    ) -> impl Stream<Item = (Hash256, Arc<BlockResult<T::EthSpec>>)> {
+        let (block_tx, block_rx) = mpsc::unbounded_channel();
+        debug!(
+            self.beacon_chain.log,
+            "Launching a BeaconBlockStreamer";
+            "blocks" => block_roots.len(),
+        );
+        executor.spawn(self.stream(block_roots, block_tx), "get_blocks_sender");
+        UnboundedReceiverStream::new(block_rx)
+    }
+}
+
+async fn send_errors<E: EthSpec>(
+    block_roots: Vec<Hash256>,
+    sender: UnboundedSender<(Hash256, Arc<BlockResult<E>>)>,
+    beacon_chain_error: BeaconChainError,
+) {
+    let result = Arc::new(Err(beacon_chain_error));
+    for root in block_roots {
+        if sender.send((root, result.clone())).is_err() {
+            break;
+        }
+    }
+}
+
+impl From<Error> for BeaconChainError {
+    fn from(value: Error) -> Self {
+        BeaconChainError::BlockStreamerError(value)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckCaches};
+    use crate::test_utils::{test_spec, BeaconChainHarness, EphemeralHarnessType};
+    use execution_layer::test_utils::{Block, DEFAULT_ENGINE_CAPABILITIES};
+    use execution_layer::EngineCapabilities;
+    use lazy_static::lazy_static;
+    use std::time::Duration;
+    use tokio::sync::mpsc;
+    use types::{ChainSpec, Epoch, EthSpec, Hash256, Keypair, MinimalEthSpec, Slot};
+
+    const VALIDATOR_COUNT: usize = 48;
+    lazy_static! {
+        /// A cached set of keys.
+        static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
+    }
+
+    fn get_harness(
+        validator_count: usize,
+        spec: ChainSpec,
+    ) -> BeaconChainHarness<EphemeralHarnessType<MinimalEthSpec>> {
+        let harness = BeaconChainHarness::builder(MinimalEthSpec)
+            .spec(spec)
+            .keypairs(KEYPAIRS[0..validator_count].to_vec())
+            .logger(logging::test_logger())
+            .fresh_ephemeral_store()
+            .mock_execution_layer()
+            .build();
+
+        harness.advance_slot();
+
+        harness
+    }
+
+    #[tokio::test]
+    async fn check_all_blocks_from_altair_to_electra() {
+        let slots_per_epoch = MinimalEthSpec::slots_per_epoch() as usize;
+        let num_epochs = 10;
+        let bellatrix_fork_epoch = 2usize;
+        let capella_fork_epoch = 4usize;
+        let deneb_fork_epoch = 6usize;
+        let electra_fork_epoch = 8usize;
+        let num_blocks_produced = num_epochs * slots_per_epoch;
+
+        let mut spec = test_spec::<MinimalEthSpec>();
+        spec.altair_fork_epoch = Some(Epoch::new(0));
+        spec.bellatrix_fork_epoch = Some(Epoch::new(bellatrix_fork_epoch as u64));
+        spec.capella_fork_epoch = Some(Epoch::new(capella_fork_epoch as u64));
+        spec.deneb_fork_epoch = Some(Epoch::new(deneb_fork_epoch as u64));
+        spec.electra_fork_epoch = Some(Epoch::new(electra_fork_epoch as u64));
+
+        let harness = get_harness(VALIDATOR_COUNT, spec.clone());
+        // go to bellatrix fork
+        harness
+            .extend_slots(bellatrix_fork_epoch * slots_per_epoch)
+            .await;
+        // extend half an epoch
+        harness.extend_slots(slots_per_epoch / 2).await;
+        // trigger merge
+        harness
+            .execution_block_generator()
+            .move_to_terminal_block()
+            .expect("should move to terminal block");
+        let timestamp = harness.get_timestamp_at_slot() + harness.spec.seconds_per_slot;
+        harness
+            .execution_block_generator()
+            .modify_last_block(|block| {
+                if let Block::PoW(terminal_block) = block {
+                    terminal_block.timestamp = timestamp;
+                }
+            });
+        // finish out merge epoch
+        harness.extend_slots(slots_per_epoch / 2).await;
+        // finish rest of epochs
+        harness
+            .extend_slots((num_epochs - 1 - bellatrix_fork_epoch) * slots_per_epoch)
+            .await;
+
+        let head = harness.chain.head_snapshot();
+        let state = &head.beacon_state;
+
+        assert_eq!(
+            state.slot(),
+            Slot::new(num_blocks_produced as u64),
+            "head should be at the current slot"
+        );
+        assert_eq!(
+            state.current_epoch(),
+            num_blocks_produced as u64 / MinimalEthSpec::slots_per_epoch(),
+            "head should be at the expected epoch"
+        );
+        assert_eq!(
+            state.current_justified_checkpoint().epoch,
+            state.current_epoch() - 1,
+            "the head should be justified one behind the current epoch"
+        );
+        assert_eq!(
+            state.finalized_checkpoint().epoch,
+            state.current_epoch() - 2,
+            "the head should be finalized two behind the current epoch"
+        );
+
+        let block_roots: Vec<Hash256> = harness
+            .chain
+            .forwards_iter_block_roots(Slot::new(0))
+            .expect("should get iter")
+            .map(Result::unwrap)
+            .map(|(root, _)| root)
+            .collect();
+
+        let mut expected_blocks = vec![];
+        // get all blocks the old fashioned way
+        for root in &block_roots {
+            let block = harness
+                .chain
+                .get_block(root)
+                .await
+                .expect("should get block")
+                .expect("block should exist");
+            expected_blocks.push(block);
+        }
+
+        for epoch in 0..num_epochs {
+            let start = epoch * slots_per_epoch;
+            let mut epoch_roots = vec![Hash256::zero(); slots_per_epoch];
+            epoch_roots[..].clone_from_slice(&block_roots[start..(start + slots_per_epoch)]);
+            let streamer = BeaconBlockStreamer::new(&harness.chain, CheckCaches::No)
+                .expect("should create streamer");
+            let (block_tx, mut block_rx) = mpsc::unbounded_channel();
+            streamer.stream(epoch_roots.clone(), block_tx).await;
+
+            for (i, expected_root) in epoch_roots.into_iter().enumerate() {
+                let (found_root, found_block_result) =
+                    block_rx.recv().await.expect("should get block");
+
+                assert_eq!(
+                    found_root, expected_root,
+                    "expected block root should match"
+                );
+                match found_block_result.as_ref() {
+                    Ok(maybe_block) => {
+                        let found_block = maybe_block.clone().expect("should have a block");
+                        let expected_block = expected_blocks
+                            .get(start + i)
+                            .expect("should get expected block");
+                        assert_eq!(
+                            found_block.as_ref(),
+                            expected_block,
+                            "expected block should match found block"
+                        );
+                    }
+                    Err(e) => panic!("Error retrieving block {}: {:?}", expected_root, e),
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn check_fallback_altair_to_electra() {
+        let slots_per_epoch = MinimalEthSpec::slots_per_epoch() as usize;
+        let num_epochs = 10;
+        let bellatrix_fork_epoch = 2usize;
+        let capella_fork_epoch = 4usize;
+        let deneb_fork_epoch = 6usize;
+        let electra_fork_epoch = 8usize;
+        let num_blocks_produced = num_epochs * slots_per_epoch;
+
+        let mut spec = test_spec::<MinimalEthSpec>();
+        spec.altair_fork_epoch = Some(Epoch::new(0));
+        spec.bellatrix_fork_epoch = Some(Epoch::new(bellatrix_fork_epoch as u64));
+        spec.capella_fork_epoch = Some(Epoch::new(capella_fork_epoch as u64));
+        spec.deneb_fork_epoch = Some(Epoch::new(deneb_fork_epoch as u64));
+        spec.electra_fork_epoch = Some(Epoch::new(electra_fork_epoch as u64));
+
+        let harness = get_harness(VALIDATOR_COUNT, spec);
+
+        // modify execution engine so it doesn't support engine_payloadBodiesBy* methods
+        let mock_execution_layer = harness.mock_execution_layer.as_ref().unwrap();
+        mock_execution_layer
+            .server
+            .set_engine_capabilities(EngineCapabilities {
+                get_payload_bodies_by_hash_v1: false,
+                get_payload_bodies_by_range_v1: false,
+                ..DEFAULT_ENGINE_CAPABILITIES
+            });
+        // refresh capabilities cache
+        harness
+            .chain
+            .execution_layer
+            .as_ref()
+            .unwrap()
+            .get_engine_capabilities(Some(Duration::ZERO))
+            .await
+            .unwrap();
+
+        // go to bellatrix fork
+        harness
+            .extend_slots(bellatrix_fork_epoch * slots_per_epoch)
+            .await;
+        // extend half an epoch
+        harness.extend_slots(slots_per_epoch / 2).await;
+        // trigger merge
+        harness
+            .execution_block_generator()
+            .move_to_terminal_block()
+            .expect("should move to terminal block");
+        let timestamp = harness.get_timestamp_at_slot() + harness.spec.seconds_per_slot;
+        harness
+            .execution_block_generator()
+            .modify_last_block(|block| {
+                if let Block::PoW(terminal_block) = block {
+                    terminal_block.timestamp = timestamp;
+                }
+            });
+        // finish out merge epoch
+        harness.extend_slots(slots_per_epoch / 2).await;
+        // finish rest of epochs
+        harness
+            .extend_slots((num_epochs - 1 - bellatrix_fork_epoch) * slots_per_epoch)
+            .await;
+
+        let head = harness.chain.head_snapshot();
+        let state = &head.beacon_state;
+
+        assert_eq!(
+            state.slot(),
+            Slot::new(num_blocks_produced as u64),
+            "head should be at the current slot"
+        );
+        assert_eq!(
+            state.current_epoch(),
+            num_blocks_produced as u64 / MinimalEthSpec::slots_per_epoch(),
+            "head should be at the expected epoch"
+        );
+        assert_eq!(
+            state.current_justified_checkpoint().epoch,
+            state.current_epoch() - 1,
+            "the head should be justified one behind the current epoch"
+        );
+        assert_eq!(
+            state.finalized_checkpoint().epoch,
+            state.current_epoch() - 2,
+            "the head should be finalized two behind the current epoch"
+        );
+
+        let block_roots: Vec<Hash256> = harness
+            .chain
+            .forwards_iter_block_roots(Slot::new(0))
+            .expect("should get iter")
+            .map(Result::unwrap)
+            .map(|(root, _)| root)
+            .collect();
+
+        let mut expected_blocks = vec![];
+        // get all blocks the old fashioned way
+        for root in &block_roots {
+            let block = harness
+                .chain
+                .get_block(root)
+                .await
+                .expect("should get block")
+                .expect("block should exist");
+            expected_blocks.push(block);
+        }
+
+        for epoch in 0..num_epochs {
+            let start = epoch * slots_per_epoch;
+            let mut epoch_roots = vec![Hash256::zero(); slots_per_epoch];
+            epoch_roots[..].clone_from_slice(&block_roots[start..(start + slots_per_epoch)]);
+            let streamer = BeaconBlockStreamer::new(&harness.chain, CheckCaches::No)
+                .expect("should create streamer");
+            let (block_tx, mut block_rx) = mpsc::unbounded_channel();
+            streamer.stream(epoch_roots.clone(), block_tx).await;
+
+            for (i, expected_root) in epoch_roots.into_iter().enumerate() {
+                let (found_root, found_block_result) =
+                    block_rx.recv().await.expect("should get block");
+
+                assert_eq!(
+                    found_root, expected_root,
+                    "expected block root should match"
+                );
+                match found_block_result.as_ref() {
+                    Ok(maybe_block) => {
+                        let found_block = maybe_block.clone().expect("should have a block");
+                        let expected_block = expected_blocks
+                            .get(start + i)
+                            .expect("should get expected block");
+                        assert_eq!(
+                            found_block.as_ref(),
+                            expected_block,
+                            "expected block should match found block"
+                        );
+                    }
+                    Err(e) => panic!("Error retrieving block {}: {:?}", expected_root, e),
+                }
+            }
+        }
+    }
+}

beacon_node/beacon_chain/src/beacon_fork_choice_store.rs

@@ -7,13 +7,25 @@
 use crate::{metrics, BeaconSnapshot};
 use derivative::Derivative;
 use fork_choice::ForkChoiceStore;
+use proto_array::JustifiedBalances;
+use safe_arith::ArithError;
 use ssz_derive::{Decode, Encode};
+use std::collections::BTreeSet;
 use std::marker::PhantomData;
 use std::sync::Arc;
 use store::{Error as StoreError, HotColdDB, ItemStore};
 use superstruct::superstruct;
 use types::{
-    BeaconBlock, BeaconState, BeaconStateError, Checkpoint, Epoch, EthSpec, Hash256, Slot,
+    AbstractExecPayload, BeaconBlockRef, BeaconState, BeaconStateError, Checkpoint, Epoch, EthSpec,
+    Hash256, Slot,
+};
+
+/// Ensure this justified checkpoint has an epoch of 0 so that it is never
+/// greater than the justified checkpoint and enshrined as the actual justified
+/// checkpoint.
+const JUNK_BEST_JUSTIFIED_CHECKPOINT: Checkpoint = Checkpoint {
+    epoch: Epoch::new(0),
+    root: Hash256::repeat_byte(0),
 };
 
 #[derive(Debug)]
@@ -29,6 +41,7 @@ pub enum Error {
     MissingState(Hash256),
     InvalidPersistedBytes(ssz::DecodeError),
     BeaconStateError(BeaconStateError),
+    Arith(ArithError),
 }
 
 impl From<BeaconStateError> for Error {
@@ -37,29 +50,17 @@ impl From<BeaconStateError> for Error {
     }
 }
 
+impl From<ArithError> for Error {
+    fn from(e: ArithError) -> Self {
+        Error::Arith(e)
+    }
+}
+
 /// The number of validator balance sets that are cached within `BalancesCache`.
 const MAX_BALANCE_CACHE_SIZE: usize = 4;
 
-/// Returns the effective balances for every validator in the given `state`.
-///
-/// Any validator who is not active in the epoch of the given `state` is assigned a balance of
-/// zero.
-pub fn get_effective_balances<T: EthSpec>(state: &BeaconState<T>) -> Vec<u64> {
-    state
-        .validators()
-        .iter()
-        .map(|validator| {
-            if validator.is_active_at(state.current_epoch()) {
-                validator.effective_balance
-            } else {
-                0
-            }
-        })
-        .collect()
-}
-
 #[superstruct(
-    variants(V1, V8),
+    variants(V8),
     variant_attributes(derive(PartialEq, Clone, Debug, Encode, Decode)),
     no_enum
 )]
@@ -73,13 +74,11 @@ pub(crate) struct CacheItem {
 pub(crate) type CacheItem = CacheItemV8;
 
 #[superstruct(
-    variants(V1, V8),
+    variants(V8),
     variant_attributes(derive(PartialEq, Clone, Default, Debug, Encode, Decode)),
     no_enum
 )]
 pub struct BalancesCache {
-    #[superstruct(only(V1))]
-    pub(crate) items: Vec<CacheItemV1>,
     #[superstruct(only(V8))]
     pub(crate) items: Vec<CacheItemV8>,
 }
@@ -113,7 +112,7 @@ impl BalancesCache {
             let item = CacheItem {
                 block_root: epoch_boundary_root,
                 epoch,
-                balances: get_effective_balances(state),
+                balances: JustifiedBalances::from_justified_state(state)?.effective_balances,
             };
 
             if self.items.len() == MAX_BALANCE_CACHE_SIZE {
@@ -152,9 +151,11 @@ pub struct BeaconForkChoiceStore<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<
     time: Slot,
     finalized_checkpoint: Checkpoint,
     justified_checkpoint: Checkpoint,
-    justified_balances: Vec<u64>,
-    best_justified_checkpoint: Checkpoint,
+    justified_balances: JustifiedBalances,
+    unrealized_justified_checkpoint: Checkpoint,
+    unrealized_finalized_checkpoint: Checkpoint,
     proposer_boost_root: Hash256,
+    equivocating_indices: BTreeSet<u64>,
     _phantom: PhantomData<E>,
 }
 
@@ -178,7 +179,7 @@ where
     pub fn get_forkchoice_store(
         store: Arc<HotColdDB<E, Hot, Cold>>,
         anchor: &BeaconSnapshot<E>,
-    ) -> Self {
+    ) -> Result<Self, Error> {
         let anchor_state = &anchor.beacon_state;
         let mut anchor_block_header = anchor_state.latest_block_header().clone();
         if anchor_block_header.state_root == Hash256::zero() {
@@ -191,18 +192,21 @@ where
             root: anchor_root,
         };
         let finalized_checkpoint = justified_checkpoint;
+        let justified_balances = JustifiedBalances::from_justified_state(anchor_state)?;
 
-        Self {
+        Ok(Self {
             store,
             balances_cache: <_>::default(),
             time: anchor_state.slot(),
             justified_checkpoint,
-            justified_balances: anchor_state.balances().clone().into(),
+            justified_balances,
             finalized_checkpoint,
-            best_justified_checkpoint: justified_checkpoint,
+            unrealized_justified_checkpoint: justified_checkpoint,
+            unrealized_finalized_checkpoint: finalized_checkpoint,
             proposer_boost_root: Hash256::zero(),
+            equivocating_indices: BTreeSet::new(),
             _phantom: PhantomData,
-        }
+        })
     }
 
     /// Save the current state of `Self` to a `PersistedForkChoiceStore` which can be stored to the
@@ -213,9 +217,11 @@ where
             time: self.time,
             finalized_checkpoint: self.finalized_checkpoint,
             justified_checkpoint: self.justified_checkpoint,
-            justified_balances: self.justified_balances.clone(),
-            best_justified_checkpoint: self.best_justified_checkpoint,
+            justified_balances: self.justified_balances.effective_balances.clone(),
+            unrealized_justified_checkpoint: self.unrealized_justified_checkpoint,
+            unrealized_finalized_checkpoint: self.unrealized_finalized_checkpoint,
             proposer_boost_root: self.proposer_boost_root,
+            equivocating_indices: self.equivocating_indices.clone(),
         }
     }
 
@@ -224,15 +230,19 @@ where
         persisted: PersistedForkChoiceStore,
         store: Arc<HotColdDB<E, Hot, Cold>>,
     ) -> Result<Self, Error> {
+        let justified_balances =
+            JustifiedBalances::from_effective_balances(persisted.justified_balances)?;
         Ok(Self {
             store,
             balances_cache: persisted.balances_cache,
             time: persisted.time,
             finalized_checkpoint: persisted.finalized_checkpoint,
             justified_checkpoint: persisted.justified_checkpoint,
-            justified_balances: persisted.justified_balances,
-            best_justified_checkpoint: persisted.best_justified_checkpoint,
+            justified_balances,
+            unrealized_justified_checkpoint: persisted.unrealized_justified_checkpoint,
+            unrealized_finalized_checkpoint: persisted.unrealized_finalized_checkpoint,
             proposer_boost_root: persisted.proposer_boost_root,
+            equivocating_indices: persisted.equivocating_indices,
             _phantom: PhantomData,
         })
     }
@@ -254,9 +264,9 @@ where
         self.time = slot
     }
 
-    fn on_verified_block(
+    fn on_verified_block<Payload: AbstractExecPayload<E>>(
         &mut self,
-        _block: &BeaconBlock<E>,
+        _block: BeaconBlockRef<E, Payload>,
         block_root: Hash256,
         state: &BeaconState<E>,
     ) -> Result<(), Self::Error> {
@@ -267,18 +277,22 @@ where
         &self.justified_checkpoint
     }
 
-    fn justified_balances(&self) -> &[u64] {
+    fn justified_balances(&self) -> &JustifiedBalances {
         &self.justified_balances
     }
 
-    fn best_justified_checkpoint(&self) -> &Checkpoint {
-        &self.best_justified_checkpoint
-    }
-
     fn finalized_checkpoint(&self) -> &Checkpoint {
         &self.finalized_checkpoint
     }
 
+    fn unrealized_justified_checkpoint(&self) -> &Checkpoint {
+        &self.unrealized_justified_checkpoint
+    }
+
+    fn unrealized_finalized_checkpoint(&self) -> &Checkpoint {
+        &self.unrealized_finalized_checkpoint
+    }
+
     fn proposer_boost_root(&self) -> Hash256 {
         self.proposer_boost_root
     }
@@ -294,57 +308,116 @@ where
             self.justified_checkpoint.root,
             self.justified_checkpoint.epoch,
         ) {
+            // NOTE: could avoid this re-calculation by introducing a `PersistedCacheItem`.
             metrics::inc_counter(&metrics::BALANCES_CACHE_HITS);
-            self.justified_balances = balances;
+            self.justified_balances = JustifiedBalances::from_effective_balances(balances)?;
         } else {
             metrics::inc_counter(&metrics::BALANCES_CACHE_MISSES);
             let justified_block = self
                 .store
-                .get_block(&self.justified_checkpoint.root)
+                .get_blinded_block(&self.justified_checkpoint.root, None)
                 .map_err(Error::FailedToReadBlock)?
                 .ok_or(Error::MissingBlock(self.justified_checkpoint.root))?
                 .deconstruct()
                 .0;
 
-            let state = self
+            let max_slot = self
+                .justified_checkpoint
+                .epoch
+                .start_slot(E::slots_per_epoch());
+            let (_, state) = self
                 .store
-                .get_state(&justified_block.state_root(), Some(justified_block.slot()))
+                .get_advanced_hot_state(
+                    self.justified_checkpoint.root,
+                    max_slot,
+                    justified_block.state_root(),
+                )
                 .map_err(Error::FailedToReadState)?
                 .ok_or_else(|| Error::MissingState(justified_block.state_root()))?;
 
-            self.justified_balances = get_effective_balances(&state);
+            self.justified_balances = JustifiedBalances::from_justified_state(&state)?;
         }
 
         Ok(())
     }
 
-    fn set_best_justified_checkpoint(&mut self, checkpoint: Checkpoint) {
-        self.best_justified_checkpoint = checkpoint
+    fn set_unrealized_justified_checkpoint(&mut self, checkpoint: Checkpoint) {
+        self.unrealized_justified_checkpoint = checkpoint;
+    }
+
+    fn set_unrealized_finalized_checkpoint(&mut self, checkpoint: Checkpoint) {
+        self.unrealized_finalized_checkpoint = checkpoint;
     }
 
     fn set_proposer_boost_root(&mut self, proposer_boost_root: Hash256) {
         self.proposer_boost_root = proposer_boost_root;
     }
+
+    fn equivocating_indices(&self) -> &BTreeSet<u64> {
+        &self.equivocating_indices
+    }
+
+    fn extend_equivocating_indices(&mut self, indices: impl IntoIterator<Item = u64>) {
+        self.equivocating_indices.extend(indices);
+    }
 }
 
+pub type PersistedForkChoiceStore = PersistedForkChoiceStoreV17;
+
 /// A container which allows persisting the `BeaconForkChoiceStore` to the on-disk database.
 #[superstruct(
-    variants(V1, V7, V8),
+    variants(V11, V17),
     variant_attributes(derive(Encode, Decode)),
     no_enum
 )]
 pub struct PersistedForkChoiceStore {
-    #[superstruct(only(V1, V7))]
-    pub balances_cache: BalancesCacheV1,
-    #[superstruct(only(V8))]
+    #[superstruct(only(V11, V17))]
     pub balances_cache: BalancesCacheV8,
     pub time: Slot,
     pub finalized_checkpoint: Checkpoint,
     pub justified_checkpoint: Checkpoint,
     pub justified_balances: Vec<u64>,
+    #[superstruct(only(V11))]
     pub best_justified_checkpoint: Checkpoint,
-    #[superstruct(only(V7, V8))]
+    #[superstruct(only(V11, V17))]
+    pub unrealized_justified_checkpoint: Checkpoint,
+    #[superstruct(only(V11, V17))]
+    pub unrealized_finalized_checkpoint: Checkpoint,
+    #[superstruct(only(V11, V17))]
     pub proposer_boost_root: Hash256,
+    #[superstruct(only(V11, V17))]
+    pub equivocating_indices: BTreeSet<u64>,
 }
 
-pub type PersistedForkChoiceStore = PersistedForkChoiceStoreV8;
+impl Into<PersistedForkChoiceStore> for PersistedForkChoiceStoreV11 {
+    fn into(self) -> PersistedForkChoiceStore {
+        PersistedForkChoiceStore {
+            balances_cache: self.balances_cache,
+            time: self.time,
+            finalized_checkpoint: self.finalized_checkpoint,
+            justified_checkpoint: self.justified_checkpoint,
+            justified_balances: self.justified_balances,
+            unrealized_justified_checkpoint: self.unrealized_justified_checkpoint,
+            unrealized_finalized_checkpoint: self.unrealized_finalized_checkpoint,
+            proposer_boost_root: self.proposer_boost_root,
+            equivocating_indices: self.equivocating_indices,
+        }
+    }
+}
+
+impl Into<PersistedForkChoiceStoreV11> for PersistedForkChoiceStore {
+    fn into(self) -> PersistedForkChoiceStoreV11 {
+        PersistedForkChoiceStoreV11 {
+            balances_cache: self.balances_cache,
+            time: self.time,
+            finalized_checkpoint: self.finalized_checkpoint,
+            justified_checkpoint: self.justified_checkpoint,
+            justified_balances: self.justified_balances,
+            best_justified_checkpoint: JUNK_BEST_JUSTIFIED_CHECKPOINT,
+            unrealized_justified_checkpoint: self.unrealized_justified_checkpoint,
+            unrealized_finalized_checkpoint: self.unrealized_finalized_checkpoint,
+            proposer_boost_root: self.proposer_boost_root,
+            equivocating_indices: self.equivocating_indices,
+        }
+    }
+}

beacon_node/beacon_chain/src/beacon_proposer_cache.rs

@@ -8,17 +8,25 @@
 //! very simple to reason about, but it might store values that are useless due to finalization. The
 //! values it stores are very small, so this should not be an issue.
 
+use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use fork_choice::ExecutionStatus;
 use lru::LruCache;
 use smallvec::SmallVec;
-use types::{BeaconStateError, Epoch, EthSpec, Fork, Hash256, Slot, Unsigned};
+use state_processing::state_advance::partial_state_advance;
+use std::cmp::Ordering;
+use std::num::NonZeroUsize;
+use types::non_zero_usize::new_non_zero_usize;
+use types::{
+    BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, Fork, Hash256, Slot, Unsigned,
+};
 
 /// The number of sets of proposer indices that should be cached.
-const CACHE_SIZE: usize = 16;
+const CACHE_SIZE: NonZeroUsize = new_non_zero_usize(16);
 
 /// This value is fairly unimportant, it's used to avoid heap allocations. The result of it being
 /// incorrect is non-substantial from a consensus perspective (and probably also from a
 /// performance perspective).
-const TYPICAL_SLOTS_PER_EPOCH: usize = 32;
+pub const TYPICAL_SLOTS_PER_EPOCH: usize = 32;
 
 /// For some given slot, this contains the proposer index (`index`) and the `fork` that should be
 /// used to verify their signature.
@@ -59,19 +67,19 @@ impl Default for BeaconProposerCache {
 impl BeaconProposerCache {
     /// If it is cached, returns the proposer for the block at `slot` where the block has the
     /// ancestor block root of `shuffling_decision_block` at `end_slot(slot.epoch() - 1)`.
-    pub fn get_slot<T: EthSpec>(
+    pub fn get_slot<E: EthSpec>(
         &mut self,
         shuffling_decision_block: Hash256,
         slot: Slot,
     ) -> Option<Proposer> {
-        let epoch = slot.epoch(T::slots_per_epoch());
+        let epoch = slot.epoch(E::slots_per_epoch());
         let key = (epoch, shuffling_decision_block);
         if let Some(cache) = self.cache.get(&key) {
             // This `if` statement is likely unnecessary, but it feels like good practice.
             if epoch == cache.epoch {
                 cache
                     .proposers
-                    .get(slot.as_usize() % T::SlotsPerEpoch::to_usize())
+                    .get(slot.as_usize() % E::SlotsPerEpoch::to_usize())
                     .map(|&index| Proposer {
                         index,
                         fork: cache.fork,
@@ -89,7 +97,7 @@ impl BeaconProposerCache {
     /// The nth slot in the returned `SmallVec` will be equal to the nth slot in the given `epoch`.
     /// E.g., if `epoch == 1` then `smallvec[0]` refers to slot 32 (assuming `SLOTS_PER_EPOCH ==
     /// 32`).
-    pub fn get_epoch<T: EthSpec>(
+    pub fn get_epoch<E: EthSpec>(
         &mut self,
         shuffling_decision_block: Hash256,
         epoch: Epoch,
@@ -125,3 +133,68 @@ impl BeaconProposerCache {
         Ok(())
     }
 }
+
+/// Compute the proposer duties using the head state without cache.
+pub fn compute_proposer_duties_from_head<T: BeaconChainTypes>(
+    request_epoch: Epoch,
+    chain: &BeaconChain<T>,
+) -> Result<(Vec<usize>, Hash256, ExecutionStatus, Fork), BeaconChainError> {
+    // Atomically collect information about the head whilst holding the canonical head `Arc` as
+    // short as possible.
+    let (mut state, head_state_root, head_block_root) = {
+        let head = chain.canonical_head.cached_head();
+        // Take a copy of the head state.
+        let head_state = head.snapshot.beacon_state.clone();
+        let head_state_root = head.head_state_root();
+        let head_block_root = head.head_block_root();
+        (head_state, head_state_root, head_block_root)
+    };
+
+    let execution_status = chain
+        .canonical_head
+        .fork_choice_read_lock()
+        .get_block_execution_status(&head_block_root)
+        .ok_or(BeaconChainError::HeadMissingFromForkChoice(head_block_root))?;
+
+    // Advance the state into the requested epoch.
+    ensure_state_is_in_epoch(&mut state, head_state_root, request_epoch, &chain.spec)?;
+
+    let indices = state
+        .get_beacon_proposer_indices(&chain.spec)
+        .map_err(BeaconChainError::from)?;
+
+    let dependent_root = state
+        // The only block which decides its own shuffling is the genesis block.
+        .proposer_shuffling_decision_root(chain.genesis_block_root)
+        .map_err(BeaconChainError::from)?;
+
+    Ok((indices, dependent_root, execution_status, state.fork()))
+}
+
+/// If required, advance `state` to `target_epoch`.
+///
+/// ## Details
+///
+/// - Returns an error if `state.current_epoch() > target_epoch`.
+/// - No-op if `state.current_epoch() == target_epoch`.
+/// - It must be the case that `state.canonical_root() == state_root`, but this function will not
+///     check that.
+pub fn ensure_state_is_in_epoch<E: EthSpec>(
+    state: &mut BeaconState<E>,
+    state_root: Hash256,
+    target_epoch: Epoch,
+    spec: &ChainSpec,
+) -> Result<(), BeaconChainError> {
+    match state.current_epoch().cmp(&target_epoch) {
+        // Protects against an inconsistent slot clock.
+        Ordering::Greater => Err(BeaconStateError::SlotOutOfBounds.into()),
+        // The state needs to be advanced.
+        Ordering::Less => {
+            let target_slot = target_epoch.start_slot(E::slots_per_epoch());
+            partial_state_advance(state, Some(state_root), target_slot, spec)
+                .map_err(BeaconChainError::from)
+        }
+        // The state is suitable, nothing to do.
+        Ordering::Equal => Ok(()),
+    }
+}

beacon_node/beacon_chain/src/beacon_snapshot.rs

@@ -1,19 +1,36 @@
-use serde_derive::Serialize;
-use types::{beacon_state::CloneConfig, BeaconState, EthSpec, Hash256, SignedBeaconBlock};
+use serde::Serialize;
+use std::sync::Arc;
+use types::{
+    AbstractExecPayload, BeaconState, EthSpec, FullPayload, Hash256, SignedBeaconBlock,
+    SignedBlindedBeaconBlock,
+};
 
 /// Represents some block and its associated state. Generally, this will be used for tracking the
 /// head, justified head and finalized head.
 #[derive(Clone, Serialize, PartialEq, Debug)]
-pub struct BeaconSnapshot<E: EthSpec> {
-    pub beacon_block: SignedBeaconBlock<E>,
+pub struct BeaconSnapshot<E: EthSpec, Payload: AbstractExecPayload<E> = FullPayload<E>> {
+    pub beacon_block: Arc<SignedBeaconBlock<E, Payload>>,
     pub beacon_block_root: Hash256,
     pub beacon_state: BeaconState<E>,
 }
 
-impl<E: EthSpec> BeaconSnapshot<E> {
+/// This snapshot is to be used for verifying a child of `self.beacon_block`.
+#[derive(Debug)]
+pub struct PreProcessingSnapshot<T: EthSpec> {
+    /// This state is equivalent to the `self.beacon_block.state_root()` state that has been
+    /// advanced forward one slot using `per_slot_processing`. This state is "primed and ready" for
+    /// the application of another block.
+    pub pre_state: BeaconState<T>,
+    /// This value is only set to `Some` if the `pre_state` was *not* advanced forward.
+    pub beacon_state_root: Option<Hash256>,
+    pub beacon_block: SignedBlindedBeaconBlock<T>,
+    pub beacon_block_root: Hash256,
+}
+
+impl<E: EthSpec, Payload: AbstractExecPayload<E>> BeaconSnapshot<E, Payload> {
     /// Create a new checkpoint.
     pub fn new(
-        beacon_block: SignedBeaconBlock<E>,
+        beacon_block: Arc<SignedBeaconBlock<E, Payload>>,
         beacon_block_root: Hash256,
         beacon_state: BeaconState<E>,
     ) -> Self {
@@ -36,7 +53,7 @@ impl<E: EthSpec> BeaconSnapshot<E> {
     /// Update all fields of the checkpoint.
     pub fn update(
         &mut self,
-        beacon_block: SignedBeaconBlock<E>,
+        beacon_block: Arc<SignedBeaconBlock<E, Payload>>,
         beacon_block_root: Hash256,
         beacon_state: BeaconState<E>,
     ) {
@@ -44,12 +61,4 @@ impl<E: EthSpec> BeaconSnapshot<E> {
         self.beacon_block_root = beacon_block_root;
         self.beacon_state = beacon_state;
     }
-
-    pub fn clone_with(&self, clone_config: CloneConfig) -> Self {
-        Self {
-            beacon_block: self.beacon_block.clone(),
-            beacon_block_root: self.beacon_block_root,
-            beacon_state: self.beacon_state.clone_with(clone_config),
-        }
-    }
 }

beacon_node/beacon_chain/src/blob_verification.rs

@@ -0,0 +1,602 @@
+use derivative::Derivative;
+use slot_clock::SlotClock;
+use std::sync::Arc;
+
+use crate::beacon_chain::{BeaconChain, BeaconChainTypes};
+use crate::block_verification::{
+    cheap_state_advance_to_obtain_committees, process_block_slash_info, BlockSlashInfo,
+};
+use crate::kzg_utils::{validate_blob, validate_blobs};
+use crate::{metrics, BeaconChainError};
+use kzg::{Error as KzgError, Kzg, KzgCommitment};
+use merkle_proof::MerkleTreeError;
+use slog::debug;
+use ssz_derive::{Decode, Encode};
+use ssz_types::VariableList;
+use tree_hash::TreeHash;
+use types::blob_sidecar::BlobIdentifier;
+use types::{BeaconStateError, BlobSidecar, EthSpec, Hash256, SignedBeaconBlockHeader, Slot};
+
+/// An error occurred while validating a gossip blob.
+#[derive(Debug)]
+pub enum GossipBlobError<E: EthSpec> {
+    /// The blob sidecar is from a slot that is later than the current slot (with respect to the
+    /// gossip clock disparity).
+    ///
+    /// ## Peer scoring
+    ///
+    /// Assuming the local clock is correct, the peer has sent an invalid message.
+    FutureSlot {
+        message_slot: Slot,
+        latest_permissible_slot: Slot,
+    },
+
+    /// There was an error whilst processing the blob. It is not known if it is
+    /// valid or invalid.
+    ///
+    /// ## Peer scoring
+    ///
+    /// We were unable to process this blob due to an internal error. It's
+    /// unclear if the blob is valid.
+    BeaconChainError(BeaconChainError),
+
+    /// The `BlobSidecar` was gossiped over an incorrect subnet.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The blob is invalid or the peer is faulty.
+    InvalidSubnet { expected: u64, received: u64 },
+
+    /// The sidecar corresponds to a slot older than the finalized head slot.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this blob is valid, but this blob is for a finalized slot and is
+    /// therefore useless to us.
+    PastFinalizedSlot {
+        blob_slot: Slot,
+        finalized_slot: Slot,
+    },
+
+    /// The proposer index specified in the sidecar does not match the locally computed
+    /// proposer index.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The blob is invalid and the peer is faulty.
+    ProposerIndexMismatch { sidecar: usize, local: usize },
+
+    /// The proposal signature in invalid.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The blob is invalid and the peer is faulty.
+    ProposalSignatureInvalid,
+
+    /// The proposal_index corresponding to blob.beacon_block_root is not known.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The blob is invalid and the peer is faulty.
+    UnknownValidator(u64),
+
+    /// The provided blob is not from a later slot than its parent.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The blob is invalid and the peer is faulty.
+    BlobIsNotLaterThanParent { blob_slot: Slot, parent_slot: Slot },
+
+    /// The provided blob's parent block is unknown.
+    ///
+    /// ## Peer scoring
+    ///
+    /// We cannot process the blob without validating its parent, the peer isn't necessarily faulty.
+    BlobParentUnknown(Arc<BlobSidecar<E>>),
+
+    /// Invalid kzg commitment inclusion proof
+    /// ## Peer scoring
+    ///
+    /// The blob sidecar is invalid and the peer is faulty
+    InvalidInclusionProof,
+
+    /// A blob has already been seen for the given `(sidecar.block_root, sidecar.index)` tuple
+    /// over gossip or no gossip sources.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer isn't faulty, but we do not forward it over gossip.
+    RepeatBlob {
+        proposer: u64,
+        slot: Slot,
+        index: u64,
+    },
+
+    /// `Kzg` struct hasn't been initialized. This is an internal error.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer isn't faulty, This is an internal error.
+    KzgNotInitialized,
+
+    /// The kzg verification failed.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The blob sidecar is invalid and the peer is faulty.
+    KzgError(kzg::Error),
+
+    /// The kzg commitment inclusion proof failed.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The blob sidecar is invalid
+    InclusionProof(MerkleTreeError),
+
+    /// The pubkey cache timed out.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The blob sidecar may be valid, this is an internal error.
+    PubkeyCacheTimeout,
+
+    /// The block conflicts with finalization, no need to propagate.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this block is valid, but it conflicts with finality and shouldn't be
+    /// imported.
+    NotFinalizedDescendant { block_parent_root: Hash256 },
+}
+
+impl<E: EthSpec> std::fmt::Display for GossipBlobError<E> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            GossipBlobError::BlobParentUnknown(blob_sidecar) => {
+                write!(
+                    f,
+                    "BlobParentUnknown(parent_root:{})",
+                    blob_sidecar.block_parent_root()
+                )
+            }
+            other => write!(f, "{:?}", other),
+        }
+    }
+}
+
+impl<E: EthSpec> From<BeaconChainError> for GossipBlobError<E> {
+    fn from(e: BeaconChainError) -> Self {
+        GossipBlobError::BeaconChainError(e)
+    }
+}
+
+impl<E: EthSpec> From<BeaconStateError> for GossipBlobError<E> {
+    fn from(e: BeaconStateError) -> Self {
+        GossipBlobError::BeaconChainError(BeaconChainError::BeaconStateError(e))
+    }
+}
+
+pub type GossipVerifiedBlobList<T> = VariableList<
+    GossipVerifiedBlob<T>,
+    <<T as BeaconChainTypes>::EthSpec as EthSpec>::MaxBlobsPerBlock,
+>;
+
+/// A wrapper around a `BlobSidecar` that indicates it has been approved for re-gossiping on
+/// the p2p network.
+#[derive(Debug)]
+pub struct GossipVerifiedBlob<T: BeaconChainTypes> {
+    block_root: Hash256,
+    blob: KzgVerifiedBlob<T::EthSpec>,
+}
+
+impl<T: BeaconChainTypes> GossipVerifiedBlob<T> {
+    pub fn new(
+        blob: Arc<BlobSidecar<T::EthSpec>>,
+        subnet_id: u64,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, GossipBlobError<T::EthSpec>> {
+        let header = blob.signed_block_header.clone();
+        // We only process slashing info if the gossip verification failed
+        // since we do not process the blob any further in that case.
+        validate_blob_sidecar_for_gossip(blob, subnet_id, chain).map_err(|e| {
+            process_block_slash_info::<_, GossipBlobError<T::EthSpec>>(
+                chain,
+                BlockSlashInfo::from_early_error_blob(header, e),
+            )
+        })
+    }
+    /// Construct a `GossipVerifiedBlob` that is assumed to be valid.
+    ///
+    /// This should ONLY be used for testing.
+    pub fn __assumed_valid(blob: Arc<BlobSidecar<T::EthSpec>>) -> Self {
+        Self {
+            block_root: blob.block_root(),
+            blob: KzgVerifiedBlob { blob },
+        }
+    }
+    pub fn id(&self) -> BlobIdentifier {
+        BlobIdentifier {
+            block_root: self.block_root,
+            index: self.blob.blob_index(),
+        }
+    }
+    pub fn block_root(&self) -> Hash256 {
+        self.block_root
+    }
+    pub fn slot(&self) -> Slot {
+        self.blob.blob.slot()
+    }
+    pub fn index(&self) -> u64 {
+        self.blob.blob.index
+    }
+    pub fn kzg_commitment(&self) -> KzgCommitment {
+        self.blob.blob.kzg_commitment
+    }
+    pub fn signed_block_header(&self) -> SignedBeaconBlockHeader {
+        self.blob.blob.signed_block_header.clone()
+    }
+    pub fn block_proposer_index(&self) -> u64 {
+        self.blob.blob.block_proposer_index()
+    }
+    pub fn into_inner(self) -> KzgVerifiedBlob<T::EthSpec> {
+        self.blob
+    }
+    pub fn as_blob(&self) -> &BlobSidecar<T::EthSpec> {
+        self.blob.as_blob()
+    }
+    /// This is cheap as we're calling clone on an Arc
+    pub fn clone_blob(&self) -> Arc<BlobSidecar<T::EthSpec>> {
+        self.blob.clone_blob()
+    }
+}
+
+/// Wrapper over a `BlobSidecar` for which we have completed kzg verification.
+/// i.e. `verify_blob_kzg_proof(blob, commitment, proof) == true`.
+#[derive(Debug, Derivative, Clone, Encode, Decode)]
+#[derivative(PartialEq, Eq)]
+#[ssz(struct_behaviour = "transparent")]
+pub struct KzgVerifiedBlob<E: EthSpec> {
+    blob: Arc<BlobSidecar<E>>,
+}
+
+impl<E: EthSpec> PartialOrd for KzgVerifiedBlob<E> {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl<E: EthSpec> Ord for KzgVerifiedBlob<E> {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.blob.cmp(&other.blob)
+    }
+}
+
+impl<E: EthSpec> KzgVerifiedBlob<E> {
+    pub fn new(blob: Arc<BlobSidecar<E>>, kzg: &Kzg) -> Result<Self, KzgError> {
+        verify_kzg_for_blob(blob, kzg)
+    }
+    pub fn to_blob(self) -> Arc<BlobSidecar<E>> {
+        self.blob
+    }
+    pub fn as_blob(&self) -> &BlobSidecar<E> {
+        &self.blob
+    }
+    /// This is cheap as we're calling clone on an Arc
+    pub fn clone_blob(&self) -> Arc<BlobSidecar<E>> {
+        self.blob.clone()
+    }
+    pub fn blob_index(&self) -> u64 {
+        self.blob.index
+    }
+    /// Construct a `KzgVerifiedBlob` that is assumed to be valid.
+    ///
+    /// This should ONLY be used for testing.
+    #[cfg(test)]
+    pub fn __assumed_valid(blob: Arc<BlobSidecar<E>>) -> Self {
+        Self { blob }
+    }
+}
+
+/// Complete kzg verification for a `BlobSidecar`.
+///
+/// Returns an error if the kzg verification check fails.
+pub fn verify_kzg_for_blob<E: EthSpec>(
+    blob: Arc<BlobSidecar<E>>,
+    kzg: &Kzg,
+) -> Result<KzgVerifiedBlob<E>, KzgError> {
+    validate_blob::<E>(kzg, &blob.blob, blob.kzg_commitment, blob.kzg_proof)?;
+    Ok(KzgVerifiedBlob { blob })
+}
+
+pub struct KzgVerifiedBlobList<E: EthSpec> {
+    verified_blobs: Vec<KzgVerifiedBlob<E>>,
+}
+
+impl<E: EthSpec> KzgVerifiedBlobList<E> {
+    pub fn new<I: IntoIterator<Item = Arc<BlobSidecar<E>>>>(
+        blob_list: I,
+        kzg: &Kzg,
+    ) -> Result<Self, KzgError> {
+        let blobs = blob_list.into_iter().collect::<Vec<_>>();
+        verify_kzg_for_blob_list(blobs.iter(), kzg)?;
+        Ok(Self {
+            verified_blobs: blobs
+                .into_iter()
+                .map(|blob| KzgVerifiedBlob { blob })
+                .collect(),
+        })
+    }
+}
+
+impl<E: EthSpec> IntoIterator for KzgVerifiedBlobList<E> {
+    type Item = KzgVerifiedBlob<E>;
+    type IntoIter = std::vec::IntoIter<Self::Item>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.verified_blobs.into_iter()
+    }
+}
+
+/// Complete kzg verification for a list of `BlobSidecar`s.
+/// Returns an error if any of the `BlobSidecar`s fails kzg verification.
+///
+/// Note: This function should be preferred over calling `verify_kzg_for_blob`
+/// in a loop since this function kzg verifies a list of blobs more efficiently.
+pub fn verify_kzg_for_blob_list<'a, E: EthSpec, I>(
+    blob_iter: I,
+    kzg: &'a Kzg,
+) -> Result<(), KzgError>
+where
+    I: Iterator<Item = &'a Arc<BlobSidecar<E>>>,
+{
+    let (blobs, (commitments, proofs)): (Vec<_>, (Vec<_>, Vec<_>)) = blob_iter
+        .map(|blob| (&blob.blob, (blob.kzg_commitment, blob.kzg_proof)))
+        .unzip();
+    validate_blobs::<E>(kzg, commitments.as_slice(), blobs, proofs.as_slice())
+}
+
+pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes>(
+    blob_sidecar: Arc<BlobSidecar<T::EthSpec>>,
+    subnet: u64,
+    chain: &BeaconChain<T>,
+) -> Result<GossipVerifiedBlob<T>, GossipBlobError<T::EthSpec>> {
+    let blob_slot = blob_sidecar.slot();
+    let blob_index = blob_sidecar.index;
+    let block_parent_root = blob_sidecar.block_parent_root();
+    let blob_proposer_index = blob_sidecar.block_proposer_index();
+    let block_root = blob_sidecar.block_root();
+    let blob_epoch = blob_slot.epoch(T::EthSpec::slots_per_epoch());
+    let signed_block_header = &blob_sidecar.signed_block_header;
+
+    // This condition is not possible if we have received the blob from the network
+    // since we only subscribe to `MaxBlobsPerBlock` subnets over gossip network.
+    // We include this check only for completeness.
+    // Getting this error would imply something very wrong with our networking decoding logic.
+    if blob_index >= T::EthSpec::max_blobs_per_block() as u64 {
+        return Err(GossipBlobError::InvalidSubnet {
+            expected: subnet,
+            received: blob_index,
+        });
+    }
+
+    // Verify that the blob_sidecar was received on the correct subnet.
+    if blob_index != subnet {
+        return Err(GossipBlobError::InvalidSubnet {
+            expected: blob_index,
+            received: subnet,
+        });
+    }
+
+    // Verify that the sidecar is not from a future slot.
+    let latest_permissible_slot = chain
+        .slot_clock
+        .now_with_future_tolerance(chain.spec.maximum_gossip_clock_disparity())
+        .ok_or(BeaconChainError::UnableToReadSlot)?;
+    if blob_slot > latest_permissible_slot {
+        return Err(GossipBlobError::FutureSlot {
+            message_slot: blob_slot,
+            latest_permissible_slot,
+        });
+    }
+
+    // Verify that the sidecar slot is greater than the latest finalized slot
+    let latest_finalized_slot = chain
+        .head()
+        .finalized_checkpoint()
+        .epoch
+        .start_slot(T::EthSpec::slots_per_epoch());
+    if blob_slot <= latest_finalized_slot {
+        return Err(GossipBlobError::PastFinalizedSlot {
+            blob_slot,
+            finalized_slot: latest_finalized_slot,
+        });
+    }
+
+    // Verify that this is the first blob sidecar received for the tuple:
+    // (block_header.slot, block_header.proposer_index, blob_sidecar.index)
+    if chain
+        .observed_blob_sidecars
+        .read()
+        .proposer_is_known(&blob_sidecar)
+        .map_err(|e| GossipBlobError::BeaconChainError(e.into()))?
+    {
+        return Err(GossipBlobError::RepeatBlob {
+            proposer: blob_proposer_index,
+            slot: blob_slot,
+            index: blob_index,
+        });
+    }
+
+    // Verify the inclusion proof in the sidecar
+    let _timer = metrics::start_timer(&metrics::BLOB_SIDECAR_INCLUSION_PROOF_VERIFICATION);
+    if !blob_sidecar
+        .verify_blob_sidecar_inclusion_proof()
+        .map_err(GossipBlobError::InclusionProof)?
+    {
+        return Err(GossipBlobError::InvalidInclusionProof);
+    }
+    drop(_timer);
+
+    let fork_choice = chain.canonical_head.fork_choice_read_lock();
+
+    // We have already verified that the blob is past finalization, so we can
+    // just check fork choice for the block's parent.
+    let Some(parent_block) = fork_choice.get_block(&block_parent_root) else {
+        return Err(GossipBlobError::BlobParentUnknown(blob_sidecar));
+    };
+
+    // Do not process a blob that does not descend from the finalized root.
+    // We just loaded the parent_block, so we can be sure that it exists in fork choice.
+    if !fork_choice.is_finalized_checkpoint_or_descendant(block_parent_root) {
+        return Err(GossipBlobError::NotFinalizedDescendant { block_parent_root });
+    }
+    drop(fork_choice);
+
+    if parent_block.slot >= blob_slot {
+        return Err(GossipBlobError::BlobIsNotLaterThanParent {
+            blob_slot,
+            parent_slot: parent_block.slot,
+        });
+    }
+
+    let proposer_shuffling_root =
+        if parent_block.slot.epoch(T::EthSpec::slots_per_epoch()) == blob_epoch {
+            parent_block
+                .next_epoch_shuffling_id
+                .shuffling_decision_block
+        } else {
+            parent_block.root
+        };
+
+    let proposer_opt = chain
+        .beacon_proposer_cache
+        .lock()
+        .get_slot::<T::EthSpec>(proposer_shuffling_root, blob_slot);
+
+    let (proposer_index, fork) = if let Some(proposer) = proposer_opt {
+        (proposer.index, proposer.fork)
+    } else {
+        debug!(
+            chain.log,
+            "Proposer shuffling cache miss for blob verification";
+            "block_root" => %block_root,
+            "index" => %blob_index,
+        );
+        let (parent_state_root, mut parent_state) = chain
+            .store
+            .get_advanced_hot_state(block_parent_root, blob_slot, parent_block.state_root)
+            .map_err(|e| GossipBlobError::BeaconChainError(e.into()))?
+            .ok_or_else(|| {
+                BeaconChainError::DBInconsistent(format!(
+                    "Missing state for parent block {block_parent_root:?}",
+                ))
+            })?;
+
+        let state = cheap_state_advance_to_obtain_committees::<_, GossipBlobError<T::EthSpec>>(
+            &mut parent_state,
+            Some(parent_state_root),
+            blob_slot,
+            &chain.spec,
+        )?;
+
+        let proposers = state.get_beacon_proposer_indices(&chain.spec)?;
+        let proposer_index = *proposers
+            .get(blob_slot.as_usize() % T::EthSpec::slots_per_epoch() as usize)
+            .ok_or_else(|| BeaconChainError::NoProposerForSlot(blob_slot))?;
+
+        // Prime the proposer shuffling cache with the newly-learned value.
+        chain.beacon_proposer_cache.lock().insert(
+            blob_epoch,
+            proposer_shuffling_root,
+            proposers,
+            state.fork(),
+        )?;
+        (proposer_index, state.fork())
+    };
+
+    // Signature verify the signed block header.
+    let signature_is_valid = {
+        let pubkey_cache = chain.validator_pubkey_cache.read();
+
+        let pubkey = pubkey_cache
+            .get(proposer_index)
+            .ok_or_else(|| GossipBlobError::UnknownValidator(proposer_index as u64))?;
+        signed_block_header.verify_signature::<T::EthSpec>(
+            pubkey,
+            &fork,
+            chain.genesis_validators_root,
+            &chain.spec,
+        )
+    };
+
+    if !signature_is_valid {
+        return Err(GossipBlobError::ProposalSignatureInvalid);
+    }
+
+    if proposer_index != blob_proposer_index as usize {
+        return Err(GossipBlobError::ProposerIndexMismatch {
+            sidecar: blob_proposer_index as usize,
+            local: proposer_index,
+        });
+    }
+
+    chain
+        .observed_slashable
+        .write()
+        .observe_slashable(
+            blob_sidecar.slot(),
+            blob_sidecar.block_proposer_index(),
+            block_root,
+        )
+        .map_err(|e| GossipBlobError::BeaconChainError(e.into()))?;
+
+    // Now the signature is valid, store the proposal so we don't accept another blob sidecar
+    // with the same `BlobIdentifier`.
+    // It's important to double-check that the proposer still hasn't been observed so we don't
+    // have a race-condition when verifying two blocks simultaneously.
+    //
+    // Note: If this BlobSidecar goes on to fail full verification, we do not evict it from the seen_cache
+    // as alternate blob_sidecars for the same identifier can still be retrieved
+    // over rpc. Evicting them from this cache would allow faster propagation over gossip. So we allow
+    // retrieval of potentially valid blocks over rpc, but try to punish the proposer for signing
+    // invalid messages. Issue for more background
+    // https://github.com/ethereum/consensus-specs/issues/3261
+    if chain
+        .observed_blob_sidecars
+        .write()
+        .observe_sidecar(&blob_sidecar)
+        .map_err(|e| GossipBlobError::BeaconChainError(e.into()))?
+    {
+        return Err(GossipBlobError::RepeatBlob {
+            proposer: proposer_index as u64,
+            slot: blob_slot,
+            index: blob_index,
+        });
+    }
+
+    // Kzg verification for gossip blob sidecar
+    let kzg = chain
+        .kzg
+        .as_ref()
+        .ok_or(GossipBlobError::KzgNotInitialized)?;
+    let kzg_verified_blob =
+        KzgVerifiedBlob::new(blob_sidecar, kzg).map_err(GossipBlobError::KzgError)?;
+
+    Ok(GossipVerifiedBlob {
+        block_root,
+        blob: kzg_verified_blob,
+    })
+}
+
+/// Returns the canonical root of the given `blob`.
+///
+/// Use this function to ensure that we report the blob hashing time Prometheus metric.
+pub fn get_blob_root<E: EthSpec>(blob: &BlobSidecar<E>) -> Hash256 {
+    let blob_root_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_BLOB_ROOT);
+
+    let blob_root = blob.tree_hash_root();
+
+    metrics::stop_timer(blob_root_timer);
+
+    blob_root
+}

beacon_node/beacon_chain/src/block_reward.rs

@@ -1,29 +1,50 @@
 use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
 use eth2::lighthouse::{AttestationRewards, BlockReward, BlockRewardMeta};
-use operation_pool::{AttMaxCover, MaxCover};
-use state_processing::per_block_processing::altair::sync_committee::compute_sync_aggregate_rewards;
-use types::{BeaconBlockRef, BeaconState, EthSpec, Hash256, RelativeEpoch};
+use operation_pool::{AttMaxCover, MaxCover, RewardCache, SplitAttestation};
+use state_processing::{
+    common::get_attesting_indices_from_state,
+    per_block_processing::altair::sync_committee::compute_sync_aggregate_rewards,
+};
+use types::{AbstractExecPayload, BeaconBlockRef, BeaconState, EthSpec, Hash256};
 
 impl<T: BeaconChainTypes> BeaconChain<T> {
-    pub fn compute_block_reward(
+    pub fn compute_block_reward<Payload: AbstractExecPayload<T::EthSpec>>(
         &self,
-        block: BeaconBlockRef<'_, T::EthSpec>,
+        block: BeaconBlockRef<'_, T::EthSpec, Payload>,
         block_root: Hash256,
         state: &BeaconState<T::EthSpec>,
+        reward_cache: &mut RewardCache,
+        include_attestations: bool,
     ) -> Result<BlockReward, BeaconChainError> {
         if block.slot() != state.slot() {
             return Err(BeaconChainError::BlockRewardSlotError);
         }
 
-        let active_indices = state.get_cached_active_validator_indices(RelativeEpoch::Current)?;
-        let total_active_balance = state.get_total_balance(active_indices, &self.spec)?;
-        let mut per_attestation_rewards = block
+        reward_cache.update(state)?;
+
+        let total_active_balance = state.get_total_active_balance()?;
+
+        let split_attestations = block
             .body()
             .attestations()
             .iter()
             .map(|att| {
-                AttMaxCover::new(att, state, total_active_balance, &self.spec)
-                    .ok_or(BeaconChainError::BlockRewardAttestationError)
+                let attesting_indices = get_attesting_indices_from_state(state, att)?;
+                Ok(SplitAttestation::new(att.clone(), attesting_indices))
+            })
+            .collect::<Result<Vec<_>, BeaconChainError>>()?;
+
+        let mut per_attestation_rewards = split_attestations
+            .iter()
+            .map(|att| {
+                AttMaxCover::new(
+                    att.as_ref(),
+                    state,
+                    reward_cache,
+                    total_active_balance,
+                    &self.spec,
+                )
+                .ok_or(BeaconChainError::BlockRewardAttestationError)
             })
             .collect::<Result<Vec<_>, _>>()?;
 
@@ -34,7 +55,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             let latest_att = &updated[i];
 
             for att in to_update {
-                att.update_covering_set(latest_att.object(), latest_att.covering_set());
+                att.update_covering_set(latest_att.intermediate(), latest_att.covering_set());
             }
         }
 
@@ -60,11 +81,24 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .map(|cover| cover.fresh_validators_rewards)
             .collect();
 
+        // Add the attestation data if desired.
+        let attestations = if include_attestations {
+            block
+                .body()
+                .attestations()
+                .iter()
+                .map(|a| a.data.clone())
+                .collect()
+        } else {
+            vec![]
+        };
+
         let attestation_rewards = AttestationRewards {
             total: attestation_total,
             prev_epoch_total,
             curr_epoch_total,
             per_attestation_rewards,
+            attestations,
         };
 
         // Sync committee rewards.

beacon_node/beacon_chain/src/block_times_cache.rs

@@ -18,14 +18,16 @@ type BlockRoot = Hash256;
 #[derive(Clone, Default)]
 pub struct Timestamps {
     pub observed: Option<Duration>,
+    pub attestable: Option<Duration>,
     pub imported: Option<Duration>,
     pub set_as_head: Option<Duration>,
 }
 
 // Helps arrange delay data so it is more relevant to metrics.
-#[derive(Default)]
+#[derive(Debug, Default)]
 pub struct BlockDelays {
     pub observed: Option<Duration>,
+    pub attestable: Option<Duration>,
     pub imported: Option<Duration>,
     pub set_as_head: Option<Duration>,
 }
@@ -35,6 +37,9 @@ impl BlockDelays {
         let observed = times
             .observed
             .and_then(|observed_time| observed_time.checked_sub(slot_start_time));
+        let attestable = times
+            .attestable
+            .and_then(|attestable_time| attestable_time.checked_sub(slot_start_time));
         let imported = times
             .imported
             .and_then(|imported_time| imported_time.checked_sub(times.observed?));
@@ -43,6 +48,7 @@ impl BlockDelays {
             .and_then(|set_as_head_time| set_as_head_time.checked_sub(times.imported?));
         BlockDelays {
             observed,
+            attestable,
             imported,
             set_as_head,
         }
@@ -51,7 +57,7 @@ impl BlockDelays {
 
 // If the block was received via gossip, we can record the client type of the peer which sent us
 // the block.
-#[derive(Clone, Default)]
+#[derive(Debug, Clone, Default, PartialEq)]
 pub struct BlockPeerInfo {
     pub id: Option<String>,
     pub client: Option<String>,
@@ -80,6 +86,8 @@ pub struct BlockTimesCache {
 
 /// Helper methods to read from and write to the cache.
 impl BlockTimesCache {
+    /// Set the observation time for `block_root` to `timestamp` if `timestamp` is less than
+    /// any previous timestamp at which this block was observed.
     pub fn set_time_observed(
         &mut self,
         block_root: BlockRoot,
@@ -92,11 +100,27 @@ impl BlockTimesCache {
             .cache
             .entry(block_root)
             .or_insert_with(|| BlockTimesCacheValue::new(slot));
-        block_times.timestamps.observed = Some(timestamp);
-        block_times.peer_info = BlockPeerInfo {
-            id: peer_id,
-            client: peer_client,
-        };
+        match block_times.timestamps.observed {
+            Some(existing_observation_time) if existing_observation_time <= timestamp => {
+                // Existing timestamp is earlier, do nothing.
+            }
+            _ => {
+                // No existing timestamp, or new timestamp is earlier.
+                block_times.timestamps.observed = Some(timestamp);
+                block_times.peer_info = BlockPeerInfo {
+                    id: peer_id,
+                    client: peer_client,
+                };
+            }
+        }
+    }
+
+    pub fn set_time_attestable(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) {
+        let block_times = self
+            .cache
+            .entry(block_root)
+            .or_insert_with(|| BlockTimesCacheValue::new(slot));
+        block_times.timestamps.attestable = Some(timestamp);
     }
 
     pub fn set_time_imported(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) {
@@ -141,3 +165,71 @@ impl BlockTimesCache {
             .retain(|_, cache| cache.slot > current_slot.saturating_sub(64_u64));
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn observed_time_uses_minimum() {
+        let mut cache = BlockTimesCache::default();
+
+        let block_root = Hash256::zero();
+        let slot = Slot::new(100);
+
+        let slot_start_time = Duration::from_secs(0);
+
+        let ts1 = Duration::from_secs(5);
+        let ts2 = Duration::from_secs(6);
+        let ts3 = Duration::from_secs(4);
+
+        let peer_info2 = BlockPeerInfo {
+            id: Some("peer2".to_string()),
+            client: Some("lighthouse".to_string()),
+        };
+
+        let peer_info3 = BlockPeerInfo {
+            id: Some("peer3".to_string()),
+            client: Some("prysm".to_string()),
+        };
+
+        cache.set_time_observed(block_root, slot, ts1, None, None);
+
+        assert_eq!(
+            cache.get_block_delays(block_root, slot_start_time).observed,
+            Some(ts1)
+        );
+        assert_eq!(cache.get_peer_info(block_root), BlockPeerInfo::default());
+
+        // Second observation with higher timestamp should not override anything, even though it has
+        // superior peer info.
+        cache.set_time_observed(
+            block_root,
+            slot,
+            ts2,
+            peer_info2.id.clone(),
+            peer_info2.client.clone(),
+        );
+
+        assert_eq!(
+            cache.get_block_delays(block_root, slot_start_time).observed,
+            Some(ts1)
+        );
+        assert_eq!(cache.get_peer_info(block_root), BlockPeerInfo::default());
+
+        // Third observation with lower timestamp should override everything.
+        cache.set_time_observed(
+            block_root,
+            slot,
+            ts3,
+            peer_info3.id.clone(),
+            peer_info3.client.clone(),
+        );
+
+        assert_eq!(
+            cache.get_block_delays(block_root, slot_start_time).observed,
+            Some(ts3)
+        );
+        assert_eq!(cache.get_peer_info(block_root), peer_info3);
+    }
+}

beacon_node/beacon_chain/src/block_verification_types.rs

@@ -0,0 +1,547 @@
+use crate::blob_verification::{GossipBlobError, GossipVerifiedBlobList};
+use crate::block_verification::BlockError;
+use crate::data_availability_checker::AvailabilityCheckError;
+pub use crate::data_availability_checker::{AvailableBlock, MaybeAvailableBlock};
+use crate::eth1_finalization_cache::Eth1FinalizationData;
+use crate::{get_block_root, GossipVerifiedBlock, PayloadVerificationOutcome};
+use derivative::Derivative;
+use ssz_types::VariableList;
+use state_processing::ConsensusContext;
+use std::fmt::{Debug, Formatter};
+use std::sync::Arc;
+use types::blob_sidecar::{BlobIdentifier, BlobSidecarError, FixedBlobSidecarList};
+use types::{
+    BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, Epoch, EthSpec, Hash256,
+    SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
+};
+
+/// A block that has been received over RPC. It has 2 internal variants:
+///
+/// 1. `BlockAndBlobs`: A fully available post deneb block with all the blobs available. This variant
+///    is only constructed after making consistency checks between blocks and blobs.
+///    Hence, it is fully self contained w.r.t verification. i.e. this block has all the required
+///    data to get verified and imported into fork choice.
+///
+/// 2. `Block`: This can be a fully available pre-deneb block **or** a post-deneb block that may or may
+///    not require blobs to be considered fully available.
+///
+/// Note: We make a distinction over blocks received over gossip because
+/// in a post-deneb world, the blobs corresponding to a given block that are received
+/// over rpc do not contain the proposer signature for dos resistance.
+#[derive(Clone, Derivative)]
+#[derivative(Hash(bound = "E: EthSpec"))]
+pub struct RpcBlock<E: EthSpec> {
+    block_root: Hash256,
+    block: RpcBlockInner<E>,
+}
+
+impl<E: EthSpec> Debug for RpcBlock<E> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "RpcBlock({:?})", self.block_root)
+    }
+}
+
+impl<E: EthSpec> RpcBlock<E> {
+    pub fn block_root(&self) -> Hash256 {
+        self.block_root
+    }
+
+    pub fn as_block(&self) -> &SignedBeaconBlock<E> {
+        match &self.block {
+            RpcBlockInner::Block(block) => block,
+            RpcBlockInner::BlockAndBlobs(block, _) => block,
+        }
+    }
+
+    pub fn block_cloned(&self) -> Arc<SignedBeaconBlock<E>> {
+        match &self.block {
+            RpcBlockInner::Block(block) => block.clone(),
+            RpcBlockInner::BlockAndBlobs(block, _) => block.clone(),
+        }
+    }
+
+    pub fn blobs(&self) -> Option<&BlobSidecarList<E>> {
+        match &self.block {
+            RpcBlockInner::Block(_) => None,
+            RpcBlockInner::BlockAndBlobs(_, blobs) => Some(blobs),
+        }
+    }
+}
+
+/// Note: This variant is intentionally private because we want to safely construct the
+/// internal variants after applying consistency checks to ensure that the block and blobs
+/// are consistent with respect to each other.
+#[derive(Debug, Clone, Derivative)]
+#[derivative(Hash(bound = "E: EthSpec"))]
+enum RpcBlockInner<E: EthSpec> {
+    /// Single block lookup response. This should potentially hit the data availability cache.
+    Block(Arc<SignedBeaconBlock<E>>),
+    /// This variant is used with parent lookups and by-range responses. It should have all blobs
+    /// ordered, all block roots matching, and the correct number of blobs for this block.
+    BlockAndBlobs(Arc<SignedBeaconBlock<E>>, BlobSidecarList<E>),
+}
+
+impl<E: EthSpec> RpcBlock<E> {
+    /// Constructs a `Block` variant.
+    pub fn new_without_blobs(
+        block_root: Option<Hash256>,
+        block: Arc<SignedBeaconBlock<E>>,
+    ) -> Self {
+        let block_root = block_root.unwrap_or_else(|| get_block_root(&block));
+
+        Self {
+            block_root,
+            block: RpcBlockInner::Block(block),
+        }
+    }
+
+    /// Constructs a new `BlockAndBlobs` variant after making consistency
+    /// checks between the provided blocks and blobs.
+    pub fn new(
+        block_root: Option<Hash256>,
+        block: Arc<SignedBeaconBlock<E>>,
+        blobs: Option<BlobSidecarList<E>>,
+    ) -> Result<Self, AvailabilityCheckError> {
+        let block_root = block_root.unwrap_or_else(|| get_block_root(&block));
+
+        if let (Some(blobs), Ok(block_commitments)) = (
+            blobs.as_ref(),
+            block.message().body().blob_kzg_commitments(),
+        ) {
+            if blobs.len() != block_commitments.len() {
+                return Err(AvailabilityCheckError::MissingBlobs);
+            }
+            for (blob, &block_commitment) in blobs.iter().zip(block_commitments.iter()) {
+                let blob_commitment = blob.kzg_commitment;
+                if blob_commitment != block_commitment {
+                    return Err(AvailabilityCheckError::KzgCommitmentMismatch {
+                        block_commitment,
+                        blob_commitment,
+                    });
+                }
+            }
+        }
+        let inner = match blobs {
+            Some(blobs) => RpcBlockInner::BlockAndBlobs(block, blobs),
+            None => RpcBlockInner::Block(block),
+        };
+        Ok(Self {
+            block_root,
+            block: inner,
+        })
+    }
+
+    pub fn new_from_fixed(
+        block_root: Hash256,
+        block: Arc<SignedBeaconBlock<E>>,
+        blobs: FixedBlobSidecarList<E>,
+    ) -> Result<Self, AvailabilityCheckError> {
+        let filtered = blobs
+            .into_iter()
+            .filter_map(|b| b.clone())
+            .collect::<Vec<_>>();
+        let blobs = if filtered.is_empty() {
+            None
+        } else {
+            Some(VariableList::from(filtered))
+        };
+        Self::new(Some(block_root), block, blobs)
+    }
+
+    pub fn deconstruct(
+        self,
+    ) -> (
+        Hash256,
+        Arc<SignedBeaconBlock<E>>,
+        Option<BlobSidecarList<E>>,
+    ) {
+        let block_root = self.block_root();
+        match self.block {
+            RpcBlockInner::Block(block) => (block_root, block, None),
+            RpcBlockInner::BlockAndBlobs(block, blobs) => (block_root, block, Some(blobs)),
+        }
+    }
+    pub fn n_blobs(&self) -> usize {
+        match &self.block {
+            RpcBlockInner::Block(_) => 0,
+            RpcBlockInner::BlockAndBlobs(_, blobs) => blobs.len(),
+        }
+    }
+}
+
+/// A block that has gone through all pre-deneb block processing checks including block processing
+/// and execution by an EL client. This block hasn't necessarily completed data availability checks.
+///
+///
+/// It contains 2 variants:
+/// 1. `Available`: This block has been executed and also contains all data to consider it a
+///    fully available block. i.e. for post-deneb, this implies that this contains all the
+///    required blobs.
+/// 2. `AvailabilityPending`: This block hasn't received all required blobs to consider it a
+///    fully available block.
+pub enum ExecutedBlock<E: EthSpec> {
+    Available(AvailableExecutedBlock<E>),
+    AvailabilityPending(AvailabilityPendingExecutedBlock<E>),
+}
+
+impl<E: EthSpec> ExecutedBlock<E> {
+    pub fn new(
+        block: MaybeAvailableBlock<E>,
+        import_data: BlockImportData<E>,
+        payload_verification_outcome: PayloadVerificationOutcome,
+    ) -> Self {
+        match block {
+            MaybeAvailableBlock::Available(available_block) => {
+                Self::Available(AvailableExecutedBlock::new(
+                    available_block,
+                    import_data,
+                    payload_verification_outcome,
+                ))
+            }
+            MaybeAvailableBlock::AvailabilityPending {
+                block_root: _,
+                block: pending_block,
+            } => Self::AvailabilityPending(AvailabilityPendingExecutedBlock::new(
+                pending_block,
+                import_data,
+                payload_verification_outcome,
+            )),
+        }
+    }
+
+    pub fn as_block(&self) -> &SignedBeaconBlock<E> {
+        match self {
+            Self::Available(available) => available.block.block(),
+            Self::AvailabilityPending(pending) => &pending.block,
+        }
+    }
+
+    pub fn block_root(&self) -> Hash256 {
+        match self {
+            ExecutedBlock::AvailabilityPending(pending) => pending.import_data.block_root,
+            ExecutedBlock::Available(available) => available.import_data.block_root,
+        }
+    }
+}
+
+/// A block that has completed all pre-deneb block processing checks including verification
+/// by an EL client **and** has all requisite blob data to be imported into fork choice.
+#[derive(PartialEq)]
+pub struct AvailableExecutedBlock<E: EthSpec> {
+    pub block: AvailableBlock<E>,
+    pub import_data: BlockImportData<E>,
+    pub payload_verification_outcome: PayloadVerificationOutcome,
+}
+
+impl<E: EthSpec> AvailableExecutedBlock<E> {
+    pub fn new(
+        block: AvailableBlock<E>,
+        import_data: BlockImportData<E>,
+        payload_verification_outcome: PayloadVerificationOutcome,
+    ) -> Self {
+        Self {
+            block,
+            import_data,
+            payload_verification_outcome,
+        }
+    }
+
+    pub fn get_all_blob_ids(&self) -> Vec<BlobIdentifier> {
+        let num_blobs_expected = self
+            .block
+            .message()
+            .body()
+            .blob_kzg_commitments()
+            .map_or(0, |commitments| commitments.len());
+        let mut blob_ids = Vec::with_capacity(num_blobs_expected);
+        for i in 0..num_blobs_expected {
+            blob_ids.push(BlobIdentifier {
+                block_root: self.import_data.block_root,
+                index: i as u64,
+            });
+        }
+        blob_ids
+    }
+}
+
+/// A block that has completed all pre-deneb block processing checks, verification
+/// by an EL client but does not have all requisite blob data to get imported into
+/// fork choice.
+pub struct AvailabilityPendingExecutedBlock<E: EthSpec> {
+    pub block: Arc<SignedBeaconBlock<E>>,
+    pub import_data: BlockImportData<E>,
+    pub payload_verification_outcome: PayloadVerificationOutcome,
+}
+
+impl<E: EthSpec> AvailabilityPendingExecutedBlock<E> {
+    pub fn new(
+        block: Arc<SignedBeaconBlock<E>>,
+        import_data: BlockImportData<E>,
+        payload_verification_outcome: PayloadVerificationOutcome,
+    ) -> Self {
+        Self {
+            block,
+            import_data,
+            payload_verification_outcome,
+        }
+    }
+
+    pub fn as_block(&self) -> &SignedBeaconBlock<E> {
+        &self.block
+    }
+
+    pub fn num_blobs_expected(&self) -> usize {
+        self.block
+            .message()
+            .body()
+            .blob_kzg_commitments()
+            .map_or(0, |commitments| commitments.len())
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct BlockImportData<E: EthSpec> {
+    pub block_root: Hash256,
+    pub state: BeaconState<E>,
+    pub parent_block: SignedBeaconBlock<E, BlindedPayload<E>>,
+    pub parent_eth1_finalization_data: Eth1FinalizationData,
+    pub confirmed_state_roots: Vec<Hash256>,
+    pub consensus_context: ConsensusContext<E>,
+}
+
+pub type GossipVerifiedBlockContents<E> =
+    (GossipVerifiedBlock<E>, Option<GossipVerifiedBlobList<E>>);
+
+#[derive(Debug)]
+pub enum BlockContentsError<E: EthSpec> {
+    BlockError(BlockError<E>),
+    BlobError(GossipBlobError<E>),
+    SidecarError(BlobSidecarError),
+}
+
+impl<E: EthSpec> From<BlockError<E>> for BlockContentsError<E> {
+    fn from(value: BlockError<E>) -> Self {
+        Self::BlockError(value)
+    }
+}
+
+impl<E: EthSpec> From<GossipBlobError<E>> for BlockContentsError<E> {
+    fn from(value: GossipBlobError<E>) -> Self {
+        Self::BlobError(value)
+    }
+}
+
+impl<E: EthSpec> std::fmt::Display for BlockContentsError<E> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BlockContentsError::BlockError(err) => {
+                write!(f, "BlockError({})", err)
+            }
+            BlockContentsError::BlobError(err) => {
+                write!(f, "BlobError({})", err)
+            }
+            BlockContentsError::SidecarError(err) => {
+                write!(f, "SidecarError({:?})", err)
+            }
+        }
+    }
+}
+
+/// Trait for common block operations.
+pub trait AsBlock<E: EthSpec> {
+    fn slot(&self) -> Slot;
+    fn epoch(&self) -> Epoch;
+    fn parent_root(&self) -> Hash256;
+    fn state_root(&self) -> Hash256;
+    fn signed_block_header(&self) -> SignedBeaconBlockHeader;
+    fn message(&self) -> BeaconBlockRef<E>;
+    fn as_block(&self) -> &SignedBeaconBlock<E>;
+    fn block_cloned(&self) -> Arc<SignedBeaconBlock<E>>;
+    fn canonical_root(&self) -> Hash256;
+    fn into_rpc_block(self) -> RpcBlock<E>;
+}
+
+impl<E: EthSpec> AsBlock<E> for Arc<SignedBeaconBlock<E>> {
+    fn slot(&self) -> Slot {
+        SignedBeaconBlock::slot(self)
+    }
+
+    fn epoch(&self) -> Epoch {
+        SignedBeaconBlock::epoch(self)
+    }
+
+    fn parent_root(&self) -> Hash256 {
+        SignedBeaconBlock::parent_root(self)
+    }
+
+    fn state_root(&self) -> Hash256 {
+        SignedBeaconBlock::state_root(self)
+    }
+
+    fn signed_block_header(&self) -> SignedBeaconBlockHeader {
+        SignedBeaconBlock::signed_block_header(self)
+    }
+
+    fn message(&self) -> BeaconBlockRef<E> {
+        SignedBeaconBlock::message(self)
+    }
+
+    fn as_block(&self) -> &SignedBeaconBlock<E> {
+        self
+    }
+
+    fn block_cloned(&self) -> Arc<SignedBeaconBlock<E>> {
+        Arc::<SignedBeaconBlock<E>>::clone(self)
+    }
+
+    fn canonical_root(&self) -> Hash256 {
+        SignedBeaconBlock::canonical_root(self)
+    }
+
+    fn into_rpc_block(self) -> RpcBlock<E> {
+        RpcBlock::new_without_blobs(None, self)
+    }
+}
+
+impl<E: EthSpec> AsBlock<E> for MaybeAvailableBlock<E> {
+    fn slot(&self) -> Slot {
+        self.as_block().slot()
+    }
+    fn epoch(&self) -> Epoch {
+        self.as_block().epoch()
+    }
+    fn parent_root(&self) -> Hash256 {
+        self.as_block().parent_root()
+    }
+    fn state_root(&self) -> Hash256 {
+        self.as_block().state_root()
+    }
+    fn signed_block_header(&self) -> SignedBeaconBlockHeader {
+        self.as_block().signed_block_header()
+    }
+    fn message(&self) -> BeaconBlockRef<E> {
+        self.as_block().message()
+    }
+    fn as_block(&self) -> &SignedBeaconBlock<E> {
+        match &self {
+            MaybeAvailableBlock::Available(block) => block.as_block(),
+            MaybeAvailableBlock::AvailabilityPending {
+                block_root: _,
+                block,
+            } => block,
+        }
+    }
+    fn block_cloned(&self) -> Arc<SignedBeaconBlock<E>> {
+        match &self {
+            MaybeAvailableBlock::Available(block) => block.block_cloned(),
+            MaybeAvailableBlock::AvailabilityPending {
+                block_root: _,
+                block,
+            } => block.clone(),
+        }
+    }
+    fn canonical_root(&self) -> Hash256 {
+        self.as_block().canonical_root()
+    }
+
+    fn into_rpc_block(self) -> RpcBlock<E> {
+        match self {
+            MaybeAvailableBlock::Available(available_block) => available_block.into_rpc_block(),
+            MaybeAvailableBlock::AvailabilityPending { block_root, block } => {
+                RpcBlock::new_without_blobs(Some(block_root), block)
+            }
+        }
+    }
+}
+
+impl<E: EthSpec> AsBlock<E> for AvailableBlock<E> {
+    fn slot(&self) -> Slot {
+        self.block().slot()
+    }
+
+    fn epoch(&self) -> Epoch {
+        self.block().epoch()
+    }
+
+    fn parent_root(&self) -> Hash256 {
+        self.block().parent_root()
+    }
+
+    fn state_root(&self) -> Hash256 {
+        self.block().state_root()
+    }
+
+    fn signed_block_header(&self) -> SignedBeaconBlockHeader {
+        self.block().signed_block_header()
+    }
+
+    fn message(&self) -> BeaconBlockRef<E> {
+        self.block().message()
+    }
+
+    fn as_block(&self) -> &SignedBeaconBlock<E> {
+        self.block()
+    }
+
+    fn block_cloned(&self) -> Arc<SignedBeaconBlock<E>> {
+        AvailableBlock::block_cloned(self)
+    }
+
+    fn canonical_root(&self) -> Hash256 {
+        self.block().canonical_root()
+    }
+
+    fn into_rpc_block(self) -> RpcBlock<E> {
+        let (block_root, block, blobs_opt) = self.deconstruct();
+        // Circumvent the constructor here, because an Available block will have already had
+        // consistency checks performed.
+        let inner = match blobs_opt {
+            None => RpcBlockInner::Block(block),
+            Some(blobs) => RpcBlockInner::BlockAndBlobs(block, blobs),
+        };
+        RpcBlock {
+            block_root,
+            block: inner,
+        }
+    }
+}
+
+impl<E: EthSpec> AsBlock<E> for RpcBlock<E> {
+    fn slot(&self) -> Slot {
+        self.as_block().slot()
+    }
+    fn epoch(&self) -> Epoch {
+        self.as_block().epoch()
+    }
+    fn parent_root(&self) -> Hash256 {
+        self.as_block().parent_root()
+    }
+    fn state_root(&self) -> Hash256 {
+        self.as_block().state_root()
+    }
+    fn signed_block_header(&self) -> SignedBeaconBlockHeader {
+        self.as_block().signed_block_header()
+    }
+    fn message(&self) -> BeaconBlockRef<E> {
+        self.as_block().message()
+    }
+    fn as_block(&self) -> &SignedBeaconBlock<E> {
+        match &self.block {
+            RpcBlockInner::Block(block) => block,
+            RpcBlockInner::BlockAndBlobs(block, _) => block,
+        }
+    }
+    fn block_cloned(&self) -> Arc<SignedBeaconBlock<E>> {
+        match &self.block {
+            RpcBlockInner::Block(block) => block.clone(),
+            RpcBlockInner::BlockAndBlobs(block, _) => block.clone(),
+        }
+    }
+    fn canonical_root(&self) -> Hash256 {
+        self.as_block().canonical_root()
+    }
+
+    fn into_rpc_block(self) -> RpcBlock<E> {
+        self
+    }
+}

beacon_node/beacon_chain/src/capella_readiness.rs

@@ -0,0 +1,121 @@
+//! Provides tools for checking if a node is ready for the Capella upgrade.
+
+use crate::{BeaconChain, BeaconChainTypes};
+use execution_layer::http::{
+    ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_GET_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V2,
+};
+use serde::{Deserialize, Serialize};
+use std::fmt;
+use std::time::Duration;
+use types::*;
+
+/// The time before the Capella fork when we will start issuing warnings about preparation.
+use super::merge_readiness::SECONDS_IN_A_WEEK;
+pub const CAPELLA_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2;
+pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300;
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(tag = "type")]
+pub enum CapellaReadiness {
+    /// The execution engine is capella-enabled (as far as we can tell)
+    Ready,
+    /// We are connected to an execution engine which doesn't support the V2 engine api methods
+    V2MethodsNotSupported { error: String },
+    /// The transition configuration with the EL failed, there might be a problem with
+    /// connectivity, authentication or a difference in configuration.
+    ExchangeCapabilitiesFailed { error: String },
+    /// The user has not configured an execution endpoint
+    NoExecutionEndpoint,
+}
+
+impl fmt::Display for CapellaReadiness {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            CapellaReadiness::Ready => {
+                write!(f, "This node appears ready for Capella.")
+            }
+            CapellaReadiness::ExchangeCapabilitiesFailed { error } => write!(
+                f,
+                "Could not exchange capabilities with the \
+                    execution endpoint: {}",
+                error
+            ),
+            CapellaReadiness::NoExecutionEndpoint => write!(
+                f,
+                "The --execution-endpoint flag is not specified, this is a \
+                    requirement post-merge"
+            ),
+            CapellaReadiness::V2MethodsNotSupported { error } => write!(
+                f,
+                "Execution endpoint does not support Capella methods: {}",
+                error
+            ),
+        }
+    }
+}
+
+impl<T: BeaconChainTypes> BeaconChain<T> {
+    /// Returns `true` if capella epoch is set and Capella fork has occurred or will
+    /// occur within `CAPELLA_READINESS_PREPARATION_SECONDS`
+    pub fn is_time_to_prepare_for_capella(&self, current_slot: Slot) -> bool {
+        if let Some(capella_epoch) = self.spec.capella_fork_epoch {
+            let capella_slot = capella_epoch.start_slot(T::EthSpec::slots_per_epoch());
+            let capella_readiness_preparation_slots =
+                CAPELLA_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot;
+            // Return `true` if Capella has happened or is within the preparation time.
+            current_slot + capella_readiness_preparation_slots > capella_slot
+        } else {
+            // The Capella fork epoch has not been defined yet, no need to prepare.
+            false
+        }
+    }
+
+    /// Attempts to connect to the EL and confirm that it is ready for capella.
+    pub async fn check_capella_readiness(&self) -> CapellaReadiness {
+        if let Some(el) = self.execution_layer.as_ref() {
+            match el
+                .get_engine_capabilities(Some(Duration::from_secs(
+                    ENGINE_CAPABILITIES_REFRESH_INTERVAL,
+                )))
+                .await
+            {
+                Err(e) => {
+                    // The EL was either unreachable or responded with an error
+                    CapellaReadiness::ExchangeCapabilitiesFailed {
+                        error: format!("{:?}", e),
+                    }
+                }
+                Ok(capabilities) => {
+                    let mut missing_methods = String::from("Required Methods Unsupported:");
+                    let mut all_good = true;
+                    if !capabilities.get_payload_v2 {
+                        missing_methods.push(' ');
+                        missing_methods.push_str(ENGINE_GET_PAYLOAD_V2);
+                        all_good = false;
+                    }
+                    if !capabilities.forkchoice_updated_v2 {
+                        missing_methods.push(' ');
+                        missing_methods.push_str(ENGINE_FORKCHOICE_UPDATED_V2);
+                        all_good = false;
+                    }
+                    if !capabilities.new_payload_v2 {
+                        missing_methods.push(' ');
+                        missing_methods.push_str(ENGINE_NEW_PAYLOAD_V2);
+                        all_good = false;
+                    }
+
+                    if all_good {
+                        CapellaReadiness::Ready
+                    } else {
+                        CapellaReadiness::V2MethodsNotSupported {
+                            error: missing_methods,
+                        }
+                    }
+                }
+            }
+        } else {
+            CapellaReadiness::NoExecutionEndpoint
+        }
+    }
+}

beacon_node/beacon_chain/src/chain_config.rs

@@ -1,10 +1,27 @@
-use serde_derive::{Deserialize, Serialize};
-use types::Checkpoint;
+pub use proto_array::{DisallowedReOrgOffsets, ReOrgThreshold};
+use serde::{Deserialize, Serialize};
+use std::time::Duration;
+use types::{Checkpoint, Epoch};
+
+pub const DEFAULT_RE_ORG_HEAD_THRESHOLD: ReOrgThreshold = ReOrgThreshold(20);
+pub const DEFAULT_RE_ORG_PARENT_THRESHOLD: ReOrgThreshold = ReOrgThreshold(160);
+pub const DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION: Epoch = Epoch::new(2);
+/// Default to 1/12th of the slot, which is 1 second on mainnet.
+pub const DEFAULT_RE_ORG_CUTOFF_DENOMINATOR: u32 = 12;
+pub const DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT: u64 = 250;
+
+/// Default fraction of a slot lookahead for payload preparation (12/3 = 4 seconds on mainnet).
+pub const DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR: u32 = 3;
+
+/// Fraction of a slot lookahead for fork choice in the state advance timer (500ms on mainnet).
+pub const FORK_CHOICE_LOOKAHEAD_FACTOR: u32 = 24;
+
+/// Cache only a small number of states in the parallel cache by default.
+pub const DEFAULT_PARALLEL_STATE_CACHE_SIZE: usize = 2;
 
 #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
 pub struct ChainConfig {
-    /// Maximum number of slots to skip when importing a consensus message (e.g., block,
-    /// attestation, etc).
+    /// Maximum number of slots to skip when importing an attestation.
     ///
     /// If `None`, there is no limit.
     pub import_max_skip_slots: Option<u64>,
@@ -18,6 +35,62 @@ pub struct ChainConfig {
     pub enable_lock_timeouts: bool,
     /// The max size of a message that can be sent over the network.
     pub max_network_size: usize,
+    /// Maximum percentage of the head committee weight at which to attempt re-orging the canonical head.
+    pub re_org_head_threshold: Option<ReOrgThreshold>,
+    /// Minimum percentage of the parent committee weight at which to attempt re-orging the canonical head.
+    pub re_org_parent_threshold: Option<ReOrgThreshold>,
+    /// Maximum number of epochs since finalization for attempting a proposer re-org.
+    pub re_org_max_epochs_since_finalization: Epoch,
+    /// Maximum delay after the start of the slot at which to propose a reorging block.
+    pub re_org_cutoff_millis: Option<u64>,
+    /// Additional epoch offsets at which re-orging block proposals are not permitted.
+    ///
+    /// By default this list is empty, but it can be useful for reacting to network conditions, e.g.
+    /// slow gossip of re-org blocks at slot 1 in the epoch.
+    pub re_org_disallowed_offsets: DisallowedReOrgOffsets,
+    /// Number of milliseconds to wait for fork choice before proposing a block.
+    ///
+    /// If set to 0 then block proposal will not wait for fork choice at all.
+    pub fork_choice_before_proposal_timeout_ms: u64,
+    /// Number of skip slots in a row before the BN refuses to use connected builders during payload construction.
+    pub builder_fallback_skips: usize,
+    /// Number of skip slots in the past `SLOTS_PER_EPOCH` before the BN refuses to use connected
+    /// builders during payload construction.
+    pub builder_fallback_skips_per_epoch: usize,
+    /// Number of epochs since finalization before the BN refuses to use connected builders during
+    /// payload construction.
+    pub builder_fallback_epochs_since_finalization: usize,
+    /// Whether any chain health checks should be considered when deciding whether to use the builder API.
+    pub builder_fallback_disable_checks: bool,
+    /// When set to `true`, forget any valid/invalid/optimistic statuses in fork choice during start
+    /// up.
+    pub always_reset_payload_statuses: bool,
+    /// Whether to apply paranoid checks to blocks proposed by this beacon node.
+    pub paranoid_block_proposal: bool,
+    /// Optionally set timeout for calls to checkpoint sync endpoint.
+    pub checkpoint_sync_url_timeout: u64,
+    /// The offset before the start of a proposal slot at which payload attributes should be sent.
+    ///
+    /// Low values are useful for execution engines which don't improve their payload after the
+    /// first call, and high values are useful for ensuring the EL is given ample notice.
+    pub prepare_payload_lookahead: Duration,
+    /// Use EL-free optimistic sync for the finalized part of the chain.
+    pub optimistic_finalized_sync: bool,
+    /// The size of the shuffling cache,
+    pub shuffling_cache_size: usize,
+    /// If using a weak-subjectivity sync, whether we should download blocks all the way back to
+    /// genesis.
+    pub genesis_backfill: bool,
+    /// Whether to send payload attributes every slot, regardless of connected proposers.
+    ///
+    /// This is useful for block builders and testing.
+    pub always_prepare_payload: bool,
+    /// Number of epochs between each migration of data from the hot database to the freezer.
+    pub epochs_per_migration: u64,
+    /// Size of the promise cache for de-duplicating parallel state requests.
+    pub parallel_state_cache_size: usize,
+    /// When set to true Light client server computes and caches state proofs for serving updates
+    pub enable_light_client_server: bool,
 }
 
 impl Default for ChainConfig {
@@ -28,6 +101,40 @@ impl Default for ChainConfig {
             reconstruct_historic_states: false,
             enable_lock_timeouts: true,
             max_network_size: 10 * 1_048_576, // 10M
+            re_org_head_threshold: Some(DEFAULT_RE_ORG_HEAD_THRESHOLD),
+            re_org_parent_threshold: Some(DEFAULT_RE_ORG_PARENT_THRESHOLD),
+            re_org_max_epochs_since_finalization: DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION,
+            re_org_cutoff_millis: None,
+            re_org_disallowed_offsets: DisallowedReOrgOffsets::default(),
+            fork_choice_before_proposal_timeout_ms: DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT,
+            // Builder fallback configs that are set in `clap` will override these.
+            builder_fallback_skips: 3,
+            builder_fallback_skips_per_epoch: 8,
+            builder_fallback_epochs_since_finalization: 3,
+            builder_fallback_disable_checks: false,
+            always_reset_payload_statuses: false,
+            paranoid_block_proposal: false,
+            checkpoint_sync_url_timeout: 60,
+            prepare_payload_lookahead: Duration::from_secs(4),
+            // This value isn't actually read except in tests.
+            optimistic_finalized_sync: true,
+            shuffling_cache_size: crate::shuffling_cache::DEFAULT_CACHE_SIZE,
+            genesis_backfill: false,
+            always_prepare_payload: false,
+            epochs_per_migration: crate::migrate::DEFAULT_EPOCHS_PER_MIGRATION,
+            parallel_state_cache_size: DEFAULT_PARALLEL_STATE_CACHE_SIZE,
+            enable_light_client_server: false,
         }
     }
 }
+
+impl ChainConfig {
+    /// The latest delay from the start of the slot at which to attempt a 1-slot re-org.
+    pub fn re_org_cutoff(&self, seconds_per_slot: u64) -> Duration {
+        self.re_org_cutoff_millis
+            .map(Duration::from_millis)
+            .unwrap_or_else(|| {
+                Duration::from_secs(seconds_per_slot) / DEFAULT_RE_ORG_CUTOFF_DENOMINATOR
+            })
+    }
+}

beacon_node/beacon_chain/src/data_availability_checker.rs

@@ -0,0 +1,639 @@
+use crate::blob_verification::{verify_kzg_for_blob_list, GossipVerifiedBlob, KzgVerifiedBlobList};
+use crate::block_verification_types::{
+    AvailabilityPendingExecutedBlock, AvailableExecutedBlock, RpcBlock,
+};
+pub use crate::data_availability_checker::availability_view::{
+    AvailabilityView, GetCommitment, GetCommitments,
+};
+pub use crate::data_availability_checker::child_components::ChildComponents;
+use crate::data_availability_checker::overflow_lru_cache::OverflowLRUCache;
+use crate::data_availability_checker::processing_cache::ProcessingCache;
+use crate::{BeaconChain, BeaconChainTypes, BeaconStore};
+use kzg::Kzg;
+use parking_lot::RwLock;
+pub use processing_cache::ProcessingComponents;
+use slasher::test_utils::E;
+use slog::{debug, error, Logger};
+use slot_clock::SlotClock;
+use ssz_types::FixedVector;
+use std::fmt;
+use std::fmt::Debug;
+use std::num::NonZeroUsize;
+use std::sync::Arc;
+use task_executor::TaskExecutor;
+use types::beacon_block_body::KzgCommitmentOpts;
+use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList};
+use types::{BlobSidecarList, ChainSpec, Epoch, EthSpec, Hash256, SignedBeaconBlock};
+
+mod availability_view;
+mod child_components;
+mod error;
+mod overflow_lru_cache;
+mod processing_cache;
+mod state_lru_cache;
+
+pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory};
+use types::non_zero_usize::new_non_zero_usize;
+
+/// The LRU Cache stores `PendingComponents` which can store up to
+/// `MAX_BLOBS_PER_BLOCK = 6` blobs each. A `BlobSidecar` is 0.131256 MB. So
+/// the maximum size of a `PendingComponents` is ~ 0.787536 MB. Setting this
+/// to 1024 means the maximum size of the cache is ~ 0.8 GB. But the cache
+/// will target a size of less than 75% of capacity.
+pub const OVERFLOW_LRU_CAPACITY: NonZeroUsize = new_non_zero_usize(1024);
+/// Until tree-states is implemented, we can't store very many states in memory :(
+pub const STATE_LRU_CAPACITY_NON_ZERO: NonZeroUsize = new_non_zero_usize(2);
+pub const STATE_LRU_CAPACITY: usize = STATE_LRU_CAPACITY_NON_ZERO.get();
+
+/// This includes a cache for any blocks or blobs that have been received over gossip or RPC
+/// and are awaiting more components before they can be imported. Additionally the
+/// `DataAvailabilityChecker` is responsible for KZG verification of block components as well as
+/// checking whether a "availability check" is required at all.
+pub struct DataAvailabilityChecker<T: BeaconChainTypes> {
+    processing_cache: RwLock<ProcessingCache<T::EthSpec>>,
+    availability_cache: Arc<OverflowLRUCache<T>>,
+    slot_clock: T::SlotClock,
+    kzg: Option<Arc<Kzg>>,
+    log: Logger,
+    spec: ChainSpec,
+}
+
+/// This type is returned after adding a block / blob to the `DataAvailabilityChecker`.
+///
+/// Indicates if the block is fully `Available` or if we need blobs or blocks
+///  to "complete" the requirements for an `AvailableBlock`.
+#[derive(PartialEq)]
+pub enum Availability<E: EthSpec> {
+    MissingComponents(Hash256),
+    Available(Box<AvailableExecutedBlock<E>>),
+}
+
+impl<E: EthSpec> Debug for Availability<E> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::MissingComponents(block_root) => {
+                write!(f, "MissingComponents({})", block_root)
+            }
+            Self::Available(block) => write!(f, "Available({:?})", block.import_data.block_root),
+        }
+    }
+}
+
+impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
+    pub fn new(
+        slot_clock: T::SlotClock,
+        kzg: Option<Arc<Kzg>>,
+        store: BeaconStore<T>,
+        log: &Logger,
+        spec: ChainSpec,
+    ) -> Result<Self, AvailabilityCheckError> {
+        let overflow_cache = OverflowLRUCache::new(OVERFLOW_LRU_CAPACITY, store, spec.clone())?;
+        Ok(Self {
+            processing_cache: <_>::default(),
+            availability_cache: Arc::new(overflow_cache),
+            slot_clock,
+            log: log.clone(),
+            kzg,
+            spec,
+        })
+    }
+
+    /// Checks if the given block root is cached.
+    pub fn has_block(&self, block_root: &Hash256) -> bool {
+        self.processing_cache.read().has_block(block_root)
+    }
+
+    /// Get the processing info for a block.
+    pub fn get_processing_components(
+        &self,
+        block_root: Hash256,
+    ) -> Option<ProcessingComponents<T::EthSpec>> {
+        self.processing_cache.read().get(&block_root).cloned()
+    }
+
+    /// If there's no block, all possible ids will be returned that don't exist in the given blobs.
+    /// If there no blobs, all possible ids will be returned.
+    pub fn get_missing_blob_ids<V>(
+        &self,
+        block_root: Hash256,
+        block: &Option<Arc<SignedBeaconBlock<T::EthSpec>>>,
+        blobs: &FixedVector<Option<V>, <T::EthSpec as EthSpec>::MaxBlobsPerBlock>,
+    ) -> MissingBlobs {
+        let Some(current_slot) = self.slot_clock.now_or_genesis() else {
+            error!(
+                self.log,
+                "Failed to read slot clock when checking for missing blob ids"
+            );
+            return MissingBlobs::BlobsNotRequired;
+        };
+
+        let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch());
+
+        if self.da_check_required_for_epoch(current_epoch) {
+            match block {
+                Some(cached_block) => {
+                    let block_commitments = cached_block.get_commitments();
+                    let blob_ids = blobs
+                        .iter()
+                        .take(block_commitments.len())
+                        .enumerate()
+                        .filter_map(|(index, blob_commitment_opt)| {
+                            blob_commitment_opt.is_none().then_some(BlobIdentifier {
+                                block_root,
+                                index: index as u64,
+                            })
+                        })
+                        .collect();
+                    MissingBlobs::KnownMissing(blob_ids)
+                }
+                None => {
+                    MissingBlobs::PossibleMissing(BlobIdentifier::get_all_blob_ids::<E>(block_root))
+                }
+            }
+        } else {
+            MissingBlobs::BlobsNotRequired
+        }
+    }
+
+    /// Get a blob from the availability cache.
+    pub fn get_blob(
+        &self,
+        blob_id: &BlobIdentifier,
+    ) -> Result<Option<Arc<BlobSidecar<T::EthSpec>>>, AvailabilityCheckError> {
+        self.availability_cache.peek_blob(blob_id)
+    }
+
+    /// Get a block from the availability cache. Includes any blocks we are currently processing.
+    pub fn get_block(&self, block_root: &Hash256) -> Option<Arc<SignedBeaconBlock<T::EthSpec>>> {
+        self.processing_cache
+            .read()
+            .get(block_root)
+            .and_then(|cached| cached.block.clone())
+    }
+
+    /// Put a list of blobs received via RPC into the availability cache. This performs KZG
+    /// verification on the blobs in the list.
+    pub fn put_rpc_blobs(
+        &self,
+        block_root: Hash256,
+        blobs: FixedBlobSidecarList<T::EthSpec>,
+    ) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
+        let Some(kzg) = self.kzg.as_ref() else {
+            return Err(AvailabilityCheckError::KzgNotInitialized);
+        };
+
+        let verified_blobs = KzgVerifiedBlobList::new(Vec::from(blobs).into_iter().flatten(), kzg)
+            .map_err(AvailabilityCheckError::Kzg)?;
+
+        self.availability_cache
+            .put_kzg_verified_blobs(block_root, verified_blobs)
+    }
+
+    /// Check if we've cached other blobs for this block. If it completes a set and we also
+    /// have a block cached, return the `Availability` variant triggering block import.
+    /// Otherwise cache the blob sidecar.
+    ///
+    /// This should only accept gossip verified blobs, so we should not have to worry about dupes.
+    pub fn put_gossip_blob(
+        &self,
+        gossip_blob: GossipVerifiedBlob<T>,
+    ) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
+        self.availability_cache
+            .put_kzg_verified_blobs(gossip_blob.block_root(), vec![gossip_blob.into_inner()])
+    }
+
+    /// Check if we have all the blobs for a block. Returns `Availability` which has information
+    /// about whether all components have been received or more are required.
+    pub fn put_pending_executed_block(
+        &self,
+        executed_block: AvailabilityPendingExecutedBlock<T::EthSpec>,
+    ) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
+        self.availability_cache
+            .put_pending_executed_block(executed_block)
+    }
+
+    /// Verifies kzg commitments for an RpcBlock, returns a `MaybeAvailableBlock` that may
+    /// include the fully available block.
+    ///
+    /// WARNING: This function assumes all required blobs are already present, it does NOT
+    ///          check if there are any missing blobs.
+    pub fn verify_kzg_for_rpc_block(
+        &self,
+        block: RpcBlock<T::EthSpec>,
+    ) -> Result<MaybeAvailableBlock<T::EthSpec>, AvailabilityCheckError> {
+        let (block_root, block, blobs) = block.deconstruct();
+        match blobs {
+            None => {
+                if self.blobs_required_for_block(&block) {
+                    Ok(MaybeAvailableBlock::AvailabilityPending { block_root, block })
+                } else {
+                    Ok(MaybeAvailableBlock::Available(AvailableBlock {
+                        block_root,
+                        block,
+                        blobs: None,
+                    }))
+                }
+            }
+            Some(blob_list) => {
+                let verified_blobs = if self.blobs_required_for_block(&block) {
+                    let kzg = self
+                        .kzg
+                        .as_ref()
+                        .ok_or(AvailabilityCheckError::KzgNotInitialized)?;
+                    verify_kzg_for_blob_list(blob_list.iter(), kzg)
+                        .map_err(AvailabilityCheckError::Kzg)?;
+                    Some(blob_list)
+                } else {
+                    None
+                };
+                Ok(MaybeAvailableBlock::Available(AvailableBlock {
+                    block_root,
+                    block,
+                    blobs: verified_blobs,
+                }))
+            }
+        }
+    }
+
+    /// Checks if a vector of blocks are available. Returns a vector of `MaybeAvailableBlock`
+    /// This is more efficient than calling `verify_kzg_for_rpc_block` in a loop as it does
+    /// all kzg verification at once
+    ///
+    /// WARNING: This function assumes all required blobs are already present, it does NOT
+    ///          check if there are any missing blobs.
+    pub fn verify_kzg_for_rpc_blocks(
+        &self,
+        blocks: Vec<RpcBlock<T::EthSpec>>,
+    ) -> Result<Vec<MaybeAvailableBlock<T::EthSpec>>, AvailabilityCheckError> {
+        let mut results = Vec::with_capacity(blocks.len());
+        let all_blobs: BlobSidecarList<T::EthSpec> = blocks
+            .iter()
+            .filter(|block| self.blobs_required_for_block(block.as_block()))
+            // this clone is cheap as it's cloning an Arc
+            .filter_map(|block| block.blobs().cloned())
+            .flatten()
+            .collect::<Vec<_>>()
+            .into();
+
+        // verify kzg for all blobs at once
+        if !all_blobs.is_empty() {
+            let kzg = self
+                .kzg
+                .as_ref()
+                .ok_or(AvailabilityCheckError::KzgNotInitialized)?;
+            verify_kzg_for_blob_list(all_blobs.iter(), kzg)?;
+        }
+
+        for block in blocks {
+            let (block_root, block, blobs) = block.deconstruct();
+            match blobs {
+                None => {
+                    if self.blobs_required_for_block(&block) {
+                        results.push(MaybeAvailableBlock::AvailabilityPending { block_root, block })
+                    } else {
+                        results.push(MaybeAvailableBlock::Available(AvailableBlock {
+                            block_root,
+                            block,
+                            blobs: None,
+                        }))
+                    }
+                }
+                Some(blob_list) => {
+                    let verified_blobs = if self.blobs_required_for_block(&block) {
+                        Some(blob_list)
+                    } else {
+                        None
+                    };
+                    // already verified kzg for all blobs
+                    results.push(MaybeAvailableBlock::Available(AvailableBlock {
+                        block_root,
+                        block,
+                        blobs: verified_blobs,
+                    }))
+                }
+            }
+        }
+
+        Ok(results)
+    }
+
+    /// Determines the blob requirements for a block. If the block is pre-deneb, no blobs are required.
+    /// If the block's epoch is from prior to the data availability boundary, no blobs are required.
+    fn blobs_required_for_block(&self, block: &SignedBeaconBlock<T::EthSpec>) -> bool {
+        block.num_expected_blobs() > 0 && self.da_check_required_for_epoch(block.epoch())
+    }
+
+    /// Adds a block to the processing cache. This block's commitments are unverified but caching
+    /// them here is useful to avoid duplicate downloads of blocks, as well as understanding
+    /// our blob download requirements. We will also serve this over RPC.
+    pub fn notify_block(&self, block_root: Hash256, block: Arc<SignedBeaconBlock<T::EthSpec>>) {
+        self.processing_cache
+            .write()
+            .entry(block_root)
+            .or_default()
+            .merge_block(block);
+    }
+
+    /// Add a single blob commitment to the processing cache. This commitment is unverified but caching
+    /// them here is useful to avoid duplicate downloads of blobs, as well as understanding
+    /// our block and blob download requirements.
+    pub fn notify_gossip_blob(&self, block_root: Hash256, blob: &GossipVerifiedBlob<T>) {
+        let index = blob.index();
+        let commitment = blob.kzg_commitment();
+        self.processing_cache
+            .write()
+            .entry(block_root)
+            .or_default()
+            .merge_single_blob(index as usize, commitment);
+    }
+
+    /// Adds blob commitments to the processing cache. These commitments are unverified but caching
+    /// them here is useful to avoid duplicate downloads of blobs, as well as understanding
+    /// our block and blob download requirements.
+    pub fn notify_rpc_blobs(&self, block_root: Hash256, blobs: &FixedBlobSidecarList<T::EthSpec>) {
+        let mut commitments = KzgCommitmentOpts::<T::EthSpec>::default();
+        for blob in blobs.iter().flatten() {
+            if let Some(commitment) = commitments.get_mut(blob.index as usize) {
+                *commitment = Some(blob.kzg_commitment);
+            }
+        }
+        self.processing_cache
+            .write()
+            .entry(block_root)
+            .or_default()
+            .merge_blobs(commitments);
+    }
+
+    /// Clears the block and all blobs from the processing cache for a give root if they exist.
+    pub fn remove_notified(&self, block_root: &Hash256) {
+        self.processing_cache.write().remove(block_root)
+    }
+
+    /// The epoch at which we require a data availability check in block processing.
+    /// `None` if the `Deneb` fork is disabled.
+    pub fn data_availability_boundary(&self) -> Option<Epoch> {
+        self.spec.deneb_fork_epoch.and_then(|fork_epoch| {
+            self.slot_clock
+                .now()
+                .map(|slot| slot.epoch(T::EthSpec::slots_per_epoch()))
+                .map(|current_epoch| {
+                    std::cmp::max(
+                        fork_epoch,
+                        current_epoch
+                            .saturating_sub(self.spec.min_epochs_for_blob_sidecars_requests),
+                    )
+                })
+        })
+    }
+
+    /// Returns true if the given epoch lies within the da boundary and false otherwise.
+    pub fn da_check_required_for_epoch(&self, block_epoch: Epoch) -> bool {
+        self.data_availability_boundary()
+            .map_or(false, |da_epoch| block_epoch >= da_epoch)
+    }
+
+    /// Returns `true` if the current epoch is greater than or equal to the `Deneb` epoch.
+    pub fn is_deneb(&self) -> bool {
+        self.slot_clock.now().map_or(false, |slot| {
+            self.spec.deneb_fork_epoch.map_or(false, |deneb_epoch| {
+                let now_epoch = slot.epoch(T::EthSpec::slots_per_epoch());
+                now_epoch >= deneb_epoch
+            })
+        })
+    }
+
+    /// Persist all in memory components to disk
+    pub fn persist_all(&self) -> Result<(), AvailabilityCheckError> {
+        self.availability_cache.write_all_to_disk()
+    }
+
+    /// Collects metrics from the data availability checker.
+    pub fn metrics(&self) -> DataAvailabilityCheckerMetrics {
+        DataAvailabilityCheckerMetrics {
+            processing_cache_size: self.processing_cache.read().len(),
+            num_store_entries: self.availability_cache.num_store_entries(),
+            state_cache_size: self.availability_cache.state_cache_size(),
+            block_cache_size: self.availability_cache.block_cache_size(),
+        }
+    }
+}
+
+/// Helper struct to group data availability checker metrics.
+pub struct DataAvailabilityCheckerMetrics {
+    pub processing_cache_size: usize,
+    pub num_store_entries: usize,
+    pub state_cache_size: usize,
+    pub block_cache_size: usize,
+}
+
+pub fn start_availability_cache_maintenance_service<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    chain: Arc<BeaconChain<T>>,
+) {
+    // this cache only needs to be maintained if deneb is configured
+    if chain.spec.deneb_fork_epoch.is_some() {
+        let overflow_cache = chain.data_availability_checker.availability_cache.clone();
+        executor.spawn(
+            async move { availability_cache_maintenance_service(chain, overflow_cache).await },
+            "availability_cache_service",
+        );
+    } else {
+        debug!(
+            chain.log,
+            "Deneb fork not configured, not starting availability cache maintenance service"
+        );
+    }
+}
+
+async fn availability_cache_maintenance_service<T: BeaconChainTypes>(
+    chain: Arc<BeaconChain<T>>,
+    overflow_cache: Arc<OverflowLRUCache<T>>,
+) {
+    let epoch_duration = chain.slot_clock.slot_duration() * T::EthSpec::slots_per_epoch() as u32;
+    loop {
+        match chain
+            .slot_clock
+            .duration_to_next_epoch(T::EthSpec::slots_per_epoch())
+        {
+            Some(duration) => {
+                // this service should run 3/4 of the way through the epoch
+                let additional_delay = (epoch_duration * 3) / 4;
+                tokio::time::sleep(duration + additional_delay).await;
+
+                let Some(deneb_fork_epoch) = chain.spec.deneb_fork_epoch else {
+                    // shutdown service if deneb fork epoch not set
+                    break;
+                };
+
+                debug!(
+                    chain.log,
+                    "Availability cache maintenance service firing";
+                );
+                let Some(current_epoch) = chain
+                    .slot_clock
+                    .now()
+                    .map(|slot| slot.epoch(T::EthSpec::slots_per_epoch()))
+                else {
+                    continue;
+                };
+
+                if current_epoch < deneb_fork_epoch {
+                    // we are not in deneb yet
+                    continue;
+                }
+
+                let finalized_epoch = chain
+                    .canonical_head
+                    .fork_choice_read_lock()
+                    .finalized_checkpoint()
+                    .epoch;
+                // any data belonging to an epoch before this should be pruned
+                let cutoff_epoch = std::cmp::max(
+                    finalized_epoch + 1,
+                    std::cmp::max(
+                        current_epoch
+                            .saturating_sub(chain.spec.min_epochs_for_blob_sidecars_requests),
+                        deneb_fork_epoch,
+                    ),
+                );
+
+                if let Err(e) = overflow_cache.do_maintenance(cutoff_epoch) {
+                    error!(chain.log, "Failed to maintain availability cache"; "error" => ?e);
+                }
+            }
+            None => {
+                error!(chain.log, "Failed to read slot clock");
+                // If we can't read the slot clock, just wait another slot.
+                tokio::time::sleep(chain.slot_clock.slot_duration()).await;
+            }
+        };
+    }
+}
+
+/// A fully available block that is ready to be imported into fork choice.
+#[derive(Clone, Debug, PartialEq)]
+pub struct AvailableBlock<E: EthSpec> {
+    block_root: Hash256,
+    block: Arc<SignedBeaconBlock<E>>,
+    blobs: Option<BlobSidecarList<E>>,
+}
+
+impl<E: EthSpec> AvailableBlock<E> {
+    pub fn __new_for_testing(
+        block_root: Hash256,
+        block: Arc<SignedBeaconBlock<E>>,
+        blobs: Option<BlobSidecarList<E>>,
+    ) -> Self {
+        Self {
+            block_root,
+            block,
+            blobs,
+        }
+    }
+
+    pub fn block(&self) -> &SignedBeaconBlock<E> {
+        &self.block
+    }
+    pub fn block_cloned(&self) -> Arc<SignedBeaconBlock<E>> {
+        self.block.clone()
+    }
+
+    pub fn blobs(&self) -> Option<&BlobSidecarList<E>> {
+        self.blobs.as_ref()
+    }
+
+    pub fn deconstruct(
+        self,
+    ) -> (
+        Hash256,
+        Arc<SignedBeaconBlock<E>>,
+        Option<BlobSidecarList<E>>,
+    ) {
+        let AvailableBlock {
+            block_root,
+            block,
+            blobs,
+        } = self;
+        (block_root, block, blobs)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum MaybeAvailableBlock<E: EthSpec> {
+    /// This variant is fully available.
+    /// i.e. for pre-deneb blocks, it contains a (`SignedBeaconBlock`, `Blobs::None`) and for
+    /// post-4844 blocks, it contains a `SignedBeaconBlock` and a Blobs variant other than `Blobs::None`.
+    Available(AvailableBlock<E>),
+    /// This variant is not fully available and requires blobs to become fully available.
+    AvailabilityPending {
+        block_root: Hash256,
+        block: Arc<SignedBeaconBlock<E>>,
+    },
+}
+
+impl<E: EthSpec> MaybeAvailableBlock<E> {
+    pub fn block_cloned(&self) -> Arc<SignedBeaconBlock<E>> {
+        match self {
+            Self::Available(block) => block.block_cloned(),
+            Self::AvailabilityPending { block, .. } => block.clone(),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum MissingBlobs {
+    /// We know for certain these blobs are missing.
+    KnownMissing(Vec<BlobIdentifier>),
+    /// We think these blobs might be missing.
+    PossibleMissing(Vec<BlobIdentifier>),
+    /// Blobs are not required.
+    BlobsNotRequired,
+}
+
+impl MissingBlobs {
+    pub fn new_without_block(block_root: Hash256, is_deneb: bool) -> Self {
+        if is_deneb {
+            MissingBlobs::PossibleMissing(BlobIdentifier::get_all_blob_ids::<E>(block_root))
+        } else {
+            MissingBlobs::BlobsNotRequired
+        }
+    }
+    pub fn is_empty(&self) -> bool {
+        match self {
+            MissingBlobs::KnownMissing(v) => v.is_empty(),
+            MissingBlobs::PossibleMissing(v) => v.is_empty(),
+            MissingBlobs::BlobsNotRequired => true,
+        }
+    }
+    pub fn contains(&self, blob_id: &BlobIdentifier) -> bool {
+        match self {
+            MissingBlobs::KnownMissing(v) => v.contains(blob_id),
+            MissingBlobs::PossibleMissing(v) => v.contains(blob_id),
+            MissingBlobs::BlobsNotRequired => false,
+        }
+    }
+    pub fn remove(&mut self, blob_id: &BlobIdentifier) {
+        match self {
+            MissingBlobs::KnownMissing(v) => v.retain(|id| id != blob_id),
+            MissingBlobs::PossibleMissing(v) => v.retain(|id| id != blob_id),
+            MissingBlobs::BlobsNotRequired => {}
+        }
+    }
+    pub fn indices(&self) -> Vec<u64> {
+        match self {
+            MissingBlobs::KnownMissing(v) => v.iter().map(|id| id.index).collect(),
+            MissingBlobs::PossibleMissing(v) => v.iter().map(|id| id.index).collect(),
+            MissingBlobs::BlobsNotRequired => vec![],
+        }
+    }
+}
+
+impl Into<Vec<BlobIdentifier>> for MissingBlobs {
+    fn into(self) -> Vec<BlobIdentifier> {
+        match self {
+            MissingBlobs::KnownMissing(v) => v,
+            MissingBlobs::PossibleMissing(v) => v,
+            MissingBlobs::BlobsNotRequired => vec![],
+        }
+    }
+}

beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs

@@ -0,0 +1,507 @@
+use super::state_lru_cache::DietAvailabilityPendingExecutedBlock;
+use crate::blob_verification::KzgVerifiedBlob;
+use crate::block_verification_types::AsBlock;
+use crate::data_availability_checker::overflow_lru_cache::PendingComponents;
+use crate::data_availability_checker::ProcessingComponents;
+use kzg::KzgCommitment;
+use ssz_types::FixedVector;
+use std::sync::Arc;
+use types::beacon_block_body::KzgCommitments;
+use types::{BlobSidecar, EthSpec, SignedBeaconBlock};
+
+/// Defines an interface for managing data availability with two key invariants:
+///
+/// 1. If we haven't seen a block yet, we will insert the first blob for a given (block_root, index)
+///    but we won't insert subsequent blobs for the same (block_root, index) if they have a different
+///    commitment.
+/// 2. On block insertion, any non-matching blob commitments are evicted.
+///
+/// Types implementing this trait can be used for validating and managing availability
+/// of blocks and blobs in a cache-like data structure.
+pub trait AvailabilityView<E: EthSpec> {
+    /// The type representing a block in the implementation.
+    type BlockType: GetCommitments<E>;
+
+    /// The type representing a blob in the implementation. Must implement `Clone`.
+    type BlobType: Clone + GetCommitment<E>;
+
+    /// Returns an immutable reference to the cached block.
+    fn get_cached_block(&self) -> &Option<Self::BlockType>;
+
+    /// Returns an immutable reference to the fixed vector of cached blobs.
+    fn get_cached_blobs(&self) -> &FixedVector<Option<Self::BlobType>, E::MaxBlobsPerBlock>;
+
+    /// Returns a mutable reference to the cached block.
+    fn get_cached_block_mut(&mut self) -> &mut Option<Self::BlockType>;
+
+    /// Returns a mutable reference to the fixed vector of cached blobs.
+    fn get_cached_blobs_mut(
+        &mut self,
+    ) -> &mut FixedVector<Option<Self::BlobType>, E::MaxBlobsPerBlock>;
+
+    /// Checks if a block exists in the cache.
+    ///
+    /// Returns:
+    /// - `true` if a block exists.
+    /// - `false` otherwise.
+    fn block_exists(&self) -> bool {
+        self.get_cached_block().is_some()
+    }
+
+    /// Checks if a blob exists at the given index in the cache.
+    ///
+    /// Returns:
+    /// - `true` if a blob exists at the given index.
+    /// - `false` otherwise.
+    fn blob_exists(&self, blob_index: usize) -> bool {
+        self.get_cached_blobs()
+            .get(blob_index)
+            .map(|b| b.is_some())
+            .unwrap_or(false)
+    }
+
+    /// Returns the number of blobs that are expected to be present. Returns `None` if we don't have a
+    /// block.
+    ///
+    /// This corresponds to the number of commitments that are present in a block.
+    fn num_expected_blobs(&self) -> Option<usize> {
+        self.get_cached_block()
+            .as_ref()
+            .map(|b| b.get_commitments().len())
+    }
+
+    /// Returns the number of blobs that have been received and are stored in the cache.
+    fn num_received_blobs(&self) -> usize {
+        self.get_cached_blobs().iter().flatten().count()
+    }
+
+    /// Inserts a block into the cache.
+    fn insert_block(&mut self, block: Self::BlockType) {
+        *self.get_cached_block_mut() = Some(block)
+    }
+
+    /// Inserts a blob at a specific index in the cache.
+    ///
+    /// Existing blob at the index will be replaced.
+    fn insert_blob_at_index(&mut self, blob_index: usize, blob: Self::BlobType) {
+        if let Some(b) = self.get_cached_blobs_mut().get_mut(blob_index) {
+            *b = Some(blob);
+        }
+    }
+
+    /// Merges a given set of blobs into the cache.
+    ///
+    /// Blobs are only inserted if:
+    /// 1. The blob entry at the index is empty and no block exists.
+    /// 2. The block exists and its commitment matches the blob's commitment.
+    fn merge_blobs(&mut self, blobs: FixedVector<Option<Self::BlobType>, E::MaxBlobsPerBlock>) {
+        for (index, blob) in blobs.iter().cloned().enumerate() {
+            let Some(blob) = blob else { continue };
+            self.merge_single_blob(index, blob);
+        }
+    }
+
+    /// Merges a single blob into the cache.
+    ///
+    /// Blobs are only inserted if:
+    /// 1. The blob entry at the index is empty and no block exists, or
+    /// 2. The block exists and its commitment matches the blob's commitment.
+    fn merge_single_blob(&mut self, index: usize, blob: Self::BlobType) {
+        if let Some(cached_block) = self.get_cached_block() {
+            let block_commitment_opt = cached_block.get_commitments().get(index).copied();
+            if let Some(block_commitment) = block_commitment_opt {
+                if block_commitment == *blob.get_commitment() {
+                    self.insert_blob_at_index(index, blob)
+                }
+            }
+        } else if !self.blob_exists(index) {
+            self.insert_blob_at_index(index, blob)
+        }
+    }
+
+    /// Inserts a new block and revalidates the existing blobs against it.
+    ///
+    /// Blobs that don't match the new block's commitments are evicted.
+    fn merge_block(&mut self, block: Self::BlockType) {
+        self.insert_block(block);
+        let reinsert = std::mem::take(self.get_cached_blobs_mut());
+        self.merge_blobs(reinsert);
+    }
+
+    /// Checks if the block and all of its expected blobs are available in the cache.
+    ///
+    /// Returns `true` if both the block exists and the number of received blobs matches the number
+    /// of expected blobs.
+    fn is_available(&self) -> bool {
+        if let Some(num_expected_blobs) = self.num_expected_blobs() {
+            num_expected_blobs == self.num_received_blobs()
+        } else {
+            false
+        }
+    }
+}
+
+/// Implements the `AvailabilityView` trait for a given struct.
+///
+/// - `$struct_name`: The name of the struct for which to implement `AvailabilityView`.
+/// - `$block_type`: The type to use for `BlockType` in the `AvailabilityView` trait.
+/// - `$blob_type`: The type to use for `BlobType` in the `AvailabilityView` trait.
+/// - `$block_field`: The field name in the struct that holds the cached block.
+/// - `$blob_field`: The field name in the struct that holds the cached blobs.
+#[macro_export]
+macro_rules! impl_availability_view {
+    ($struct_name:ident, $block_type:ty, $blob_type:ty, $block_field:ident, $blob_field:ident) => {
+        impl<E: EthSpec> AvailabilityView<E> for $struct_name<E> {
+            type BlockType = $block_type;
+            type BlobType = $blob_type;
+
+            fn get_cached_block(&self) -> &Option<Self::BlockType> {
+                &self.$block_field
+            }
+
+            fn get_cached_blobs(
+                &self,
+            ) -> &FixedVector<Option<Self::BlobType>, E::MaxBlobsPerBlock> {
+                &self.$blob_field
+            }
+
+            fn get_cached_block_mut(&mut self) -> &mut Option<Self::BlockType> {
+                &mut self.$block_field
+            }
+
+            fn get_cached_blobs_mut(
+                &mut self,
+            ) -> &mut FixedVector<Option<Self::BlobType>, E::MaxBlobsPerBlock> {
+                &mut self.$blob_field
+            }
+        }
+    };
+}
+
+impl_availability_view!(
+    ProcessingComponents,
+    Arc<SignedBeaconBlock<E>>,
+    KzgCommitment,
+    block,
+    blob_commitments
+);
+
+impl_availability_view!(
+    PendingComponents,
+    DietAvailabilityPendingExecutedBlock<E>,
+    KzgVerifiedBlob<E>,
+    executed_block,
+    verified_blobs
+);
+
+pub trait GetCommitments<E: EthSpec> {
+    fn get_commitments(&self) -> KzgCommitments<E>;
+}
+
+pub trait GetCommitment<E: EthSpec> {
+    fn get_commitment(&self) -> &KzgCommitment;
+}
+
+impl<E: EthSpec> GetCommitment<E> for KzgCommitment {
+    fn get_commitment(&self) -> &KzgCommitment {
+        self
+    }
+}
+
+// These implementations are required to implement `AvailabilityView` for `PendingComponents`.
+impl<E: EthSpec> GetCommitments<E> for DietAvailabilityPendingExecutedBlock<E> {
+    fn get_commitments(&self) -> KzgCommitments<E> {
+        self.as_block()
+            .message()
+            .body()
+            .blob_kzg_commitments()
+            .cloned()
+            .unwrap_or_default()
+    }
+}
+
+impl<E: EthSpec> GetCommitment<E> for KzgVerifiedBlob<E> {
+    fn get_commitment(&self) -> &KzgCommitment {
+        &self.as_blob().kzg_commitment
+    }
+}
+
+// These implementations are required to implement `AvailabilityView` for `ChildComponents`.
+impl<E: EthSpec> GetCommitments<E> for Arc<SignedBeaconBlock<E>> {
+    fn get_commitments(&self) -> KzgCommitments<E> {
+        self.message()
+            .body()
+            .blob_kzg_commitments()
+            .ok()
+            .cloned()
+            .unwrap_or_default()
+    }
+}
+impl<E: EthSpec> GetCommitment<E> for Arc<BlobSidecar<E>> {
+    fn get_commitment(&self) -> &KzgCommitment {
+        &self.kzg_commitment
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+    use crate::block_verification_types::BlockImportData;
+    use crate::eth1_finalization_cache::Eth1FinalizationData;
+    use crate::test_utils::{generate_rand_block_and_blobs, NumBlobs};
+    use crate::AvailabilityPendingExecutedBlock;
+    use crate::PayloadVerificationOutcome;
+    use fork_choice::PayloadVerificationStatus;
+    use rand::rngs::StdRng;
+    use rand::SeedableRng;
+    use state_processing::ConsensusContext;
+    use types::test_utils::TestRandom;
+    use types::{BeaconState, ChainSpec, ForkName, MainnetEthSpec, Slot};
+
+    type E = MainnetEthSpec;
+
+    type Setup<E> = (
+        SignedBeaconBlock<E>,
+        FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
+        FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
+    );
+
+    pub fn pre_setup() -> Setup<E> {
+        let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64);
+        let (block, blobs_vec) =
+            generate_rand_block_and_blobs::<E>(ForkName::Deneb, NumBlobs::Random, &mut rng);
+        let mut blobs: FixedVector<_, <E as EthSpec>::MaxBlobsPerBlock> = FixedVector::default();
+
+        for blob in blobs_vec {
+            if let Some(b) = blobs.get_mut(blob.index as usize) {
+                *b = Some(Arc::new(blob));
+            }
+        }
+
+        let mut invalid_blobs: FixedVector<
+            Option<Arc<BlobSidecar<E>>>,
+            <E as EthSpec>::MaxBlobsPerBlock,
+        > = FixedVector::default();
+        for (index, blob) in blobs.iter().enumerate() {
+            if let Some(invalid_blob) = blob {
+                let mut blob_copy = invalid_blob.as_ref().clone();
+                blob_copy.kzg_commitment = KzgCommitment::random_for_test(&mut rng);
+                *invalid_blobs.get_mut(index).unwrap() = Some(Arc::new(blob_copy));
+            }
+        }
+
+        (block, blobs, invalid_blobs)
+    }
+
+    type ProcessingViewSetup<E> = (
+        Arc<SignedBeaconBlock<E>>,
+        FixedVector<Option<KzgCommitment>, <E as EthSpec>::MaxBlobsPerBlock>,
+        FixedVector<Option<KzgCommitment>, <E as EthSpec>::MaxBlobsPerBlock>,
+    );
+
+    pub fn setup_processing_components(
+        block: SignedBeaconBlock<E>,
+        valid_blobs: FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
+        invalid_blobs: FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
+    ) -> ProcessingViewSetup<E> {
+        let blobs = FixedVector::from(
+            valid_blobs
+                .iter()
+                .map(|blob_opt| blob_opt.as_ref().map(|blob| blob.kzg_commitment))
+                .collect::<Vec<_>>(),
+        );
+        let invalid_blobs = FixedVector::from(
+            invalid_blobs
+                .iter()
+                .map(|blob_opt| blob_opt.as_ref().map(|blob| blob.kzg_commitment))
+                .collect::<Vec<_>>(),
+        );
+        (Arc::new(block), blobs, invalid_blobs)
+    }
+
+    type PendingComponentsSetup<E> = (
+        DietAvailabilityPendingExecutedBlock<E>,
+        FixedVector<Option<KzgVerifiedBlob<E>>, <E as EthSpec>::MaxBlobsPerBlock>,
+        FixedVector<Option<KzgVerifiedBlob<E>>, <E as EthSpec>::MaxBlobsPerBlock>,
+    );
+
+    pub fn setup_pending_components(
+        block: SignedBeaconBlock<E>,
+        valid_blobs: FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
+        invalid_blobs: FixedVector<Option<Arc<BlobSidecar<E>>>, <E as EthSpec>::MaxBlobsPerBlock>,
+    ) -> PendingComponentsSetup<E> {
+        let blobs = FixedVector::from(
+            valid_blobs
+                .iter()
+                .map(|blob_opt| {
+                    blob_opt
+                        .as_ref()
+                        .map(|blob| KzgVerifiedBlob::__assumed_valid(blob.clone()))
+                })
+                .collect::<Vec<_>>(),
+        );
+        let invalid_blobs = FixedVector::from(
+            invalid_blobs
+                .iter()
+                .map(|blob_opt| {
+                    blob_opt
+                        .as_ref()
+                        .map(|blob| KzgVerifiedBlob::__assumed_valid(blob.clone()))
+                })
+                .collect::<Vec<_>>(),
+        );
+        let dummy_parent = block.clone_as_blinded();
+        let block = AvailabilityPendingExecutedBlock {
+            block: Arc::new(block),
+            import_data: BlockImportData {
+                block_root: Default::default(),
+                state: BeaconState::new(0, Default::default(), &ChainSpec::minimal()),
+                parent_block: dummy_parent,
+                parent_eth1_finalization_data: Eth1FinalizationData {
+                    eth1_data: Default::default(),
+                    eth1_deposit_index: 0,
+                },
+                confirmed_state_roots: vec![],
+                consensus_context: ConsensusContext::new(Slot::new(0)),
+            },
+            payload_verification_outcome: PayloadVerificationOutcome {
+                payload_verification_status: PayloadVerificationStatus::Verified,
+                is_valid_merge_transition_block: false,
+            },
+        };
+        (block.into(), blobs, invalid_blobs)
+    }
+
+    pub fn assert_cache_consistent<V: AvailabilityView<E>>(cache: V) {
+        if let Some(cached_block) = cache.get_cached_block() {
+            let cached_block_commitments = cached_block.get_commitments();
+            for index in 0..E::max_blobs_per_block() {
+                let block_commitment = cached_block_commitments.get(index).copied();
+                let blob_commitment_opt = cache.get_cached_blobs().get(index).unwrap();
+                let blob_commitment = blob_commitment_opt.as_ref().map(|b| *b.get_commitment());
+                assert_eq!(block_commitment, blob_commitment);
+            }
+        } else {
+            panic!("No cached block")
+        }
+    }
+
+    pub fn assert_empty_blob_cache<V: AvailabilityView<E>>(cache: V) {
+        for blob in cache.get_cached_blobs().iter() {
+            assert!(blob.is_none());
+        }
+    }
+
+    #[macro_export]
+    macro_rules! generate_tests {
+        ($module_name:ident, $type_name:ty, $block_field:ident, $blob_field:ident, $setup_fn:ident) => {
+            mod $module_name {
+                use super::*;
+                use types::Hash256;
+
+                #[test]
+                fn valid_block_invalid_blobs_valid_blobs() {
+                    let (block_commitments, blobs, random_blobs) = pre_setup();
+                    let (block_commitments, blobs, random_blobs) =
+                        $setup_fn(block_commitments, blobs, random_blobs);
+                    let block_root = Hash256::zero();
+                    let mut cache = <$type_name>::empty(block_root);
+                    cache.merge_block(block_commitments);
+                    cache.merge_blobs(random_blobs);
+                    cache.merge_blobs(blobs);
+
+                    assert_cache_consistent(cache);
+                }
+
+                #[test]
+                fn invalid_blobs_block_valid_blobs() {
+                    let (block_commitments, blobs, random_blobs) = pre_setup();
+                    let (block_commitments, blobs, random_blobs) =
+                        $setup_fn(block_commitments, blobs, random_blobs);
+                    let block_root = Hash256::zero();
+                    let mut cache = <$type_name>::empty(block_root);
+                    cache.merge_blobs(random_blobs);
+                    cache.merge_block(block_commitments);
+                    cache.merge_blobs(blobs);
+
+                    assert_cache_consistent(cache);
+                }
+
+                #[test]
+                fn invalid_blobs_valid_blobs_block() {
+                    let (block_commitments, blobs, random_blobs) = pre_setup();
+                    let (block_commitments, blobs, random_blobs) =
+                        $setup_fn(block_commitments, blobs, random_blobs);
+
+                    let block_root = Hash256::zero();
+                    let mut cache = <$type_name>::empty(block_root);
+                    cache.merge_blobs(random_blobs);
+                    cache.merge_blobs(blobs);
+                    cache.merge_block(block_commitments);
+
+                    assert_empty_blob_cache(cache);
+                }
+
+                #[test]
+                fn block_valid_blobs_invalid_blobs() {
+                    let (block_commitments, blobs, random_blobs) = pre_setup();
+                    let (block_commitments, blobs, random_blobs) =
+                        $setup_fn(block_commitments, blobs, random_blobs);
+
+                    let block_root = Hash256::zero();
+                    let mut cache = <$type_name>::empty(block_root);
+                    cache.merge_block(block_commitments);
+                    cache.merge_blobs(blobs);
+                    cache.merge_blobs(random_blobs);
+
+                    assert_cache_consistent(cache);
+                }
+
+                #[test]
+                fn valid_blobs_block_invalid_blobs() {
+                    let (block_commitments, blobs, random_blobs) = pre_setup();
+                    let (block_commitments, blobs, random_blobs) =
+                        $setup_fn(block_commitments, blobs, random_blobs);
+
+                    let block_root = Hash256::zero();
+                    let mut cache = <$type_name>::empty(block_root);
+                    cache.merge_blobs(blobs);
+                    cache.merge_block(block_commitments);
+                    cache.merge_blobs(random_blobs);
+
+                    assert_cache_consistent(cache);
+                }
+
+                #[test]
+                fn valid_blobs_invalid_blobs_block() {
+                    let (block_commitments, blobs, random_blobs) = pre_setup();
+                    let (block_commitments, blobs, random_blobs) =
+                        $setup_fn(block_commitments, blobs, random_blobs);
+
+                    let block_root = Hash256::zero();
+                    let mut cache = <$type_name>::empty(block_root);
+                    cache.merge_blobs(blobs);
+                    cache.merge_blobs(random_blobs);
+                    cache.merge_block(block_commitments);
+
+                    assert_cache_consistent(cache);
+                }
+            }
+        };
+    }
+
+    generate_tests!(
+        processing_components_tests,
+        ProcessingComponents::<E>,
+        kzg_commitments,
+        processing_blobs,
+        setup_processing_components
+    );
+    generate_tests!(
+        pending_components_tests,
+        PendingComponents<E>,
+        executed_block,
+        verified_blobs,
+        setup_pending_components
+    );
+}

beacon_node/beacon_chain/src/data_availability_checker/child_components.rs

@@ -0,0 +1,69 @@
+use crate::block_verification_types::RpcBlock;
+use bls::Hash256;
+use std::sync::Arc;
+use types::blob_sidecar::FixedBlobSidecarList;
+use types::{BlobSidecar, EthSpec, SignedBeaconBlock};
+
+/// For requests triggered by an `UnknownBlockParent` or `UnknownBlobParent`, this struct
+/// is used to cache components as they are sent to the network service. We can't use the
+/// data availability cache currently because any blocks or blobs without parents
+/// won't pass validation and therefore won't make it into the cache.
+pub struct ChildComponents<E: EthSpec> {
+    pub block_root: Hash256,
+    pub downloaded_block: Option<Arc<SignedBeaconBlock<E>>>,
+    pub downloaded_blobs: FixedBlobSidecarList<E>,
+}
+
+impl<E: EthSpec> From<RpcBlock<E>> for ChildComponents<E> {
+    fn from(value: RpcBlock<E>) -> Self {
+        let (block_root, block, blobs) = value.deconstruct();
+        let fixed_blobs = blobs.map(|blobs| {
+            FixedBlobSidecarList::from(blobs.into_iter().map(Some).collect::<Vec<_>>())
+        });
+        Self::new(block_root, Some(block), fixed_blobs)
+    }
+}
+
+impl<E: EthSpec> ChildComponents<E> {
+    pub fn empty(block_root: Hash256) -> Self {
+        Self {
+            block_root,
+            downloaded_block: None,
+            downloaded_blobs: <_>::default(),
+        }
+    }
+    pub fn new(
+        block_root: Hash256,
+        block: Option<Arc<SignedBeaconBlock<E>>>,
+        blobs: Option<FixedBlobSidecarList<E>>,
+    ) -> Self {
+        let mut cache = Self::empty(block_root);
+        if let Some(block) = block {
+            cache.merge_block(block);
+        }
+        if let Some(blobs) = blobs {
+            cache.merge_blobs(blobs);
+        }
+        cache
+    }
+
+    pub fn merge_block(&mut self, block: Arc<SignedBeaconBlock<E>>) {
+        self.downloaded_block = Some(block);
+    }
+
+    pub fn merge_blob(&mut self, blob: Arc<BlobSidecar<E>>) {
+        if let Some(blob_ref) = self.downloaded_blobs.get_mut(blob.index as usize) {
+            *blob_ref = Some(blob);
+        }
+    }
+
+    pub fn merge_blobs(&mut self, blobs: FixedBlobSidecarList<E>) {
+        for blob in blobs.iter().flatten() {
+            self.merge_blob(blob.clone());
+        }
+    }
+
+    pub fn clear_blobs(&mut self) {
+        self.downloaded_blobs = FixedBlobSidecarList::default();
+    }
+}

beacon_node/beacon_chain/src/data_availability_checker/error.rs

@@ -0,0 +1,79 @@
+use kzg::{Error as KzgError, KzgCommitment};
+use types::{BeaconStateError, Hash256};
+
+#[derive(Debug)]
+pub enum Error {
+    Kzg(KzgError),
+    KzgNotInitialized,
+    KzgVerificationFailed,
+    KzgCommitmentMismatch {
+        blob_commitment: KzgCommitment,
+        block_commitment: KzgCommitment,
+    },
+    Unexpected,
+    SszTypes(ssz_types::Error),
+    MissingBlobs,
+    BlobIndexInvalid(u64),
+    StoreError(store::Error),
+    DecodeError(ssz::DecodeError),
+    ParentStateMissing(Hash256),
+    BlockReplayError(state_processing::BlockReplayError),
+    RebuildingStateCaches(BeaconStateError),
+}
+
+pub enum ErrorCategory {
+    /// Internal Errors (not caused by peers)
+    Internal,
+    /// Errors caused by faulty / malicious peers
+    Malicious,
+}
+
+impl Error {
+    pub fn category(&self) -> ErrorCategory {
+        match self {
+            Error::KzgNotInitialized
+            | Error::SszTypes(_)
+            | Error::MissingBlobs
+            | Error::StoreError(_)
+            | Error::DecodeError(_)
+            | Error::Unexpected
+            | Error::ParentStateMissing(_)
+            | Error::BlockReplayError(_)
+            | Error::RebuildingStateCaches(_) => ErrorCategory::Internal,
+            Error::Kzg(_)
+            | Error::BlobIndexInvalid(_)
+            | Error::KzgCommitmentMismatch { .. }
+            | Error::KzgVerificationFailed => ErrorCategory::Malicious,
+        }
+    }
+}
+
+impl From<ssz_types::Error> for Error {
+    fn from(value: ssz_types::Error) -> Self {
+        Self::SszTypes(value)
+    }
+}
+
+impl From<store::Error> for Error {
+    fn from(value: store::Error) -> Self {
+        Self::StoreError(value)
+    }
+}
+
+impl From<ssz::DecodeError> for Error {
+    fn from(value: ssz::DecodeError) -> Self {
+        Self::DecodeError(value)
+    }
+}
+
+impl From<state_processing::BlockReplayError> for Error {
+    fn from(value: state_processing::BlockReplayError) -> Self {
+        Self::BlockReplayError(value)
+    }
+}
+
+impl From<KzgError> for Error {
+    fn from(value: KzgError) -> Self {
+        Self::Kzg(value)
+    }
+}

beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs

@@ -0,0 +1,63 @@
+use crate::data_availability_checker::AvailabilityView;
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+use std::sync::Arc;
+use types::beacon_block_body::KzgCommitmentOpts;
+use types::{EthSpec, Hash256, SignedBeaconBlock};
+
+/// This cache is used only for gossip blocks/blobs and single block/blob lookups, to give req/resp
+/// a view of what we have and what we require. This cache serves a slightly different purpose than
+/// gossip caches because it allows us to process duplicate blobs that are valid in gossip.
+/// See `AvailabilityView`'s trait definition.
+#[derive(Default)]
+pub struct ProcessingCache<E: EthSpec> {
+    processing_cache: HashMap<Hash256, ProcessingComponents<E>>,
+}
+
+impl<E: EthSpec> ProcessingCache<E> {
+    pub fn get(&self, block_root: &Hash256) -> Option<&ProcessingComponents<E>> {
+        self.processing_cache.get(block_root)
+    }
+    pub fn entry(&mut self, block_root: Hash256) -> Entry<'_, Hash256, ProcessingComponents<E>> {
+        self.processing_cache.entry(block_root)
+    }
+    pub fn remove(&mut self, block_root: &Hash256) {
+        self.processing_cache.remove(block_root);
+    }
+    pub fn has_block(&self, block_root: &Hash256) -> bool {
+        self.processing_cache
+            .get(block_root)
+            .map_or(false, |b| b.block_exists())
+    }
+    pub fn len(&self) -> usize {
+        self.processing_cache.len()
+    }
+}
+
+#[derive(Default, Debug, Clone)]
+pub struct ProcessingComponents<E: EthSpec> {
+    /// Blobs required for a block can only be known if we have seen the block. So `Some` here
+    /// means we've seen it, a `None` means we haven't. The `kzg_commitments` value helps us figure
+    /// out whether incoming blobs actually match the block.
+    pub block: Option<Arc<SignedBeaconBlock<E>>>,
+    /// `KzgCommitments` for blobs are always known, even if we haven't seen the block. See
+    /// `AvailabilityView`'s trait definition for more details.
+    pub blob_commitments: KzgCommitmentOpts<E>,
+}
+
+impl<E: EthSpec> ProcessingComponents<E> {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+// Not safe for use outside of tests as this always required a slot.
+#[cfg(test)]
+impl<E: EthSpec> ProcessingComponents<E> {
+    pub fn empty(_block_root: Hash256) -> Self {
+        Self {
+            block: None,
+            blob_commitments: KzgCommitmentOpts::<E>::default(),
+        }
+    }
+}

beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs

@@ -0,0 +1,229 @@
+use crate::block_verification_types::AsBlock;
+use crate::{
+    block_verification_types::BlockImportData,
+    data_availability_checker::{AvailabilityCheckError, STATE_LRU_CAPACITY_NON_ZERO},
+    eth1_finalization_cache::Eth1FinalizationData,
+    AvailabilityPendingExecutedBlock, BeaconChainTypes, BeaconStore, PayloadVerificationOutcome,
+};
+use lru::LruCache;
+use parking_lot::RwLock;
+use ssz_derive::{Decode, Encode};
+use state_processing::{BlockReplayer, ConsensusContext, StateProcessingStrategy};
+use std::sync::Arc;
+use types::{ssz_tagged_signed_beacon_block, ssz_tagged_signed_beacon_block_arc};
+use types::{BeaconState, BlindedPayload, ChainSpec, Epoch, EthSpec, Hash256, SignedBeaconBlock};
+
+/// This mirrors everything in the `AvailabilityPendingExecutedBlock`, except
+/// that it is much smaller because it contains only a state root instead of
+/// a full `BeaconState`.
+#[derive(Encode, Decode, Clone)]
+pub struct DietAvailabilityPendingExecutedBlock<E: EthSpec> {
+    #[ssz(with = "ssz_tagged_signed_beacon_block_arc")]
+    block: Arc<SignedBeaconBlock<E>>,
+    state_root: Hash256,
+    #[ssz(with = "ssz_tagged_signed_beacon_block")]
+    parent_block: SignedBeaconBlock<E, BlindedPayload<E>>,
+    parent_eth1_finalization_data: Eth1FinalizationData,
+    confirmed_state_roots: Vec<Hash256>,
+    consensus_context: ConsensusContext<E>,
+    payload_verification_outcome: PayloadVerificationOutcome,
+}
+
+/// just implementing the same methods as `AvailabilityPendingExecutedBlock`
+impl<E: EthSpec> DietAvailabilityPendingExecutedBlock<E> {
+    pub fn as_block(&self) -> &SignedBeaconBlock<E> {
+        &self.block
+    }
+
+    pub fn num_blobs_expected(&self) -> usize {
+        self.block
+            .message()
+            .body()
+            .blob_kzg_commitments()
+            .map_or(0, |commitments| commitments.len())
+    }
+}
+
+/// This LRU cache holds BeaconStates used for block import. If the cache overflows,
+/// the least recently used state will be dropped. If the dropped state is needed
+/// later on, it will be recovered from the parent state and replaying the block.
+///
+/// WARNING: This cache assumes the parent block of any `AvailabilityPendingExecutedBlock`
+/// has already been imported into ForkChoice. If this is not the case, the cache
+/// will fail to recover the state when the cache overflows because it can't load
+/// the parent state!
+pub struct StateLRUCache<T: BeaconChainTypes> {
+    states: RwLock<LruCache<Hash256, BeaconState<T::EthSpec>>>,
+    store: BeaconStore<T>,
+    spec: ChainSpec,
+}
+
+impl<T: BeaconChainTypes> StateLRUCache<T> {
+    pub fn new(store: BeaconStore<T>, spec: ChainSpec) -> Self {
+        Self {
+            states: RwLock::new(LruCache::new(STATE_LRU_CAPACITY_NON_ZERO)),
+            store,
+            spec,
+        }
+    }
+
+    /// This will store the state in the LRU cache and return a
+    /// `DietAvailabilityPendingExecutedBlock` which is much cheaper to
+    /// keep around in memory.
+    pub fn register_pending_executed_block(
+        &self,
+        executed_block: AvailabilityPendingExecutedBlock<T::EthSpec>,
+    ) -> DietAvailabilityPendingExecutedBlock<T::EthSpec> {
+        let state = executed_block.import_data.state;
+        let state_root = executed_block.block.state_root();
+        self.states.write().put(state_root, state);
+
+        DietAvailabilityPendingExecutedBlock {
+            block: executed_block.block,
+            state_root,
+            parent_block: executed_block.import_data.parent_block,
+            parent_eth1_finalization_data: executed_block.import_data.parent_eth1_finalization_data,
+            confirmed_state_roots: executed_block.import_data.confirmed_state_roots,
+            consensus_context: executed_block.import_data.consensus_context,
+            payload_verification_outcome: executed_block.payload_verification_outcome,
+        }
+    }
+
+    /// Recover the `AvailabilityPendingExecutedBlock` from the diet version.
+    /// This method will first check the cache and if the state is not found
+    /// it will reconstruct the state by loading the parent state from disk and
+    /// replaying the block.
+    pub fn recover_pending_executed_block(
+        &self,
+        diet_executed_block: DietAvailabilityPendingExecutedBlock<T::EthSpec>,
+    ) -> Result<AvailabilityPendingExecutedBlock<T::EthSpec>, AvailabilityCheckError> {
+        let maybe_state = self.states.write().pop(&diet_executed_block.state_root);
+        if let Some(state) = maybe_state {
+            let block_root = diet_executed_block.block.canonical_root();
+            Ok(AvailabilityPendingExecutedBlock {
+                block: diet_executed_block.block,
+                import_data: BlockImportData {
+                    block_root,
+                    state,
+                    parent_block: diet_executed_block.parent_block,
+                    parent_eth1_finalization_data: diet_executed_block
+                        .parent_eth1_finalization_data,
+                    confirmed_state_roots: diet_executed_block.confirmed_state_roots,
+                    consensus_context: diet_executed_block.consensus_context,
+                },
+                payload_verification_outcome: diet_executed_block.payload_verification_outcome,
+            })
+        } else {
+            self.reconstruct_pending_executed_block(diet_executed_block)
+        }
+    }
+
+    /// Reconstruct the `AvailabilityPendingExecutedBlock` by loading the parent
+    /// state from disk and replaying the block. This function does NOT check the
+    /// LRU cache.
+    pub fn reconstruct_pending_executed_block(
+        &self,
+        diet_executed_block: DietAvailabilityPendingExecutedBlock<T::EthSpec>,
+    ) -> Result<AvailabilityPendingExecutedBlock<T::EthSpec>, AvailabilityCheckError> {
+        let block_root = diet_executed_block.block.canonical_root();
+        let state = self.reconstruct_state(&diet_executed_block)?;
+        Ok(AvailabilityPendingExecutedBlock {
+            block: diet_executed_block.block,
+            import_data: BlockImportData {
+                block_root,
+                state,
+                parent_block: diet_executed_block.parent_block,
+                parent_eth1_finalization_data: diet_executed_block.parent_eth1_finalization_data,
+                confirmed_state_roots: diet_executed_block.confirmed_state_roots,
+                consensus_context: diet_executed_block.consensus_context,
+            },
+            payload_verification_outcome: diet_executed_block.payload_verification_outcome,
+        })
+    }
+
+    /// Reconstruct the state by loading the parent state from disk and replaying
+    /// the block.
+    fn reconstruct_state(
+        &self,
+        diet_executed_block: &DietAvailabilityPendingExecutedBlock<T::EthSpec>,
+    ) -> Result<BeaconState<T::EthSpec>, AvailabilityCheckError> {
+        let parent_block_root = diet_executed_block.parent_block.canonical_root();
+        let parent_block_state_root = diet_executed_block.parent_block.state_root();
+        let (parent_state_root, parent_state) = self
+            .store
+            .get_advanced_hot_state(
+                parent_block_root,
+                diet_executed_block.parent_block.slot(),
+                parent_block_state_root,
+            )
+            .map_err(AvailabilityCheckError::StoreError)?
+            .ok_or(AvailabilityCheckError::ParentStateMissing(
+                parent_block_state_root,
+            ))?;
+
+        let state_roots = vec![
+            Ok((parent_state_root, diet_executed_block.parent_block.slot())),
+            Ok((
+                diet_executed_block.state_root,
+                diet_executed_block.block.slot(),
+            )),
+        ];
+
+        let block_replayer: BlockReplayer<'_, T::EthSpec, AvailabilityCheckError, _> =
+            BlockReplayer::new(parent_state, &self.spec)
+                .no_signature_verification()
+                .state_processing_strategy(StateProcessingStrategy::Accurate)
+                .state_root_iter(state_roots.into_iter())
+                .minimal_block_root_verification();
+
+        block_replayer
+            .apply_blocks(vec![diet_executed_block.block.clone_as_blinded()], None)
+            .map(|block_replayer| block_replayer.into_state())
+            .and_then(|mut state| {
+                state
+                    .build_exit_cache(&self.spec)
+                    .map_err(AvailabilityCheckError::RebuildingStateCaches)?;
+                state
+                    .update_tree_hash_cache()
+                    .map_err(AvailabilityCheckError::RebuildingStateCaches)?;
+                Ok(state)
+            })
+    }
+
+    /// returns the state cache for inspection
+    pub fn lru_cache(&self) -> &RwLock<LruCache<Hash256, BeaconState<T::EthSpec>>> {
+        &self.states
+    }
+
+    /// remove any states from the cache from before the given epoch
+    pub fn do_maintenance(&self, cutoff_epoch: Epoch) {
+        let mut write_lock = self.states.write();
+        while let Some((_, state)) = write_lock.peek_lru() {
+            if state.slot().epoch(T::EthSpec::slots_per_epoch()) < cutoff_epoch {
+                write_lock.pop_lru();
+            } else {
+                break;
+            }
+        }
+    }
+}
+
+/// This can only be used during testing. The intended way to
+/// obtain a `DietAvailabilityPendingExecutedBlock` is to call
+/// `register_pending_executed_block` on the `StateLRUCache`.
+#[cfg(test)]
+impl<E: EthSpec> From<AvailabilityPendingExecutedBlock<E>>
+    for DietAvailabilityPendingExecutedBlock<E>
+{
+    fn from(value: AvailabilityPendingExecutedBlock<E>) -> Self {
+        Self {
+            block: value.block,
+            state_root: value.import_data.state.canonical_root(),
+            parent_block: value.import_data.parent_block,
+            parent_eth1_finalization_data: value.import_data.parent_eth1_finalization_data,
+            confirmed_state_roots: value.import_data.confirmed_state_roots,
+            consensus_context: value.import_data.consensus_context,
+            payload_verification_outcome: value.payload_verification_outcome,
+        }
+    }
+}

beacon_node/beacon_chain/src/deneb_readiness.rs

@@ -0,0 +1,121 @@
+//! Provides tools for checking if a node is ready for the Deneb upgrade.
+
+use crate::{BeaconChain, BeaconChainTypes};
+use execution_layer::http::{
+    ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V3,
+};
+use serde::{Deserialize, Serialize};
+use std::fmt;
+use std::time::Duration;
+use types::*;
+
+/// The time before the Deneb fork when we will start issuing warnings about preparation.
+use super::merge_readiness::SECONDS_IN_A_WEEK;
+pub const DENEB_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2;
+pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300;
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(tag = "type")]
+pub enum DenebReadiness {
+    /// The execution engine is deneb-enabled (as far as we can tell)
+    Ready,
+    /// We are connected to an execution engine which doesn't support the V3 engine api methods
+    V3MethodsNotSupported { error: String },
+    /// The transition configuration with the EL failed, there might be a problem with
+    /// connectivity, authentication or a difference in configuration.
+    ExchangeCapabilitiesFailed { error: String },
+    /// The user has not configured an execution endpoint
+    NoExecutionEndpoint,
+}
+
+impl fmt::Display for DenebReadiness {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            DenebReadiness::Ready => {
+                write!(f, "This node appears ready for Deneb.")
+            }
+            DenebReadiness::ExchangeCapabilitiesFailed { error } => write!(
+                f,
+                "Could not exchange capabilities with the \
+                    execution endpoint: {}",
+                error
+            ),
+            DenebReadiness::NoExecutionEndpoint => write!(
+                f,
+                "The --execution-endpoint flag is not specified, this is a \
+                    requirement post-merge"
+            ),
+            DenebReadiness::V3MethodsNotSupported { error } => write!(
+                f,
+                "Execution endpoint does not support Deneb methods: {}",
+                error
+            ),
+        }
+    }
+}
+
+impl<T: BeaconChainTypes> BeaconChain<T> {
+    /// Returns `true` if deneb epoch is set and Deneb fork has occurred or will
+    /// occur within `DENEB_READINESS_PREPARATION_SECONDS`
+    pub fn is_time_to_prepare_for_deneb(&self, current_slot: Slot) -> bool {
+        if let Some(deneb_epoch) = self.spec.deneb_fork_epoch {
+            let deneb_slot = deneb_epoch.start_slot(T::EthSpec::slots_per_epoch());
+            let deneb_readiness_preparation_slots =
+                DENEB_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot;
+            // Return `true` if Deneb has happened or is within the preparation time.
+            current_slot + deneb_readiness_preparation_slots > deneb_slot
+        } else {
+            // The Deneb fork epoch has not been defined yet, no need to prepare.
+            false
+        }
+    }
+
+    /// Attempts to connect to the EL and confirm that it is ready for capella.
+    pub async fn check_deneb_readiness(&self) -> DenebReadiness {
+        if let Some(el) = self.execution_layer.as_ref() {
+            match el
+                .get_engine_capabilities(Some(Duration::from_secs(
+                    ENGINE_CAPABILITIES_REFRESH_INTERVAL,
+                )))
+                .await
+            {
+                Err(e) => {
+                    // The EL was either unreachable or responded with an error
+                    DenebReadiness::ExchangeCapabilitiesFailed {
+                        error: format!("{:?}", e),
+                    }
+                }
+                Ok(capabilities) => {
+                    let mut missing_methods = String::from("Required Methods Unsupported:");
+                    let mut all_good = true;
+                    if !capabilities.get_payload_v3 {
+                        missing_methods.push(' ');
+                        missing_methods.push_str(ENGINE_GET_PAYLOAD_V3);
+                        all_good = false;
+                    }
+                    if !capabilities.forkchoice_updated_v3 {
+                        missing_methods.push(' ');
+                        missing_methods.push_str(ENGINE_FORKCHOICE_UPDATED_V3);
+                        all_good = false;
+                    }
+                    if !capabilities.new_payload_v3 {
+                        missing_methods.push(' ');
+                        missing_methods.push_str(ENGINE_NEW_PAYLOAD_V3);
+                        all_good = false;
+                    }
+
+                    if all_good {
+                        DenebReadiness::Ready
+                    } else {
+                        DenebReadiness::V3MethodsNotSupported {
+                            error: missing_methods,
+                        }
+                    }
+                }
+            }
+        } else {
+            DenebReadiness::NoExecutionEndpoint
+        }
+    }
+}

beacon_node/beacon_chain/src/early_attester_cache.rs

@@ -1,9 +1,11 @@
+use crate::data_availability_checker::AvailableBlock;
 use crate::{
     attester_cache::{CommitteeLengths, Error},
     metrics,
 };
 use parking_lot::RwLock;
 use proto_array::Block as ProtoBlock;
+use std::sync::Arc;
 use types::*;
 
 pub struct CacheItem<E: EthSpec> {
@@ -18,7 +20,8 @@ pub struct CacheItem<E: EthSpec> {
     /*
      * Values used to make the block available.
      */
-    block: SignedBeaconBlock<E>,
+    block: Arc<SignedBeaconBlock<E>>,
+    blobs: Option<BlobSidecarList<E>>,
     proto_block: ProtoBlock,
 }
 
@@ -48,7 +51,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
     pub fn add_head_block(
         &self,
         beacon_block_root: Hash256,
-        block: SignedBeaconBlock<E>,
+        block: AvailableBlock<E>,
         proto_block: ProtoBlock,
         state: &BeaconState<E>,
         spec: &ChainSpec,
@@ -66,6 +69,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
             },
         };
 
+        let (_, block, blobs) = block.deconstruct();
         let item = CacheItem {
             epoch,
             committee_lengths,
@@ -73,6 +77,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
             source,
             target,
             block,
+            blobs,
             proto_block,
         };
 
@@ -85,7 +90,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
     ///
     /// - There is a cache `item` present.
     /// - If `request_slot` is in the same epoch as `item.epoch`.
-    /// - If `request_index` does not exceed `item.comittee_count`.
+    /// - If `request_index` does not exceed `item.committee_count`.
     pub fn try_attest(
         &self,
         request_slot: Slot,
@@ -93,9 +98,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
         spec: &ChainSpec,
     ) -> Result<Option<Attestation<E>>, Error> {
         let lock = self.item.read();
-        let item = if let Some(item) = lock.as_ref() {
-            item
-        } else {
+        let Some(item) = lock.as_ref() else {
             return Ok(None);
         };
 
@@ -104,6 +107,10 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
             return Ok(None);
         }
 
+        if request_slot < item.block.slot() {
+            return Ok(None);
+        }
+
         let committee_count = item
             .committee_lengths
             .get_committee_count_per_slot::<E>(spec)?;
@@ -142,7 +149,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
     }
 
     /// Returns the block, if `block_root` matches the cached item.
-    pub fn get_block(&self, block_root: Hash256) -> Option<SignedBeaconBlock<E>> {
+    pub fn get_block(&self, block_root: Hash256) -> Option<Arc<SignedBeaconBlock<E>>> {
         self.item
             .read()
             .as_ref()
@@ -150,6 +157,15 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
             .map(|item| item.block.clone())
     }
 
+    /// Returns the blobs, if `block_root` matches the cached item.
+    pub fn get_blobs(&self, block_root: Hash256) -> Option<BlobSidecarList<E>> {
+        self.item
+            .read()
+            .as_ref()
+            .filter(|item| item.beacon_block_root == block_root)
+            .and_then(|item| item.blobs.clone())
+    }
+
     /// Returns the proto-array block, if `block_root` matches the cached item.
     pub fn get_proto_block(&self, block_root: Hash256) -> Option<ProtoBlock> {
         self.item

beacon_node/beacon_chain/src/electra_readiness.rs

@@ -0,0 +1,123 @@
+//! Provides tools for checking if a node is ready for the Electra upgrade and following merge
+//! transition.
+
+use crate::{BeaconChain, BeaconChainTypes};
+use execution_layer::http::{
+    ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V3,
+};
+use serde::{Deserialize, Serialize};
+use std::fmt;
+use std::time::Duration;
+use types::*;
+
+/// The time before the Electra fork when we will start issuing warnings about preparation.
+use super::merge_readiness::SECONDS_IN_A_WEEK;
+pub const ELECTRA_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2;
+pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300;
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(tag = "type")]
+pub enum ElectraReadiness {
+    /// The execution engine is electra-enabled (as far as we can tell)
+    Ready,
+    /// We are connected to an execution engine which doesn't support the V3 engine api methods
+    V3MethodsNotSupported { error: String },
+    /// The transition configuration with the EL failed, there might be a problem with
+    /// connectivity, authentication or a difference in configuration.
+    ExchangeCapabilitiesFailed { error: String },
+    /// The user has not configured an execution endpoint
+    NoExecutionEndpoint,
+}
+
+impl fmt::Display for ElectraReadiness {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ElectraReadiness::Ready => {
+                write!(f, "This node appears ready for Electra.")
+            }
+            ElectraReadiness::ExchangeCapabilitiesFailed { error } => write!(
+                f,
+                "Could not exchange capabilities with the \
+                    execution endpoint: {}",
+                error
+            ),
+            ElectraReadiness::NoExecutionEndpoint => write!(
+                f,
+                "The --execution-endpoint flag is not specified, this is a \
+                    requirement post-merge"
+            ),
+            ElectraReadiness::V3MethodsNotSupported { error } => write!(
+                f,
+                "Execution endpoint does not support Electra methods: {}",
+                error
+            ),
+        }
+    }
+}
+
+impl<T: BeaconChainTypes> BeaconChain<T> {
+    /// Returns `true` if electra epoch is set and Electra fork has occurred or will
+    /// occur within `ELECTRA_READINESS_PREPARATION_SECONDS`
+    pub fn is_time_to_prepare_for_electra(&self, current_slot: Slot) -> bool {
+        if let Some(electra_epoch) = self.spec.electra_fork_epoch {
+            let electra_slot = electra_epoch.start_slot(T::EthSpec::slots_per_epoch());
+            let electra_readiness_preparation_slots =
+                ELECTRA_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot;
+            // Return `true` if Electra has happened or is within the preparation time.
+            current_slot + electra_readiness_preparation_slots > electra_slot
+        } else {
+            // The Electra fork epoch has not been defined yet, no need to prepare.
+            false
+        }
+    }
+
+    /// Attempts to connect to the EL and confirm that it is ready for electra.
+    pub async fn check_electra_readiness(&self) -> ElectraReadiness {
+        if let Some(el) = self.execution_layer.as_ref() {
+            match el
+                .get_engine_capabilities(Some(Duration::from_secs(
+                    ENGINE_CAPABILITIES_REFRESH_INTERVAL,
+                )))
+                .await
+            {
+                Err(e) => {
+                    // The EL was either unreachable or responded with an error
+                    ElectraReadiness::ExchangeCapabilitiesFailed {
+                        error: format!("{:?}", e),
+                    }
+                }
+                Ok(capabilities) => {
+                    // TODO(electra): Update in the event we get V4s.
+                    let mut missing_methods = String::from("Required Methods Unsupported:");
+                    let mut all_good = true;
+                    if !capabilities.get_payload_v3 {
+                        missing_methods.push(' ');
+                        missing_methods.push_str(ENGINE_GET_PAYLOAD_V3);
+                        all_good = false;
+                    }
+                    if !capabilities.forkchoice_updated_v3 {
+                        missing_methods.push(' ');
+                        missing_methods.push_str(ENGINE_FORKCHOICE_UPDATED_V3);
+                        all_good = false;
+                    }
+                    if !capabilities.new_payload_v3 {
+                        missing_methods.push(' ');
+                        missing_methods.push_str(ENGINE_NEW_PAYLOAD_V3);
+                        all_good = false;
+                    }
+
+                    if all_good {
+                        ElectraReadiness::Ready
+                    } else {
+                        ElectraReadiness::V3MethodsNotSupported {
+                            error: missing_methods,
+                        }
+                    }
+                }
+            }
+        } else {
+            ElectraReadiness::NoExecutionEndpoint
+        }
+    }
+}

beacon_node/beacon_chain/src/errors.rs

@@ -1,13 +1,18 @@
 use crate::attester_cache::Error as AttesterCacheError;
+use crate::beacon_block_streamer::Error as BlockStreamerError;
 use crate::beacon_chain::ForkChoiceError;
 use crate::beacon_fork_choice_store::Error as ForkChoiceStoreError;
+use crate::data_availability_checker::AvailabilityCheckError;
 use crate::eth1_chain::Error as Eth1ChainError;
 use crate::historical_blocks::HistoricalBlockError;
 use crate::migrate::PruningError;
 use crate::naive_aggregation_pool::Error as NaiveAggregationError;
 use crate::observed_aggregates::Error as ObservedAttestationsError;
 use crate::observed_attesters::Error as ObservedAttestersError;
+use crate::observed_blob_sidecars::Error as ObservedBlobSidecarsError;
 use crate::observed_block_producers::Error as ObservedBlockProducersError;
+use execution_layer::PayloadStatus;
+use fork_choice::ExecutionStatus;
 use futures::channel::mpsc::TrySendError;
 use operation_pool::OpPoolError;
 use safe_arith::ArithError;
@@ -15,15 +20,17 @@ use ssz_types::Error as SszTypesError;
 use state_processing::{
     block_signature_verifier::Error as BlockSignatureVerifierError,
     per_block_processing::errors::{
-        AttestationValidationError, AttesterSlashingValidationError, ExitValidationError,
-        ProposerSlashingValidationError, SyncCommitteeMessageValidationError,
+        AttestationValidationError, AttesterSlashingValidationError,
+        BlsExecutionChangeValidationError, ExitValidationError, ProposerSlashingValidationError,
+        SyncCommitteeMessageValidationError,
     },
     signature_sets::Error as SignatureSetError,
     state_advance::Error as StateAdvanceError,
-    BlockProcessingError, BlockReplayError, SlotProcessingError,
+    BlockProcessingError, BlockReplayError, EpochProcessingError, SlotProcessingError,
 };
 use std::time::Duration;
 use task_executor::ShutdownReason;
+use tokio::task::JoinError;
 use types::*;
 
 macro_rules! easy_from_to {
@@ -42,13 +49,13 @@ pub enum BeaconChainError {
     UnableToReadSlot,
     UnableToComputeTimeAtSlot,
     RevertedFinalizedEpoch {
-        previous_epoch: Epoch,
-        new_epoch: Epoch,
+        old: Checkpoint,
+        new: Checkpoint,
     },
     SlotClockDidNotStart,
     NoStateForSlot(Slot),
-    UnableToFindTargetRoot(Slot),
     BeaconStateError(BeaconStateError),
+    EpochCacheError(EpochCacheError),
     DBInconsistent(String),
     DBError(store::Error),
     ForkChoiceError(ForkChoiceError),
@@ -56,6 +63,7 @@ pub enum BeaconChainError {
     MissingBeaconBlock(Hash256),
     MissingBeaconState(Hash256),
     SlotProcessingError(SlotProcessingError),
+    EpochProcessingError(EpochProcessingError),
     StateAdvanceError(StateAdvanceError),
     UnableToAdvanceState(String),
     NoStateForAttestation {
@@ -67,6 +75,7 @@ pub enum BeaconChainError {
     ExitValidationError(ExitValidationError),
     ProposerSlashingValidationError(ProposerSlashingValidationError),
     AttesterSlashingValidationError(AttesterSlashingValidationError),
+    BlsExecutionChangeValidationError(BlsExecutionChangeValidationError),
     StateSkipTooLarge {
         start_slot: Slot,
         requested_slot: Slot,
@@ -82,13 +91,10 @@ pub enum BeaconChainError {
     ValidatorPubkeyCacheLockTimeout,
     SnapshotCacheLockTimeout,
     IncorrectStateForAttestation(RelativeEpochError),
-    InvalidValidatorPubkeyBytes(bls::Error),
     ValidatorPubkeyCacheIncomplete(usize),
     SignatureSetError(SignatureSetError),
     BlockSignatureVerifierError(state_processing::block_signature_verifier::Error),
     BlockReplayError(BlockReplayError),
-    DuplicateValidatorPublicKey,
-    ValidatorPubkeyCacheFileError(String),
     ValidatorIndexUnknown(usize),
     ValidatorPubkeyUnknown(PublicKeyBytes),
     OpPoolError(OpPoolError),
@@ -96,6 +102,7 @@ pub enum BeaconChainError {
     ObservedAttestationsError(ObservedAttestationsError),
     ObservedAttestersError(ObservedAttestersError),
     ObservedBlockProducersError(ObservedBlockProducersError),
+    ObservedBlobSidecarsError(ObservedBlobSidecarsError),
     AttesterCacheError(AttesterCacheError),
     PruningError(PruningError),
     ArithError(ArithError),
@@ -135,28 +142,101 @@ pub enum BeaconChainError {
         new_slot: Slot,
     },
     AltairForkDisabled,
+    BuilderMissing,
     ExecutionLayerMissing,
+    BlockVariantLacksExecutionPayload(Hash256),
+    ExecutionLayerErrorPayloadReconstruction(ExecutionBlockHash, Box<execution_layer::Error>),
+    EngineGetCapabilititesFailed(Box<execution_layer::Error>),
+    ExecutionLayerGetBlockByNumberFailed(Box<execution_layer::Error>),
+    ExecutionLayerGetBlockByHashFailed(Box<execution_layer::Error>),
+    BlockHashMissingFromExecutionLayer(ExecutionBlockHash),
+    InconsistentPayloadReconstructed {
+        slot: Slot,
+        exec_block_hash: ExecutionBlockHash,
+        canonical_transactions_root: Hash256,
+        reconstructed_transactions_root: Hash256,
+    },
+    BlockStreamerError(BlockStreamerError),
+    AddPayloadLogicError,
     ExecutionForkChoiceUpdateFailed(execution_layer::Error),
+    PrepareProposerFailed(BlockProcessingError),
+    ExecutionForkChoiceUpdateInvalid {
+        status: PayloadStatus,
+    },
+    BlockRewardError,
     BlockRewardSlotError,
     BlockRewardAttestationError,
     BlockRewardSyncError,
+    SyncCommitteeRewardsSyncError,
+    AttestationRewardsError,
     HeadMissingFromForkChoice(Hash256),
     FinalizedBlockMissingFromForkChoice(Hash256),
+    HeadBlockMissingFromForkChoice(Hash256),
+    InvalidFinalizedPayload {
+        finalized_root: Hash256,
+        execution_block_hash: ExecutionBlockHash,
+    },
     InvalidFinalizedPayloadShutdownError(TrySendError<ShutdownReason>),
+    JustifiedPayloadInvalid {
+        justified_root: Hash256,
+        execution_block_hash: Option<ExecutionBlockHash>,
+    },
+    ForkchoiceUpdate(execution_layer::Error),
+    FinalizedCheckpointMismatch {
+        head_state: Checkpoint,
+        fork_choice: Hash256,
+    },
+    InvalidSlot(Slot),
+    HeadBlockNotFullyVerified {
+        beacon_block_root: Hash256,
+        execution_status: ExecutionStatus,
+    },
+    CannotAttestToFinalizedBlock {
+        beacon_block_root: Hash256,
+    },
+    SyncContributionDataReferencesFinalizedBlock {
+        beacon_block_root: Hash256,
+    },
+    RuntimeShutdown,
+    TokioJoin(tokio::task::JoinError),
+    ProcessInvalidExecutionPayload(JoinError),
+    ForkChoiceSignalOutOfOrder {
+        current: Slot,
+        latest: Slot,
+    },
+    ForkchoiceUpdateParamsMissing,
+    HeadHasInvalidPayload {
+        block_root: Hash256,
+        execution_status: ExecutionStatus,
+    },
+    AttestationHeadNotInForkChoice(Hash256),
+    MissingPersistedForkChoice,
+    ShufflingCacheError(promise_cache::PromiseCacheError),
+    BlsToExecutionPriorToCapella,
+    BlsToExecutionConflictsWithPool,
+    InconsistentFork(InconsistentFork),
+    ProposerHeadForkChoiceError(fork_choice::Error<proto_array::Error>),
+    UnableToPublish,
+    AvailabilityCheckError(AvailabilityCheckError),
+    LightClientError(LightClientError),
+    UnsupportedFork,
 }
 
 easy_from_to!(SlotProcessingError, BeaconChainError);
+easy_from_to!(EpochProcessingError, BeaconChainError);
 easy_from_to!(AttestationValidationError, BeaconChainError);
 easy_from_to!(SyncCommitteeMessageValidationError, BeaconChainError);
 easy_from_to!(ExitValidationError, BeaconChainError);
 easy_from_to!(ProposerSlashingValidationError, BeaconChainError);
 easy_from_to!(AttesterSlashingValidationError, BeaconChainError);
+easy_from_to!(BlsExecutionChangeValidationError, BeaconChainError);
 easy_from_to!(SszTypesError, BeaconChainError);
 easy_from_to!(OpPoolError, BeaconChainError);
 easy_from_to!(NaiveAggregationError, BeaconChainError);
 easy_from_to!(ObservedAttestationsError, BeaconChainError);
 easy_from_to!(ObservedAttestersError, BeaconChainError);
 easy_from_to!(ObservedBlockProducersError, BeaconChainError);
+easy_from_to!(ObservedBlobSidecarsError, BeaconChainError);
 easy_from_to!(AttesterCacheError, BeaconChainError);
 easy_from_to!(BlockSignatureVerifierError, BeaconChainError);
 easy_from_to!(PruningError, BeaconChainError);
@@ -165,15 +245,20 @@ easy_from_to!(ForkChoiceStoreError, BeaconChainError);
 easy_from_to!(HistoricalBlockError, BeaconChainError);
 easy_from_to!(StateAdvanceError, BeaconChainError);
 easy_from_to!(BlockReplayError, BeaconChainError);
+easy_from_to!(InconsistentFork, BeaconChainError);
+easy_from_to!(AvailabilityCheckError, BeaconChainError);
+easy_from_to!(EpochCacheError, BeaconChainError);
+easy_from_to!(LightClientError, BeaconChainError);
 
 #[derive(Debug)]
 pub enum BlockProductionError {
-    UnableToGetHeadInfo(BeaconChainError),
     UnableToGetBlockRootFromState,
     UnableToReadSlot,
     UnableToProduceAtSlot(Slot),
     SlotProcessingError(SlotProcessingError),
     BlockProcessingError(BlockProcessingError),
+    EpochCacheError(EpochCacheError),
+    ForkChoiceError(ForkChoiceError),
     Eth1ChainError(Eth1ChainError),
     BeaconStateError(BeaconStateError),
     StateAdvanceError(StateAdvanceError),
@@ -190,8 +275,21 @@ pub enum BlockProductionError {
     TerminalPoWBlockLookupFailed(execution_layer::Error),
     GetPayloadFailed(execution_layer::Error),
     FailedToReadFinalizedBlock(store::Error),
+    FailedToLoadState(store::Error),
     MissingFinalizedBlock(Hash256),
     BlockTooLarge(usize),
+    ShuttingDown,
+    MissingBlobs,
+    MissingSyncAggregate,
+    MissingExecutionPayload,
+    MissingKzgCommitment(String),
+    TokioJoin(JoinError),
+    BeaconChain(BeaconChainError),
+    InvalidPayloadFork,
+    TrustedSetupNotInitialized,
+    InvalidBlockVariant(String),
+    KzgError(kzg::Error),
+    FailedToBuildBlobSidecars(String),
 }
 
 easy_from_to!(BlockProcessingError, BlockProductionError);
@@ -199,3 +297,5 @@ easy_from_to!(BeaconStateError, BlockProductionError);
 easy_from_to!(SlotProcessingError, BlockProductionError);
 easy_from_to!(Eth1ChainError, BlockProductionError);
 easy_from_to!(StateAdvanceError, BlockProductionError);
+easy_from_to!(ForkChoiceError, BlockProductionError);
+easy_from_to!(EpochCacheError, BlockProductionError);

beacon_node/beacon_chain/src/eth1_chain.rs

@@ -1,7 +1,7 @@
 use crate::metrics;
 use eth1::{Config as Eth1Config, Eth1Block, Service as HttpService};
 use eth2::lighthouse::Eth1SyncStatusData;
-use eth2_hashing::hash;
+use ethereum_hashing::hash;
 use int_to_bytes::int_to_bytes32;
 use slog::{debug, error, trace, Logger};
 use ssz::{Decode, Encode};
@@ -9,14 +9,12 @@ use ssz_derive::{Decode, Encode};
 use state_processing::per_block_processing::get_new_eth1_data;
 use std::cmp::Ordering;
 use std::collections::HashMap;
-use std::iter::DoubleEndedIterator;
 use std::marker::PhantomData;
 use std::time::{SystemTime, UNIX_EPOCH};
 use store::{DBColumn, Error as StoreError, StoreItem};
 use task_executor::TaskExecutor;
 use types::{
     BeaconState, BeaconStateError, ChainSpec, Deposit, Eth1Data, EthSpec, Hash256, Slot, Unsigned,
-    DEPOSIT_TREE_DEPTH,
 };
 
 type BlockNumber = u64;
@@ -68,7 +66,7 @@ impl From<safe_arith::ArithError> for Error {
 /// - `genesis_time`: beacon chain genesis time.
 /// - `current_slot`: current beacon chain slot.
 /// - `spec`: current beacon chain specification.
-fn get_sync_status<T: EthSpec>(
+fn get_sync_status<E: EthSpec>(
     latest_cached_block: Option<&Eth1Block>,
     head_block: Option<&Eth1Block>,
     genesis_time: u64,
@@ -86,10 +84,10 @@ fn get_sync_status<T: EthSpec>(
     // that are *before* genesis, so that we can indicate to users that we're actually adequately
     // cached for where they are in time.
     let voting_target_timestamp = if let Some(current_slot) = current_slot {
-        let period = T::SlotsPerEth1VotingPeriod::to_u64();
+        let period = E::SlotsPerEth1VotingPeriod::to_u64();
         let voting_period_start_slot = (current_slot / period) * period;
 
-        let period_start = slot_start_seconds::<T>(
+        let period_start = slot_start_seconds(
             genesis_time,
             spec.seconds_per_slot,
             voting_period_start_slot,
@@ -99,7 +97,7 @@ fn get_sync_status<T: EthSpec>(
     } else {
         // The number of seconds in an eth1 voting period.
         let voting_period_duration =
-            T::slots_per_eth1_voting_period() as u64 * spec.seconds_per_slot;
+            E::slots_per_eth1_voting_period() as u64 * spec.seconds_per_slot;
 
         let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs();
 
@@ -170,8 +168,8 @@ fn get_sync_status<T: EthSpec>(
 
 #[derive(Encode, Decode, Clone)]
 pub struct SszEth1 {
-    use_dummy_backend: bool,
-    backend_bytes: Vec<u8>,
+    pub use_dummy_backend: bool,
+    pub backend_bytes: Vec<u8>,
 }
 
 impl StoreItem for SszEth1 {
@@ -305,16 +303,22 @@ where
         }
     }
 
+    /// Set in motion the finalization of `Eth1Data`. This method is called during block import
+    /// so it should be fast.
+    pub fn finalize_eth1_data(&self, eth1_data: Eth1Data) {
+        self.backend.finalize_eth1_data(eth1_data);
+    }
+
     /// Consumes `self`, returning the backend.
     pub fn into_backend(self) -> T {
         self.backend
     }
 }
 
-pub trait Eth1ChainBackend<T: EthSpec>: Sized + Send + Sync {
+pub trait Eth1ChainBackend<E: EthSpec>: Sized + Send + Sync {
     /// Returns the `Eth1Data` that should be included in a block being produced for the given
     /// `state`.
-    fn eth1_data(&self, beacon_state: &BeaconState<T>, spec: &ChainSpec)
+    fn eth1_data(&self, beacon_state: &BeaconState<E>, spec: &ChainSpec)
         -> Result<Eth1Data, Error>;
 
     /// Returns all `Deposits` between `state.eth1_deposit_index` and
@@ -326,7 +330,7 @@ pub trait Eth1ChainBackend<T: EthSpec>: Sized + Send + Sync {
     /// be more than `MAX_DEPOSIT_COUNT` or the churn may be too high.
     fn queued_deposits(
         &self,
-        beacon_state: &BeaconState<T>,
+        beacon_state: &BeaconState<E>,
         eth1_data_vote: &Eth1Data,
         spec: &ChainSpec,
     ) -> Result<Vec<Deposit>, Error>;
@@ -335,6 +339,10 @@ pub trait Eth1ChainBackend<T: EthSpec>: Sized + Send + Sync {
     /// beacon node eth1 cache is.
     fn latest_cached_block(&self) -> Option<Eth1Block>;
 
+    /// Set in motion the finalization of `Eth1Data`. This method is called during block import
+    /// so it should be fast.
+    fn finalize_eth1_data(&self, eth1_data: Eth1Data);
+
     /// Returns the block at the head of the chain (ignoring follow distance, etc). Used to obtain
     /// an idea of how up-to-date the remote eth1 node is.
     fn head_block(&self) -> Option<Eth1Block>;
@@ -356,13 +364,13 @@ pub trait Eth1ChainBackend<T: EthSpec>: Sized + Send + Sync {
 /// Never creates deposits, therefore the validator set is static.
 ///
 /// This was used in the 2019 Canada interop workshops.
-pub struct DummyEth1ChainBackend<T: EthSpec>(PhantomData<T>);
+pub struct DummyEth1ChainBackend<E: EthSpec>(PhantomData<E>);
 
-impl<T: EthSpec> Eth1ChainBackend<T> for DummyEth1ChainBackend<T> {
+impl<E: EthSpec> Eth1ChainBackend<E> for DummyEth1ChainBackend<E> {
     /// Produce some deterministic junk based upon the current epoch.
-    fn eth1_data(&self, state: &BeaconState<T>, _spec: &ChainSpec) -> Result<Eth1Data, Error> {
+    fn eth1_data(&self, state: &BeaconState<E>, _spec: &ChainSpec) -> Result<Eth1Data, Error> {
         let current_epoch = state.current_epoch();
-        let slots_per_voting_period = T::slots_per_eth1_voting_period() as u64;
+        let slots_per_voting_period = E::slots_per_eth1_voting_period() as u64;
         let current_voting_period: u64 = current_epoch.as_u64() / slots_per_voting_period;
 
         let deposit_root = hash(&int_to_bytes32(current_voting_period));
@@ -378,7 +386,7 @@ impl<T: EthSpec> Eth1ChainBackend<T> for DummyEth1ChainBackend<T> {
     /// The dummy back-end never produces deposits.
     fn queued_deposits(
         &self,
-        _: &BeaconState<T>,
+        _: &BeaconState<E>,
         _: &Eth1Data,
         _: &ChainSpec,
     ) -> Result<Vec<Deposit>, Error> {
@@ -389,6 +397,8 @@ impl<T: EthSpec> Eth1ChainBackend<T> for DummyEth1ChainBackend<T> {
         None
     }
 
+    fn finalize_eth1_data(&self, _eth1_data: Eth1Data) {}
+
     fn head_block(&self) -> Option<Eth1Block> {
         None
     }
@@ -409,7 +419,7 @@ impl<T: EthSpec> Eth1ChainBackend<T> for DummyEth1ChainBackend<T> {
     }
 }
 
-impl<T: EthSpec> Default for DummyEth1ChainBackend<T> {
+impl<E: EthSpec> Default for DummyEth1ChainBackend<E> {
     fn default() -> Self {
         Self(PhantomData)
     }
@@ -421,22 +431,23 @@ impl<T: EthSpec> Default for DummyEth1ChainBackend<T> {
 /// The `core` connects to some external eth1 client (e.g., Parity/Geth) and polls it for
 /// information.
 #[derive(Clone)]
-pub struct CachingEth1Backend<T: EthSpec> {
+pub struct CachingEth1Backend<E: EthSpec> {
     pub core: HttpService,
     log: Logger,
-    _phantom: PhantomData<T>,
+    _phantom: PhantomData<E>,
 }
 
-impl<T: EthSpec> CachingEth1Backend<T> {
+impl<E: EthSpec> CachingEth1Backend<E> {
     /// Instantiates `self` with empty caches.
     ///
     /// Does not connect to the eth1 node or start any tasks to keep the cache updated.
-    pub fn new(config: Eth1Config, log: Logger, spec: ChainSpec) -> Self {
-        Self {
-            core: HttpService::new(config, log.clone(), spec),
+    pub fn new(config: Eth1Config, log: Logger, spec: ChainSpec) -> Result<Self, String> {
+        Ok(Self {
+            core: HttpService::new(config, log.clone(), spec)
+                .map_err(|e| format!("Failed to create eth1 http service: {:?}", e))?,
             log,
             _phantom: PhantomData,
-        }
+        })
     }
 
     /// Starts the routine which connects to the external eth1 node and updates the caches.
@@ -454,11 +465,11 @@ impl<T: EthSpec> CachingEth1Backend<T> {
     }
 }
 
-impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
-    fn eth1_data(&self, state: &BeaconState<T>, spec: &ChainSpec) -> Result<Eth1Data, Error> {
-        let period = T::SlotsPerEth1VotingPeriod::to_u64();
+impl<E: EthSpec> Eth1ChainBackend<E> for CachingEth1Backend<E> {
+    fn eth1_data(&self, state: &BeaconState<E>, spec: &ChainSpec) -> Result<Eth1Data, Error> {
+        let period = E::SlotsPerEth1VotingPeriod::to_u64();
         let voting_period_start_slot = (state.slot() / period) * period;
-        let voting_period_start_seconds = slot_start_seconds::<T>(
+        let voting_period_start_seconds = slot_start_seconds(
             state.genesis_time(),
             spec.seconds_per_slot,
             voting_period_start_slot,
@@ -524,7 +535,7 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
 
     fn queued_deposits(
         &self,
-        state: &BeaconState<T>,
+        state: &BeaconState<E>,
         eth1_data_vote: &Eth1Data,
         _spec: &ChainSpec,
     ) -> Result<Vec<Deposit>, Error> {
@@ -540,13 +551,13 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
             Ordering::Equal => Ok(vec![]),
             Ordering::Less => {
                 let next = deposit_index;
-                let last = std::cmp::min(deposit_count, next + T::MaxDeposits::to_u64());
+                let last = std::cmp::min(deposit_count, next + E::MaxDeposits::to_u64());
 
                 self.core
                     .deposits()
                     .read()
                     .cache
-                    .get_deposits(next, last, deposit_count, DEPOSIT_TREE_DEPTH)
+                    .get_deposits(next, last, deposit_count)
                     .map_err(|e| Error::BackendError(format!("Failed to get deposits: {:?}", e)))
                     .map(|(_deposit_root, deposits)| deposits)
             }
@@ -557,6 +568,12 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
         self.core.latest_cached_block()
     }
 
+    /// This only writes the eth1_data to a temporary cache so that the service
+    /// thread can later do the actual finalizing of the deposit tree.
+    fn finalize_eth1_data(&self, eth1_data: Eth1Data) {
+        self.core.set_to_finalize(Some(eth1_data));
+    }
+
     fn head_block(&self) -> Option<Eth1Block> {
         self.core.head_block()
     }
@@ -609,8 +626,8 @@ where
 
 /// Collect all valid votes that are cast during the current voting period.
 /// Return hashmap with count of each vote cast.
-fn collect_valid_votes<T: EthSpec>(
-    state: &BeaconState<T>,
+fn collect_valid_votes<E: EthSpec>(
+    state: &BeaconState<E>,
     votes_to_consider: &HashMap<Eth1Data, BlockNumber>,
 ) -> Eth1DataVoteCount {
     let mut valid_votes = HashMap::new();
@@ -640,11 +657,7 @@ fn find_winning_vote(valid_votes: Eth1DataVoteCount) -> Option<Eth1Data> {
 }
 
 /// Returns the unix-epoch seconds at the start of the given `slot`.
-fn slot_start_seconds<T: EthSpec>(
-    genesis_unix_seconds: u64,
-    seconds_per_slot: u64,
-    slot: Slot,
-) -> u64 {
+fn slot_start_seconds(genesis_unix_seconds: u64, seconds_per_slot: u64, slot: Slot) -> u64 {
     genesis_unix_seconds + slot.as_u64() * seconds_per_slot
 }
 
@@ -680,7 +693,7 @@ mod test {
     fn get_voting_period_start_seconds(state: &BeaconState<E>, spec: &ChainSpec) -> u64 {
         let period = <E as EthSpec>::SlotsPerEth1VotingPeriod::to_u64();
         let voting_period_start_slot = (state.slot() / period) * period;
-        slot_start_seconds::<E>(
+        slot_start_seconds(
             state.genesis_time(),
             spec.seconds_per_slot,
             voting_period_start_slot,
@@ -690,23 +703,23 @@ mod test {
     #[test]
     fn slot_start_time() {
         let zero_sec = 0;
-        assert_eq!(slot_start_seconds::<E>(100, zero_sec, Slot::new(2)), 100);
+        assert_eq!(slot_start_seconds(100, zero_sec, Slot::new(2)), 100);
 
         let one_sec = 1;
-        assert_eq!(slot_start_seconds::<E>(100, one_sec, Slot::new(0)), 100);
-        assert_eq!(slot_start_seconds::<E>(100, one_sec, Slot::new(1)), 101);
-        assert_eq!(slot_start_seconds::<E>(100, one_sec, Slot::new(2)), 102);
+        assert_eq!(slot_start_seconds(100, one_sec, Slot::new(0)), 100);
+        assert_eq!(slot_start_seconds(100, one_sec, Slot::new(1)), 101);
+        assert_eq!(slot_start_seconds(100, one_sec, Slot::new(2)), 102);
 
         let three_sec = 3;
-        assert_eq!(slot_start_seconds::<E>(100, three_sec, Slot::new(0)), 100);
-        assert_eq!(slot_start_seconds::<E>(100, three_sec, Slot::new(1)), 103);
-        assert_eq!(slot_start_seconds::<E>(100, three_sec, Slot::new(2)), 106);
+        assert_eq!(slot_start_seconds(100, three_sec, Slot::new(0)), 100);
+        assert_eq!(slot_start_seconds(100, three_sec, Slot::new(1)), 103);
+        assert_eq!(slot_start_seconds(100, three_sec, Slot::new(2)), 106);
 
         let five_sec = 5;
-        assert_eq!(slot_start_seconds::<E>(100, five_sec, Slot::new(0)), 100);
-        assert_eq!(slot_start_seconds::<E>(100, five_sec, Slot::new(1)), 105);
-        assert_eq!(slot_start_seconds::<E>(100, five_sec, Slot::new(2)), 110);
-        assert_eq!(slot_start_seconds::<E>(100, five_sec, Slot::new(3)), 115);
+        assert_eq!(slot_start_seconds(100, five_sec, Slot::new(0)), 100);
+        assert_eq!(slot_start_seconds(100, five_sec, Slot::new(1)), 105);
+        assert_eq!(slot_start_seconds(100, five_sec, Slot::new(2)), 110);
+        assert_eq!(slot_start_seconds(100, five_sec, Slot::new(3)), 115);
     }
 
     fn get_eth1_block(timestamp: u64, number: u64) -> Eth1Block {
@@ -722,7 +735,7 @@ mod test {
     mod eth1_chain_json_backend {
         use super::*;
         use eth1::DepositLog;
-        use types::{test_utils::generate_deterministic_keypair, EthSpec, MainnetEthSpec};
+        use types::{test_utils::generate_deterministic_keypair, MainnetEthSpec};
 
         fn get_eth1_chain() -> Eth1Chain<CachingEth1Backend<E>, E> {
             let eth1_config = Eth1Config {
@@ -730,11 +743,9 @@ mod test {
             };
 
             let log = null_logger().unwrap();
-            Eth1Chain::new(CachingEth1Backend::new(
-                eth1_config,
-                log,
-                MainnetEthSpec::default_spec(),
-            ))
+            Eth1Chain::new(
+                CachingEth1Backend::new(eth1_config, log, MainnetEthSpec::default_spec()).unwrap(),
+            )
         }
 
         fn get_deposit_log(i: u64, spec: &ChainSpec) -> DepositLog {
@@ -955,7 +966,7 @@ mod test {
             let spec = &E::default_spec();
             let state: BeaconState<E> = BeaconState::new(0, get_eth1_data(0), spec);
 
-            let blocks = vec![];
+            let blocks = [];
 
             assert_eq!(
                 get_votes_to_consider(
@@ -1009,6 +1020,7 @@ mod test {
 
     mod collect_valid_votes {
         use super::*;
+        use types::List;
 
         fn get_eth1_data_vec(n: u64, block_number_offset: u64) -> Vec<(Eth1Data, BlockNumber)> {
             (0..n)
@@ -1056,12 +1068,14 @@ mod test {
 
             let votes_to_consider = get_eth1_data_vec(slots, 0);
 
-            *state.eth1_data_votes_mut() = votes_to_consider[0..slots as usize / 4]
-                .iter()
-                .map(|(eth1_data, _)| eth1_data)
-                .cloned()
-                .collect::<Vec<_>>()
-                .into();
+            *state.eth1_data_votes_mut() = List::new(
+                votes_to_consider[0..slots as usize / 4]
+                    .iter()
+                    .map(|(eth1_data, _)| eth1_data)
+                    .cloned()
+                    .collect::<Vec<_>>(),
+            )
+            .unwrap();
 
             let votes =
                 collect_valid_votes(&state, &votes_to_consider.clone().into_iter().collect());
@@ -1085,12 +1099,14 @@ mod test {
                 .expect("should have some eth1 data")
                 .clone();
 
-            *state.eth1_data_votes_mut() = vec![duplicate_eth1_data.clone(); 4]
-                .iter()
-                .map(|(eth1_data, _)| eth1_data)
-                .cloned()
-                .collect::<Vec<_>>()
-                .into();
+            *state.eth1_data_votes_mut() = List::new(
+                vec![duplicate_eth1_data.clone(); 4]
+                    .iter()
+                    .map(|(eth1_data, _)| eth1_data)
+                    .cloned()
+                    .collect::<Vec<_>>(),
+            )
+            .unwrap();
 
             let votes = collect_valid_votes(&state, &votes_to_consider.into_iter().collect());
             assert_votes!(

beacon_node/beacon_chain/src/eth1_finalization_cache.rs

@@ -0,0 +1,499 @@
+use slog::{debug, Logger};
+use ssz_derive::{Decode, Encode};
+use std::cmp;
+use std::collections::BTreeMap;
+use types::{Checkpoint, Epoch, Eth1Data, Hash256 as Root};
+
+/// The default size of the cache.
+/// The beacon chain only looks at the last 4 epochs for finalization.
+/// Add 1 for current epoch and 4 earlier epochs.
+pub const DEFAULT_ETH1_CACHE_SIZE: usize = 5;
+
+/// These fields are named the same as the corresponding fields in the `BeaconState`
+/// as this structure stores these values from the `BeaconState` at a `Checkpoint`
+#[derive(Clone, Debug, PartialEq, Encode, Decode)]
+pub struct Eth1FinalizationData {
+    pub eth1_data: Eth1Data,
+    pub eth1_deposit_index: u64,
+}
+
+impl Eth1FinalizationData {
+    /// Ensures the deposit finalization conditions have been met. See:
+    /// https://eips.ethereum.org/EIPS/eip-4881#deposit-finalization-conditions
+    fn fully_imported(&self) -> bool {
+        self.eth1_deposit_index >= self.eth1_data.deposit_count
+    }
+}
+
+/// Implements map from Checkpoint -> Eth1CacheData
+pub struct CheckpointMap {
+    capacity: usize,
+    // There shouldn't be more than a couple of potential checkpoints at the same
+    // epoch. Searching through a vector for the matching Root should be faster
+    // than using another map from Root->Eth1CacheData
+    store: BTreeMap<Epoch, Vec<(Root, Eth1FinalizationData)>>,
+}
+
+impl Default for CheckpointMap {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Provides a map of `Eth1CacheData` referenced by `Checkpoint`
+///
+/// ## Cache Queuing
+///
+/// The cache keeps a maximum number of (`capacity`) epochs. Because there may be
+/// forks at the epoch boundary, it's possible that there exists more than one
+/// `Checkpoint` for the same `Epoch`. This cache will store all checkpoints for
+/// a given `Epoch`. When adding data for a new `Checkpoint` would cause the number
+/// of `Epoch`s stored to exceed `capacity`, the data for oldest `Epoch` is dropped
+impl CheckpointMap {
+    pub fn new() -> Self {
+        CheckpointMap {
+            capacity: DEFAULT_ETH1_CACHE_SIZE,
+            store: BTreeMap::new(),
+        }
+    }
+
+    pub fn with_capacity(capacity: usize) -> Self {
+        CheckpointMap {
+            capacity: cmp::max(1, capacity),
+            store: BTreeMap::new(),
+        }
+    }
+
+    pub fn insert(&mut self, checkpoint: Checkpoint, eth1_finalization_data: Eth1FinalizationData) {
+        self.store
+            .entry(checkpoint.epoch)
+            .or_default()
+            .push((checkpoint.root, eth1_finalization_data));
+
+        // faster to reduce size after the fact than do pre-checking to see
+        // if the current data would increase the size of the BTreeMap
+        while self.store.len() > self.capacity {
+            let oldest_stored_epoch = self.store.keys().next().cloned().unwrap();
+            self.store.remove(&oldest_stored_epoch);
+        }
+    }
+
+    pub fn get(&self, checkpoint: &Checkpoint) -> Option<&Eth1FinalizationData> {
+        match self.store.get(&checkpoint.epoch) {
+            Some(vec) => {
+                for (root, data) in vec {
+                    if *root == checkpoint.root {
+                        return Some(data);
+                    }
+                }
+                None
+            }
+            None => None,
+        }
+    }
+
+    #[cfg(test)]
+    pub fn len(&self) -> usize {
+        self.store.len()
+    }
+}
+
+/// This cache stores `Eth1CacheData` that could potentially be finalized within 4
+/// future epochs.
+pub struct Eth1FinalizationCache {
+    by_checkpoint: CheckpointMap,
+    pending_eth1: BTreeMap<u64, Eth1Data>,
+    last_finalized: Option<Eth1Data>,
+    log: Logger,
+}
+
+/// Provides a cache of `Eth1CacheData` at epoch boundaries. This is used to
+/// finalize deposits when a new epoch is finalized.
+///
+impl Eth1FinalizationCache {
+    pub fn new(log: Logger) -> Self {
+        Eth1FinalizationCache {
+            by_checkpoint: CheckpointMap::new(),
+            pending_eth1: BTreeMap::new(),
+            last_finalized: None,
+            log,
+        }
+    }
+
+    pub fn with_capacity(log: Logger, capacity: usize) -> Self {
+        Eth1FinalizationCache {
+            by_checkpoint: CheckpointMap::with_capacity(capacity),
+            pending_eth1: BTreeMap::new(),
+            last_finalized: None,
+            log,
+        }
+    }
+
+    pub fn insert(&mut self, checkpoint: Checkpoint, eth1_finalization_data: Eth1FinalizationData) {
+        if !eth1_finalization_data.fully_imported() {
+            self.pending_eth1.insert(
+                eth1_finalization_data.eth1_data.deposit_count,
+                eth1_finalization_data.eth1_data.clone(),
+            );
+            debug!(
+                self.log,
+                "Eth1Cache: inserted pending eth1";
+                "eth1_data.deposit_count" => eth1_finalization_data.eth1_data.deposit_count,
+                "eth1_deposit_index" => eth1_finalization_data.eth1_deposit_index,
+            );
+        }
+        self.by_checkpoint
+            .insert(checkpoint, eth1_finalization_data);
+    }
+
+    pub fn finalize(&mut self, checkpoint: &Checkpoint) -> Option<Eth1Data> {
+        if let Some(eth1_finalized_data) = self.by_checkpoint.get(checkpoint) {
+            let finalized_deposit_index = eth1_finalized_data.eth1_deposit_index;
+            let mut result = None;
+            while let Some(pending_count) = self.pending_eth1.keys().next().cloned() {
+                if finalized_deposit_index >= pending_count {
+                    result = self.pending_eth1.remove(&pending_count);
+                    debug!(
+                        self.log,
+                        "Eth1Cache: dropped pending eth1";
+                        "pending_count" => pending_count,
+                        "finalized_deposit_index" => finalized_deposit_index,
+                    );
+                } else {
+                    break;
+                }
+            }
+            if eth1_finalized_data.fully_imported() {
+                result = Some(eth1_finalized_data.eth1_data.clone())
+            }
+            if result.is_some() {
+                self.last_finalized = result;
+            }
+            self.last_finalized.clone()
+        } else {
+            debug!(
+                self.log,
+                "Eth1Cache: cache miss";
+                "epoch" => checkpoint.epoch,
+            );
+            None
+        }
+    }
+
+    #[cfg(test)]
+    pub fn by_checkpoint(&self) -> &CheckpointMap {
+        &self.by_checkpoint
+    }
+
+    #[cfg(test)]
+    pub fn pending_eth1(&self) -> &BTreeMap<u64, Eth1Data> {
+        &self.pending_eth1
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+    use sloggers::null::NullLoggerBuilder;
+    use sloggers::Build;
+    use std::collections::HashMap;
+
+    const SLOTS_PER_EPOCH: u64 = 32;
+    const MAX_DEPOSITS: u64 = 16;
+    const EPOCHS_PER_ETH1_VOTING_PERIOD: u64 = 64;
+
+    fn eth1cache() -> Eth1FinalizationCache {
+        let log_builder = NullLoggerBuilder;
+        Eth1FinalizationCache::new(log_builder.build().expect("should build log"))
+    }
+
+    fn random_eth1_data(deposit_count: u64) -> Eth1Data {
+        Eth1Data {
+            deposit_root: Root::random(),
+            deposit_count,
+            block_hash: Root::random(),
+        }
+    }
+
+    fn random_checkpoint(epoch: u64) -> Checkpoint {
+        Checkpoint {
+            epoch: epoch.into(),
+            root: Root::random(),
+        }
+    }
+
+    fn random_checkpoints(n: usize) -> Vec<Checkpoint> {
+        let mut result = Vec::with_capacity(n);
+        for epoch in 0..n {
+            result.push(random_checkpoint(epoch as u64))
+        }
+        result
+    }
+
+    #[test]
+    fn fully_imported_deposits() {
+        let epochs = 16;
+        let deposits_imported = 128;
+
+        let eth1data = random_eth1_data(deposits_imported);
+        let checkpoints = random_checkpoints(epochs as usize);
+        let mut eth1cache = eth1cache();
+
+        for epoch in 4..epochs {
+            assert_eq!(
+                eth1cache.by_checkpoint().len(),
+                cmp::min((epoch - 4) as usize, DEFAULT_ETH1_CACHE_SIZE),
+                "Unexpected cache size"
+            );
+
+            let checkpoint = checkpoints
+                .get(epoch as usize)
+                .expect("should get checkpoint");
+            eth1cache.insert(
+                *checkpoint,
+                Eth1FinalizationData {
+                    eth1_data: eth1data.clone(),
+                    eth1_deposit_index: deposits_imported,
+                },
+            );
+
+            let finalized_checkpoint = checkpoints
+                .get((epoch - 4) as usize)
+                .expect("should get finalized checkpoint");
+            assert!(
+                eth1cache.pending_eth1().is_empty(),
+                "Deposits are fully imported so pending cache should be empty"
+            );
+            if epoch < 8 {
+                assert_eq!(
+                    eth1cache.finalize(finalized_checkpoint),
+                    None,
+                    "Should have cache miss"
+                );
+            } else {
+                assert_eq!(
+                    eth1cache.finalize(finalized_checkpoint),
+                    Some(eth1data.clone()),
+                    "Should have cache hit"
+                )
+            }
+        }
+    }
+
+    #[test]
+    fn partially_imported_deposits() {
+        let epochs = 16;
+        let initial_deposits_imported = 1024;
+        let deposits_imported_per_epoch = MAX_DEPOSITS * SLOTS_PER_EPOCH;
+        let full_import_epoch = 13;
+        let total_deposits =
+            initial_deposits_imported + deposits_imported_per_epoch * full_import_epoch;
+
+        let eth1data = random_eth1_data(total_deposits);
+        let checkpoints = random_checkpoints(epochs as usize);
+        let mut eth1cache = eth1cache();
+
+        for epoch in 0..epochs {
+            assert_eq!(
+                eth1cache.by_checkpoint().len(),
+                cmp::min(epoch as usize, DEFAULT_ETH1_CACHE_SIZE),
+                "Unexpected cache size"
+            );
+
+            let checkpoint = checkpoints
+                .get(epoch as usize)
+                .expect("should get checkpoint");
+            let deposits_imported = cmp::min(
+                total_deposits,
+                initial_deposits_imported + deposits_imported_per_epoch * epoch,
+            );
+            eth1cache.insert(
+                *checkpoint,
+                Eth1FinalizationData {
+                    eth1_data: eth1data.clone(),
+                    eth1_deposit_index: deposits_imported,
+                },
+            );
+
+            if epoch >= 4 {
+                let finalized_epoch = epoch - 4;
+                let finalized_checkpoint = checkpoints
+                    .get(finalized_epoch as usize)
+                    .expect("should get finalized checkpoint");
+                if finalized_epoch < full_import_epoch {
+                    assert_eq!(
+                        eth1cache.finalize(finalized_checkpoint),
+                        None,
+                        "Deposits not fully finalized so cache should return no Eth1Data",
+                    );
+                    assert_eq!(
+                        eth1cache.pending_eth1().len(),
+                        1,
+                        "Deposits not fully finalized. Pending eth1 cache should have 1 entry"
+                    );
+                } else {
+                    assert_eq!(
+                        eth1cache.finalize(finalized_checkpoint),
+                        Some(eth1data.clone()),
+                        "Deposits fully imported and finalized. Cache should return Eth1Data. finalized_deposits[{}]",
+                        (initial_deposits_imported + deposits_imported_per_epoch * finalized_epoch),
+                    );
+                    assert!(
+                        eth1cache.pending_eth1().is_empty(),
+                        "Deposits fully imported and finalized. Pending cache should be empty"
+                    );
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn fork_at_epoch_boundary() {
+        let epochs = 12;
+        let deposits_imported = 128;
+
+        let eth1data = random_eth1_data(deposits_imported);
+        let checkpoints = random_checkpoints(epochs as usize);
+        let mut forks = HashMap::new();
+        let mut eth1cache = eth1cache();
+
+        for epoch in 0..epochs {
+            assert_eq!(
+                eth1cache.by_checkpoint().len(),
+                cmp::min(epoch as usize, DEFAULT_ETH1_CACHE_SIZE),
+                "Unexpected cache size"
+            );
+
+            let checkpoint = checkpoints
+                .get(epoch as usize)
+                .expect("should get checkpoint");
+            eth1cache.insert(
+                *checkpoint,
+                Eth1FinalizationData {
+                    eth1_data: eth1data.clone(),
+                    eth1_deposit_index: deposits_imported,
+                },
+            );
+            // lets put a fork at every third epoch
+            if epoch % 3 == 0 {
+                let fork = random_checkpoint(epoch);
+                eth1cache.insert(
+                    fork,
+                    Eth1FinalizationData {
+                        eth1_data: eth1data.clone(),
+                        eth1_deposit_index: deposits_imported,
+                    },
+                );
+                forks.insert(epoch as usize, fork);
+            }
+
+            assert!(
+                eth1cache.pending_eth1().is_empty(),
+                "Deposits are fully imported so pending cache should be empty"
+            );
+            if epoch >= 4 {
+                let finalized_epoch = (epoch - 4) as usize;
+                let finalized_checkpoint = if finalized_epoch % 3 == 0 {
+                    forks.get(&finalized_epoch).expect("should get fork")
+                } else {
+                    checkpoints
+                        .get(finalized_epoch)
+                        .expect("should get checkpoint")
+                };
+                assert_eq!(
+                    eth1cache.finalize(finalized_checkpoint),
+                    Some(eth1data.clone()),
+                    "Should have cache hit"
+                );
+                if finalized_epoch >= 3 {
+                    let dropped_epoch = finalized_epoch - 3;
+                    if let Some(dropped_checkpoint) = forks.get(&dropped_epoch) {
+                        // got checkpoint for an old fork that should no longer
+                        // be in the cache because it is from too long ago
+                        assert_eq!(
+                            eth1cache.finalize(dropped_checkpoint),
+                            None,
+                            "Should have cache miss"
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn massive_deposit_queue() {
+        // Simulating a situation where deposits don't get imported within an eth1 voting period
+        let eth1_voting_periods = 8;
+        let initial_deposits_imported = 1024;
+        let deposits_imported_per_epoch = MAX_DEPOSITS * SLOTS_PER_EPOCH;
+        let initial_deposit_queue =
+            deposits_imported_per_epoch * EPOCHS_PER_ETH1_VOTING_PERIOD * 2 + 32;
+        let new_deposits_per_voting_period =
+            EPOCHS_PER_ETH1_VOTING_PERIOD * deposits_imported_per_epoch / 2;
+
+        let mut epoch_data = BTreeMap::new();
+        let mut eth1s_by_count = BTreeMap::new();
+        let mut eth1cache = eth1cache();
+        let mut last_period_deposits = initial_deposits_imported;
+        for period in 0..eth1_voting_periods {
+            let period_deposits = initial_deposits_imported
+                + initial_deposit_queue
+                + period * new_deposits_per_voting_period;
+            let period_eth1_data = random_eth1_data(period_deposits);
+            eth1s_by_count.insert(period_eth1_data.deposit_count, period_eth1_data.clone());
+
+            for epoch_mod_period in 0..EPOCHS_PER_ETH1_VOTING_PERIOD {
+                let epoch = period * EPOCHS_PER_ETH1_VOTING_PERIOD + epoch_mod_period;
+                let checkpoint = random_checkpoint(epoch);
+                let deposits_imported = cmp::min(
+                    period_deposits,
+                    last_period_deposits + deposits_imported_per_epoch * epoch_mod_period,
+                );
+                eth1cache.insert(
+                    checkpoint,
+                    Eth1FinalizationData {
+                        eth1_data: period_eth1_data.clone(),
+                        eth1_deposit_index: deposits_imported,
+                    },
+                );
+                epoch_data.insert(epoch, (checkpoint, deposits_imported));
+
+                if epoch >= 4 {
+                    let finalized_epoch = epoch - 4;
+                    let (finalized_checkpoint, finalized_deposits) = epoch_data
+                        .get(&finalized_epoch)
+                        .expect("should get epoch data");
+
+                    let pending_eth1s = eth1s_by_count.range((finalized_deposits + 1)..).count();
+                    let last_finalized_eth1 = eth1s_by_count
+                        .range(0..(finalized_deposits + 1))
+                        .map(|(_, eth1)| eth1)
+                        .last()
+                        .cloned();
+                    assert_eq!(
+                        eth1cache.finalize(finalized_checkpoint),
+                        last_finalized_eth1,
+                        "finalized checkpoint mismatch",
+                    );
+                    assert_eq!(
+                        eth1cache.pending_eth1().len(),
+                        pending_eth1s,
+                        "pending eth1 mismatch"
+                    );
+                }
+            }
+
+            // remove unneeded stuff from old epochs
+            while epoch_data.len() > DEFAULT_ETH1_CACHE_SIZE {
+                let oldest_stored_epoch = epoch_data
+                    .keys()
+                    .next()
+                    .cloned()
+                    .expect("should get oldest epoch");
+                epoch_data.remove(&oldest_stored_epoch);
+            }
+            last_period_deposits = period_deposits;
+        }
+    }
+}

beacon_node/beacon_chain/src/events.rs

@@ -6,111 +6,181 @@ use types::EthSpec;
 
 const DEFAULT_CHANNEL_CAPACITY: usize = 16;
 
-pub struct ServerSentEventHandler<T: EthSpec> {
-    attestation_tx: Sender<EventKind<T>>,
-    block_tx: Sender<EventKind<T>>,
-    finalized_tx: Sender<EventKind<T>>,
-    head_tx: Sender<EventKind<T>>,
-    exit_tx: Sender<EventKind<T>>,
-    chain_reorg_tx: Sender<EventKind<T>>,
-    contribution_tx: Sender<EventKind<T>>,
-    late_head: Sender<EventKind<T>>,
-    block_reward_tx: Sender<EventKind<T>>,
+pub struct ServerSentEventHandler<E: EthSpec> {
+    attestation_tx: Sender<EventKind<E>>,
+    block_tx: Sender<EventKind<E>>,
+    blob_sidecar_tx: Sender<EventKind<E>>,
+    finalized_tx: Sender<EventKind<E>>,
+    head_tx: Sender<EventKind<E>>,
+    exit_tx: Sender<EventKind<E>>,
+    chain_reorg_tx: Sender<EventKind<E>>,
+    contribution_tx: Sender<EventKind<E>>,
+    payload_attributes_tx: Sender<EventKind<E>>,
+    late_head: Sender<EventKind<E>>,
+    light_client_finality_update_tx: Sender<EventKind<E>>,
+    light_client_optimistic_update_tx: Sender<EventKind<E>>,
+    block_reward_tx: Sender<EventKind<E>>,
     log: Logger,
 }
 
-impl<T: EthSpec> ServerSentEventHandler<T> {
-    pub fn new(log: Logger) -> Self {
-        Self::new_with_capacity(log, DEFAULT_CHANNEL_CAPACITY)
+impl<E: EthSpec> ServerSentEventHandler<E> {
+    pub fn new(log: Logger, capacity_multiplier: usize) -> Self {
+        Self::new_with_capacity(
+            log,
+            capacity_multiplier.saturating_mul(DEFAULT_CHANNEL_CAPACITY),
+        )
     }
 
     pub fn new_with_capacity(log: Logger, capacity: usize) -> Self {
         let (attestation_tx, _) = broadcast::channel(capacity);
         let (block_tx, _) = broadcast::channel(capacity);
+        let (blob_sidecar_tx, _) = broadcast::channel(capacity);
         let (finalized_tx, _) = broadcast::channel(capacity);
         let (head_tx, _) = broadcast::channel(capacity);
         let (exit_tx, _) = broadcast::channel(capacity);
         let (chain_reorg_tx, _) = broadcast::channel(capacity);
         let (contribution_tx, _) = broadcast::channel(capacity);
+        let (payload_attributes_tx, _) = broadcast::channel(capacity);
         let (late_head, _) = broadcast::channel(capacity);
+        let (light_client_finality_update_tx, _) = broadcast::channel(capacity);
+        let (light_client_optimistic_update_tx, _) = broadcast::channel(capacity);
         let (block_reward_tx, _) = broadcast::channel(capacity);
 
         Self {
             attestation_tx,
             block_tx,
+            blob_sidecar_tx,
             finalized_tx,
             head_tx,
             exit_tx,
             chain_reorg_tx,
             contribution_tx,
+            payload_attributes_tx,
             late_head,
+            light_client_finality_update_tx,
+            light_client_optimistic_update_tx,
             block_reward_tx,
             log,
         }
     }
 
-    pub fn register(&self, kind: EventKind<T>) {
-        let result = match kind {
-            EventKind::Attestation(attestation) => self
+    pub fn register(&self, kind: EventKind<E>) {
+        let log_count = |name, count| {
+            trace!(
+                self.log,
+                "Registering server-sent event";
+                "kind" => name,
+                "receiver_count" => count
+            );
+        };
+        let result = match &kind {
+            EventKind::Attestation(_) => self
                 .attestation_tx
-                .send(EventKind::Attestation(attestation))
-                .map(|count| trace!(self.log, "Registering server-sent attestation event"; "receiver_count" => count)),
-            EventKind::Block(block) => self.block_tx.send(EventKind::Block(block))
-                .map(|count| trace!(self.log, "Registering server-sent block event"; "receiver_count" => count)),
-            EventKind::FinalizedCheckpoint(checkpoint) => self.finalized_tx
-                .send(EventKind::FinalizedCheckpoint(checkpoint))
-                .map(|count| trace!(self.log, "Registering server-sent finalized checkpoint event"; "receiver_count" => count)),
-            EventKind::Head(head) => self.head_tx.send(EventKind::Head(head))
-                .map(|count| trace!(self.log, "Registering server-sent head event"; "receiver_count" => count)),
-            EventKind::VoluntaryExit(exit) => self.exit_tx.send(EventKind::VoluntaryExit(exit))
-                .map(|count| trace!(self.log, "Registering server-sent voluntary exit event"; "receiver_count" => count)),
-            EventKind::ChainReorg(reorg) => self.chain_reorg_tx.send(EventKind::ChainReorg(reorg))
-                .map(|count| trace!(self.log, "Registering server-sent chain reorg event"; "receiver_count" => count)),
-            EventKind::ContributionAndProof(contribution_and_proof) => self.contribution_tx.send(EventKind::ContributionAndProof(contribution_and_proof))
-                .map(|count| trace!(self.log, "Registering server-sent contribution and proof event"; "receiver_count" => count)),
-            EventKind::LateHead(late_head) => self.late_head.send(EventKind::LateHead(late_head))
-                .map(|count| trace!(self.log, "Registering server-sent late head event"; "receiver_count" => count)),
-            EventKind::BlockReward(block_reward) => self.block_reward_tx.send(EventKind::BlockReward(block_reward))
-                .map(|count| trace!(self.log, "Registering server-sent contribution and proof event"; "receiver_count" => count)),
+                .send(kind)
+                .map(|count| log_count("attestation", count)),
+            EventKind::Block(_) => self
+                .block_tx
+                .send(kind)
+                .map(|count| log_count("block", count)),
+            EventKind::BlobSidecar(_) => self
+                .blob_sidecar_tx
+                .send(kind)
+                .map(|count| log_count("blob sidecar", count)),
+            EventKind::FinalizedCheckpoint(_) => self
+                .finalized_tx
+                .send(kind)
+                .map(|count| log_count("finalized checkpoint", count)),
+            EventKind::Head(_) => self
+                .head_tx
+                .send(kind)
+                .map(|count| log_count("head", count)),
+            EventKind::VoluntaryExit(_) => self
+                .exit_tx
+                .send(kind)
+                .map(|count| log_count("exit", count)),
+            EventKind::ChainReorg(_) => self
+                .chain_reorg_tx
+                .send(kind)
+                .map(|count| log_count("chain reorg", count)),
+            EventKind::ContributionAndProof(_) => self
+                .contribution_tx
+                .send(kind)
+                .map(|count| log_count("contribution and proof", count)),
+            EventKind::PayloadAttributes(_) => self
+                .payload_attributes_tx
+                .send(kind)
+                .map(|count| log_count("payload attributes", count)),
+            EventKind::LateHead(_) => self
+                .late_head
+                .send(kind)
+                .map(|count| log_count("late head", count)),
+            EventKind::LightClientFinalityUpdate(_) => self
+                .light_client_finality_update_tx
+                .send(kind)
+                .map(|count| log_count("light client finality update", count)),
+            EventKind::LightClientOptimisticUpdate(_) => self
+                .light_client_optimistic_update_tx
+                .send(kind)
+                .map(|count| log_count("light client optimistic update", count)),
+            EventKind::BlockReward(_) => self
+                .block_reward_tx
+                .send(kind)
+                .map(|count| log_count("block reward", count)),
         };
         if let Err(SendError(event)) = result {
             trace!(self.log, "No receivers registered to listen for event"; "event" => ?event);
         }
     }
 
-    pub fn subscribe_attestation(&self) -> Receiver<EventKind<T>> {
+    pub fn subscribe_attestation(&self) -> Receiver<EventKind<E>> {
         self.attestation_tx.subscribe()
     }
 
-    pub fn subscribe_block(&self) -> Receiver<EventKind<T>> {
+    pub fn subscribe_block(&self) -> Receiver<EventKind<E>> {
         self.block_tx.subscribe()
     }
 
-    pub fn subscribe_finalized(&self) -> Receiver<EventKind<T>> {
+    pub fn subscribe_blob_sidecar(&self) -> Receiver<EventKind<E>> {
+        self.blob_sidecar_tx.subscribe()
+    }
+
+    pub fn subscribe_finalized(&self) -> Receiver<EventKind<E>> {
         self.finalized_tx.subscribe()
     }
 
-    pub fn subscribe_head(&self) -> Receiver<EventKind<T>> {
+    pub fn subscribe_head(&self) -> Receiver<EventKind<E>> {
         self.head_tx.subscribe()
     }
 
-    pub fn subscribe_exit(&self) -> Receiver<EventKind<T>> {
+    pub fn subscribe_exit(&self) -> Receiver<EventKind<E>> {
         self.exit_tx.subscribe()
     }
 
-    pub fn subscribe_reorgs(&self) -> Receiver<EventKind<T>> {
+    pub fn subscribe_reorgs(&self) -> Receiver<EventKind<E>> {
         self.chain_reorg_tx.subscribe()
     }
 
-    pub fn subscribe_contributions(&self) -> Receiver<EventKind<T>> {
+    pub fn subscribe_contributions(&self) -> Receiver<EventKind<E>> {
         self.contribution_tx.subscribe()
     }
 
-    pub fn subscribe_late_head(&self) -> Receiver<EventKind<T>> {
+    pub fn subscribe_payload_attributes(&self) -> Receiver<EventKind<E>> {
+        self.payload_attributes_tx.subscribe()
+    }
+
+    pub fn subscribe_late_head(&self) -> Receiver<EventKind<E>> {
         self.late_head.subscribe()
     }
 
-    pub fn subscribe_block_reward(&self) -> Receiver<EventKind<T>> {
+    pub fn subscribe_light_client_finality_update(&self) -> Receiver<EventKind<E>> {
+        self.light_client_finality_update_tx.subscribe()
+    }
+
+    pub fn subscribe_light_client_optimistic_update(&self) -> Receiver<EventKind<E>> {
+        self.light_client_optimistic_update_tx.subscribe()
+    }
+
+    pub fn subscribe_block_reward(&self) -> Receiver<EventKind<E>> {
         self.block_reward_tx.subscribe()
     }
 
@@ -122,6 +192,10 @@ impl<T: EthSpec> ServerSentEventHandler<T> {
         self.block_tx.receiver_count() > 0
     }
 
+    pub fn has_blob_sidecar_subscribers(&self) -> bool {
+        self.blob_sidecar_tx.receiver_count() > 0
+    }
+
     pub fn has_finalized_subscribers(&self) -> bool {
         self.finalized_tx.receiver_count() > 0
     }
@@ -142,6 +216,10 @@ impl<T: EthSpec> ServerSentEventHandler<T> {
         self.contribution_tx.receiver_count() > 0
     }
 
+    pub fn has_payload_attributes_subscribers(&self) -> bool {
+        self.payload_attributes_tx.receiver_count() > 0
+    }
+
     pub fn has_late_head_subscribers(&self) -> bool {
         self.late_head.receiver_count() > 0
     }

beacon_node/beacon_chain/src/execution_payload.rs

@@ -7,65 +7,217 @@
 //! So, this module contains functions that one might expect to find in other crates, but they live
 //! here for good reason.
 
+use crate::otb_verification_service::OptimisticTransitionBlock;
 use crate::{
     BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, BlockProductionError,
     ExecutionPayloadError,
 };
-use execution_layer::ExecutePayloadResponseStatus;
-use fork_choice::PayloadVerificationStatus;
+use execution_layer::{
+    BlockProposalContents, BlockProposalContentsType, BuilderParams, NewPayloadRequest,
+    PayloadAttributes, PayloadStatus,
+};
+use fork_choice::{InvalidationOperation, PayloadVerificationStatus};
 use proto_array::{Block as ProtoBlock, ExecutionStatus};
-use slog::debug;
+use slog::{debug, warn};
 use slot_clock::SlotClock;
 use state_processing::per_block_processing::{
-    compute_timestamp_at_slot, is_execution_enabled, is_merge_transition_complete,
-    partially_verify_execution_payload,
+    compute_timestamp_at_slot, get_expected_withdrawals, is_execution_enabled,
+    is_merge_transition_complete, partially_verify_execution_payload,
 };
+use std::sync::Arc;
+use tokio::task::JoinHandle;
+use tree_hash::TreeHash;
+use types::payload::BlockProductionVersion;
 use types::*;
 
+pub type PreparePayloadResult<E> = Result<BlockProposalContentsType<E>, BlockProductionError>;
+pub type PreparePayloadHandle<E> = JoinHandle<Option<PreparePayloadResult<E>>>;
+
+#[derive(PartialEq)]
+pub enum AllowOptimisticImport {
+    Yes,
+    No,
+}
+
+/// Signal whether the execution payloads of new blocks should be
+/// immediately verified with the EL or imported optimistically without
+/// any EL communication.
+#[derive(Default, Clone, Copy)]
+pub enum NotifyExecutionLayer {
+    #[default]
+    Yes,
+    No,
+}
+
+/// Used to await the result of executing payload with a remote EE.
+pub struct PayloadNotifier<T: BeaconChainTypes> {
+    pub chain: Arc<BeaconChain<T>>,
+    pub block: Arc<SignedBeaconBlock<T::EthSpec>>,
+    payload_verification_status: Option<PayloadVerificationStatus>,
+}
+
+impl<T: BeaconChainTypes> PayloadNotifier<T> {
+    pub fn new(
+        chain: Arc<BeaconChain<T>>,
+        block: Arc<SignedBeaconBlock<T::EthSpec>>,
+        state: &BeaconState<T::EthSpec>,
+        notify_execution_layer: NotifyExecutionLayer,
+    ) -> Result<Self, BlockError<T::EthSpec>> {
+        let payload_verification_status = if is_execution_enabled(state, block.message().body()) {
+            // Perform the initial stages of payload verification.
+            //
+            // We will duplicate these checks again during `per_block_processing`, however these
+            // checks are cheap and doing them here ensures we have verified them before marking
+            // the block as optimistically imported. This is particularly relevant in the case
+            // where we do not send the block to the EL at all.
+            let block_message = block.message();
+            partially_verify_execution_payload::<_, FullPayload<_>>(
+                state,
+                block.slot(),
+                block_message.body(),
+                &chain.spec,
+            )
+            .map_err(BlockError::PerBlockProcessingError)?;
+
+            match notify_execution_layer {
+                NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => {
+                    // Create a NewPayloadRequest (no clones required) and check optimistic sync verifications
+                    let new_payload_request: NewPayloadRequest<T::EthSpec> =
+                        block_message.try_into()?;
+                    if let Err(e) = new_payload_request.perform_optimistic_sync_verifications() {
+                        warn!(
+                            chain.log,
+                            "Falling back to slow block hash verification";
+                            "block_number" => ?block_message.execution_payload().map(|payload| payload.block_number()),
+                            "info" => "you can silence this warning with --disable-optimistic-finalized-sync",
+                            "error" => ?e,
+                        );
+                        None
+                    } else {
+                        Some(PayloadVerificationStatus::Optimistic)
+                    }
+                }
+                _ => None,
+            }
+        } else {
+            Some(PayloadVerificationStatus::Irrelevant)
+        };
+
+        Ok(Self {
+            chain,
+            block,
+            payload_verification_status,
+        })
+    }
+
+    pub async fn notify_new_payload(
+        self,
+    ) -> Result<PayloadVerificationStatus, BlockError<T::EthSpec>> {
+        if let Some(precomputed_status) = self.payload_verification_status {
+            Ok(precomputed_status)
+        } else {
+            notify_new_payload(&self.chain, self.block.message()).await
+        }
+    }
+}
+
 /// Verify that `execution_payload` contained by `block` is considered valid by an execution
 /// engine.
 ///
 /// ## Specification
 ///
-/// Equivalent to the `execute_payload` function in the merge Beacon Chain Changes, although it
+/// Equivalent to the `notify_new_payload` function in the merge Beacon Chain Changes, although it
 /// contains a few extra checks by running `partially_verify_execution_payload` first:
 ///
-/// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/beacon-chain.md#execute_payload
-pub fn execute_payload<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
-    state: &BeaconState<T::EthSpec>,
-    block: BeaconBlockRef<T::EthSpec>,
+/// https://github.com/ethereum/consensus-specs/blob/v1.1.9/specs/bellatrix/beacon-chain.md#notify_new_payload
+async fn notify_new_payload<'a, T: BeaconChainTypes>(
+    chain: &Arc<BeaconChain<T>>,
+    block: BeaconBlockRef<'a, T::EthSpec>,
 ) -> Result<PayloadVerificationStatus, BlockError<T::EthSpec>> {
-    if !is_execution_enabled(state, block.body()) {
-        return Ok(PayloadVerificationStatus::Irrelevant);
-    }
-
-    let execution_payload = block.execution_payload()?;
-
-    // Perform the initial stages of payload verification.
-    //
-    // We will duplicate these checks again during `per_block_processing`, however these checks
-    // are cheap and doing them here ensures we protect the execution payload from junk.
-    partially_verify_execution_payload(state, execution_payload, &chain.spec)
-        .map_err(BlockError::PerBlockProcessingError)?;
-
     let execution_layer = chain
         .execution_layer
         .as_ref()
         .ok_or(ExecutionPayloadError::NoExecutionConnection)?;
-    let execute_payload_response = execution_layer
-        .block_on(|execution_layer| execution_layer.execute_payload(execution_payload));
 
-    match execute_payload_response {
-        Ok((status, _latest_valid_hash)) => match status {
-            ExecutePayloadResponseStatus::Valid => Ok(PayloadVerificationStatus::Verified),
-            // TODO(merge): invalidate any invalid ancestors of this block in fork choice.
-            ExecutePayloadResponseStatus::Invalid => {
-                Err(ExecutionPayloadError::RejectedByExecutionEngine.into())
+    let execution_block_hash = block.execution_payload()?.block_hash();
+    let new_payload_response = execution_layer.notify_new_payload(block.try_into()?).await;
+
+    match new_payload_response {
+        Ok(status) => match status {
+            PayloadStatus::Valid => Ok(PayloadVerificationStatus::Verified),
+            PayloadStatus::Syncing | PayloadStatus::Accepted => {
+                Ok(PayloadVerificationStatus::Optimistic)
+            }
+            PayloadStatus::Invalid {
+                latest_valid_hash,
+                ref validation_error,
+            } => {
+                warn!(
+                    chain.log,
+                    "Invalid execution payload";
+                    "validation_error" => ?validation_error,
+                    "latest_valid_hash" => ?latest_valid_hash,
+                    "execution_block_hash" => ?execution_block_hash,
+                    "root" => ?block.tree_hash_root(),
+                    "graffiti" => block.body().graffiti().as_utf8_lossy(),
+                    "proposer_index" => block.proposer_index(),
+                    "slot" => block.slot(),
+                    "method" => "new_payload",
+                );
+
+                // Only trigger payload invalidation in fork choice if the
+                // `latest_valid_hash` is `Some` and non-zero.
+                //
+                // A `None` latest valid hash indicates that the EE was unable
+                // to determine the most recent valid ancestor. Since `block`
+                // has not yet been applied to fork choice, there's nothing to
+                // invalidate.
+                //
+                // An all-zeros payload indicates that an EIP-3675 check has
+                // failed regarding the validity of the terminal block. Rather
+                // than iterating back in the chain to find the terminal block
+                // and invalidating that, we simply reject this block without
+                // invalidating anything else.
+                if let Some(latest_valid_hash) =
+                    latest_valid_hash.filter(|hash| *hash != ExecutionBlockHash::zero())
+                {
+                    // This block has not yet been applied to fork choice, so the latest block that was
+                    // imported to fork choice was the parent.
+                    let latest_root = block.parent_root();
+
+                    chain
+                        .process_invalid_execution_payload(&InvalidationOperation::InvalidateMany {
+                            head_block_root: latest_root,
+                            always_invalidate_head: false,
+                            latest_valid_ancestor: latest_valid_hash,
+                        })
+                        .await?;
+                }
+
+                Err(ExecutionPayloadError::RejectedByExecutionEngine { status }.into())
+            }
+            PayloadStatus::InvalidBlockHash {
+                ref validation_error,
+            } => {
+                warn!(
+                    chain.log,
+                    "Invalid execution payload block hash";
+                    "validation_error" => ?validation_error,
+                    "execution_block_hash" => ?execution_block_hash,
+                    "root" => ?block.tree_hash_root(),
+                    "graffiti" => block.body().graffiti().as_utf8_lossy(),
+                    "proposer_index" => block.proposer_index(),
+                    "slot" => block.slot(),
+                    "method" => "new_payload",
+                );
+
+                // Returning an error here should be sufficient to invalidate the block. We have no
+                // information to indicate its parent is invalid, so no need to run
+                // `BeaconChain::process_invalid_execution_payload`.
+                Err(ExecutionPayloadError::RejectedByExecutionEngine { status }.into())
             }
-            ExecutePayloadResponseStatus::Syncing => Ok(PayloadVerificationStatus::NotVerified),
         },
-        Err(_) => Err(ExecutionPayloadError::RejectedByExecutionEngine.into()),
+        Err(e) => Err(ExecutionPayloadError::RequestFailed(e).into()),
     }
 }
 
@@ -81,15 +233,16 @@ pub fn execute_payload<T: BeaconChainTypes>(
 /// Equivalent to the `validate_merge_block` function in the merge Fork Choice Changes:
 ///
 /// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/fork-choice.md#validate_merge_block
-pub fn validate_merge_block<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
-    block: BeaconBlockRef<T::EthSpec>,
+pub async fn validate_merge_block<'a, T: BeaconChainTypes>(
+    chain: &Arc<BeaconChain<T>>,
+    block: BeaconBlockRef<'a, T::EthSpec>,
+    allow_optimistic_import: AllowOptimisticImport,
 ) -> Result<(), BlockError<T::EthSpec>> {
     let spec = &chain.spec;
     let block_epoch = block.slot().epoch(T::EthSpec::slots_per_epoch());
     let execution_payload = block.execution_payload()?;
 
-    if spec.terminal_block_hash != Hash256::zero() {
+    if spec.terminal_block_hash != ExecutionBlockHash::zero() {
         if block_epoch < spec.terminal_block_hash_activation_epoch {
             return Err(ExecutionPayloadError::InvalidActivationEpoch {
                 activation_epoch: spec.terminal_block_hash_activation_epoch,
@@ -98,10 +251,10 @@ pub fn validate_merge_block<T: BeaconChainTypes>(
             .into());
         }
 
-        if execution_payload.parent_hash != spec.terminal_block_hash {
+        if execution_payload.parent_hash() != spec.terminal_block_hash {
             return Err(ExecutionPayloadError::InvalidTerminalBlockHash {
                 terminal_block_hash: spec.terminal_block_hash,
-                payload_parent_hash: execution_payload.parent_hash,
+                payload_parent_hash: execution_payload.parent_hash(),
             }
             .into());
         }
@@ -115,29 +268,67 @@ pub fn validate_merge_block<T: BeaconChainTypes>(
         .ok_or(ExecutionPayloadError::NoExecutionConnection)?;
 
     let is_valid_terminal_pow_block = execution_layer
-        .block_on(|execution_layer| {
-            execution_layer.is_valid_terminal_pow_block_hash(execution_payload.parent_hash, spec)
-        })
+        .is_valid_terminal_pow_block_hash(execution_payload.parent_hash(), spec)
+        .await
         .map_err(ExecutionPayloadError::from)?;
 
     match is_valid_terminal_pow_block {
         Some(true) => Ok(()),
         Some(false) => Err(ExecutionPayloadError::InvalidTerminalPoWBlock {
-            parent_hash: execution_payload.parent_hash,
+            parent_hash: execution_payload.parent_hash(),
         }
         .into()),
         None => {
-            debug!(
-                chain.log,
-                "Optimistically accepting terminal block";
-                "block_hash" => ?execution_payload.parent_hash,
-                "msg" => "the terminal block/parent was unavailable"
-            );
-            Ok(())
+            if allow_optimistic_import == AllowOptimisticImport::Yes
+                && is_optimistic_candidate_block(chain, block.slot(), block.parent_root()).await?
+            {
+                debug!(
+                    chain.log,
+                    "Optimistically importing merge transition block";
+                    "block_hash" => ?execution_payload.parent_hash(),
+                    "msg" => "the terminal block/parent was unavailable"
+                );
+                // Store Optimistic Transition Block in Database for later Verification
+                OptimisticTransitionBlock::from_block(block)
+                    .persist_in_store::<T, _>(&chain.store)?;
+                Ok(())
+            } else {
+                Err(ExecutionPayloadError::UnverifiedNonOptimisticCandidate.into())
+            }
         }
     }
 }
 
+/// Check to see if a block with the given parameters is valid to be imported optimistically.
+pub async fn is_optimistic_candidate_block<T: BeaconChainTypes>(
+    chain: &Arc<BeaconChain<T>>,
+    block_slot: Slot,
+    block_parent_root: Hash256,
+) -> Result<bool, BeaconChainError> {
+    let current_slot = chain.slot()?;
+    let inner_chain = chain.clone();
+
+    // Use a blocking task to check if the block is an optimistic candidate. Interacting
+    // with the `fork_choice` lock in an async task can block the core executor.
+    chain
+        .spawn_blocking_handle(
+            move || {
+                inner_chain
+                    .canonical_head
+                    .fork_choice_read_lock()
+                    .is_optimistic_candidate_block(
+                        current_slot,
+                        block_slot,
+                        &block_parent_root,
+                        &inner_chain.spec,
+                    )
+            },
+            "validate_merge_block_optimistic_candidate",
+        )
+        .await?
+        .map_err(BeaconChainError::from)
+}
+
 /// Validate the gossip block's execution_payload according to the checks described here:
 /// https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/p2p-interface.md#beacon_block
 pub fn validate_execution_payload_for_gossip<T: BeaconChainTypes>(
@@ -152,7 +343,7 @@ pub fn validate_execution_payload_for_gossip<T: BeaconChainTypes>(
 
         let is_merge_transition_complete = match parent_block.execution_status {
             // Optimistically declare that an "unknown" status block has completed the merge.
-            ExecutionStatus::Valid(_) | ExecutionStatus::Unknown(_) => true,
+            ExecutionStatus::Valid(_) | ExecutionStatus::Optimistic(_) => true,
             // It's impossible for an irrelevant block to have completed the merge. It is pre-merge
             // by definition.
             ExecutionStatus::Irrelevant(_) => false,
@@ -165,7 +356,7 @@ pub fn validate_execution_payload_for_gossip<T: BeaconChainTypes>(
             }
         };
 
-        if is_merge_transition_complete || execution_payload != &<_>::default() {
+        if is_merge_transition_complete || !execution_payload.is_default_with_empty_roots() {
             let expected_timestamp = chain
                 .slot_clock
                 .start_of(block.slot())
@@ -175,11 +366,11 @@ pub fn validate_execution_payload_for_gossip<T: BeaconChainTypes>(
                 ))?;
 
             // The block's execution payload timestamp is correct with respect to the slot
-            if execution_payload.timestamp != expected_timestamp {
+            if execution_payload.timestamp() != expected_timestamp {
                 return Err(BlockError::ExecutionPayloadError(
                     ExecutionPayloadError::InvalidPayloadTimestamp {
                         expected: expected_timestamp,
-                        found: execution_payload.timestamp,
+                        found: execution_payload.timestamp(),
                     },
                 ));
             }
@@ -202,29 +393,66 @@ pub fn validate_execution_payload_for_gossip<T: BeaconChainTypes>(
 ///
 /// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/validator.md#block-proposal
 pub fn get_execution_payload<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
+    chain: Arc<BeaconChain<T>>,
     state: &BeaconState<T::EthSpec>,
+    parent_block_root: Hash256,
     proposer_index: u64,
-) -> Result<ExecutionPayload<T::EthSpec>, BlockProductionError> {
-    Ok(prepare_execution_payload_blocking(chain, state, proposer_index)?.unwrap_or_default())
-}
+    builder_params: BuilderParams,
+    builder_boost_factor: Option<u64>,
+    block_production_version: BlockProductionVersion,
+) -> Result<PreparePayloadHandle<T::EthSpec>, BlockProductionError> {
+    // Compute all required values from the `state` now to avoid needing to pass it into a spawned
+    // task.
+    let spec = &chain.spec;
+    let current_epoch = state.current_epoch();
+    let is_merge_transition_complete = is_merge_transition_complete(state);
+    let timestamp =
+        compute_timestamp_at_slot(state, state.slot(), spec).map_err(BeaconStateError::from)?;
+    let random = *state.get_randao_mix(current_epoch)?;
+    let latest_execution_payload_header_block_hash =
+        state.latest_execution_payload_header()?.block_hash();
+    let withdrawals = match state {
+        &BeaconState::Capella(_) | &BeaconState::Deneb(_) | &BeaconState::Electra(_) => {
+            Some(get_expected_withdrawals(state, spec)?.into())
+        }
+        &BeaconState::Merge(_) => None,
+        // These shouldn't happen but they're here to make the pattern irrefutable
+        &BeaconState::Base(_) | &BeaconState::Altair(_) => None,
+    };
+    let parent_beacon_block_root = match state {
+        BeaconState::Deneb(_) | BeaconState::Electra(_) => Some(parent_block_root),
+        BeaconState::Merge(_) | BeaconState::Capella(_) => None,
+        // These shouldn't happen but they're here to make the pattern irrefutable
+        BeaconState::Base(_) | BeaconState::Altair(_) => None,
+    };
 
-/// Wraps the async `prepare_execution_payload` function as a blocking task.
-pub fn prepare_execution_payload_blocking<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
-    state: &BeaconState<T::EthSpec>,
-    proposer_index: u64,
-) -> Result<Option<ExecutionPayload<T::EthSpec>>, BlockProductionError> {
-    let execution_layer = chain
-        .execution_layer
-        .as_ref()
-        .ok_or(BlockProductionError::ExecutionLayerMissing)?;
+    // Spawn a task to obtain the execution payload from the EL via a series of async calls. The
+    // `join_handle` can be used to await the result of the function.
+    let join_handle = chain
+        .task_executor
+        .clone()
+        .spawn_handle(
+            async move {
+                prepare_execution_payload::<T>(
+                    &chain,
+                    is_merge_transition_complete,
+                    timestamp,
+                    random,
+                    proposer_index,
+                    latest_execution_payload_header_block_hash,
+                    builder_params,
+                    withdrawals,
+                    parent_beacon_block_root,
+                    builder_boost_factor,
+                    block_production_version,
+                )
+                .await
+            },
+            "get_execution_payload",
+        )
+        .ok_or(BlockProductionError::ShuttingDown)?;
 
-    execution_layer
-        .block_on_generic(|_| async {
-            prepare_execution_payload(chain, state, proposer_index).await
-        })
-        .map_err(BlockProductionError::BlockingFailed)?
+    Ok(join_handle)
 }
 
 /// Prepares an execution payload for inclusion in a block.
@@ -241,74 +469,110 @@ pub fn prepare_execution_payload_blocking<T: BeaconChainTypes>(
 /// Equivalent to the `prepare_execution_payload` function in the Validator Guide:
 ///
 /// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/validator.md#block-proposal
-pub async fn prepare_execution_payload<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
-    state: &BeaconState<T::EthSpec>,
+#[allow(clippy::too_many_arguments)]
+pub async fn prepare_execution_payload<T>(
+    chain: &Arc<BeaconChain<T>>,
+    is_merge_transition_complete: bool,
+    timestamp: u64,
+    random: Hash256,
     proposer_index: u64,
-) -> Result<Option<ExecutionPayload<T::EthSpec>>, BlockProductionError> {
+    latest_execution_payload_header_block_hash: ExecutionBlockHash,
+    builder_params: BuilderParams,
+    withdrawals: Option<Vec<Withdrawal>>,
+    parent_beacon_block_root: Option<Hash256>,
+    builder_boost_factor: Option<u64>,
+    block_production_version: BlockProductionVersion,
+) -> Result<BlockProposalContentsType<T::EthSpec>, BlockProductionError>
+where
+    T: BeaconChainTypes,
+{
+    let current_epoch = builder_params.slot.epoch(T::EthSpec::slots_per_epoch());
     let spec = &chain.spec;
+    let fork = spec.fork_name_at_slot::<T::EthSpec>(builder_params.slot);
     let execution_layer = chain
         .execution_layer
         .as_ref()
         .ok_or(BlockProductionError::ExecutionLayerMissing)?;
 
-    let parent_hash = if !is_merge_transition_complete(state) {
-        let is_terminal_block_hash_set = spec.terminal_block_hash != Hash256::zero();
+    let parent_hash = if !is_merge_transition_complete {
+        let is_terminal_block_hash_set = spec.terminal_block_hash != ExecutionBlockHash::zero();
         let is_activation_epoch_reached =
-            state.current_epoch() >= spec.terminal_block_hash_activation_epoch;
+            current_epoch >= spec.terminal_block_hash_activation_epoch;
 
         if is_terminal_block_hash_set && !is_activation_epoch_reached {
-            return Ok(None);
+            // Use the "empty" payload if there's a terminal block hash, but we haven't reached the
+            // terminal block epoch yet.
+            return Ok(BlockProposalContentsType::Full(
+                BlockProposalContents::Payload {
+                    payload: FullPayload::default_at_fork(fork)?,
+                    block_value: Uint256::zero(),
+                },
+            ));
         }
 
         let terminal_pow_block_hash = execution_layer
-            .get_terminal_pow_block_hash(spec)
+            .get_terminal_pow_block_hash(spec, timestamp)
             .await
             .map_err(BlockProductionError::TerminalPoWBlockLookupFailed)?;
 
         if let Some(terminal_pow_block_hash) = terminal_pow_block_hash {
             terminal_pow_block_hash
         } else {
-            return Ok(None);
+            // If the merge transition hasn't occurred yet and the EL hasn't found the terminal
+            // block, return an "empty" payload.
+            return Ok(BlockProposalContentsType::Full(
+                BlockProposalContents::Payload {
+                    payload: FullPayload::default_at_fork(fork)?,
+                    block_value: Uint256::zero(),
+                },
+            ));
         }
     } else {
-        state.latest_execution_payload_header()?.block_hash
+        latest_execution_payload_header_block_hash
     };
 
-    let timestamp = compute_timestamp_at_slot(state, spec).map_err(BeaconStateError::from)?;
-    let random = *state.get_randao_mix(state.current_epoch())?;
-    let finalized_root = state.finalized_checkpoint().root;
+    // Try to obtain the fork choice update parameters from the cached head.
+    //
+    // Use a blocking task to interact with the `canonical_head` lock otherwise we risk blocking the
+    // core `tokio` executor.
+    let inner_chain = chain.clone();
+    let forkchoice_update_params = chain
+        .spawn_blocking_handle(
+            move || {
+                inner_chain
+                    .canonical_head
+                    .cached_head()
+                    .forkchoice_update_parameters()
+            },
+            "prepare_execution_payload_forkchoice_update_params",
+        )
+        .await
+        .map_err(BlockProductionError::BeaconChain)?;
 
-    // The finalized block hash is not included in the specification, however we provide this
-    // parameter so that the execution layer can produce a payload id if one is not already known
-    // (e.g., due to a recent reorg).
-    let finalized_block_hash =
-        if let Some(block) = chain.fork_choice.read().get_block(&finalized_root) {
-            block.execution_status.block_hash()
-        } else {
-            chain
-                .store
-                .get_block(&finalized_root)
-                .map_err(BlockProductionError::FailedToReadFinalizedBlock)?
-                .ok_or(BlockProductionError::MissingFinalizedBlock(finalized_root))?
-                .message()
-                .body()
-                .execution_payload()
-                .ok()
-                .map(|ep| ep.block_hash)
-        };
+    let suggested_fee_recipient = execution_layer
+        .get_suggested_fee_recipient(proposer_index)
+        .await;
+    let payload_attributes = PayloadAttributes::new(
+        timestamp,
+        random,
+        suggested_fee_recipient,
+        withdrawals,
+        parent_beacon_block_root,
+    );
 
-    // Note: the suggested_fee_recipient is stored in the `execution_layer`, it will add this parameter.
-    let execution_payload = execution_layer
+    let block_contents = execution_layer
         .get_payload(
             parent_hash,
-            timestamp,
-            random,
-            finalized_block_hash.unwrap_or_else(Hash256::zero),
-            proposer_index,
+            &payload_attributes,
+            forkchoice_update_params,
+            builder_params,
+            fork,
+            &chain.spec,
+            builder_boost_factor,
+            block_production_version,
         )
         .await
         .map_err(BlockProductionError::GetPayloadFailed)?;
 
-    Ok(Some(execution_payload))
+    Ok(block_contents)
 }

beacon_node/beacon_chain/src/fork_choice_signal.rs

@@ -0,0 +1,97 @@
+//! Concurrency helpers for synchronising block proposal with fork choice.
+//!
+//! The transmitter provides a way for a thread runnning fork choice on a schedule to signal
+//! to the receiver that fork choice has been updated for a given slot.
+use crate::BeaconChainError;
+use parking_lot::{Condvar, Mutex};
+use std::sync::Arc;
+use std::time::Duration;
+use types::Slot;
+
+/// Sender, for use by the per-slot task timer.
+pub struct ForkChoiceSignalTx {
+    pair: Arc<(Mutex<Slot>, Condvar)>,
+}
+
+/// Receiver, for use by the beacon chain waiting on fork choice to complete.
+pub struct ForkChoiceSignalRx {
+    pair: Arc<(Mutex<Slot>, Condvar)>,
+}
+
+pub enum ForkChoiceWaitResult {
+    /// Successfully reached a slot greater than or equal to the awaited slot.
+    Success(Slot),
+    /// Fork choice was updated to a lower slot, indicative of lag or processing delays.
+    Behind(Slot),
+    /// Timed out waiting for the fork choice update from the sender.
+    TimeOut,
+}
+
+impl ForkChoiceSignalTx {
+    pub fn new() -> Self {
+        let pair = Arc::new((Mutex::new(Slot::new(0)), Condvar::new()));
+        Self { pair }
+    }
+
+    pub fn get_receiver(&self) -> ForkChoiceSignalRx {
+        ForkChoiceSignalRx {
+            pair: self.pair.clone(),
+        }
+    }
+
+    /// Signal to the receiver that fork choice has been updated to `slot`.
+    ///
+    /// Return an error if the provided `slot` is strictly less than any previously provided slot.
+    pub fn notify_fork_choice_complete(&self, slot: Slot) -> Result<(), BeaconChainError> {
+        let (lock, condvar) = &*self.pair;
+
+        let mut current_slot = lock.lock();
+
+        if slot < *current_slot {
+            return Err(BeaconChainError::ForkChoiceSignalOutOfOrder {
+                current: *current_slot,
+                latest: slot,
+            });
+        } else {
+            *current_slot = slot;
+        }
+
+        // We use `notify_all` because there may be multiple block proposals waiting simultaneously.
+        // Usually there'll be 0-1.
+        condvar.notify_all();
+
+        Ok(())
+    }
+}
+
+impl Default for ForkChoiceSignalTx {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ForkChoiceSignalRx {
+    pub fn wait_for_fork_choice(&self, slot: Slot, timeout: Duration) -> ForkChoiceWaitResult {
+        let (lock, condvar) = &*self.pair;
+
+        let mut current_slot = lock.lock();
+
+        // Wait for `current_slot >= slot`.
+        //
+        // Do not loop and wait, if we receive an update for the wrong slot then something is
+        // quite out of whack and we shouldn't waste more time waiting.
+        if *current_slot < slot {
+            let timeout_result = condvar.wait_for(&mut current_slot, timeout);
+
+            if timeout_result.timed_out() {
+                return ForkChoiceWaitResult::TimeOut;
+            }
+        }
+
+        if *current_slot >= slot {
+            ForkChoiceWaitResult::Success(*current_slot)
+        } else {
+            ForkChoiceWaitResult::Behind(*current_slot)
+        }
+    }
+}

beacon_node/beacon_chain/src/fork_revert.rs

@@ -4,7 +4,8 @@ use itertools::process_results;
 use slog::{info, warn, Logger};
 use state_processing::state_advance::complete_state_advance;
 use state_processing::{
-    per_block_processing, per_block_processing::BlockSignatureStrategy, VerifyBlockRoot,
+    per_block_processing, per_block_processing::BlockSignatureStrategy, ConsensusContext,
+    StateProcessingStrategy, VerifyBlockRoot,
 };
 use std::sync::Arc;
 use std::time::Duration;
@@ -48,7 +49,7 @@ pub fn revert_to_fork_boundary<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>
     );
     let block_iter = ParentRootBlockIterator::fork_tolerant(&store, head_block_root);
 
-    process_results(block_iter, |mut iter| {
+    let (block_root, blinded_block) = process_results(block_iter, |mut iter| {
         iter.find_map(|(block_root, block)| {
             if block.slot() < fork_epoch.start_slot(E::slots_per_epoch()) {
                 Some((block_root, block))
@@ -69,7 +70,13 @@ pub fn revert_to_fork_boundary<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>
             e, CORRUPT_DB_MESSAGE
         )
     })?
-    .ok_or_else(|| format!("No pre-fork blocks found. {}", CORRUPT_DB_MESSAGE))
+    .ok_or_else(|| format!("No pre-fork blocks found. {}", CORRUPT_DB_MESSAGE))?;
+
+    let block = store
+        .make_full_block(&block_root, blinded_block)
+        .map_err(|e| format!("Unable to add payload to new head block: {:?}", e))?;
+
+    Ok((block_root, block))
 }
 
 /// Reset fork choice to the finalized checkpoint of the supplied head state.
@@ -91,13 +98,14 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
     head_block_root: Hash256,
     head_state: &BeaconState<E>,
     store: Arc<HotColdDB<E, Hot, Cold>>,
+    current_slot: Option<Slot>,
     spec: &ChainSpec,
 ) -> Result<ForkChoice<BeaconForkChoiceStore<E, Hot, Cold>, E>, String> {
     // Fetch finalized block.
     let finalized_checkpoint = head_state.finalized_checkpoint();
     let finalized_block_root = finalized_checkpoint.root;
     let finalized_block = store
-        .get_block(&finalized_block_root)
+        .get_full_block(&finalized_block_root, None)
         .map_err(|e| format!("Error loading finalized block: {:?}", e))?
         .ok_or_else(|| {
             format!(
@@ -132,17 +140,20 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
     })?;
     let finalized_snapshot = BeaconSnapshot {
         beacon_block_root: finalized_block_root,
-        beacon_block: finalized_block,
+        beacon_block: Arc::new(finalized_block),
         beacon_state: finalized_state,
     };
 
-    let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), &finalized_snapshot);
+    let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), &finalized_snapshot)
+        .map_err(|e| format!("Unable to reset fork choice store for revert: {e:?}"))?;
 
     let mut fork_choice = ForkChoice::from_anchor(
         fc_store,
         finalized_block_root,
         &finalized_snapshot.beacon_block,
         &finalized_snapshot.beacon_state,
+        current_slot,
+        spec,
     )
     .map_err(|e| format!("Unable to reset fork choice for revert: {:?}", e))?;
 
@@ -158,12 +169,15 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
         complete_state_advance(&mut state, None, block.slot(), spec)
             .map_err(|e| format!("State advance failed: {:?}", e))?;
 
+        let mut ctxt = ConsensusContext::new(block.slot())
+            .set_proposer_index(block.message().proposer_index());
         per_block_processing(
             &mut state,
             &block,
-            None,
             BlockSignatureStrategy::NoVerification,
+            StateProcessingStrategy::Accurate,
             VerifyBlockRoot::True,
+            &mut ctxt,
             spec,
         )
         .map_err(|e| format!("Error replaying block: {:?}", e))?;
@@ -172,13 +186,12 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
         // retro-actively determine if they were valid or not.
         //
         // This scenario is so rare that it seems OK to double-verify some blocks.
-        let payload_verification_status = PayloadVerificationStatus::NotVerified;
+        let payload_verification_status = PayloadVerificationStatus::Optimistic;
 
-        let (block, _) = block.deconstruct();
         fork_choice
             .on_block(
                 block.slot(),
-                &block,
+                block.message(),
                 block.canonical_root(),
                 // Reward proposer boost. We are reinforcing the canonical chain.
                 Duration::from_secs(0),

beacon_node/beacon_chain/src/head_tracker.rs

@@ -1,4 +1,4 @@
-use parking_lot::RwLock;
+use parking_lot::{RwLock, RwLockReadGuard};
 use ssz_derive::{Decode, Encode};
 use std::collections::HashMap;
 use types::{Hash256, Slot};
@@ -16,6 +16,8 @@ pub enum Error {
 #[derive(Default, Debug)]
 pub struct HeadTracker(pub RwLock<HashMap<Hash256, Slot>>);
 
+pub type HeadTrackerReader<'a> = RwLockReadGuard<'a, HashMap<Hash256, Slot>>;
+
 impl HeadTracker {
     /// Register a block with `Self`, so it may or may not be included in a `Self::heads` call.
     ///
@@ -44,8 +46,13 @@ impl HeadTracker {
 
     /// Returns a `SszHeadTracker`, which contains all necessary information to restore the state
     /// of `Self` at some later point.
+    ///
+    /// Should ONLY be used for tests, due to the potential for database races.
+    ///
+    /// See <https://github.com/sigp/lighthouse/issues/4773>
+    #[cfg(test)]
     pub fn to_ssz_container(&self) -> SszHeadTracker {
-        SszHeadTracker::from_map(&*self.0.read())
+        SszHeadTracker::from_map(&self.0.read())
     }
 
     /// Creates a new `Self` from the given `SszHeadTracker`, restoring `Self` to the same state of
@@ -83,8 +90,8 @@ impl PartialEq<HeadTracker> for HeadTracker {
 /// This is used when persisting the state of the `BeaconChain` to disk.
 #[derive(Encode, Decode, Clone)]
 pub struct SszHeadTracker {
-    roots: Vec<Hash256>,
-    slots: Vec<Slot>,
+    pub roots: Vec<Hash256>,
+    pub slots: Vec<Slot>,
 }
 
 impl SszHeadTracker {

beacon_node/beacon_chain/src/historical_blocks.rs

@@ -1,3 +1,4 @@
+use crate::data_availability_checker::AvailableBlock;
 use crate::{errors::BeaconChainError as Error, metrics, BeaconChain, BeaconChainTypes};
 use itertools::Itertools;
 use slog::debug;
@@ -8,8 +9,8 @@ use state_processing::{
 use std::borrow::Cow;
 use std::iter;
 use std::time::Duration;
-use store::{chunked_vector::BlockRoots, AnchorInfo, ChunkWriter, KeyValueStore};
-use types::{Hash256, SignedBeaconBlock, Slot};
+use store::{get_key_for_col, AnchorInfo, BlobInfo, DBColumn, KeyValueStore, KeyValueStoreOp};
+use types::{Hash256, Slot};
 
 /// Use a longer timeout on the pubkey cache.
 ///
@@ -58,27 +59,30 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Return the number of blocks successfully imported.
     pub fn import_historical_block_batch(
         &self,
-        blocks: &[SignedBeaconBlock<T::EthSpec>],
+        mut blocks: Vec<AvailableBlock<T::EthSpec>>,
     ) -> Result<usize, Error> {
         let anchor_info = self
             .store
             .get_anchor_info()
             .ok_or(HistoricalBlockError::NoAnchorInfo)?;
+        let blob_info = self.store.get_blob_info();
 
         // Take all blocks with slots less than the oldest block slot.
-        let num_relevant =
-            blocks.partition_point(|block| block.slot() < anchor_info.oldest_block_slot);
-        let blocks_to_import = &blocks
-            .get(..num_relevant)
-            .ok_or(HistoricalBlockError::IndexOutOfBounds)?;
+        let num_relevant = blocks.partition_point(|available_block| {
+            available_block.block().slot() < anchor_info.oldest_block_slot
+        });
 
-        if blocks_to_import.len() != blocks.len() {
+        let total_blocks = blocks.len();
+        blocks.truncate(num_relevant);
+        let blocks_to_import = blocks;
+
+        if blocks_to_import.len() != total_blocks {
             debug!(
                 self.log,
                 "Ignoring some historic blocks";
                 "oldest_block_slot" => anchor_info.oldest_block_slot,
-                "total_blocks" => blocks.len(),
-                "ignored" => blocks.len().saturating_sub(blocks_to_import.len()),
+                "total_blocks" => total_blocks,
+                "ignored" => total_blocks.saturating_sub(blocks_to_import.len()),
             );
         }
 
@@ -86,17 +90,22 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             return Ok(0);
         }
 
+        let n_blobs_lists_to_import = blocks_to_import
+            .iter()
+            .filter(|available_block| available_block.blobs().is_some())
+            .count();
+
         let mut expected_block_root = anchor_info.oldest_block_parent;
         let mut prev_block_slot = anchor_info.oldest_block_slot;
-        let mut chunk_writer =
-            ChunkWriter::<BlockRoots, _, _>::new(&self.store.cold_db, prev_block_slot.as_usize())?;
 
-        let mut cold_batch = Vec::with_capacity(blocks.len());
-        let mut hot_batch = Vec::with_capacity(blocks.len());
+        let mut new_oldest_blob_slot = blob_info.oldest_blob_slot;
 
-        for block in blocks_to_import.iter().rev() {
-            // Check chain integrity.
-            let block_root = block.canonical_root();
+        let mut blob_batch = Vec::with_capacity(n_blobs_lists_to_import);
+        let mut cold_batch = Vec::with_capacity(blocks_to_import.len());
+        let mut signed_blocks = Vec::with_capacity(blocks_to_import.len());
+
+        for available_block in blocks_to_import.into_iter().rev() {
+            let (block_root, block, maybe_blobs) = available_block.deconstruct();
 
             if block_root != expected_block_root {
                 return Err(HistoricalBlockError::MismatchedBlockRoot {
@@ -106,32 +115,52 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 .into());
             }
 
-            // Store block in the hot database.
-            hot_batch.push(self.store.block_as_kv_store_op(&block_root, block));
+            let blinded_block = block.clone_as_blinded();
+            // Store block in the freezer database without payload.
+            self.store.blinded_block_as_cold_kv_store_ops(
+                &block_root,
+                &blinded_block,
+                &mut cold_batch,
+            )?;
+            // Store the blobs too
+            if let Some(blobs) = maybe_blobs {
+                new_oldest_blob_slot = Some(block.slot());
+                self.store
+                    .blobs_as_kv_store_ops(&block_root, blobs, &mut blob_batch);
+            }
 
             // Store block roots, including at all skip slots in the freezer DB.
-            for slot in (block.slot().as_usize()..prev_block_slot.as_usize()).rev() {
-                chunk_writer.set(slot, block_root, &mut cold_batch)?;
+            // The block root mapping for `block.slot()` itself was already written when the block
+            // was stored, above.
+            for slot in (block.slot().as_usize() + 1..prev_block_slot.as_usize()).rev() {
+                cold_batch.push(KeyValueStoreOp::PutKeyValue(
+                    get_key_for_col(DBColumn::BeaconBlockRoots.into(), &slot.to_be_bytes()),
+                    block_root.as_bytes().to_vec(),
+                ));
             }
 
             prev_block_slot = block.slot();
             expected_block_root = block.message().parent_root();
+            signed_blocks.push(block);
 
-            // If we've reached genesis, add the genesis block root to the batch and set the
-            // anchor slot to 0 to indicate completion.
+            // If we've reached genesis, add the genesis block root to the batch for all slots
+            // between 0 and the first block slot, and set the anchor slot to 0 to indicate
+            // completion.
             if expected_block_root == self.genesis_block_root {
                 let genesis_slot = self.spec.genesis_slot;
-                chunk_writer.set(
-                    genesis_slot.as_usize(),
-                    self.genesis_block_root,
-                    &mut cold_batch,
-                )?;
+                for slot in genesis_slot.as_u64()..prev_block_slot.as_u64() {
+                    cold_batch.push(KeyValueStoreOp::PutKeyValue(
+                        get_key_for_col(DBColumn::BeaconBlockRoots.into(), &slot.to_be_bytes()),
+                        self.genesis_block_root.as_bytes().to_vec(),
+                    ));
+                }
                 prev_block_slot = genesis_slot;
                 expected_block_root = Hash256::zero();
                 break;
             }
         }
-        chunk_writer.write(&mut cold_batch)?;
+        // these were pushed in reverse order so we reverse again
+        signed_blocks.reverse();
 
         // Verify signatures in one batch, holding the pubkey cache lock for the shortest duration
         // possible. For each block fetch the parent root from its successor. Slicing from index 1
@@ -142,15 +171,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .validator_pubkey_cache
             .try_read_for(PUBKEY_CACHE_LOCK_TIMEOUT)
             .ok_or(HistoricalBlockError::ValidatorPubkeyCacheTimeout)?;
-        let block_roots = blocks_to_import
+        let block_roots = signed_blocks
             .get(1..)
             .ok_or(HistoricalBlockError::IndexOutOfBounds)?
             .iter()
             .map(|block| block.parent_root())
             .chain(iter::once(anchor_info.oldest_block_parent));
-        let signature_set = blocks_to_import
+        let signature_set = signed_blocks
             .iter()
             .zip_eq(block_roots)
+            .filter(|&(_block, block_root)| (block_root != self.genesis_block_root))
             .map(|(block, block_root)| {
                 block_proposal_signature_set_from_parts(
                     block,
@@ -175,11 +205,31 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         drop(verify_timer);
         drop(sig_timer);
 
-        // Write the I/O batches to disk, writing the blocks themselves first, as it's better
-        // for the hot DB to contain extra blocks than for the cold DB to point to blocks that
-        // do not exist.
-        self.store.hot_db.do_atomically(hot_batch)?;
+        // Write the I/O batches to disk.
+        // We fsync after each write because we need the writes to the blob and freezer DB to
+        // be persisted if the writes to the hot DB are persisted. Without fsync we could end up
+        // in a situation where the hot DB's anchor is updated but the actual blocks are forgotten
+        // from disk.
+        self.store.blobs_db.do_atomically(blob_batch)?;
+        self.store.blobs_db.sync()?;
         self.store.cold_db.do_atomically(cold_batch)?;
+        self.store.cold_db.sync()?;
+
+        let mut anchor_and_blob_batch = Vec::with_capacity(2);
+
+        // Update the blob info.
+        if new_oldest_blob_slot != blob_info.oldest_blob_slot {
+            if let Some(oldest_blob_slot) = new_oldest_blob_slot {
+                let new_blob_info = BlobInfo {
+                    oldest_blob_slot: Some(oldest_blob_slot),
+                    ..blob_info.clone()
+                };
+                anchor_and_blob_batch.push(
+                    self.store
+                        .compare_and_set_blob_info(blob_info, new_blob_info)?,
+                );
+            }
+        }
 
         // Update the anchor.
         let new_anchor = AnchorInfo {
@@ -187,16 +237,24 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             oldest_block_parent: expected_block_root,
             ..anchor_info
         };
-        let backfill_complete = new_anchor.block_backfill_complete();
-        self.store
-            .compare_and_set_anchor_info_with_write(Some(anchor_info), Some(new_anchor))?;
+        let backfill_complete = new_anchor.block_backfill_complete(self.genesis_backfill_slot);
+        anchor_and_blob_batch.push(
+            self.store
+                .compare_and_set_anchor_info(Some(anchor_info), Some(new_anchor))?,
+        );
+        self.store.hot_db.do_atomically(anchor_and_blob_batch)?;
+        self.store.hot_db.sync()?;
 
         // If backfill has completed and the chain is configured to reconstruct historic states,
         // send a message to the background migrator instructing it to begin reconstruction.
-        if backfill_complete && self.config.reconstruct_historic_states {
+        // This can only happen if we have backfilled all the way to genesis.
+        if backfill_complete
+            && self.genesis_backfill_slot == Slot::new(0)
+            && self.config.reconstruct_historic_states
+        {
             self.store_migrator.process_reconstruction();
         }
 
-        Ok(blocks_to_import.len())
+        Ok(num_relevant)
     }
 }

beacon_node/beacon_chain/src/kzg_utils.rs

@@ -0,0 +1,78 @@
+use kzg::{Blob as KzgBlob, Error as KzgError, Kzg};
+use types::{Blob, EthSpec, Hash256, KzgCommitment, KzgProof};
+
+/// Converts a blob ssz List object to an array to be used with the kzg
+/// crypto library.
+fn ssz_blob_to_crypto_blob<E: EthSpec>(blob: &Blob<E>) -> Result<KzgBlob, KzgError> {
+    KzgBlob::from_bytes(blob.as_ref()).map_err(Into::into)
+}
+
+/// Validate a single blob-commitment-proof triplet from a `BlobSidecar`.
+pub fn validate_blob<E: EthSpec>(
+    kzg: &Kzg,
+    blob: &Blob<E>,
+    kzg_commitment: KzgCommitment,
+    kzg_proof: KzgProof,
+) -> Result<(), KzgError> {
+    let _timer = crate::metrics::start_timer(&crate::metrics::KZG_VERIFICATION_SINGLE_TIMES);
+    let kzg_blob = ssz_blob_to_crypto_blob::<E>(blob)?;
+    kzg.verify_blob_kzg_proof(&kzg_blob, kzg_commitment, kzg_proof)
+}
+
+/// Validate a batch of blob-commitment-proof triplets from multiple `BlobSidecars`.
+pub fn validate_blobs<E: EthSpec>(
+    kzg: &Kzg,
+    expected_kzg_commitments: &[KzgCommitment],
+    blobs: Vec<&Blob<E>>,
+    kzg_proofs: &[KzgProof],
+) -> Result<(), KzgError> {
+    let _timer = crate::metrics::start_timer(&crate::metrics::KZG_VERIFICATION_BATCH_TIMES);
+    let blobs = blobs
+        .into_iter()
+        .map(|blob| ssz_blob_to_crypto_blob::<E>(blob))
+        .collect::<Result<Vec<_>, KzgError>>()?;
+
+    kzg.verify_blob_kzg_proof_batch(&blobs, expected_kzg_commitments, kzg_proofs)
+}
+
+/// Compute the kzg proof given an ssz blob and its kzg commitment.
+pub fn compute_blob_kzg_proof<E: EthSpec>(
+    kzg: &Kzg,
+    blob: &Blob<E>,
+    kzg_commitment: KzgCommitment,
+) -> Result<KzgProof, KzgError> {
+    let kzg_blob = ssz_blob_to_crypto_blob::<E>(blob)?;
+    kzg.compute_blob_kzg_proof(&kzg_blob, kzg_commitment)
+}
+
+/// Compute the kzg commitment for a given blob.
+pub fn blob_to_kzg_commitment<E: EthSpec>(
+    kzg: &Kzg,
+    blob: &Blob<E>,
+) -> Result<KzgCommitment, KzgError> {
+    let kzg_blob = ssz_blob_to_crypto_blob::<E>(blob)?;
+    kzg.blob_to_kzg_commitment(&kzg_blob)
+}
+
+/// Compute the kzg proof for a given blob and an evaluation point z.
+pub fn compute_kzg_proof<E: EthSpec>(
+    kzg: &Kzg,
+    blob: &Blob<E>,
+    z: Hash256,
+) -> Result<(KzgProof, Hash256), KzgError> {
+    let z = z.0.into();
+    let kzg_blob = ssz_blob_to_crypto_blob::<E>(blob)?;
+    kzg.compute_kzg_proof(&kzg_blob, &z)
+        .map(|(proof, z)| (proof, Hash256::from_slice(&z.to_vec())))
+}
+
+/// Verify a `kzg_proof` for a `kzg_commitment` that evaluating a polynomial at `z` results in `y`
+pub fn verify_kzg_proof<E: EthSpec>(
+    kzg: &Kzg,
+    kzg_commitment: KzgCommitment,
+    kzg_proof: KzgProof,
+    z: Hash256,
+    y: Hash256,
+) -> Result<bool, KzgError> {
+    kzg.verify_kzg_proof(kzg_commitment, &z.0.into(), &y.0.into(), kzg_proof)
+}

beacon_node/beacon_chain/src/lib.rs

@@ -1,47 +1,70 @@
-#![recursion_limit = "128"] // For lazy-static
+pub mod attestation_rewards;
+pub mod attestation_simulator;
 pub mod attestation_verification;
 mod attester_cache;
+pub mod beacon_block_reward;
+mod beacon_block_streamer;
 mod beacon_chain;
 mod beacon_fork_choice_store;
-mod beacon_proposer_cache;
+pub mod beacon_proposer_cache;
 mod beacon_snapshot;
+pub mod blob_verification;
 pub mod block_reward;
 mod block_times_cache;
 mod block_verification;
+pub mod block_verification_types;
 pub mod builder;
+pub mod canonical_head;
+pub mod capella_readiness;
 pub mod chain_config;
+pub mod data_availability_checker;
+pub mod deneb_readiness;
 mod early_attester_cache;
+pub mod electra_readiness;
 mod errors;
 pub mod eth1_chain;
+mod eth1_finalization_cache;
 pub mod events;
-mod execution_payload;
+pub mod execution_payload;
+pub mod fork_choice_signal;
 pub mod fork_revert;
 mod head_tracker;
 pub mod historical_blocks;
-mod metrics;
+pub mod kzg_utils;
+pub mod light_client_finality_update_verification;
+pub mod light_client_optimistic_update_verification;
+mod light_client_server_cache;
+pub mod merge_readiness;
+pub mod metrics;
 pub mod migrate;
 mod naive_aggregation_pool;
 mod observed_aggregates;
 mod observed_attesters;
-mod observed_block_producers;
+mod observed_blob_sidecars;
+pub mod observed_block_producers;
 pub mod observed_operations;
+mod observed_slashable;
+pub mod otb_verification_service;
+mod parallel_state_cache;
 mod persisted_beacon_chain;
 mod persisted_fork_choice;
 mod pre_finalization_cache;
+pub mod proposer_prep_service;
 pub mod schema_change;
-mod shuffling_cache;
-mod snapshot_cache;
+pub mod shuffling_cache;
 pub mod state_advance_timer;
+pub mod sync_committee_rewards;
 pub mod sync_committee_verification;
 pub mod test_utils;
 mod timeout_rw_lock;
 pub mod validator_monitor;
-mod validator_pubkey_cache;
 
 pub use self::beacon_chain::{
-    AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult,
-    ForkChoiceError, HeadInfo, HeadSafetyStatus, StateSkipConfig, WhenSlotSkipped,
-    MAXIMUM_GOSSIP_CLOCK_DISPARITY,
+    AttestationProcessingOutcome, AvailabilityProcessingStatus, BeaconBlockResponse,
+    BeaconBlockResponseWrapper, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult,
+    ForkChoiceError, LightClientProducerEvent, OverrideForkchoiceUpdate, ProduceBlockVerification,
+    StateSkipConfig, WhenSlotSkipped, INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON,
+    INVALID_JUSTIFIED_PAYLOAD_SHUTDOWN_REASON,
 };
 pub use self::beacon_snapshot::BeaconSnapshot;
 pub use self::chain_config::ChainConfig;
@@ -49,10 +72,22 @@ pub use self::errors::{BeaconChainError, BlockProductionError};
 pub use self::historical_blocks::HistoricalBlockError;
 pub use attestation_verification::Error as AttestationError;
 pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError};
-pub use block_verification::{BlockError, ExecutionPayloadError, GossipVerifiedBlock};
+pub use block_verification::{
+    get_block_root, BlockError, ExecutionPayloadError, ExecutionPendingBlock, GossipVerifiedBlock,
+    IntoExecutionPendingBlock, IntoGossipVerifiedBlockContents, PayloadVerificationOutcome,
+    PayloadVerificationStatus,
+};
+pub use block_verification_types::AvailabilityPendingExecutedBlock;
+pub use block_verification_types::ExecutedBlock;
+pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock};
 pub use eth1_chain::{Eth1Chain, Eth1ChainBackend};
 pub use events::ServerSentEventHandler;
+pub use execution_layer::EngineState;
+pub use execution_payload::NotifyExecutionLayer;
+pub use fork_choice::{ExecutionStatus, ForkchoiceUpdateParameters};
+pub use kzg::TrustedSetup;
 pub use metrics::scrape_for_metrics;
+pub use migrate::MigratorConfig;
 pub use parking_lot;
 pub use slot_clock;
 pub use state_processing::per_block_processing::errors::{
@@ -62,3 +97,13 @@ pub use state_processing::per_block_processing::errors::{
 pub use store;
 pub use timeout_rw_lock::TimeoutRwLock;
 pub use types;
+
+pub mod validator_pubkey_cache {
+    use crate::BeaconChainTypes;
+
+    pub type ValidatorPubkeyCache<T> = store::ValidatorPubkeyCache<
+        <T as BeaconChainTypes>::EthSpec,
+        <T as BeaconChainTypes>::HotStore,
+        <T as BeaconChainTypes>::ColdStore,
+    >;
+}

beacon_node/beacon_chain/src/light_client_finality_update_verification.rs

@@ -0,0 +1,75 @@
+use crate::{BeaconChain, BeaconChainTypes};
+use derivative::Derivative;
+use slot_clock::SlotClock;
+use std::time::Duration;
+use strum::AsRefStr;
+use types::LightClientFinalityUpdate;
+
+/// Returned when a light client finality update was not successfully verified. It might not have been verified for
+/// two reasons:
+///
+/// - The light client finality message is malformed or inappropriate for the context (indicated by all variants
+///   other than `BeaconChainError`).
+/// - The application encountered an internal error whilst attempting to determine validity
+///   (the `BeaconChainError` variant)
+#[derive(Debug, AsRefStr)]
+pub enum Error {
+    /// The light client finality message was received is prior to one-third of slot duration passage. (with
+    /// respect to the gossip clock disparity and slot clock duration).
+    ///
+    /// ## Peer scoring
+    ///
+    /// Assuming the local clock is correct, the peer has sent an invalid message.
+    TooEarly,
+    /// Light client finality update message does not match the locally constructed one.
+    InvalidLightClientFinalityUpdate,
+    /// Signature slot start time is none.
+    SigSlotStartIsNone,
+    /// Failed to construct a LightClientFinalityUpdate from state.
+    FailedConstructingUpdate,
+}
+
+/// Wraps a `LightClientFinalityUpdate` that has been verified for propagation on the gossip network.
+#[derive(Derivative)]
+#[derivative(Clone(bound = "T: BeaconChainTypes"))]
+pub struct VerifiedLightClientFinalityUpdate<T: BeaconChainTypes> {
+    light_client_finality_update: LightClientFinalityUpdate<T::EthSpec>,
+    seen_timestamp: Duration,
+}
+
+impl<T: BeaconChainTypes> VerifiedLightClientFinalityUpdate<T> {
+    /// Returns `Ok(Self)` if the `light_client_finality_update` is valid to be (re)published on the gossip
+    /// network.
+    pub fn verify(
+        rcv_finality_update: LightClientFinalityUpdate<T::EthSpec>,
+        chain: &BeaconChain<T>,
+        seen_timestamp: Duration,
+    ) -> Result<Self, Error> {
+        // verify that enough time has passed for the block to have been propagated
+        let start_time = chain
+            .slot_clock
+            .start_of(*rcv_finality_update.signature_slot())
+            .ok_or(Error::SigSlotStartIsNone)?;
+        let one_third_slot_duration = Duration::new(chain.spec.seconds_per_slot / 3, 0);
+        if seen_timestamp + chain.spec.maximum_gossip_clock_disparity()
+            < start_time + one_third_slot_duration
+        {
+            return Err(Error::TooEarly);
+        }
+
+        let latest_finality_update = chain
+            .light_client_server_cache
+            .get_latest_finality_update()
+            .ok_or(Error::FailedConstructingUpdate)?;
+
+        // verify that the gossiped finality update is the same as the locally constructed one.
+        if latest_finality_update != rcv_finality_update {
+            return Err(Error::InvalidLightClientFinalityUpdate);
+        }
+
+        Ok(Self {
+            light_client_finality_update: rcv_finality_update,
+            seen_timestamp,
+        })
+    }
+}

beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs

@@ -0,0 +1,91 @@
+use crate::{BeaconChain, BeaconChainTypes};
+use derivative::Derivative;
+use eth2::types::Hash256;
+use slot_clock::SlotClock;
+use std::time::Duration;
+use strum::AsRefStr;
+use types::LightClientOptimisticUpdate;
+
+/// Returned when a light client optimistic update was not successfully verified. It might not have been verified for
+/// two reasons:
+///
+/// - The light client optimistic message is malformed or inappropriate for the context (indicated by all variants
+///   other than `BeaconChainError`).
+/// - The application encountered an internal error whilst attempting to determine validity
+///   (the `BeaconChainError` variant)
+#[derive(Debug, AsRefStr)]
+pub enum Error {
+    /// The light client optimistic message was received is prior to one-third of slot duration passage. (with
+    /// respect to the gossip clock disparity and slot clock duration).
+    ///
+    /// ## Peer scoring
+    ///
+    /// Assuming the local clock is correct, the peer has sent an invalid message.
+    TooEarly,
+    /// Light client optimistic update message does not match the locally constructed one.
+    InvalidLightClientOptimisticUpdate,
+    /// Signature slot start time is none.
+    SigSlotStartIsNone,
+    /// Failed to construct a LightClientOptimisticUpdate from state.
+    FailedConstructingUpdate,
+    /// Unknown block with parent root.
+    UnknownBlockParentRoot(Hash256),
+}
+
+/// Wraps a `LightClientOptimisticUpdate` that has been verified for propagation on the gossip network.
+#[derive(Derivative)]
+#[derivative(Clone(bound = "T: BeaconChainTypes"))]
+pub struct VerifiedLightClientOptimisticUpdate<T: BeaconChainTypes> {
+    light_client_optimistic_update: LightClientOptimisticUpdate<T::EthSpec>,
+    pub parent_root: Hash256,
+    seen_timestamp: Duration,
+}
+
+impl<T: BeaconChainTypes> VerifiedLightClientOptimisticUpdate<T> {
+    /// Returns `Ok(Self)` if the `light_client_optimistic_update` is valid to be (re)published on the gossip
+    /// network.
+    pub fn verify(
+        rcv_optimistic_update: LightClientOptimisticUpdate<T::EthSpec>,
+        chain: &BeaconChain<T>,
+        seen_timestamp: Duration,
+    ) -> Result<Self, Error> {
+        // verify that enough time has passed for the block to have been propagated
+        let start_time = chain
+            .slot_clock
+            .start_of(*rcv_optimistic_update.signature_slot())
+            .ok_or(Error::SigSlotStartIsNone)?;
+        let one_third_slot_duration = Duration::new(chain.spec.seconds_per_slot / 3, 0);
+        if seen_timestamp + chain.spec.maximum_gossip_clock_disparity()
+            < start_time + one_third_slot_duration
+        {
+            return Err(Error::TooEarly);
+        }
+
+        let head = chain.canonical_head.cached_head();
+        let head_block = &head.snapshot.beacon_block;
+        // check if we can process the optimistic update immediately
+        // otherwise queue
+        let canonical_root = rcv_optimistic_update.get_canonical_root();
+
+        if canonical_root != head_block.message().parent_root() {
+            return Err(Error::UnknownBlockParentRoot(canonical_root));
+        }
+
+        let latest_optimistic_update = chain
+            .light_client_server_cache
+            .get_latest_optimistic_update()
+            .ok_or(Error::FailedConstructingUpdate)?;
+
+        // verify that the gossiped optimistic update is the same as the locally constructed one.
+        if latest_optimistic_update != rcv_optimistic_update {
+            return Err(Error::InvalidLightClientOptimisticUpdate);
+        }
+
+        let parent_root = rcv_optimistic_update.get_parent_root();
+        Ok(Self {
+            light_client_optimistic_update: rcv_optimistic_update,
+            parent_root,
+            seen_timestamp,
+        })
+    }
+}

beacon_node/beacon_chain/src/light_client_server_cache.rs

@@ -0,0 +1,250 @@
+use crate::errors::BeaconChainError;
+use crate::{metrics, BeaconChainTypes, BeaconStore};
+use parking_lot::{Mutex, RwLock};
+use slog::{debug, Logger};
+use ssz_types::FixedVector;
+use std::num::NonZeroUsize;
+use types::light_client_update::{FinalizedRootProofLen, FINALIZED_ROOT_INDEX};
+use types::non_zero_usize::new_non_zero_usize;
+use types::{
+    BeaconBlockRef, BeaconState, ChainSpec, EthSpec, ForkName, Hash256, LightClientFinalityUpdate,
+    LightClientOptimisticUpdate, Slot, SyncAggregate,
+};
+
+/// A prev block cache miss requires to re-generate the state of the post-parent block. Items in the
+/// prev block cache are very small 32 * (6 + 1) = 224 bytes. 32 is an arbitrary number that
+/// represents unlikely re-orgs, while keeping the cache very small.
+const PREV_BLOCK_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(32);
+
+/// This cache computes light client messages ahead of time, required to satisfy p2p and API
+/// requests. These messages include proofs on historical states, so on-demand computation is
+/// expensive.
+///
+pub struct LightClientServerCache<T: BeaconChainTypes> {
+    /// Tracks a single global latest finality update out of all imported blocks.
+    ///
+    /// TODO: Active discussion with @etan-status if this cache should be fork aware to return
+    /// latest canonical (update with highest signature slot, where its attested header is part of
+    /// the head chain) instead of global latest (update with highest signature slot, out of all
+    /// branches).
+    latest_finality_update: RwLock<Option<LightClientFinalityUpdate<T::EthSpec>>>,
+    /// Tracks a single global latest optimistic update out of all imported blocks.
+    latest_optimistic_update: RwLock<Option<LightClientOptimisticUpdate<T::EthSpec>>>,
+    /// Caches state proofs by block root
+    prev_block_cache: Mutex<lru::LruCache<Hash256, LightClientCachedData>>,
+}
+
+impl<T: BeaconChainTypes> LightClientServerCache<T> {
+    pub fn new() -> Self {
+        Self {
+            latest_finality_update: None.into(),
+            latest_optimistic_update: None.into(),
+            prev_block_cache: lru::LruCache::new(PREV_BLOCK_CACHE_SIZE).into(),
+        }
+    }
+
+    /// Compute and cache state proofs for latter production of light-client messages. Does not
+    /// trigger block replay.
+    pub fn cache_state_data(
+        &self,
+        spec: &ChainSpec,
+        block: BeaconBlockRef<T::EthSpec>,
+        block_root: Hash256,
+        block_post_state: &mut BeaconState<T::EthSpec>,
+    ) -> Result<(), BeaconChainError> {
+        let _timer = metrics::start_timer(&metrics::LIGHT_CLIENT_SERVER_CACHE_STATE_DATA_TIMES);
+
+        // Only post-altair
+        if spec.fork_name_at_slot::<T::EthSpec>(block.slot()) == ForkName::Base {
+            return Ok(());
+        }
+
+        // Persist in memory cache for a descendent block
+
+        let cached_data = LightClientCachedData::from_state(block_post_state)?;
+        self.prev_block_cache.lock().put(block_root, cached_data);
+
+        Ok(())
+    }
+
+    /// Given a block with a SyncAggregte computes better or more recent light client updates. The
+    /// results are cached either on disk or memory to be served via p2p and rest API
+    pub fn recompute_and_cache_updates(
+        &self,
+        store: BeaconStore<T>,
+        block_parent_root: &Hash256,
+        block_slot: Slot,
+        sync_aggregate: &SyncAggregate<T::EthSpec>,
+        log: &Logger,
+        chain_spec: &ChainSpec,
+    ) -> Result<(), BeaconChainError> {
+        let _timer =
+            metrics::start_timer(&metrics::LIGHT_CLIENT_SERVER_CACHE_RECOMPUTE_UPDATES_TIMES);
+
+        let signature_slot = block_slot;
+        let attested_block_root = block_parent_root;
+
+        let attested_block = store.get_full_block(attested_block_root, None)?.ok_or(
+            BeaconChainError::DBInconsistent(format!(
+                "Block not available {:?}",
+                attested_block_root
+            )),
+        )?;
+
+        let cached_parts = self.get_or_compute_prev_block_cache(
+            store.clone(),
+            attested_block_root,
+            &attested_block.state_root(),
+            attested_block.slot(),
+        )?;
+
+        let attested_slot = attested_block.slot();
+
+        // Spec: Full nodes SHOULD provide the LightClientOptimisticUpdate with the highest
+        // attested_header.beacon.slot (if multiple, highest signature_slot) as selected by fork choice
+        let is_latest_optimistic = match &self.latest_optimistic_update.read().clone() {
+            Some(latest_optimistic_update) => {
+                is_latest_optimistic_update(latest_optimistic_update, attested_slot, signature_slot)
+            }
+            None => true,
+        };
+        if is_latest_optimistic {
+            // can create an optimistic update, that is more recent
+            *self.latest_optimistic_update.write() = Some(LightClientOptimisticUpdate::new(
+                &attested_block,
+                sync_aggregate.clone(),
+                signature_slot,
+                chain_spec,
+            )?);
+        };
+
+        // Spec: Full nodes SHOULD provide the LightClientFinalityUpdate with the highest
+        // attested_header.beacon.slot (if multiple, highest signature_slot) as selected by fork choice
+        let is_latest_finality = match &self.latest_finality_update.read().clone() {
+            Some(latest_finality_update) => {
+                is_latest_finality_update(latest_finality_update, attested_slot, signature_slot)
+            }
+            None => true,
+        };
+        if is_latest_finality & !cached_parts.finalized_block_root.is_zero() {
+            // Immediately after checkpoint sync the finalized block may not be available yet.
+            if let Some(finalized_block) =
+                store.get_full_block(&cached_parts.finalized_block_root, None)?
+            {
+                *self.latest_finality_update.write() = Some(LightClientFinalityUpdate::new(
+                    &attested_block,
+                    &finalized_block,
+                    cached_parts.finality_branch.clone(),
+                    sync_aggregate.clone(),
+                    signature_slot,
+                    chain_spec,
+                )?);
+            } else {
+                debug!(
+                    log,
+                    "Finalized block not available in store for light_client server";
+                    "finalized_block_root" => format!("{}", cached_parts.finalized_block_root),
+                );
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Retrieves prev block cached data from cache. If not present re-computes by retrieving the
+    /// parent state, and inserts an entry to the cache.
+    ///
+    /// In separate function since FnOnce of get_or_insert can not be fallible.
+    fn get_or_compute_prev_block_cache(
+        &self,
+        store: BeaconStore<T>,
+        block_root: &Hash256,
+        block_state_root: &Hash256,
+        block_slot: Slot,
+    ) -> Result<LightClientCachedData, BeaconChainError> {
+        // Attempt to get the value from the cache first.
+        if let Some(cached_parts) = self.prev_block_cache.lock().get(block_root) {
+            return Ok(cached_parts.clone());
+        }
+        metrics::inc_counter(&metrics::LIGHT_CLIENT_SERVER_CACHE_PREV_BLOCK_CACHE_MISS);
+
+        // Compute the value, handling potential errors.
+        let mut state = store
+            .get_state(block_state_root, Some(block_slot))?
+            .ok_or_else(|| {
+                BeaconChainError::DBInconsistent(format!("Missing state {:?}", block_state_root))
+            })?;
+        let new_value = LightClientCachedData::from_state(&mut state)?;
+
+        // Insert value and return owned
+        self.prev_block_cache
+            .lock()
+            .put(*block_root, new_value.clone());
+        Ok(new_value)
+    }
+
+    pub fn get_latest_finality_update(&self) -> Option<LightClientFinalityUpdate<T::EthSpec>> {
+        self.latest_finality_update.read().clone()
+    }
+
+    pub fn get_latest_optimistic_update(&self) -> Option<LightClientOptimisticUpdate<T::EthSpec>> {
+        self.latest_optimistic_update.read().clone()
+    }
+}
+
+impl<T: BeaconChainTypes> Default for LightClientServerCache<T> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+type FinalityBranch = FixedVector<Hash256, FinalizedRootProofLen>;
+
+#[derive(Clone)]
+struct LightClientCachedData {
+    finality_branch: FinalityBranch,
+    finalized_block_root: Hash256,
+}
+
+impl LightClientCachedData {
+    fn from_state<E: EthSpec>(state: &mut BeaconState<E>) -> Result<Self, BeaconChainError> {
+        Ok(Self {
+            finality_branch: state.compute_merkle_proof(FINALIZED_ROOT_INDEX)?.into(),
+            finalized_block_root: state.finalized_checkpoint().root,
+        })
+    }
+}
+
+// Implements spec prioritization rules:
+// > Full nodes SHOULD provide the LightClientFinalityUpdate with the highest attested_header.beacon.slot (if multiple, highest signature_slot)
+//
+// ref: https://github.com/ethereum/consensus-specs/blob/113c58f9bf9c08867f6f5f633c4d98e0364d612a/specs/altair/light-client/full-node.md#create_light_client_finality_update
+fn is_latest_finality_update<E: EthSpec>(
+    prev: &LightClientFinalityUpdate<E>,
+    attested_slot: Slot,
+    signature_slot: Slot,
+) -> bool {
+    let prev_slot = prev.get_attested_header_slot();
+    if attested_slot > prev_slot {
+        true
+    } else {
+        attested_slot == prev_slot && signature_slot > *prev.signature_slot()
+    }
+}
+
+// Implements spec prioritization rules:
+// > Full nodes SHOULD provide the LightClientOptimisticUpdate with the highest attested_header.beacon.slot (if multiple, highest signature_slot)
+//
+// ref: https://github.com/ethereum/consensus-specs/blob/113c58f9bf9c08867f6f5f633c4d98e0364d612a/specs/altair/light-client/full-node.md#create_light_client_optimistic_update
+fn is_latest_optimistic_update<E: EthSpec>(
+    prev: &LightClientOptimisticUpdate<E>,
+    attested_slot: Slot,
+    signature_slot: Slot,
+) -> bool {
+    let prev_slot = prev.get_slot();
+    if attested_slot > prev_slot {
+        true
+    } else {
+        attested_slot == prev_slot && signature_slot > *prev.signature_slot()
+    }
+}

beacon_node/beacon_chain/src/merge_readiness.rs

@@ -0,0 +1,281 @@
+//! Provides tools for checking if a node is ready for the Bellatrix upgrade and following merge
+//! transition.
+
+use crate::{BeaconChain, BeaconChainError as Error, BeaconChainTypes};
+use execution_layer::BlockByNumberQuery;
+use serde::{Deserialize, Serialize, Serializer};
+use slog::debug;
+use std::fmt;
+use std::fmt::Write;
+use types::*;
+
+/// The time before the Bellatrix fork when we will start issuing warnings about preparation.
+pub const SECONDS_IN_A_WEEK: u64 = 604800;
+pub const MERGE_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2;
+
+#[derive(Default, Debug, Serialize, Deserialize)]
+pub struct MergeConfig {
+    #[serde(serialize_with = "serialize_uint256")]
+    pub terminal_total_difficulty: Option<Uint256>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub terminal_block_hash: Option<ExecutionBlockHash>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub terminal_block_hash_epoch: Option<Epoch>,
+}
+
+impl fmt::Display for MergeConfig {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.terminal_block_hash.is_none()
+            && self.terminal_block_hash_epoch.is_none()
+            && self.terminal_total_difficulty.is_none()
+        {
+            return write!(
+                f,
+                "Merge terminal difficulty parameters not configured, check your config"
+            );
+        }
+        let mut display_string = String::new();
+        if let Some(terminal_total_difficulty) = self.terminal_total_difficulty {
+            write!(
+                display_string,
+                "terminal_total_difficulty: {},",
+                terminal_total_difficulty
+            )?;
+        }
+        if let Some(terminal_block_hash) = self.terminal_block_hash {
+            write!(
+                display_string,
+                "terminal_block_hash: {},",
+                terminal_block_hash
+            )?;
+        }
+        if let Some(terminal_block_hash_epoch) = self.terminal_block_hash_epoch {
+            write!(
+                display_string,
+                "terminal_block_hash_epoch: {},",
+                terminal_block_hash_epoch
+            )?;
+        }
+        write!(f, "{}", display_string.trim_end_matches(','))?;
+        Ok(())
+    }
+}
+impl MergeConfig {
+    /// Instantiate `self` from the values in a `ChainSpec`.
+    pub fn from_chainspec(spec: &ChainSpec) -> Self {
+        let mut params = MergeConfig::default();
+        if spec.terminal_total_difficulty != Uint256::max_value() {
+            params.terminal_total_difficulty = Some(spec.terminal_total_difficulty);
+        }
+        if spec.terminal_block_hash != ExecutionBlockHash::zero() {
+            params.terminal_block_hash = Some(spec.terminal_block_hash);
+        }
+        if spec.terminal_block_hash_activation_epoch != Epoch::max_value() {
+            params.terminal_block_hash_epoch = Some(spec.terminal_block_hash_activation_epoch);
+        }
+        params
+    }
+}
+
+/// Indicates if a node is ready for the Bellatrix upgrade and subsequent merge transition.
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(tag = "type")]
+pub enum MergeReadiness {
+    /// The node is ready, as far as we can tell.
+    Ready {
+        config: MergeConfig,
+        #[serde(serialize_with = "serialize_uint256")]
+        current_difficulty: Option<Uint256>,
+    },
+    /// The EL can be reached and has the correct configuration, however it's not yet synced.
+    NotSynced,
+    /// The user has not configured this node to use an execution endpoint.
+    NoExecutionEndpoint,
+}
+
+impl fmt::Display for MergeReadiness {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            MergeReadiness::Ready {
+                config: params,
+                current_difficulty,
+            } => {
+                write!(
+                    f,
+                    "This node appears ready for the merge. \
+                        Params: {}, current_difficulty: {:?}",
+                    params, current_difficulty
+                )
+            }
+            MergeReadiness::NotSynced => write!(
+                f,
+                "The execution endpoint is connected and configured, \
+                    however it is not yet synced"
+            ),
+            MergeReadiness::NoExecutionEndpoint => write!(
+                f,
+                "The --execution-endpoint flag is not specified, this is a \
+                    requirement for the merge"
+            ),
+        }
+    }
+}
+
+pub enum GenesisExecutionPayloadStatus {
+    Correct(ExecutionBlockHash),
+    BlockHashMismatch {
+        got: ExecutionBlockHash,
+        expected: ExecutionBlockHash,
+    },
+    TransactionsRootMismatch {
+        got: Hash256,
+        expected: Hash256,
+    },
+    WithdrawalsRootMismatch {
+        got: Hash256,
+        expected: Hash256,
+    },
+    OtherMismatch,
+    Irrelevant,
+    AlreadyHappened,
+}
+
+impl<T: BeaconChainTypes> BeaconChain<T> {
+    /// Returns `true` if user has an EL configured, or if the Bellatrix fork has occurred or will
+    /// occur within `MERGE_READINESS_PREPARATION_SECONDS`.
+    pub fn is_time_to_prepare_for_bellatrix(&self, current_slot: Slot) -> bool {
+        if let Some(bellatrix_epoch) = self.spec.bellatrix_fork_epoch {
+            let bellatrix_slot = bellatrix_epoch.start_slot(T::EthSpec::slots_per_epoch());
+            let merge_readiness_preparation_slots =
+                MERGE_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot;
+
+            if self.execution_layer.is_some() {
+                // The user has already configured an execution layer, start checking for readiness
+                // right away.
+                true
+            } else {
+                // Return `true` if Bellatrix has happened or is within the preparation time.
+                current_slot + merge_readiness_preparation_slots > bellatrix_slot
+            }
+        } else {
+            // The Bellatrix fork epoch has not been defined yet, no need to prepare.
+            false
+        }
+    }
+
+    /// Attempts to connect to the EL and confirm that it is ready for the merge.
+    pub async fn check_merge_readiness(&self, current_slot: Slot) -> MergeReadiness {
+        if let Some(el) = self.execution_layer.as_ref() {
+            if !el.is_synced_for_notifier(current_slot).await {
+                // The EL is not synced.
+                return MergeReadiness::NotSynced;
+            }
+            let params = MergeConfig::from_chainspec(&self.spec);
+            let current_difficulty = el.get_current_difficulty().await.ok();
+            MergeReadiness::Ready {
+                config: params,
+                current_difficulty,
+            }
+        } else {
+            // There is no EL configured.
+            MergeReadiness::NoExecutionEndpoint
+        }
+    }
+
+    /// Check that the execution payload embedded in the genesis state matches the EL's genesis
+    /// block.
+    pub async fn check_genesis_execution_payload_is_correct(
+        &self,
+    ) -> Result<GenesisExecutionPayloadStatus, Error> {
+        let head_snapshot = self.head_snapshot();
+        let genesis_state = &head_snapshot.beacon_state;
+
+        if genesis_state.slot() != 0 {
+            return Ok(GenesisExecutionPayloadStatus::AlreadyHappened);
+        }
+
+        let Ok(latest_execution_payload_header) = genesis_state.latest_execution_payload_header()
+        else {
+            return Ok(GenesisExecutionPayloadStatus::Irrelevant);
+        };
+        let fork = self.spec.fork_name_at_epoch(Epoch::new(0));
+
+        let execution_layer = self
+            .execution_layer
+            .as_ref()
+            .ok_or(Error::ExecutionLayerMissing)?;
+        let exec_block_hash = latest_execution_payload_header.block_hash();
+
+        // Use getBlockByNumber(0) to check that the block hash matches.
+        // At present, Geth does not respond to engine_getPayloadBodiesByRange before genesis.
+        let execution_block = execution_layer
+            .get_block_by_number(BlockByNumberQuery::Tag("0x0"))
+            .await
+            .map_err(|e| Error::ExecutionLayerGetBlockByNumberFailed(Box::new(e)))?
+            .ok_or(Error::BlockHashMissingFromExecutionLayer(exec_block_hash))?;
+
+        if execution_block.block_hash != exec_block_hash {
+            return Ok(GenesisExecutionPayloadStatus::BlockHashMismatch {
+                got: execution_block.block_hash,
+                expected: exec_block_hash,
+            });
+        }
+
+        // Double-check the block by reconstructing it.
+        let execution_payload = execution_layer
+            .get_payload_by_hash_legacy(exec_block_hash, fork)
+            .await
+            .map_err(|e| Error::ExecutionLayerGetBlockByHashFailed(Box::new(e)))?
+            .ok_or(Error::BlockHashMissingFromExecutionLayer(exec_block_hash))?;
+
+        // Verify payload integrity.
+        let header_from_payload = ExecutionPayloadHeader::from(execution_payload.to_ref());
+
+        let got_transactions_root = header_from_payload.transactions_root();
+        let expected_transactions_root = latest_execution_payload_header.transactions_root();
+        let got_withdrawals_root = header_from_payload.withdrawals_root().ok();
+        let expected_withdrawals_root = latest_execution_payload_header.withdrawals_root().ok();
+
+        if got_transactions_root != expected_transactions_root {
+            return Ok(GenesisExecutionPayloadStatus::TransactionsRootMismatch {
+                got: got_transactions_root,
+                expected: expected_transactions_root,
+            });
+        }
+
+        if let Some(&expected) = expected_withdrawals_root {
+            if let Some(&got) = got_withdrawals_root {
+                if got != expected {
+                    return Ok(GenesisExecutionPayloadStatus::WithdrawalsRootMismatch {
+                        got,
+                        expected,
+                    });
+                }
+            }
+        }
+
+        if header_from_payload.to_ref() != latest_execution_payload_header {
+            debug!(
+                self.log,
+                "Genesis execution payload reconstruction failure";
+                "consensus_node_header" => ?latest_execution_payload_header,
+                "execution_node_header" => ?header_from_payload
+            );
+            return Ok(GenesisExecutionPayloadStatus::OtherMismatch);
+        }
+
+        Ok(GenesisExecutionPayloadStatus::Correct(exec_block_hash))
+    }
+}
+
+/// Utility function to serialize a Uint256 as a decimal string.
+fn serialize_uint256<S>(val: &Option<Uint256>, s: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    match val {
+        Some(v) => v.to_string().serialize(s),
+        None => s.serialize_none(),
+    }
+}

beacon_node/beacon_chain/src/metrics.rs

@@ -4,11 +4,21 @@ use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
 use lazy_static::lazy_static;
 pub use lighthouse_metrics::*;
 use slot_clock::SlotClock;
-use std::time::Duration;
 use types::{BeaconState, Epoch, EthSpec, Hash256, Slot};
 
-/// The maximum time to wait for the snapshot cache lock during a metrics scrape.
-const SNAPSHOT_CACHE_TIMEOUT: Duration = Duration::from_millis(100);
+// Attestation simulator metrics
+pub const VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_HEAD_ATTESTER_HIT_TOTAL: &str =
+    "validator_monitor_attestation_simulator_head_attester_hit_total";
+pub const VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_HEAD_ATTESTER_MISS_TOTAL: &str =
+    "validator_monitor_attestation_simulator_head_attester_miss_total";
+pub const VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_TARGET_ATTESTER_HIT_TOTAL: &str =
+    "validator_monitor_attestation_simulator_target_attester_hit_total";
+pub const VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_TARGET_ATTESTER_MISS_TOTAL: &str =
+    "validator_monitor_attestation_simulator_target_attester_miss_total";
+pub const VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_SOURCE_ATTESTER_HIT_TOTAL: &str =
+    "validator_monitor_attestation_simulator_source_attester_hit_total";
+pub const VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_SOURCE_ATTESTER_MISS_TOTAL: &str =
+    "validator_monitor_attestation_simulator_source_attester_miss_total";
 
 lazy_static! {
     /*
@@ -40,6 +50,14 @@ lazy_static! {
         "beacon_block_processing_block_root_seconds",
         "Time spent calculating the block root when processing a block."
     );
+    pub static ref BLOCK_HEADER_PROCESSING_BLOCK_ROOT: Result<Histogram> = try_create_histogram(
+        "beacon_block_header_processing_block_root_seconds",
+        "Time spent calculating the block root for a beacon block header."
+    );
+    pub static ref BLOCK_PROCESSING_BLOB_ROOT: Result<Histogram> = try_create_histogram(
+        "beacon_block_processing_blob_root_seconds",
+        "Time spent calculating the blob root when processing a block."
+    );
     pub static ref BLOCK_PROCESSING_DB_READ: Result<Histogram> = try_create_histogram(
         "beacon_block_processing_db_read_seconds",
         "Time spent loading block and state from DB for block processing"
@@ -64,6 +82,11 @@ lazy_static! {
         "beacon_block_processing_state_root_seconds",
         "Time spent calculating the state root when processing a block."
     );
+    pub static ref BLOCK_PROCESSING_POST_EXEC_PROCESSING: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_block_processing_post_exec_pre_attestable_seconds",
+        "Time between finishing execution processing and the block becoming attestable",
+        linear_buckets(5e-3, 5e-3, 10)
+    );
     pub static ref BLOCK_PROCESSING_DB_WRITE: Result<Histogram> = try_create_histogram(
         "beacon_block_processing_db_write_seconds",
         "Time spent writing a newly processed block and state to DB"
@@ -72,6 +95,11 @@ lazy_static! {
         "beacon_block_processing_attestation_observation_seconds",
         "Time spent hashing and remembering all the attestations in the block"
     );
+    pub static ref BLOCK_PROCESSING_FORK_CHOICE: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_block_processing_fork_choice_seconds",
+        "Time spent running fork choice's `get_head` during block import",
+        exponential_buckets(1e-3, 2.0, 8)
+    );
     pub static ref BLOCK_SYNC_AGGREGATE_SET_BITS: Result<IntGauge> = try_create_int_gauge(
         "block_sync_aggregate_set_bits",
         "The number of true bits in the last sync aggregate in a block"
@@ -90,6 +118,15 @@ lazy_static! {
     );
     pub static ref BLOCK_PRODUCTION_TIMES: Result<Histogram> =
         try_create_histogram("beacon_block_production_seconds", "Full runtime of block production");
+    pub static ref BLOCK_PRODUCTION_FORK_CHOICE_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_fork_choice_seconds",
+        "Time taken to run fork choice before block production"
+    );
+    pub static ref BLOCK_PRODUCTION_GET_PROPOSER_HEAD_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_block_production_get_proposer_head_times",
+        "Time taken for fork choice to compute the proposer head before block production",
+        exponential_buckets(1e-3, 2.0, 8)
+    );
     pub static ref BLOCK_PRODUCTION_STATE_LOAD_TIMES: Result<Histogram> = try_create_histogram(
         "beacon_block_production_state_load_seconds",
         "Time taken to load the base state for block production"
@@ -118,14 +155,17 @@ lazy_static! {
     /*
      * Block Statistics
      */
-    pub static ref OPERATIONS_PER_BLOCK_ATTESTATION: Result<Histogram> = try_create_histogram(
+    pub static ref OPERATIONS_PER_BLOCK_ATTESTATION: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_operations_per_block_attestation_total",
-        "Number of attestations in a block"
+        "Number of attestations in a block",
+        // Full block is 128.
+        Ok(vec![0_f64, 1_f64, 3_f64, 15_f64, 31_f64, 63_f64, 127_f64, 255_f64])
     );
 
-    pub static ref BLOCK_SIZE: Result<Histogram> = try_create_histogram(
+    pub static ref BLOCK_SIZE: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_total_size",
-        "Size of a signed beacon block"
+        "Size of a signed beacon block",
+        linear_buckets(5120_f64,5120_f64,10)
     );
 
     /*
@@ -247,6 +287,10 @@ lazy_static! {
         try_create_int_counter("beacon_shuffling_cache_hits_total", "Count of times shuffling cache fulfils request");
     pub static ref SHUFFLING_CACHE_MISSES: Result<IntCounter> =
         try_create_int_counter("beacon_shuffling_cache_misses_total", "Count of times shuffling cache fulfils request");
+    pub static ref SHUFFLING_CACHE_PROMISE_HITS: Result<IntCounter> =
+        try_create_int_counter("beacon_shuffling_cache_promise_hits_total", "Count of times shuffling cache returns a promise to future shuffling");
+    pub static ref SHUFFLING_CACHE_PROMISE_FAILS: Result<IntCounter> =
+        try_create_int_counter("beacon_shuffling_cache_promise_fails_total", "Count of times shuffling cache detects a failed promise");
 
     /*
      * Early attester cache
@@ -256,6 +300,11 @@ lazy_static! {
         "Count of times the early attester cache returns an attestation"
     );
 
+}
+
+// Second lazy-static block is used to account for macro recursion limit.
+lazy_static! {
+
     /*
      * Attestation Production
      */
@@ -275,10 +324,7 @@ lazy_static! {
         "attestation_production_cache_prime_seconds",
         "Time spent loading a new state from the disk due to a cache miss"
     );
-}
 
-// Second lazy-static block is used to account for macro recursion limit.
-lazy_static! {
     /*
      * Fork Choice
      */
@@ -298,14 +344,34 @@ lazy_static! {
         "beacon_fork_choice_reorg_total",
         "Count of occasions fork choice has switched to a different chain"
     );
+    pub static ref FORK_CHOICE_REORG_DISTANCE: Result<IntGauge> = try_create_int_gauge(
+        "beacon_fork_choice_reorg_distance",
+        "The distance of each re-org of the fork choice algorithm"
+    );
     pub static ref FORK_CHOICE_REORG_COUNT_INTEROP: Result<IntCounter> = try_create_int_counter(
         "beacon_reorgs_total",
         "Count of occasions fork choice has switched to a different chain"
     );
-    pub static ref FORK_CHOICE_TIMES: Result<Histogram> =
-        try_create_histogram("beacon_fork_choice_seconds", "Full runtime of fork choice");
-    pub static ref FORK_CHOICE_FIND_HEAD_TIMES: Result<Histogram> =
-        try_create_histogram("beacon_fork_choice_find_head_seconds", "Full runtime of fork choice find_head function");
+    pub static ref FORK_CHOICE_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_fork_choice_seconds",
+        "Full runtime of fork choice",
+        linear_buckets(10e-3, 20e-3, 10)
+    );
+    pub static ref FORK_CHOICE_OVERRIDE_FCU_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_fork_choice_override_fcu_seconds",
+        "Time taken to compute the optional forkchoiceUpdated override",
+        exponential_buckets(1e-3, 2.0, 8)
+    );
+    pub static ref FORK_CHOICE_AFTER_NEW_HEAD_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_fork_choice_after_new_head_seconds",
+        "Time taken to run `after_new_head`",
+        exponential_buckets(1e-3, 2.0, 10)
+    );
+    pub static ref FORK_CHOICE_AFTER_FINALIZATION_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_fork_choice_after_finalization_seconds",
+        "Time taken to run `after_finalization`",
+        exponential_buckets(1e-3, 2.0, 10)
+    );
     pub static ref FORK_CHOICE_PROCESS_BLOCK_TIMES: Result<Histogram> = try_create_histogram(
         "beacon_fork_choice_process_block_seconds",
         "Time taken to add a block and all attestations to fork choice"
@@ -321,7 +387,7 @@ lazy_static! {
     pub static ref BALANCES_CACHE_HITS: Result<IntCounter> =
         try_create_int_counter("beacon_balances_cache_hits_total", "Count of times balances cache fulfils request");
     pub static ref BALANCES_CACHE_MISSES: Result<IntCounter> =
-        try_create_int_counter("beacon_balances_cache_misses_total", "Count of times balances cache fulfils request");
+        try_create_int_counter("beacon_balances_cache_misses_total", "Count of times balances cache misses request");
 
     /*
      * Persisting BeaconChain components to disk
@@ -334,6 +400,8 @@ lazy_static! {
         try_create_histogram("beacon_persist_eth1_cache", "Time taken to persist the eth1 caches");
     pub static ref PERSIST_FORK_CHOICE: Result<Histogram> =
         try_create_histogram("beacon_persist_fork_choice", "Time taken to persist the fork choice struct");
+    pub static ref PERSIST_DATA_AVAILABILITY_CHECKER: Result<Histogram> =
+        try_create_histogram("beacon_persist_data_availability_checker", "Time taken to persist the data availability checker");
 
     /*
      * Eth1
@@ -767,31 +835,62 @@ lazy_static! {
         "Number of attester slashings seen",
         &["src", "validator"]
     );
+}
 
+// Fourth lazy-static block is used to account for macro recursion limit.
+lazy_static! {
     /*
      * Block Delay Metrics
      */
-    pub static ref BEACON_BLOCK_OBSERVED_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram(
+    pub static ref BEACON_BLOCK_OBSERVED_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_observed_slot_start_delay_time",
         "Duration between the start of the block's slot and the time the block was observed.",
+        // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]
+        decimal_buckets(-1,2)
     );
-    pub static ref BEACON_BLOCK_IMPORTED_OBSERVED_DELAY_TIME: Result<Histogram> = try_create_histogram(
+    pub static ref BEACON_BLOCK_IMPORTED_OBSERVED_DELAY_TIME: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_imported_observed_delay_time",
         "Duration between the time the block was observed and the time when it was imported.",
+        // [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5]
+        decimal_buckets(-2,0)
     );
-    pub static ref BEACON_BLOCK_HEAD_IMPORTED_DELAY_TIME: Result<Histogram> = try_create_histogram(
+    pub static ref BEACON_BLOCK_HEAD_IMPORTED_DELAY_TIME: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_head_imported_delay_time",
         "Duration between the time the block was imported and the time when it was set as head.",
+        // [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5]
+        decimal_buckets(-2,-1)
     );
-    pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram(
+    pub static ref BEACON_BLOCK_HEAD_ATTESTABLE_DELAY_TIME: Result<Histogram> = try_create_histogram(
+        "beacon_block_head_attestable_delay_time",
+        "Duration between the start of the slot and the time at which the block could be attested to.",
+    );
+    pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_head_slot_start_delay_time",
         "Duration between the start of the block's slot and the time when it was set as head.",
+        // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]
+        decimal_buckets(-1,2)
     );
     pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_EXCEEDED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_block_head_slot_start_delay_exceeded_total",
         "Triggered when the duration between the start of the block's slot and the current time \
         will result in failed attestations.",
     );
+    pub static ref BEACON_BLOCK_HEAD_MISSED_ATT_DEADLINE_LATE: Result<IntCounter> = try_create_int_counter(
+        "beacon_block_head_missed_att_deadline_late",
+        "Total number of delayed head blocks that arrived late"
+    );
+    pub static ref BEACON_BLOCK_HEAD_MISSED_ATT_DEADLINE_BORDERLINE: Result<IntCounter> = try_create_int_counter(
+        "beacon_block_head_missed_att_deadline_borderline",
+        "Total number of delayed head blocks that arrived very close to the deadline"
+    );
+    pub static ref BEACON_BLOCK_HEAD_MISSED_ATT_DEADLINE_SLOW: Result<IntCounter> = try_create_int_counter(
+        "beacon_block_head_missed_att_deadline_slow",
+        "Total number of delayed head blocks that arrived on time but not processed in time"
+    );
+    pub static ref BEACON_BLOCK_HEAD_MISSED_ATT_DEADLINE_OTHER: Result<IntCounter> = try_create_int_counter(
+        "beacon_block_head_missed_att_deadline_other",
+        "Total number of delayed head blocks that were not late and not slow to process"
+    );
 
     /*
      * General block metrics
@@ -801,10 +900,7 @@ lazy_static! {
             "gossip_beacon_block_skipped_slots",
             "For each gossip blocks, the number of skip slots between it and its parent"
         );
-}
 
-// Fourth lazy-static block is used to account for macro recursion limit.
-lazy_static! {
     /*
      * Sync Committee Message Verification
      */
@@ -820,6 +916,14 @@ lazy_static! {
         "beacon_sync_committee_message_gossip_verification_seconds",
         "Full runtime of sync contribution gossip verification"
     );
+    pub static ref SYNC_MESSAGE_EQUIVOCATIONS: Result<IntCounter> = try_create_int_counter(
+        "sync_message_equivocations_total",
+        "Number of sync messages with the same validator index for different blocks"
+    );
+    pub static ref SYNC_MESSAGE_EQUIVOCATIONS_TO_HEAD: Result<IntCounter> = try_create_int_counter(
+        "sync_message_equivocations_to_head_total",
+        "Number of sync message which conflict with a previous message but elect the head"
+    );
 
     /*
      * Sync Committee Contribution Verification
@@ -918,6 +1022,169 @@ lazy_static! {
             "beacon_pre_finalization_block_lookup_count",
             "Number of block roots subject to single block lookups"
         );
+
+    /*
+     * Blob sidecar Verification
+     */
+    pub static ref BLOBS_SIDECAR_PROCESSING_REQUESTS: Result<IntCounter> = try_create_int_counter(
+        "beacon_blobs_sidecar_processing_requests_total",
+        "Count of all blob sidecars submitted for processing"
+    );
+    pub static ref BLOBS_SIDECAR_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
+        "beacon_blobs_sidecar_processing_successes_total",
+        "Number of blob sidecars verified for gossip"
+    );
+    pub static ref BLOBS_SIDECAR_GOSSIP_VERIFICATION_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_blobs_sidecar_gossip_verification_seconds",
+        "Full runtime of blob sidecars gossip verification"
+    );
+    pub static ref BLOB_SIDECAR_INCLUSION_PROOF_VERIFICATION: Result<Histogram> = try_create_histogram(
+        "blob_sidecar_inclusion_proof_verification_seconds",
+        "Time taken to verify blob sidecar inclusion proof"
+    );
+    pub static ref BLOB_SIDECAR_INCLUSION_PROOF_COMPUTATION: Result<Histogram> = try_create_histogram(
+        "blob_sidecar_inclusion_proof_computation_seconds",
+        "Time taken to compute blob sidecar inclusion proof"
+    );
+}
+
+// Fifth lazy-static block is used to account for macro recursion limit.
+lazy_static! {
+    /*
+    * Light server message verification
+    */
+    pub static ref FINALITY_UPDATE_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
+        "light_client_finality_update_verification_success_total",
+        "Number of light client finality updates verified for gossip"
+    );
+    /*
+    * Light server message verification
+    */
+    pub static ref OPTIMISTIC_UPDATE_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
+        "light_client_optimistic_update_verification_success_total",
+        "Number of light client optimistic updates verified for gossip"
+    );
+    /*
+    * Aggregate subset metrics
+     */
+    pub static ref SYNC_CONTRIBUTION_SUBSETS: Result<IntCounter> = try_create_int_counter(
+        "beacon_sync_contribution_subsets_total",
+        "Count of new sync contributions that are subsets of already known aggregates"
+    );
+    pub static ref AGGREGATED_ATTESTATION_SUBSETS: Result<IntCounter> = try_create_int_counter(
+        "beacon_aggregated_attestation_subsets_total",
+        "Count of new aggregated attestations that are subsets of already known aggregates"
+    );
+    /*
+    * Attestation simulator metrics
+     */
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_HEAD_ATTESTER_HIT: Result<IntCounter> =
+    try_create_int_counter(
+        VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_HEAD_ATTESTER_HIT_TOTAL,
+        "Incremented if a validator is flagged as a previous slot head attester \
+        during per slot processing",
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_HEAD_ATTESTER_MISS: Result<IntCounter> =
+    try_create_int_counter(
+        VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_HEAD_ATTESTER_MISS_TOTAL,
+        "Incremented if a validator is not flagged as a previous slot head attester \
+        during per slot processing",
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_TARGET_ATTESTER_HIT: Result<IntCounter> =
+    try_create_int_counter(
+        VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_TARGET_ATTESTER_HIT_TOTAL,
+        "Incremented if a validator is flagged as a previous slot target attester \
+        during per slot processing",
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_TARGET_ATTESTER_MISS: Result<IntCounter> =
+    try_create_int_counter(
+        VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_TARGET_ATTESTER_MISS_TOTAL,
+        "Incremented if a validator is not flagged as a previous slot target attester \
+        during per slot processing",
+    );
+        pub static ref VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_SOURCE_ATTESTER_HIT: Result<IntCounter> =
+    try_create_int_counter(
+        VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_SOURCE_ATTESTER_HIT_TOTAL,
+        "Incremented if a validator is flagged as a previous slot source attester \
+        during per slot processing",
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_SOURCE_ATTESTER_MISS: Result<IntCounter> =
+    try_create_int_counter(
+        VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_SOURCE_ATTESTER_MISS_TOTAL,
+        "Incremented if a validator is not flagged as a previous slot source attester \
+        during per slot processing",
+    );
+    /*
+     * Missed block metrics
+     */
+    pub static ref VALIDATOR_MONITOR_MISSED_BLOCKS_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_missed_blocks_total",
+        "Number of non-finalized blocks missed",
+        &["validator"]
+    );
+
+    /*
+    * Kzg related metrics
+    */
+    pub static ref KZG_VERIFICATION_SINGLE_TIMES: Result<Histogram> =
+        try_create_histogram("kzg_verification_single_seconds", "Runtime of single kzg verification");
+    pub static ref KZG_VERIFICATION_BATCH_TIMES: Result<Histogram> =
+        try_create_histogram("kzg_verification_batch_seconds", "Runtime of batched kzg verification");
+
+    pub static ref BLOCK_PRODUCTION_BLOBS_VERIFICATION_TIMES: Result<Histogram> = try_create_histogram(
+            "beacon_block_production_blobs_verification_seconds",
+            "Time taken to verify blobs against commitments and creating BlobSidecar objects in block production"
+    );
+    /*
+    * Availability related metrics
+    */
+    pub static ref BLOCK_AVAILABILITY_DELAY: Result<Histogram> = try_create_histogram_with_buckets(
+        "block_availability_delay",
+        "Duration between start of the slot and the time at which all components of the block are available.",
+        // Create a custom bucket list for greater granularity in block delay
+        Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0])
+    );
+
+
+    /*
+    * Data Availability cache metrics
+    */
+    pub static ref DATA_AVAILABILITY_PROCESSING_CACHE_SIZE: Result<IntGauge> =
+        try_create_int_gauge(
+            "data_availability_processing_cache_size",
+            "Number of entries in the data availability processing cache."
+        );
+    pub static ref DATA_AVAILABILITY_OVERFLOW_MEMORY_BLOCK_CACHE_SIZE: Result<IntGauge> =
+        try_create_int_gauge(
+            "data_availability_overflow_memory_block_cache_size",
+            "Number of entries in the data availability overflow block memory cache."
+        );
+    pub static ref DATA_AVAILABILITY_OVERFLOW_MEMORY_STATE_CACHE_SIZE: Result<IntGauge> =
+        try_create_int_gauge(
+            "data_availability_overflow_memory_state_cache_size",
+            "Number of entries in the data availability overflow state memory cache."
+        );
+    pub static ref DATA_AVAILABILITY_OVERFLOW_STORE_CACHE_SIZE: Result<IntGauge> =
+        try_create_int_gauge(
+            "data_availability_overflow_store_cache_size",
+            "Number of entries in the data availability overflow store cache."
+        );
+
+    /*
+    * light_client server metrics
+    */
+    pub static ref LIGHT_CLIENT_SERVER_CACHE_STATE_DATA_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_light_client_server_cache_state_data_seconds",
+        "Time taken to produce and cache state data",
+    );
+    pub static ref LIGHT_CLIENT_SERVER_CACHE_RECOMPUTE_UPDATES_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_light_client_server_cache_recompute_updates_seconds",
+        "Time taken to recompute and cache updates",
+    );
+    pub static ref LIGHT_CLIENT_SERVER_CACHE_PREV_BLOCK_CACHE_MISS: Result<IntCounter> = try_create_int_counter(
+        "beacon_light_client_server_cache_prev_block_cache_miss",
+        "Count of prev block cache misses",
+    );
 }
 
 /// Scrape the `beacon_chain` for metrics that are not constantly updated (e.g., the present slot,
@@ -935,15 +1202,28 @@ pub fn scrape_for_metrics<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
 
     let attestation_stats = beacon_chain.op_pool.attestation_stats();
 
-    if let Some(snapshot_cache) = beacon_chain
-        .snapshot_cache
-        .try_write_for(SNAPSHOT_CACHE_TIMEOUT)
-    {
-        set_gauge(
-            &BLOCK_PROCESSING_SNAPSHOT_CACHE_SIZE,
-            snapshot_cache.len() as i64,
-        )
-    }
+    set_gauge_by_usize(
+        &BLOCK_PROCESSING_SNAPSHOT_CACHE_SIZE,
+        beacon_chain.store.state_cache_len(),
+    );
+
+    let da_checker_metrics = beacon_chain.data_availability_checker.metrics();
+    set_gauge_by_usize(
+        &DATA_AVAILABILITY_PROCESSING_CACHE_SIZE,
+        da_checker_metrics.processing_cache_size,
+    );
+    set_gauge_by_usize(
+        &DATA_AVAILABILITY_OVERFLOW_MEMORY_BLOCK_CACHE_SIZE,
+        da_checker_metrics.block_cache_size,
+    );
+    set_gauge_by_usize(
+        &DATA_AVAILABILITY_OVERFLOW_MEMORY_STATE_CACHE_SIZE,
+        da_checker_metrics.state_cache_size,
+    );
+    set_gauge_by_usize(
+        &DATA_AVAILABILITY_OVERFLOW_STORE_CACHE_SIZE,
+        da_checker_metrics.num_store_entries,
+    );
 
     if let Some((size, num_lookups)) = beacon_chain.pre_finalization_block_cache.metrics() {
         set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_CACHE_SIZE, size);
@@ -986,7 +1266,7 @@ pub fn scrape_for_metrics<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
 }
 
 /// Scrape the given `state` assuming it's the head state, updating the `DEFAULT_REGISTRY`.
-fn scrape_head_state<T: EthSpec>(state: &BeaconState<T>, state_root: Hash256) {
+fn scrape_head_state<E: EthSpec>(state: &BeaconState<E>, state_root: Hash256) {
     set_gauge_by_slot(&HEAD_STATE_SLOT, state.slot());
     set_gauge_by_slot(&HEAD_STATE_SLOT_INTEROP, state.slot());
     set_gauge_by_hash(&HEAD_STATE_ROOT, state_root);
@@ -1055,7 +1335,7 @@ fn scrape_head_state<T: EthSpec>(state: &BeaconState<T>, state_root: Hash256) {
             num_active += 1;
         }
 
-        if v.slashed {
+        if v.slashed() {
             num_slashed += 1;
         }
 

beacon_node/beacon_chain/src/migrate.rs

@@ -6,7 +6,7 @@ use parking_lot::Mutex;
 use slog::{debug, error, info, warn, Logger};
 use std::collections::{HashMap, HashSet};
 use std::mem;
-use std::sync::{mpsc, Arc};
+use std::sync::Arc;
 use std::thread;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use store::hot_cold_store::{migrate_database, HotColdDBError};
@@ -24,21 +24,44 @@ const MAX_COMPACTION_PERIOD_SECONDS: u64 = 604800;
 const MIN_COMPACTION_PERIOD_SECONDS: u64 = 7200;
 /// Compact after a large finality gap, if we respect `MIN_COMPACTION_PERIOD_SECONDS`.
 const COMPACTION_FINALITY_DISTANCE: u64 = 1024;
+const BLOCKS_PER_RECONSTRUCTION: usize = 8192 * 4;
+
+/// Default number of epochs to wait between finalization migrations.
+pub const DEFAULT_EPOCHS_PER_MIGRATION: u64 = 1;
 
 /// The background migrator runs a thread to perform pruning and migrate state from the hot
 /// to the cold database.
 pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
     db: Arc<HotColdDB<E, Hot, Cold>>,
-    #[allow(clippy::type_complexity)]
-    tx_thread: Option<Mutex<(mpsc::Sender<Notification>, thread::JoinHandle<()>)>>,
+    /// Record of when the last migration ran, for enforcing `epochs_per_migration`.
+    prev_migration: Arc<Mutex<PrevMigration>>,
+    tx_thread: Option<
+        Mutex<(
+            crossbeam_channel::Sender<Notification>,
+            thread::JoinHandle<()>,
+        )>,
+    >,
     /// Genesis block root, for persisting the `PersistedBeaconChain`.
     genesis_block_root: Hash256,
     log: Logger,
 }
 
-#[derive(Debug, Default, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct MigratorConfig {
     pub blocking: bool,
+    /// Run migrations at most once per `epochs_per_migration`.
+    ///
+    /// If set to 0 or 1, then run every finalization.
+    pub epochs_per_migration: u64,
+}
+
+impl Default for MigratorConfig {
+    fn default() -> Self {
+        Self {
+            blocking: false,
+            epochs_per_migration: DEFAULT_EPOCHS_PER_MIGRATION,
+        }
+    }
 }
 
 impl MigratorConfig {
@@ -46,6 +69,20 @@ impl MigratorConfig {
         self.blocking = true;
         self
     }
+
+    pub fn epochs_per_migration(mut self, epochs_per_migration: u64) -> Self {
+        self.epochs_per_migration = epochs_per_migration;
+        self
+    }
+}
+
+/// Record of when the last migration ran.
+#[derive(Debug)]
+pub struct PrevMigration {
+    /// The epoch at which the last finalization migration ran.
+    epoch: Epoch,
+    /// The number of epochs to wait between runs.
+    epochs_per_migration: u64,
 }
 
 /// Pruning can be successful, or in rare cases deferred to a later point.
@@ -55,7 +92,13 @@ pub enum PruningOutcome {
     Successful {
         old_finalized_checkpoint: Checkpoint,
     },
-    DeferredConcurrentMutation,
+    /// The run was aborted because the new finalized checkpoint is older than the previous one.
+    OutOfOrderFinalization {
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+    },
+    /// The run was aborted due to a concurrent mutation of the head tracker.
+    DeferredConcurrentHeadTrackerMutation,
 }
 
 /// Logic errors that can occur during pruning, none of these should ever happen.
@@ -68,23 +111,46 @@ pub enum PruningError {
     MissingInfoForCanonicalChain {
         slot: Slot,
     },
+    FinalizedStateOutOfOrder {
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+    },
     UnexpectedEqualStateRoots,
     UnexpectedUnequalStateRoots,
 }
 
 /// Message sent to the migration thread containing the information it needs to run.
+#[derive(Debug)]
 pub enum Notification {
     Finalization(FinalizationNotification),
     Reconstruction,
+    PruneBlobs(Epoch),
 }
 
+#[derive(Clone, Debug)]
 pub struct FinalizationNotification {
     finalized_state_root: BeaconStateHash,
     finalized_checkpoint: Checkpoint,
     head_tracker: Arc<HeadTracker>,
+    prev_migration: Arc<Mutex<PrevMigration>>,
     genesis_block_root: Hash256,
 }
 
+/*
+impl Notification {
+    pub fn epoch(&self) -> Option<Epoch> {
+        match self {
+            Notification::Finalization(FinalizationNotification {
+                finalized_checkpoint,
+                ..
+            }) => Some(finalized_checkpoint.epoch),
+            Notification::Reconstruction => None,
+            Notification::PruneBlobs => None,
+        }
+    }
+}
+*/
+
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
     /// Create a new `BackgroundMigrator` and spawn its thread if necessary.
     pub fn new(
@@ -93,6 +159,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         genesis_block_root: Hash256,
         log: Logger,
     ) -> Self {
+        // Estimate last migration run from DB split slot.
+        let prev_migration = Arc::new(Mutex::new(PrevMigration {
+            epoch: db.get_split_slot().epoch(E::slots_per_epoch()),
+            epochs_per_migration: config.epochs_per_migration,
+        }));
         let tx_thread = if config.blocking {
             None
         } else {
@@ -101,6 +172,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         Self {
             db,
             tx_thread,
+            prev_migration,
             genesis_block_root,
             log,
         }
@@ -121,6 +193,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             finalized_state_root,
             finalized_checkpoint,
             head_tracker,
+            prev_migration: self.prev_migration.clone(),
             genesis_block_root: self.genesis_block_root,
         };
 
@@ -142,8 +215,16 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         }
     }
 
+    pub fn process_prune_blobs(&self, data_availability_boundary: Epoch) {
+        if let Some(Notification::PruneBlobs(data_availability_boundary)) =
+            self.send_background_notification(Notification::PruneBlobs(data_availability_boundary))
+        {
+            Self::run_prune_blobs(self.db.clone(), data_availability_boundary, &self.log);
+        }
+    }
+
     pub fn run_reconstruction(db: Arc<HotColdDB<E, Hot, Cold>>, log: &Logger) {
-        if let Err(e) = db.reconstruct_historic_states() {
+        if let Err(e) = db.reconstruct_historic_states(None) {
             error!(
                 log,
                 "State reconstruction failed";
@@ -152,6 +233,20 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         }
     }
 
+    pub fn run_prune_blobs(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        data_availability_boundary: Epoch,
+        log: &Logger,
+    ) {
+        if let Err(e) = db.try_prune_blobs(false, data_availability_boundary) {
+            error!(
+                log,
+                "Blob pruning failed";
+                "error" => ?e,
+            );
+        }
+    }
+
     /// If configured to run in the background, send `notif` to the background thread.
     ///
     /// Return `None` if the message was sent to the background thread, `Some(notif)` otherwise.
@@ -194,9 +289,31 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         notif: FinalizationNotification,
         log: &Logger,
     ) {
+        // Do not run too frequently.
+        let epoch = notif.finalized_checkpoint.epoch;
+        let mut prev_migration = notif.prev_migration.lock();
+        if epoch < prev_migration.epoch + prev_migration.epochs_per_migration {
+            debug!(
+                log,
+                "Database consolidation deferred";
+                "last_finalized_epoch" => prev_migration.epoch,
+                "new_finalized_epoch" => epoch,
+                "epochs_per_migration" => prev_migration.epochs_per_migration,
+            );
+            return;
+        }
+
+        // Update the previous migration epoch immediately to avoid holding the lock. If the
+        // migration doesn't succeed then the next migration will be retried at the next scheduled
+        // run.
+        prev_migration.epoch = epoch;
+        drop(prev_migration);
+
         debug!(log, "Database consolidation started");
+        let timer = std::time::Instant::now();
 
         let finalized_state_root = notif.finalized_state_root;
+        let finalized_block_root = notif.finalized_checkpoint.root;
 
         let finalized_state = match db.get_state(&finalized_state_root.into(), None) {
             Ok(Some(state)) => state,
@@ -223,7 +340,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             Ok(PruningOutcome::Successful {
                 old_finalized_checkpoint,
             }) => old_finalized_checkpoint,
-            Ok(PruningOutcome::DeferredConcurrentMutation) => {
+            Ok(PruningOutcome::DeferredConcurrentHeadTrackerMutation) => {
                 warn!(
                     log,
                     "Pruning deferred because of a concurrent mutation";
@@ -231,18 +348,36 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                 );
                 return;
             }
+            Ok(PruningOutcome::OutOfOrderFinalization {
+                old_finalized_checkpoint,
+                new_finalized_checkpoint,
+            }) => {
+                warn!(
+                    log,
+                    "Ignoring out of order finalization request";
+                    "old_finalized_epoch" => old_finalized_checkpoint.epoch,
+                    "new_finalized_epoch" => new_finalized_checkpoint.epoch,
+                    "message" => "this is expected occasionally due to a (harmless) race condition"
+                );
+                return;
+            }
             Err(e) => {
-                warn!(log, "Block pruning failed"; "error" => format!("{:?}", e));
+                warn!(log, "Block pruning failed"; "error" => ?e);
                 return;
             }
         };
 
-        match migrate_database(db.clone(), finalized_state_root.into(), &finalized_state) {
+        match migrate_database(
+            db.clone(),
+            finalized_state_root.into(),
+            finalized_block_root,
+            &finalized_state,
+        ) {
             Ok(()) => {}
             Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
                 debug!(
                     log,
-                    "Database migration postponed, unaligned finalized block";
+                    "Database migration postponed due to unaligned finalized block";
                     "slot" => slot.as_u64()
                 );
             }
@@ -250,7 +385,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                 warn!(
                     log,
                     "Database migration failed";
-                    "error" => format!("{:?}", e)
+                    "error" => ?e
                 );
                 return;
             }
@@ -263,10 +398,14 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             notif.finalized_checkpoint.epoch,
             log,
         ) {
-            warn!(log, "Database compaction failed"; "error" => format!("{:?}", e));
+            warn!(log, "Database compaction failed"; "error" => ?e);
         }
 
-        debug!(log, "Database consolidation complete");
+        debug!(
+            log,
+            "Database consolidation complete";
+            "running_time_ms" => timer.elapsed().as_millis()
+        );
     }
 
     /// Spawn a new child thread to run the migration process.
@@ -275,33 +414,85 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
     fn spawn_thread(
         db: Arc<HotColdDB<E, Hot, Cold>>,
         log: Logger,
-    ) -> (mpsc::Sender<Notification>, thread::JoinHandle<()>) {
-        let (tx, rx) = mpsc::channel();
+    ) -> (
+        crossbeam_channel::Sender<Notification>,
+        thread::JoinHandle<()>,
+    ) {
+        let (tx, rx) = crossbeam_channel::unbounded();
+        let tx_thread = tx.clone();
         let thread = thread::spawn(move || {
-            while let Ok(notif) = rx.recv() {
-                // Read the rest of the messages in the channel, preferring any reconstruction
-                // notification, or the finalization notification with the greatest finalized epoch.
-                let notif =
-                    rx.try_iter()
-                        .fold(notif, |best, other: Notification| match (&best, &other) {
-                            (Notification::Reconstruction, _)
-                            | (_, Notification::Reconstruction) => Notification::Reconstruction,
-                            (
-                                Notification::Finalization(fin1),
-                                Notification::Finalization(fin2),
-                            ) => {
-                                if fin2.finalized_checkpoint.epoch > fin1.finalized_checkpoint.epoch
-                                {
-                                    other
-                                } else {
-                                    best
-                                }
-                            }
-                        });
+            let mut sel = crossbeam_channel::Select::new();
+            sel.recv(&rx);
 
-                match notif {
-                    Notification::Reconstruction => Self::run_reconstruction(db.clone(), &log),
-                    Notification::Finalization(fin) => Self::run_migration(db.clone(), fin, &log),
+            loop {
+                // Block until sth is in queue
+                let _queue_size = sel.ready();
+                let queue: Vec<Notification> = rx.try_iter().collect();
+                debug!(
+                    log,
+                    "New worker thread poll";
+                    "queue" => ?queue
+                );
+
+                // Find a reconstruction notification and best finalization notification.
+                let reconstruction_notif = queue
+                    .iter()
+                    .find(|n| matches!(n, Notification::Reconstruction));
+                let migrate_notif = queue
+                    .iter()
+                    .filter_map(|n| match n {
+                        Notification::Finalization(f) => Some(f),
+                        _ => None,
+                    })
+                    .max_by_key(|f| f.finalized_checkpoint.epoch)
+                    .cloned();
+                let prune_blobs_notif = queue
+                    .iter()
+                    .filter_map(|n| match n {
+                        Notification::PruneBlobs(dab) => Some(*dab),
+                        _ => None,
+                    })
+                    .max();
+
+                // Do a bit of state reconstruction first if required.
+                if reconstruction_notif.is_some() {
+                    let timer = std::time::Instant::now();
+
+                    match db.reconstruct_historic_states(Some(BLOCKS_PER_RECONSTRUCTION)) {
+                        Err(Error::StateReconstructionDidNotComplete) => {
+                            info!(
+                                log,
+                                "Finished reconstruction batch";
+                                "batch_time_ms" => timer.elapsed().as_millis()
+                            );
+                            // Handle send error
+                            let _ = tx_thread.send(Notification::Reconstruction);
+                        }
+                        Err(e) => {
+                            error!(
+                                log,
+                                "State reconstruction failed";
+                                "error" => ?e,
+                            );
+                        }
+                        Ok(()) => {
+                            info!(
+                                log,
+                                "Finished state reconstruction";
+                                "batch_time_ms" => timer.elapsed().as_millis()
+                            );
+                        }
+                    }
+                }
+
+                // Do the finalization migration.
+                if let Some(notif) = migrate_notif {
+                    Self::run_migration(db.clone(), notif, &log);
+                }
+
+                // Prune blobs.
+                if let Some(dab) = prune_blobs_notif {
+                    Self::run_prune_blobs(db.clone(), dab, &log);
                 }
             }
         });
@@ -321,13 +512,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         genesis_block_root: Hash256,
         log: &Logger,
     ) -> Result<PruningOutcome, BeaconChainError> {
-        let old_finalized_checkpoint =
-            store
-                .load_pruning_checkpoint()?
-                .unwrap_or_else(|| Checkpoint {
-                    epoch: Epoch::new(0),
-                    root: Hash256::zero(),
-                });
+        let old_finalized_checkpoint = store.get_pruning_checkpoint();
 
         let old_finalized_slot = old_finalized_checkpoint
             .epoch
@@ -347,6 +532,16 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             .into());
         }
 
+        // The new finalized state must be newer than the previous finalized state.
+        // I think this can happen sometimes currently due to `fork_choice` running in parallel
+        // with itself and sending us notifications out of order.
+        if old_finalized_slot > new_finalized_slot {
+            return Ok(PruningOutcome::OutOfOrderFinalization {
+                old_finalized_checkpoint,
+                new_finalized_checkpoint,
+            });
+        }
+
         debug!(
             log,
             "Starting database pruning";
@@ -371,18 +566,32 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             })
             .collect::<Result<_, _>>()?;
 
+        // Quick sanity check. If the canonical block & state roots are incorrect then we could
+        // incorrectly delete canonical states, which would corrupt the database.
+        let expected_canonical_block_roots = new_finalized_slot
+            .saturating_sub(old_finalized_slot)
+            .as_usize()
+            .saturating_add(1);
+        if newly_finalized_chain.len() != expected_canonical_block_roots {
+            return Err(BeaconChainError::DBInconsistent(format!(
+                "canonical chain iterator is corrupt; \
+                 expected {} but got {} block roots",
+                expected_canonical_block_roots,
+                newly_finalized_chain.len()
+            )));
+        }
+
         // We don't know which blocks are shared among abandoned chains, so we buffer and delete
         // everything in one fell swoop.
         let mut abandoned_blocks: HashSet<SignedBeaconBlockHash> = HashSet::new();
-        let mut abandoned_states: HashSet<(Slot, BeaconStateHash)> = HashSet::new();
         let mut abandoned_heads: HashSet<Hash256> = HashSet::new();
 
         let heads = head_tracker.heads();
         debug!(
             log,
             "Extra pruning information";
-            "old_finalized_root" => format!("{:?}", old_finalized_checkpoint.root),
-            "new_finalized_root" => format!("{:?}", new_finalized_checkpoint.root),
+            "old_finalized_root" => ?old_finalized_checkpoint.root,
+            "new_finalized_root" => ?new_finalized_checkpoint.root,
             "head_count" => heads.len(),
         );
 
@@ -391,7 +600,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             // so delete it from the head tracker but leave it and its states in the database
             // This is suboptimal as it wastes disk space, but it's difficult to fix. A re-sync
             // can be used to reclaim the space.
-            let head_state_root = match store.get_block(&head_hash) {
+            let head_state_root = match store.get_blinded_block(&head_hash, Some(head_slot)) {
                 Ok(Some(block)) => block.state_root(),
                 Ok(None) => {
                     return Err(BeaconStateError::MissingBeaconBlock(head_hash.into()).into())
@@ -410,14 +619,15 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             };
 
             let mut potentially_abandoned_head = Some(head_hash);
-            let mut potentially_abandoned_blocks = vec![];
+            let mut potentially_abandoned_blocks = HashSet::new();
 
-            // Iterate backwards from this head, staging blocks and states for deletion.
+            // Iterate backwards from this head, staging blocks for deletion.
             let iter = std::iter::once(Ok((head_hash, head_state_root, head_slot)))
                 .chain(RootsIterator::from_block(&store, head_hash)?);
 
             for maybe_tuple in iter {
                 let (block_root, state_root, slot) = maybe_tuple?;
+
                 let block_root = SignedBeaconBlockHash::from(block_root);
                 let state_root = BeaconStateHash::from(state_root);
 
@@ -427,11 +637,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                     // the fork's block and state for possible deletion.
                     None => {
                         if slot > new_finalized_slot {
-                            potentially_abandoned_blocks.push((
-                                slot,
-                                Some(block_root),
-                                Some(state_root),
-                            ));
+                            potentially_abandoned_blocks.insert(block_root);
                         } else if slot >= old_finalized_slot {
                             return Err(PruningError::MissingInfoForCanonicalChain { slot }.into());
                         } else {
@@ -441,7 +647,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                             warn!(
                                 log,
                                 "Found a chain that should already have been pruned";
-                                "head_block_root" => format!("{:?}", head_hash),
+                                "head_block_root" => ?head_hash,
                                 "head_slot" => head_slot,
                             );
                             potentially_abandoned_head.take();
@@ -469,26 +675,14 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                             // we will have just staged the common block for deletion.
                             // Unstage it.
                             else {
-                                for (_, block_root, _) in
-                                    potentially_abandoned_blocks.iter_mut().rev()
-                                {
-                                    if block_root.as_ref() == Some(finalized_block_root) {
-                                        *block_root = None;
-                                    } else {
-                                        break;
-                                    }
-                                }
+                                potentially_abandoned_blocks.remove(finalized_block_root);
                             }
                             break;
                         } else {
                             if state_root == *finalized_state_root {
                                 return Err(PruningError::UnexpectedEqualStateRoots.into());
                             }
-                            potentially_abandoned_blocks.push((
-                                slot,
-                                Some(block_root),
-                                Some(state_root),
-                            ));
+                            potentially_abandoned_blocks.insert(block_root);
                         }
                     }
                 }
@@ -498,18 +692,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                 debug!(
                     log,
                     "Pruning head";
-                    "head_block_root" => format!("{:?}", abandoned_head),
+                    "head_block_root" => ?abandoned_head,
                     "head_slot" => head_slot,
                 );
                 abandoned_heads.insert(abandoned_head);
-                abandoned_blocks.extend(
-                    potentially_abandoned_blocks
-                        .iter()
-                        .filter_map(|(_, maybe_block_hash, _)| *maybe_block_hash),
-                );
-                abandoned_states.extend(potentially_abandoned_blocks.iter().filter_map(
-                    |(slot, _, maybe_state_hash)| maybe_state_hash.map(|sr| (*slot, sr)),
-                ));
+                abandoned_blocks.extend(potentially_abandoned_blocks);
             }
         }
 
@@ -523,7 +710,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         // later.
         for head_hash in &abandoned_heads {
             if !head_tracker_lock.contains_key(head_hash) {
-                return Ok(PruningOutcome::DeferredConcurrentMutation);
+                return Ok(PruningOutcome::DeferredConcurrentHeadTrackerMutation);
             }
         }
 
@@ -532,34 +719,76 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             head_tracker_lock.remove(&head_hash);
         }
 
-        let batch: Vec<StoreOp<E>> = abandoned_blocks
+        let num_deleted_blocks = abandoned_blocks.len();
+        let mut batch: Vec<StoreOp<E>> = abandoned_blocks
             .into_iter()
             .map(Into::into)
-            .map(StoreOp::DeleteBlock)
-            .chain(
-                abandoned_states
-                    .into_iter()
-                    .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash.into(), Some(slot))),
-            )
+            .flat_map(|block_root: Hash256| {
+                [
+                    StoreOp::DeleteBlock(block_root),
+                    StoreOp::DeleteExecutionPayload(block_root),
+                    StoreOp::DeleteBlobs(block_root),
+                ]
+            })
             .collect();
 
-        let mut kv_batch = store.convert_to_kv_batch(&batch)?;
-
         // Persist the head in case the process is killed or crashes here. This prevents
         // the head tracker reverting after our mutation above.
         let persisted_head = PersistedBeaconChain {
             _canonical_head_block_root: DUMMY_CANONICAL_HEAD_BLOCK_ROOT,
             genesis_block_root,
-            ssz_head_tracker: SszHeadTracker::from_map(&*head_tracker_lock),
+            ssz_head_tracker: SszHeadTracker::from_map(&head_tracker_lock),
         };
         drop(head_tracker_lock);
-        kv_batch.push(persisted_head.as_kv_store_op(BEACON_CHAIN_DB_KEY));
+        batch.push(StoreOp::KeyValueOp(
+            persisted_head.as_kv_store_op(BEACON_CHAIN_DB_KEY),
+        ));
 
-        // Persist the new finalized checkpoint as the pruning checkpoint.
-        kv_batch.push(store.pruning_checkpoint_store_op(new_finalized_checkpoint));
+        store.do_atomically_with_block_and_blobs_cache(batch)?;
+        debug!(
+            log,
+            "Database block pruning complete";
+            "num_deleted_blocks" => num_deleted_blocks,
+        );
 
-        store.hot_db.do_atomically(kv_batch)?;
-        debug!(log, "Database pruning complete");
+        // Do a separate pass to clean up irrelevant states.
+        let mut state_delete_batch = vec![];
+        for res in store.iter_hot_state_summaries() {
+            let (state_root, summary) = res?;
+
+            if summary.slot <= new_finalized_slot {
+                // If state root doesn't match state root from canonical chain, then delete.
+                // We may also find older states here that should have been deleted by `migrate_db`
+                // but weren't due to wonky I/O atomicity.
+                if newly_finalized_chain
+                    .get(&summary.slot)
+                    .map_or(true, |(_, canonical_state_root)| {
+                        state_root != Hash256::from(*canonical_state_root)
+                    })
+                {
+                    let reason = if summary.slot < old_finalized_slot {
+                        "old dangling state"
+                    } else {
+                        "non-canonical"
+                    };
+                    debug!(
+                        log,
+                        "Deleting state";
+                        "state_root" => ?state_root,
+                        "slot" => summary.slot,
+                        "reason" => reason,
+                    );
+                    state_delete_batch.push(StoreOp::DeleteState(state_root, Some(summary.slot)));
+                }
+            }
+        }
+        let num_deleted_states = state_delete_batch.len();
+        store.do_atomically_with_block_and_blobs_cache(state_delete_batch)?;
+        debug!(
+            log,
+            "Database state pruning complete";
+            "num_deleted_states" => num_deleted_states,
+        );
 
         Ok(PruningOutcome::Successful {
             old_finalized_checkpoint,

beacon_node/beacon_chain/src/naive_aggregation_pool.rs

@@ -402,7 +402,7 @@ impl<T: AggregateMap> NaiveAggregationPool<T> {
 
     /// Returns the total number of items stored in `self`.
     pub fn num_items(&self) -> usize {
-        self.maps.iter().map(|(_, map)| map.len()).sum()
+        self.maps.values().map(T::len).sum()
     }
 
     /// Returns an aggregated `T::Value` with the given `T::Data`, if any.
@@ -421,10 +421,7 @@ impl<T: AggregateMap> NaiveAggregationPool<T> {
 
     /// Iterate all items in all slots of `self`.
     pub fn iter(&self) -> impl Iterator<Item = &T::Value> {
-        self.maps
-            .iter()
-            .map(|(_slot, map)| map.get_map().iter().map(|(_key, value)| value))
-            .flatten()
+        self.maps.values().flat_map(|map| map.get_map().values())
     }
 
     /// Removes any items with a slot lower than `current_slot` and bars any future
@@ -451,11 +448,7 @@ impl<T: AggregateMap> NaiveAggregationPool<T> {
         // If we have too many maps, remove the lowest amount to ensure we only have
         // `SLOTS_RETAINED` left.
         if self.maps.len() > SLOTS_RETAINED {
-            let mut slots = self
-                .maps
-                .iter()
-                .map(|(slot, _map)| *slot)
-                .collect::<Vec<_>>();
+            let mut slots = self.maps.keys().copied().collect::<Vec<_>>();
             // Sort is generally pretty slow, however `SLOTS_RETAINED` is quite low so it should be
             // negligible.
             slots.sort_unstable();

beacon_node/beacon_chain/src/observed_aggregates.rs

@@ -1,7 +1,9 @@
 //! Provides an `ObservedAggregates` struct which allows us to reject aggregated attestations or
 //! sync committee contributions if we've already seen them.
 
-use std::collections::HashSet;
+use crate::sync_committee_verification::SyncCommitteeData;
+use ssz_types::{BitList, BitVector};
+use std::collections::HashMap;
 use std::marker::PhantomData;
 use tree_hash::TreeHash;
 use types::consts::altair::{
@@ -10,8 +12,16 @@ use types::consts::altair::{
 use types::slot_data::SlotData;
 use types::{Attestation, EthSpec, Hash256, Slot, SyncCommitteeContribution};
 
-pub type ObservedSyncContributions<E> = ObservedAggregates<SyncCommitteeContribution<E>, E>;
-pub type ObservedAggregateAttestations<E> = ObservedAggregates<Attestation<E>, E>;
+pub type ObservedSyncContributions<E> = ObservedAggregates<
+    SyncCommitteeContribution<E>,
+    E,
+    BitVector<<E as types::EthSpec>::SyncSubcommitteeSize>,
+>;
+pub type ObservedAggregateAttestations<E> = ObservedAggregates<
+    Attestation<E>,
+    E,
+    BitList<<E as types::EthSpec>::MaxValidatorsPerCommittee>,
+>;
 
 /// A trait use to associate capacity constants with the type being stored in `ObservedAggregates`.
 pub trait Consts {
@@ -25,7 +35,7 @@ pub trait Consts {
     fn max_per_slot_capacity() -> usize;
 }
 
-impl<T: EthSpec> Consts for Attestation<T> {
+impl<E: EthSpec> Consts for Attestation<E> {
     /// Use 128 as it's the target committee size for the mainnet spec. This is perhaps a little
     /// wasteful for the minimal spec, but considering it's approx. 128 * 32 bytes we're not wasting
     /// much.
@@ -33,7 +43,7 @@ impl<T: EthSpec> Consts for Attestation<T> {
 
     /// We need to keep attestations for each slot of the current epoch.
     fn max_slot_capacity() -> usize {
-        T::slots_per_epoch() as usize
+        2 * E::slots_per_epoch() as usize
     }
 
     /// As a DoS protection measure, the maximum number of distinct `Attestations` or
@@ -52,7 +62,7 @@ impl<T: EthSpec> Consts for Attestation<T> {
     }
 }
 
-impl<T: EthSpec> Consts for SyncCommitteeContribution<T> {
+impl<E: EthSpec> Consts for SyncCommitteeContribution<E> {
     /// Set to `TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE * SYNC_COMMITTEE_SUBNET_COUNT`. This is the
     /// expected number of aggregators per slot across all subcommittees.
     const DEFAULT_PER_SLOT_CAPACITY: usize =
@@ -65,14 +75,85 @@ impl<T: EthSpec> Consts for SyncCommitteeContribution<T> {
 
     /// We should never receive more aggregates than there are sync committee participants.
     fn max_per_slot_capacity() -> usize {
-        T::sync_committee_size()
+        E::sync_committee_size()
+    }
+}
+
+/// A trait for types that implement a behaviour where one object of that type
+/// can be a subset/superset of another.
+/// This trait allows us to be generic over the aggregate item that we store in the cache that
+/// we want to prevent duplicates/subsets for.
+pub trait SubsetItem {
+    /// The item that is stored for later comparison with new incoming aggregate items.
+    type Item;
+
+    /// Returns `true` if `self` is a non-strict subset of `other` and `false` otherwise.
+    fn is_subset(&self, other: &Self::Item) -> bool;
+
+    /// Returns `true` if `self` is a non-strict superset of `other` and `false` otherwise.
+    fn is_superset(&self, other: &Self::Item) -> bool;
+
+    /// Returns the item that gets stored in `ObservedAggregates` for later subset
+    /// comparison with incoming aggregates.
+    fn get_item(&self) -> Self::Item;
+
+    /// Returns a unique value that keys the object to the item that is being stored
+    /// in `ObservedAggregates`.
+    fn root(&self) -> Hash256;
+}
+
+impl<E: EthSpec> SubsetItem for Attestation<E> {
+    type Item = BitList<E::MaxValidatorsPerCommittee>;
+    fn is_subset(&self, other: &Self::Item) -> bool {
+        self.aggregation_bits.is_subset(other)
+    }
+
+    fn is_superset(&self, other: &Self::Item) -> bool {
+        other.is_subset(&self.aggregation_bits)
+    }
+
+    /// Returns the sync contribution aggregation bits.
+    fn get_item(&self) -> Self::Item {
+        self.aggregation_bits.clone()
+    }
+
+    /// Returns the hash tree root of the attestation data.
+    fn root(&self) -> Hash256 {
+        self.data.tree_hash_root()
+    }
+}
+
+impl<E: EthSpec> SubsetItem for SyncCommitteeContribution<E> {
+    type Item = BitVector<E::SyncSubcommitteeSize>;
+    fn is_subset(&self, other: &Self::Item) -> bool {
+        self.aggregation_bits.is_subset(other)
+    }
+
+    fn is_superset(&self, other: &Self::Item) -> bool {
+        other.is_subset(&self.aggregation_bits)
+    }
+
+    /// Returns the sync contribution aggregation bits.
+    fn get_item(&self) -> Self::Item {
+        self.aggregation_bits.clone()
+    }
+
+    /// Returns the hash tree root of the root, slot and subcommittee index
+    /// of the sync contribution.
+    fn root(&self) -> Hash256 {
+        SyncCommitteeData {
+            root: self.beacon_block_root,
+            slot: self.slot,
+            subcommittee_index: self.subcommittee_index,
+        }
+        .tree_hash_root()
     }
 }
 
 #[derive(Debug, PartialEq)]
 pub enum ObserveOutcome {
-    /// This item was already known.
-    AlreadyKnown,
+    /// This item is a non-strict subset of an already known item.
+    Subset,
     /// This was the first time this item was observed.
     New,
 }
@@ -94,26 +175,28 @@ pub enum Error {
     },
 }
 
-/// A `HashSet` that contains entries related to some `Slot`.
-struct SlotHashSet {
-    set: HashSet<Hash256>,
+/// A `HashMap` that contains entries related to some `Slot`.
+struct SlotHashSet<I> {
+    /// Contains a vector of maximally-sized aggregation bitfields/bitvectors
+    /// such that no bitfield/bitvector is a subset of any other in the list.
+    map: HashMap<Hash256, Vec<I>>,
     slot: Slot,
     max_capacity: usize,
 }
 
-impl SlotHashSet {
+impl<I> SlotHashSet<I> {
     pub fn new(slot: Slot, initial_capacity: usize, max_capacity: usize) -> Self {
         Self {
             slot,
-            set: HashSet::with_capacity(initial_capacity),
+            map: HashMap::with_capacity(initial_capacity),
             max_capacity,
         }
     }
 
     /// Store the items in self so future observations recognise its existence.
-    pub fn observe_item<T: SlotData>(
+    pub fn observe_item<S: SlotData + SubsetItem<Item = I>>(
         &mut self,
-        item: &T,
+        item: &S,
         root: Hash256,
     ) -> Result<ObserveOutcome, Error> {
         if item.get_slot() != self.slot {
@@ -123,29 +206,45 @@ impl SlotHashSet {
             });
         }
 
-        if self.set.contains(&root) {
-            Ok(ObserveOutcome::AlreadyKnown)
-        } else {
-            // Here we check to see if this slot has reached the maximum observation count.
-            //
-            // The resulting behaviour is that we are no longer able to successfully observe new
-            // items, however we will continue to return `is_known` values. We could also
-            // disable `is_known`, however then we would stop forwarding items across the
-            // gossip network and I think that this is a worse case than sending some invalid ones.
-            // The underlying libp2p network is responsible for removing duplicate messages, so
-            // this doesn't risk a broadcast loop.
-            if self.set.len() >= self.max_capacity {
-                return Err(Error::ReachedMaxObservationsPerSlot(self.max_capacity));
+        if let Some(aggregates) = self.map.get_mut(&root) {
+            for existing in aggregates {
+                // Check if `item` is a subset of any of the observed aggregates
+                if item.is_subset(existing) {
+                    return Ok(ObserveOutcome::Subset);
+                // Check if `item` is a superset of any of the observed aggregates
+                // If true, we replace the new item with its existing subset. This allows us
+                // to hold fewer items in the list.
+                } else if item.is_superset(existing) {
+                    *existing = item.get_item();
+                    return Ok(ObserveOutcome::New);
+                }
             }
-
-            self.set.insert(root);
-
-            Ok(ObserveOutcome::New)
         }
+
+        // Here we check to see if this slot has reached the maximum observation count.
+        //
+        // The resulting behaviour is that we are no longer able to successfully observe new
+        // items, however we will continue to return `is_known_subset` values. We could also
+        // disable `is_known_subset`, however then we would stop forwarding items across the
+        // gossip network and I think that this is a worse case than sending some invalid ones.
+        // The underlying libp2p network is responsible for removing duplicate messages, so
+        // this doesn't risk a broadcast loop.
+        if self.map.len() >= self.max_capacity {
+            return Err(Error::ReachedMaxObservationsPerSlot(self.max_capacity));
+        }
+
+        let item = item.get_item();
+        self.map.entry(root).or_default().push(item);
+        Ok(ObserveOutcome::New)
     }
 
-    /// Indicates if `item` has been observed before.
-    pub fn is_known<T: SlotData>(&self, item: &T, root: Hash256) -> Result<bool, Error> {
+    /// Check if `item` is a non-strict subset of any of the already observed aggregates for
+    /// the given root and slot.
+    pub fn is_known_subset<S: SlotData + SubsetItem<Item = I>>(
+        &self,
+        item: &S,
+        root: Hash256,
+    ) -> Result<bool, Error> {
         if item.get_slot() != self.slot {
             return Err(Error::IncorrectSlot {
                 expected: self.slot,
@@ -153,25 +252,28 @@ impl SlotHashSet {
             });
         }
 
-        Ok(self.set.contains(&root))
+        Ok(self
+            .map
+            .get(&root)
+            .map_or(false, |agg| agg.iter().any(|val| item.is_subset(val))))
     }
 
     /// The number of observed items in `self`.
     pub fn len(&self) -> usize {
-        self.set.len()
+        self.map.len()
     }
 }
 
 /// Stores the roots of objects for some number of `Slots`, so we can determine if
 /// these have previously been seen on the network.
-pub struct ObservedAggregates<T: TreeHash + SlotData + Consts, E: EthSpec> {
+pub struct ObservedAggregates<T: SlotData + Consts, E: EthSpec, I> {
     lowest_permissible_slot: Slot,
-    sets: Vec<SlotHashSet>,
+    sets: Vec<SlotHashSet<I>>,
     _phantom_spec: PhantomData<E>,
     _phantom_tree_hash: PhantomData<T>,
 }
 
-impl<T: TreeHash + SlotData + Consts, E: EthSpec> Default for ObservedAggregates<T, E> {
+impl<T: SlotData + Consts, E: EthSpec, I> Default for ObservedAggregates<T, E, I> {
     fn default() -> Self {
         Self {
             lowest_permissible_slot: Slot::new(0),
@@ -182,17 +284,17 @@ impl<T: TreeHash + SlotData + Consts, E: EthSpec> Default for ObservedAggregates
     }
 }
 
-impl<T: TreeHash + SlotData + Consts, E: EthSpec> ObservedAggregates<T, E> {
-    /// Store the root of `item` in `self`.
+impl<T: SlotData + Consts + SubsetItem<Item = I>, E: EthSpec, I> ObservedAggregates<T, E, I> {
+    /// Store `item` in `self` keyed at `root`.
     ///
-    /// `root` must equal `item.tree_hash_root()`.
+    /// `root` must equal `item.root::<SubsetItem>()`.
     pub fn observe_item(
         &mut self,
         item: &T,
         root_opt: Option<Hash256>,
     ) -> Result<ObserveOutcome, Error> {
         let index = self.get_set_index(item.get_slot())?;
-        let root = root_opt.unwrap_or_else(|| item.tree_hash_root());
+        let root = root_opt.unwrap_or_else(|| item.root());
 
         self.sets
             .get_mut(index)
@@ -200,16 +302,18 @@ impl<T: TreeHash + SlotData + Consts, E: EthSpec> ObservedAggregates<T, E> {
             .and_then(|set| set.observe_item(item, root))
     }
 
-    /// Check to see if the `root` of `item` is in self.
+    /// Check if `item` is a non-strict subset of any of the already observed aggregates for
+    /// the given root and slot.
     ///
-    /// `root` must equal `a.tree_hash_root()`.
-    pub fn is_known(&mut self, item: &T, root: Hash256) -> Result<bool, Error> {
+    /// `root` must equal `item.root::<SubsetItem>()`.
+    #[allow(clippy::wrong_self_convention)]
+    pub fn is_known_subset(&mut self, item: &T, root: Hash256) -> Result<bool, Error> {
         let index = self.get_set_index(item.get_slot())?;
 
         self.sets
             .get(index)
             .ok_or(Error::InvalidSetIndex(index))
-            .and_then(|set| set.is_known(item, root))
+            .and_then(|set| set.is_known_subset(item, root))
     }
 
     /// The maximum number of slots that items are stored for.
@@ -295,7 +399,6 @@ impl<T: TreeHash + SlotData + Consts, E: EthSpec> ObservedAggregates<T, E> {
 #[cfg(not(debug_assertions))]
 mod tests {
     use super::*;
-    use tree_hash::TreeHash;
     use types::{test_utils::test_random_instance, Hash256};
 
     type E = types::MainnetEthSpec;
@@ -329,7 +432,7 @@ mod tests {
 
                     for a in &items {
                         assert_eq!(
-                            store.is_known(a, a.tree_hash_root()),
+                            store.is_known_subset(a, a.root()),
                             Ok(false),
                             "should indicate an unknown attestation is unknown"
                         );
@@ -342,13 +445,13 @@ mod tests {
 
                     for a in &items {
                         assert_eq!(
-                            store.is_known(a, a.tree_hash_root()),
+                            store.is_known_subset(a, a.root()),
                             Ok(true),
                             "should indicate a known attestation is known"
                         );
                         assert_eq!(
-                            store.observe_item(a, Some(a.tree_hash_root())),
-                            Ok(ObserveOutcome::AlreadyKnown),
+                            store.observe_item(a, Some(a.root())),
+                            Ok(ObserveOutcome::Subset),
                             "should acknowledge an existing attestation"
                         );
                     }

beacon_node/beacon_chain/src/observed_attesters.rs

@@ -20,29 +20,27 @@ use std::collections::{HashMap, HashSet};
 use std::hash::Hash;
 use std::marker::PhantomData;
 use types::slot_data::SlotData;
-use types::{Epoch, EthSpec, Slot, Unsigned};
+use types::{Epoch, EthSpec, Hash256, Slot, Unsigned};
 
 /// The maximum capacity of the `AutoPruningEpochContainer`.
 ///
-/// Fits the next, current and previous epochs. We require the next epoch due to the
-/// `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. We require the previous epoch since the specification
-/// declares:
+/// If the current epoch is N, this fits epoch N + 1, N, N - 1, and N - 2. We require the next epoch due
+/// to the `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. We require the N - 2 epoch since the specification declares:
 ///
 /// ```ignore
-/// aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE
-///      >= current_slot >= aggregate.data.slot
+/// the epoch of `aggregate.data.slot` is either the current or previous epoch
 /// ```
 ///
-/// This means that during the current epoch we will always accept an attestation
-/// from at least one slot in the previous epoch.
-pub const MAX_CACHED_EPOCHS: u64 = 3;
+/// This means that during the current epoch we will always accept an attestation from
+/// at least one slot in the epoch prior to the previous epoch.
+pub const MAX_CACHED_EPOCHS: u64 = 4;
 
 pub type ObservedAttesters<E> = AutoPruningEpochContainer<EpochBitfield, E>;
 pub type ObservedSyncContributors<E> =
-    AutoPruningSlotContainer<SlotSubcommitteeIndex, SyncContributorSlotHashSet<E>, E>;
+    AutoPruningSlotContainer<SlotSubcommitteeIndex, Hash256, SyncContributorSlotHashSet<E>, E>;
 pub type ObservedAggregators<E> = AutoPruningEpochContainer<EpochHashSet, E>;
 pub type ObservedSyncAggregators<E> =
-    AutoPruningSlotContainer<SlotSubcommitteeIndex, SyncAggregatorSlotHashSet, E>;
+    AutoPruningSlotContainer<SlotSubcommitteeIndex, (), SyncAggregatorSlotHashSet, E>;
 
 #[derive(Debug, PartialEq)]
 pub enum Error {
@@ -62,7 +60,7 @@ pub enum Error {
 }
 
 /// Implemented on an item in an `AutoPruningContainer`.
-pub trait Item {
+pub trait Item<T> {
     /// Instantiate `Self` with the given `capacity`.
     fn with_capacity(capacity: usize) -> Self;
 
@@ -75,11 +73,11 @@ pub trait Item {
     /// Returns the number of validators that have been observed by `self`.
     fn validator_count(&self) -> usize;
 
-    /// Store `validator_index` in `self`.
-    fn insert(&mut self, validator_index: usize) -> bool;
+    /// Store `validator_index` and `value` in `self`.
+    fn insert(&mut self, validator_index: usize, value: T) -> bool;
 
-    /// Returns `true` if `validator_index` has been stored in `self`.
-    fn contains(&self, validator_index: usize) -> bool;
+    /// Returns `Some(T)` if there is an entry for `validator_index`.
+    fn get(&self, validator_index: usize) -> Option<T>;
 }
 
 /// Stores a `BitVec` that represents which validator indices have attested or sent sync committee
@@ -88,7 +86,7 @@ pub struct EpochBitfield {
     bitfield: BitVec,
 }
 
-impl Item for EpochBitfield {
+impl Item<()> for EpochBitfield {
     fn with_capacity(capacity: usize) -> Self {
         Self {
             bitfield: BitVec::with_capacity(capacity),
@@ -108,7 +106,7 @@ impl Item for EpochBitfield {
         self.bitfield.iter().filter(|bit| **bit).count()
     }
 
-    fn insert(&mut self, validator_index: usize) -> bool {
+    fn insert(&mut self, validator_index: usize, _value: ()) -> bool {
         self.bitfield
             .get_mut(validator_index)
             .map(|mut bit| {
@@ -129,8 +127,11 @@ impl Item for EpochBitfield {
             })
     }
 
-    fn contains(&self, validator_index: usize) -> bool {
-        self.bitfield.get(validator_index).map_or(false, |bit| *bit)
+    fn get(&self, validator_index: usize) -> Option<()> {
+        self.bitfield
+            .get(validator_index)
+            .map_or(false, |bit| *bit)
+            .then_some(())
     }
 }
 
@@ -140,7 +141,7 @@ pub struct EpochHashSet {
     set: HashSet<usize>,
 }
 
-impl Item for EpochHashSet {
+impl Item<()> for EpochHashSet {
     fn with_capacity(capacity: usize) -> Self {
         Self {
             set: HashSet::with_capacity(capacity),
@@ -163,27 +164,27 @@ impl Item for EpochHashSet {
 
     /// Inserts the `validator_index` in the set. Returns `true` if the `validator_index` was
     /// already in the set.
-    fn insert(&mut self, validator_index: usize) -> bool {
+    fn insert(&mut self, validator_index: usize, _value: ()) -> bool {
         !self.set.insert(validator_index)
     }
 
     /// Returns `true` if the `validator_index` is in the set.
-    fn contains(&self, validator_index: usize) -> bool {
-        self.set.contains(&validator_index)
+    fn get(&self, validator_index: usize) -> Option<()> {
+        self.set.contains(&validator_index).then_some(())
     }
 }
 
 /// Stores a `HashSet` of which validator indices have created a sync aggregate during a
 /// slot.
 pub struct SyncContributorSlotHashSet<E> {
-    set: HashSet<usize>,
+    map: HashMap<usize, Hash256>,
     phantom: PhantomData<E>,
 }
 
-impl<E: EthSpec> Item for SyncContributorSlotHashSet<E> {
+impl<E: EthSpec> Item<Hash256> for SyncContributorSlotHashSet<E> {
     fn with_capacity(capacity: usize) -> Self {
         Self {
-            set: HashSet::with_capacity(capacity),
+            map: HashMap::with_capacity(capacity),
             phantom: PhantomData,
         }
     }
@@ -194,22 +195,24 @@ impl<E: EthSpec> Item for SyncContributorSlotHashSet<E> {
     }
 
     fn len(&self) -> usize {
-        self.set.len()
+        self.map.len()
     }
 
     fn validator_count(&self) -> usize {
-        self.set.len()
+        self.map.len()
     }
 
     /// Inserts the `validator_index` in the set. Returns `true` if the `validator_index` was
     /// already in the set.
-    fn insert(&mut self, validator_index: usize) -> bool {
-        !self.set.insert(validator_index)
+    fn insert(&mut self, validator_index: usize, beacon_block_root: Hash256) -> bool {
+        self.map
+            .insert(validator_index, beacon_block_root)
+            .is_some()
     }
 
     /// Returns `true` if the `validator_index` is in the set.
-    fn contains(&self, validator_index: usize) -> bool {
-        self.set.contains(&validator_index)
+    fn get(&self, validator_index: usize) -> Option<Hash256> {
+        self.map.get(&validator_index).copied()
     }
 }
 
@@ -219,7 +222,7 @@ pub struct SyncAggregatorSlotHashSet {
     set: HashSet<usize>,
 }
 
-impl Item for SyncAggregatorSlotHashSet {
+impl Item<()> for SyncAggregatorSlotHashSet {
     fn with_capacity(capacity: usize) -> Self {
         Self {
             set: HashSet::with_capacity(capacity),
@@ -241,13 +244,13 @@ impl Item for SyncAggregatorSlotHashSet {
 
     /// Inserts the `validator_index` in the set. Returns `true` if the `validator_index` was
     /// already in the set.
-    fn insert(&mut self, validator_index: usize) -> bool {
+    fn insert(&mut self, validator_index: usize, _value: ()) -> bool {
         !self.set.insert(validator_index)
     }
 
     /// Returns `true` if the `validator_index` is in the set.
-    fn contains(&self, validator_index: usize) -> bool {
-        self.set.contains(&validator_index)
+    fn get(&self, validator_index: usize) -> Option<()> {
+        self.set.contains(&validator_index).then_some(())
     }
 }
 
@@ -275,7 +278,7 @@ impl<T, E: EthSpec> Default for AutoPruningEpochContainer<T, E> {
     }
 }
 
-impl<T: Item, E: EthSpec> AutoPruningEpochContainer<T, E> {
+impl<T: Item<()>, E: EthSpec> AutoPruningEpochContainer<T, E> {
     /// Observe that `validator_index` has produced attestation `a`. Returns `Ok(true)` if `a` has
     /// previously been observed for `validator_index`.
     ///
@@ -293,7 +296,7 @@ impl<T: Item, E: EthSpec> AutoPruningEpochContainer<T, E> {
         self.prune(epoch);
 
         if let Some(item) = self.items.get_mut(&epoch) {
-            Ok(item.insert(validator_index))
+            Ok(item.insert(validator_index, ()))
         } else {
             // To avoid re-allocations, try and determine a rough initial capacity for the new item
             // by obtaining the mean size of all items in earlier epoch.
@@ -309,7 +312,7 @@ impl<T: Item, E: EthSpec> AutoPruningEpochContainer<T, E> {
             let initial_capacity = sum.checked_div(count).unwrap_or_else(T::default_capacity);
 
             let mut item = T::with_capacity(initial_capacity);
-            item.insert(validator_index);
+            item.insert(validator_index, ());
             self.items.insert(epoch, item);
 
             Ok(false)
@@ -333,7 +336,7 @@ impl<T: Item, E: EthSpec> AutoPruningEpochContainer<T, E> {
         let exists = self
             .items
             .get(&epoch)
-            .map_or(false, |item| item.contains(validator_index));
+            .map_or(false, |item| item.get(validator_index).is_some());
 
         Ok(exists)
     }
@@ -392,7 +395,7 @@ impl<T: Item, E: EthSpec> AutoPruningEpochContainer<T, E> {
     pub fn index_seen_at_epoch(&self, index: usize, epoch: Epoch) -> bool {
         self.items
             .get(&epoch)
-            .map(|item| item.contains(index))
+            .map(|item| item.get(index).is_some())
             .unwrap_or(false)
     }
 }
@@ -405,23 +408,63 @@ impl<T: Item, E: EthSpec> AutoPruningEpochContainer<T, E> {
 /// sync contributions with an epoch prior to `data.slot - 3` will be cleared from the cache.
 ///
 /// `V` should be set to a `SyncAggregatorSlotHashSet` or a `SyncContributorSlotHashSet`.
-pub struct AutoPruningSlotContainer<K: SlotData + Eq + Hash, V, E: EthSpec> {
+pub struct AutoPruningSlotContainer<K: SlotData + Eq + Hash, S, V, E: EthSpec> {
     lowest_permissible_slot: Slot,
     items: HashMap<K, V>,
-    _phantom: PhantomData<E>,
+    _phantom_e: PhantomData<E>,
+    _phantom_s: PhantomData<S>,
 }
 
-impl<K: SlotData + Eq + Hash, V, E: EthSpec> Default for AutoPruningSlotContainer<K, V, E> {
+impl<K: SlotData + Eq + Hash, S, V, E: EthSpec> Default for AutoPruningSlotContainer<K, S, V, E> {
     fn default() -> Self {
         Self {
             lowest_permissible_slot: Slot::new(0),
             items: HashMap::new(),
-            _phantom: PhantomData,
+            _phantom_e: PhantomData,
+            _phantom_s: PhantomData,
         }
     }
 }
 
-impl<K: SlotData + Eq + Hash, V: Item, E: EthSpec> AutoPruningSlotContainer<K, V, E> {
+impl<K: SlotData + Eq + Hash + Copy, S, V: Item<S>, E: EthSpec>
+    AutoPruningSlotContainer<K, S, V, E>
+{
+    /// Observes the given `value` for the given `validator_index`.
+    ///
+    /// The `override_observation` function is supplied `previous_observation`
+    /// and `value`. If it returns `true`, then any existing observation will be
+    /// overridden.
+    ///
+    /// This function returns `Some` if:
+    /// - An observation already existed for the validator, AND,
+    /// - The `override_observation` function returned `false`.
+    ///
+    /// Alternatively, it returns `None` if:
+    /// - An observation did not already exist for the given validator, OR,
+    /// - The `override_observation` function returned `true`.
+    pub fn observe_validator_with_override<F>(
+        &mut self,
+        key: K,
+        validator_index: usize,
+        value: S,
+        override_observation: F,
+    ) -> Result<Option<S>, Error>
+    where
+        F: Fn(&S, &S) -> bool,
+    {
+        if let Some(prev_observation) = self.observation_for_validator(key, validator_index)? {
+            if override_observation(&prev_observation, &value) {
+                self.observe_validator(key, validator_index, value)?;
+                Ok(None)
+            } else {
+                Ok(Some(prev_observation))
+            }
+        } else {
+            self.observe_validator(key, validator_index, value)?;
+            Ok(None)
+        }
+    }
+
     /// Observe that `validator_index` has produced a sync committee message. Returns `Ok(true)` if
     /// the sync committee message  has previously been observed for `validator_index`.
     ///
@@ -429,14 +472,19 @@ impl<K: SlotData + Eq + Hash, V: Item, E: EthSpec> AutoPruningSlotContainer<K, V
     ///
     /// - `validator_index` is higher than `VALIDATOR_REGISTRY_LIMIT`.
     /// - `key.slot` is earlier than `self.lowest_permissible_slot`.
-    pub fn observe_validator(&mut self, key: K, validator_index: usize) -> Result<bool, Error> {
+    pub fn observe_validator(
+        &mut self,
+        key: K,
+        validator_index: usize,
+        value: S,
+    ) -> Result<bool, Error> {
         let slot = key.get_slot();
         self.sanitize_request(slot, validator_index)?;
 
         self.prune(slot);
 
         if let Some(item) = self.items.get_mut(&key) {
-            Ok(item.insert(validator_index))
+            Ok(item.insert(validator_index, value))
         } else {
             // To avoid re-allocations, try and determine a rough initial capacity for the new item
             // by obtaining the mean size of all items in earlier slot.
@@ -452,32 +500,45 @@ impl<K: SlotData + Eq + Hash, V: Item, E: EthSpec> AutoPruningSlotContainer<K, V
             let initial_capacity = sum.checked_div(count).unwrap_or_else(V::default_capacity);
 
             let mut item = V::with_capacity(initial_capacity);
-            item.insert(validator_index);
+            item.insert(validator_index, value);
             self.items.insert(key, item);
 
             Ok(false)
         }
     }
 
-    /// Returns `Ok(true)` if the `validator_index` has already produced a conflicting sync committee message.
-    ///
-    /// ## Errors
-    ///
-    /// - `validator_index` is higher than `VALIDATOR_REGISTRY_LIMIT`.
-    /// - `key.slot` is earlier than `self.lowest_permissible_slot`.
+    // Identical to `Self::observation_for_validator` but discards the
+    // observation, simply returning `true` if the validator has been observed
+    // at all.
     pub fn validator_has_been_observed(
         &self,
         key: K,
         validator_index: usize,
     ) -> Result<bool, Error> {
+        self.observation_for_validator(key, validator_index)
+            .map(|observation| observation.is_some())
+    }
+
+    /// Returns `Ok(Some)` if the `validator_index` has already produced a
+    /// conflicting sync committee message.
+    ///
+    /// ## Errors
+    ///
+    /// - `validator_index` is higher than `VALIDATOR_REGISTRY_LIMIT`.
+    /// - `key.slot` is earlier than `self.lowest_permissible_slot`.
+    pub fn observation_for_validator(
+        &self,
+        key: K,
+        validator_index: usize,
+    ) -> Result<Option<S>, Error> {
         self.sanitize_request(key.get_slot(), validator_index)?;
 
-        let exists = self
+        let observation = self
             .items
             .get(&key)
-            .map_or(false, |item| item.contains(validator_index));
+            .and_then(|item| item.get(validator_index));
 
-        Ok(exists)
+        Ok(observation)
     }
 
     /// Returns the number of validators that have been observed at the given `slot`. Returns
@@ -561,6 +622,116 @@ mod tests {
 
     type E = types::MainnetEthSpec;
 
+    #[test]
+    fn value_storage() {
+        type Container = AutoPruningSlotContainer<Slot, Hash256, SyncContributorSlotHashSet<E>, E>;
+
+        let mut store: Container = <_>::default();
+        let key = Slot::new(0);
+        let validator_index = 0;
+        let value = Hash256::zero();
+
+        // Assert there is no entry.
+        assert!(store
+            .observation_for_validator(key, validator_index)
+            .unwrap()
+            .is_none());
+        assert!(!store
+            .validator_has_been_observed(key, validator_index)
+            .unwrap());
+
+        // Add an entry.
+        assert!(!store
+            .observe_validator(key, validator_index, value)
+            .unwrap());
+
+        // Assert there is a correct entry.
+        assert_eq!(
+            store
+                .observation_for_validator(key, validator_index)
+                .unwrap(),
+            Some(value)
+        );
+        assert!(store
+            .validator_has_been_observed(key, validator_index)
+            .unwrap());
+
+        let alternate_value = Hash256::from_low_u64_be(1);
+
+        // Assert that override false does not override.
+        assert_eq!(
+            store
+                .observe_validator_with_override(key, validator_index, alternate_value, |_, _| {
+                    false
+                })
+                .unwrap(),
+            Some(value)
+        );
+
+        // Assert that override true overrides and acts as if there was never an
+        // entry there.
+        assert_eq!(
+            store
+                .observe_validator_with_override(key, validator_index, alternate_value, |_, _| {
+                    true
+                })
+                .unwrap(),
+            None
+        );
+        assert_eq!(
+            store
+                .observation_for_validator(key, validator_index)
+                .unwrap(),
+            Some(alternate_value)
+        );
+
+        // Reset the store.
+        let mut store: Container = <_>::default();
+
+        // Asset that a new entry with override = false is inserted
+        assert_eq!(
+            store
+                .observation_for_validator(key, validator_index)
+                .unwrap(),
+            None
+        );
+        assert_eq!(
+            store
+                .observe_validator_with_override(key, validator_index, value, |_, _| { false })
+                .unwrap(),
+            None,
+        );
+        assert_eq!(
+            store
+                .observation_for_validator(key, validator_index)
+                .unwrap(),
+            Some(value)
+        );
+
+        // Reset the store.
+        let mut store: Container = <_>::default();
+
+        // Asset that a new entry with override = true is inserted
+        assert_eq!(
+            store
+                .observation_for_validator(key, validator_index)
+                .unwrap(),
+            None
+        );
+        assert_eq!(
+            store
+                .observe_validator_with_override(key, validator_index, value, |_, _| { true })
+                .unwrap(),
+            None,
+        );
+        assert_eq!(
+            store
+                .observation_for_validator(key, validator_index)
+                .unwrap(),
+            Some(value)
+        );
+    }
+
     macro_rules! test_suite_epoch {
         ($mod_name: ident, $type: ident) => {
             #[cfg(test)]
@@ -668,7 +839,7 @@ mod tests {
                     let mut store = $type::default();
                     let max_cap = store.max_capacity();
 
-                    let to_skip = vec![1_u64, 3, 4, 5];
+                    let to_skip = [1_u64, 3, 4, 5];
                     let periods = (0..max_cap * 3)
                         .into_iter()
                         .filter(|i| !to_skip.contains(i))
@@ -722,7 +893,7 @@ mod tests {
     test_suite_epoch!(observed_aggregators, ObservedAggregators);
 
     macro_rules! test_suite_slot {
-        ($mod_name: ident, $type: ident) => {
+        ($mod_name: ident, $type: ident, $value: expr) => {
             #[cfg(test)]
             mod $mod_name {
                 use super::*;
@@ -737,7 +908,7 @@ mod tests {
                             "should indicate an unknown item is unknown"
                         );
                         assert_eq!(
-                            store.observe_validator(key, i),
+                            store.observe_validator(key, i, $value),
                             Ok(false),
                             "should observe new item"
                         );
@@ -750,7 +921,7 @@ mod tests {
                             "should indicate a known item is known"
                         );
                         assert_eq!(
-                            store.observe_validator(key, i),
+                            store.observe_validator(key, i, $value),
                             Ok(true),
                             "should acknowledge an existing item"
                         );
@@ -839,7 +1010,7 @@ mod tests {
                     let mut store = $type::default();
                     let max_cap = store.max_capacity();
 
-                    let to_skip = vec![1_u64, 3, 4, 5];
+                    let to_skip = [1_u64, 3, 4, 5];
                     let periods = (0..max_cap * 3)
                         .into_iter()
                         .filter(|i| !to_skip.contains(i))
@@ -948,7 +1119,7 @@ mod tests {
                     let mut store = $type::default();
                     let max_cap = store.max_capacity();
 
-                    let to_skip = vec![1_u64, 3, 4, 5];
+                    let to_skip = [1_u64, 3, 4, 5];
                     let periods = (0..max_cap * 3)
                         .into_iter()
                         .filter(|i| !to_skip.contains(i))
@@ -997,6 +1168,10 @@ mod tests {
             }
         };
     }
-    test_suite_slot!(observed_sync_contributors, ObservedSyncContributors);
-    test_suite_slot!(observed_sync_aggregators, ObservedSyncAggregators);
+    test_suite_slot!(
+        observed_sync_contributors,
+        ObservedSyncContributors,
+        Hash256::zero()
+    );
+    test_suite_slot!(observed_sync_aggregators, ObservedSyncAggregators, ());
 }

beacon_node/beacon_chain/src/observed_blob_sidecars.rs

@@ -0,0 +1,430 @@
+//! Provides the `ObservedBlobSidecars` struct which allows for rejecting `BlobSidecar`s
+//! that we have already seen over the gossip network.
+//! Only `BlobSidecar`s that have completed proposer signature verification can be added
+//! to this cache to reduce DoS risks.
+
+use crate::observed_block_producers::ProposalKey;
+use std::collections::{HashMap, HashSet};
+use std::marker::PhantomData;
+use types::{BlobSidecar, EthSpec, Slot};
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    /// The slot of the provided `BlobSidecar` is prior to finalization and should not have been provided
+    /// to this function. This is an internal error.
+    FinalizedBlob { slot: Slot, finalized_slot: Slot },
+    /// The blob sidecar contains an invalid blob index, the blob sidecar is invalid.
+    /// Note: The invalid blob should have been caught and flagged as an error much before reaching
+    /// here.
+    InvalidBlobIndex(u64),
+}
+
+/// Maintains a cache of seen `BlobSidecar`s that are received over gossip
+/// and have been gossip verified.
+///
+/// The cache supports pruning based upon the finalized epoch. It does not automatically prune, you
+/// must call `Self::prune` manually.
+///
+/// Note: To prevent DoS attacks, this cache must include only items that have received some DoS resistance
+/// like checking the proposer signature.
+pub struct ObservedBlobSidecars<E: EthSpec> {
+    finalized_slot: Slot,
+    /// Stores all received blob indices for a given `(ValidatorIndex, Slot)` tuple.
+    items: HashMap<ProposalKey, HashSet<u64>>,
+    _phantom: PhantomData<E>,
+}
+
+impl<E: EthSpec> Default for ObservedBlobSidecars<E> {
+    /// Instantiates `Self` with `finalized_slot == 0`.
+    fn default() -> Self {
+        Self {
+            finalized_slot: Slot::new(0),
+            items: HashMap::new(),
+            _phantom: PhantomData,
+        }
+    }
+}
+
+impl<E: EthSpec> ObservedBlobSidecars<E> {
+    /// Observe the `blob_sidecar` at (`blob_sidecar.block_proposer_index, blob_sidecar.slot`).
+    /// This will update `self` so future calls to it indicate that this `blob_sidecar` is known.
+    ///
+    /// The supplied `blob_sidecar` **MUST** have completed proposer signature verification.
+    pub fn observe_sidecar(&mut self, blob_sidecar: &BlobSidecar<E>) -> Result<bool, Error> {
+        self.sanitize_blob_sidecar(blob_sidecar)?;
+
+        let blob_indices = self
+            .items
+            .entry(ProposalKey {
+                slot: blob_sidecar.slot(),
+                proposer: blob_sidecar.block_proposer_index(),
+            })
+            .or_insert_with(|| HashSet::with_capacity(E::max_blobs_per_block()));
+        let did_not_exist = blob_indices.insert(blob_sidecar.index);
+
+        Ok(!did_not_exist)
+    }
+
+    /// Returns `true` if the `blob_sidecar` has already been observed in the cache within the prune window.
+    pub fn proposer_is_known(&self, blob_sidecar: &BlobSidecar<E>) -> Result<bool, Error> {
+        self.sanitize_blob_sidecar(blob_sidecar)?;
+        let is_known = self
+            .items
+            .get(&ProposalKey {
+                slot: blob_sidecar.slot(),
+                proposer: blob_sidecar.block_proposer_index(),
+            })
+            .map_or(false, |blob_indices| {
+                blob_indices.contains(&blob_sidecar.index)
+            });
+        Ok(is_known)
+    }
+
+    fn sanitize_blob_sidecar(&self, blob_sidecar: &BlobSidecar<E>) -> Result<(), Error> {
+        if blob_sidecar.index >= E::max_blobs_per_block() as u64 {
+            return Err(Error::InvalidBlobIndex(blob_sidecar.index));
+        }
+        let finalized_slot = self.finalized_slot;
+        if finalized_slot > 0 && blob_sidecar.slot() <= finalized_slot {
+            return Err(Error::FinalizedBlob {
+                slot: blob_sidecar.slot(),
+                finalized_slot,
+            });
+        }
+
+        Ok(())
+    }
+
+    /// Prune `blob_sidecar` observations for slots less than or equal to the given slot.
+    pub fn prune(&mut self, finalized_slot: Slot) {
+        if finalized_slot == 0 {
+            return;
+        }
+
+        self.finalized_slot = finalized_slot;
+        self.items.retain(|k, _| k.slot > finalized_slot);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bls::Hash256;
+    use std::sync::Arc;
+    use types::MainnetEthSpec;
+
+    type E = MainnetEthSpec;
+
+    fn get_blob_sidecar(slot: u64, proposer_index: u64, index: u64) -> Arc<BlobSidecar<E>> {
+        let mut blob_sidecar = BlobSidecar::empty();
+        blob_sidecar.signed_block_header.message.slot = slot.into();
+        blob_sidecar.signed_block_header.message.proposer_index = proposer_index;
+        blob_sidecar.index = index;
+        Arc::new(blob_sidecar)
+    }
+
+    #[test]
+    fn pruning() {
+        let mut cache = ObservedBlobSidecars::default();
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 0, "no slots should be present");
+
+        // Slot 0, index 0
+        let proposer_index_a = 420;
+        let sidecar_a = get_blob_sidecar(0, proposer_index_a, 0);
+
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_a),
+            Ok(false),
+            "can observe proposer, indicates proposer unobserved"
+        );
+
+        /*
+         * Preconditions.
+         */
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(
+            cache.items.len(),
+            1,
+            "only one (validator_index, slot) tuple should be present"
+        );
+
+        let cached_blob_indices = cache
+            .items
+            .get(&ProposalKey::new(proposer_index_a, Slot::new(0)))
+            .expect("slot zero should be present");
+        assert_eq!(
+            cached_blob_indices.len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        /*
+         * Check that a prune at the genesis slot does nothing.
+         */
+
+        cache.prune(Slot::new(0));
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        let cached_blob_indices = cache
+            .items
+            .get(&ProposalKey::new(proposer_index_a, Slot::new(0)))
+            .expect("slot zero should be present");
+        assert_eq!(
+            cached_blob_indices.len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        /*
+         * Check that a prune empties the cache
+         */
+
+        cache.prune(E::slots_per_epoch().into());
+        assert_eq!(
+            cache.finalized_slot,
+            Slot::from(E::slots_per_epoch()),
+            "finalized slot is updated"
+        );
+        assert_eq!(cache.items.len(), 0, "no items left");
+
+        /*
+         * Check that we can't insert a finalized sidecar
+         */
+
+        // First slot of finalized epoch
+        let block_b = get_blob_sidecar(E::slots_per_epoch(), 419, 0);
+
+        assert_eq!(
+            cache.observe_sidecar(&block_b),
+            Err(Error::FinalizedBlob {
+                slot: E::slots_per_epoch().into(),
+                finalized_slot: E::slots_per_epoch().into(),
+            }),
+            "cant insert finalized sidecar"
+        );
+
+        assert_eq!(cache.items.len(), 0, "sidecar was not added");
+
+        /*
+         * Check that we _can_ insert a non-finalized block
+         */
+
+        let three_epochs = E::slots_per_epoch() * 3;
+
+        // First slot of finalized epoch
+        let proposer_index_b = 421;
+        let block_b = get_blob_sidecar(three_epochs, proposer_index_b, 0);
+
+        assert_eq!(
+            cache.observe_sidecar(&block_b),
+            Ok(false),
+            "can insert non-finalized block"
+        );
+
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        let cached_blob_indices = cache
+            .items
+            .get(&ProposalKey::new(proposer_index_b, Slot::new(three_epochs)))
+            .expect("the three epochs slot should be present");
+        assert_eq!(
+            cached_blob_indices.len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        /*
+         * Check that a prune doesnt wipe later blocks
+         */
+
+        let two_epochs = E::slots_per_epoch() * 2;
+        cache.prune(two_epochs.into());
+
+        assert_eq!(
+            cache.finalized_slot,
+            Slot::from(two_epochs),
+            "finalized slot is updated"
+        );
+
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        let cached_blob_indices = cache
+            .items
+            .get(&ProposalKey::new(proposer_index_b, Slot::new(three_epochs)))
+            .expect("the three epochs slot should be present");
+        assert_eq!(
+            cached_blob_indices.len(),
+            1,
+            "only one proposer should be present"
+        );
+    }
+
+    #[test]
+    fn simple_observations() {
+        let mut cache = ObservedBlobSidecars::default();
+
+        // Slot 0, index 0
+        let proposer_index_a = 420;
+        let sidecar_a = get_blob_sidecar(0, proposer_index_a, 0);
+
+        assert_eq!(
+            cache.proposer_is_known(&sidecar_a),
+            Ok(false),
+            "no observation in empty cache"
+        );
+
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_a),
+            Ok(false),
+            "can observe proposer, indicates proposer unobserved"
+        );
+
+        assert_eq!(
+            cache.proposer_is_known(&sidecar_a),
+            Ok(true),
+            "observed block is indicated as true"
+        );
+
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_a),
+            Ok(true),
+            "observing again indicates true"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        let cached_blob_indices = cache
+            .items
+            .get(&ProposalKey::new(proposer_index_a, Slot::new(0)))
+            .expect("slot zero should be present");
+        assert_eq!(
+            cached_blob_indices.len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        // Slot 1, proposer 0
+
+        let proposer_index_b = 421;
+        let sidecar_b = get_blob_sidecar(1, proposer_index_b, 0);
+
+        assert_eq!(
+            cache.proposer_is_known(&sidecar_b),
+            Ok(false),
+            "no observation for new slot"
+        );
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_b),
+            Ok(false),
+            "can observe proposer for new slot, indicates proposer unobserved"
+        );
+        assert_eq!(
+            cache.proposer_is_known(&sidecar_b),
+            Ok(true),
+            "observed block in slot 1 is indicated as true"
+        );
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_b),
+            Ok(true),
+            "observing slot 1 again indicates true"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
+        let cached_blob_indices = cache
+            .items
+            .get(&ProposalKey::new(proposer_index_a, Slot::new(0)))
+            .expect("slot zero should be present");
+        assert_eq!(
+            cached_blob_indices.len(),
+            1,
+            "only one proposer should be present in slot 0"
+        );
+        let cached_blob_indices = cache
+            .items
+            .get(&ProposalKey::new(proposer_index_b, Slot::new(1)))
+            .expect("slot zero should be present");
+        assert_eq!(
+            cached_blob_indices.len(),
+            1,
+            "only one proposer should be present in slot 1"
+        );
+
+        // Slot 0, index 1
+        let sidecar_c = get_blob_sidecar(0, proposer_index_a, 1);
+
+        assert_eq!(
+            cache.proposer_is_known(&sidecar_c),
+            Ok(false),
+            "no observation for new index"
+        );
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_c),
+            Ok(false),
+            "can observe new index, indicates sidecar unobserved for new index"
+        );
+        assert_eq!(
+            cache.proposer_is_known(&sidecar_c),
+            Ok(true),
+            "observed new sidecar is indicated as true"
+        );
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_c),
+            Ok(true),
+            "observing new sidecar again indicates true"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
+        let cached_blob_indices = cache
+            .items
+            .get(&ProposalKey::new(proposer_index_a, Slot::new(0)))
+            .expect("slot zero should be present");
+        assert_eq!(
+            cached_blob_indices.len(),
+            2,
+            "two blob indices should be present in slot 0"
+        );
+
+        // Create a sidecar sharing slot and proposer but with a different block root.
+        let mut sidecar_d: BlobSidecar<E> = BlobSidecar {
+            index: sidecar_c.index,
+            blob: sidecar_c.blob.clone(),
+            kzg_commitment: sidecar_c.kzg_commitment,
+            kzg_proof: sidecar_c.kzg_proof,
+            signed_block_header: sidecar_c.signed_block_header.clone(),
+            kzg_commitment_inclusion_proof: sidecar_c.kzg_commitment_inclusion_proof.clone(),
+        };
+        sidecar_d.signed_block_header.message.body_root = Hash256::repeat_byte(7);
+        assert_eq!(
+            cache.proposer_is_known(&sidecar_d),
+            Ok(true),
+            "there has been an observation for this proposer index"
+        );
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_d),
+            Ok(true),
+            "indicates sidecar proposer was observed"
+        );
+        let cached_blob_indices = cache
+            .items
+            .get(&ProposalKey::new(proposer_index_a, Slot::new(0)))
+            .expect("slot zero should be present");
+        assert_eq!(
+            cached_blob_indices.len(),
+            2,
+            "two blob indices should be present in slot 0"
+        );
+
+        // Try adding an out of bounds index
+        let invalid_index = E::max_blobs_per_block() as u64;
+        let sidecar_d = get_blob_sidecar(0, proposer_index_a, invalid_index);
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_d),
+            Err(Error::InvalidBlobIndex(invalid_index)),
+            "cannot add an index > MaxBlobsPerBlock"
+        );
+    }
+}

beacon_node/beacon_chain/src/observed_block_producers.rs

@@ -1,9 +1,10 @@
 //! Provides the `ObservedBlockProducers` struct which allows for rejecting gossip blocks from
 //! validators that have already produced a block.
 
+use std::collections::hash_map::Entry;
 use std::collections::{HashMap, HashSet};
 use std::marker::PhantomData;
-use types::{BeaconBlockRef, Epoch, EthSpec, Slot, Unsigned};
+use types::{BeaconBlockRef, Epoch, EthSpec, Hash256, Slot, Unsigned};
 
 #[derive(Debug, PartialEq)]
 pub enum Error {
@@ -14,6 +15,18 @@ pub enum Error {
     ValidatorIndexTooHigh(u64),
 }
 
+#[derive(Eq, Hash, PartialEq, Debug, Default)]
+pub struct ProposalKey {
+    pub slot: Slot,
+    pub proposer: u64,
+}
+
+impl ProposalKey {
+    pub fn new(proposer: u64, slot: Slot) -> Self {
+        Self { slot, proposer }
+    }
+}
+
 /// Maintains a cache of observed `(block.slot, block.proposer)`.
 ///
 /// The cache supports pruning based upon the finalized epoch. It does not automatically prune, you
@@ -27,7 +40,7 @@ pub enum Error {
 /// known_distinct_shufflings` which is much smaller.
 pub struct ObservedBlockProducers<E: EthSpec> {
     finalized_slot: Slot,
-    items: HashMap<Slot, HashSet<u64>>,
+    items: HashMap<ProposalKey, HashSet<Hash256>>,
     _phantom: PhantomData<E>,
 }
 
@@ -42,6 +55,24 @@ impl<E: EthSpec> Default for ObservedBlockProducers<E> {
     }
 }
 
+pub enum SeenBlock {
+    Duplicate,
+    Slashable,
+    UniqueNonSlashable,
+}
+
+impl SeenBlock {
+    pub fn proposer_previously_observed(self) -> bool {
+        match self {
+            Self::Duplicate | Self::Slashable => true,
+            Self::UniqueNonSlashable => false,
+        }
+    }
+    pub fn is_slashable(&self) -> bool {
+        matches!(self, Self::Slashable)
+    }
+}
+
 impl<E: EthSpec> ObservedBlockProducers<E> {
     /// Observe that the `block` was produced by `block.proposer_index` at `block.slot`. This will
     /// update `self` so future calls to it indicate that this block is known.
@@ -52,16 +83,44 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
     ///
     /// - `block.proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`.
     /// - `block.slot` is equal to or less than the latest pruned `finalized_slot`.
-    pub fn observe_proposer(&mut self, block: BeaconBlockRef<'_, E>) -> Result<bool, Error> {
+    pub fn observe_proposal(
+        &mut self,
+        block_root: Hash256,
+        block: BeaconBlockRef<'_, E>,
+    ) -> Result<SeenBlock, Error> {
         self.sanitize_block(block)?;
 
-        let did_not_exist = self
-            .items
-            .entry(block.slot())
-            .or_insert_with(|| HashSet::with_capacity(E::SlotsPerEpoch::to_usize()))
-            .insert(block.proposer_index());
+        let key = ProposalKey {
+            slot: block.slot(),
+            proposer: block.proposer_index(),
+        };
 
-        Ok(!did_not_exist)
+        let entry = self.items.entry(key);
+
+        let slashable_proposal = match entry {
+            Entry::Occupied(mut occupied_entry) => {
+                let block_roots = occupied_entry.get_mut();
+                let newly_inserted = block_roots.insert(block_root);
+
+                let is_equivocation = block_roots.len() > 1;
+
+                if is_equivocation {
+                    SeenBlock::Slashable
+                } else if !newly_inserted {
+                    SeenBlock::Duplicate
+                } else {
+                    SeenBlock::UniqueNonSlashable
+                }
+            }
+            Entry::Vacant(vacant_entry) => {
+                let block_roots = HashSet::from([block_root]);
+                vacant_entry.insert(block_roots);
+
+                SeenBlock::UniqueNonSlashable
+            }
+        };
+
+        Ok(slashable_proposal)
     }
 
     /// Returns `Ok(true)` if the `block` has been observed before, `Ok(false)` if not. Does not
@@ -72,15 +131,33 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
     ///
     /// - `block.proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`.
     /// - `block.slot` is equal to or less than the latest pruned `finalized_slot`.
-    pub fn proposer_has_been_observed(&self, block: BeaconBlockRef<'_, E>) -> Result<bool, Error> {
+    pub fn proposer_has_been_observed(
+        &self,
+        block: BeaconBlockRef<'_, E>,
+        block_root: Hash256,
+    ) -> Result<SeenBlock, Error> {
         self.sanitize_block(block)?;
 
-        let exists = self
-            .items
-            .get(&block.slot())
-            .map_or(false, |set| set.contains(&block.proposer_index()));
+        let key = ProposalKey {
+            slot: block.slot(),
+            proposer: block.proposer_index(),
+        };
 
-        Ok(exists)
+        if let Some(block_roots) = self.items.get(&key) {
+            let block_already_known = block_roots.contains(&block_root);
+            let no_prev_known_blocks =
+                block_roots.difference(&HashSet::from([block_root])).count() == 0;
+
+            if !no_prev_known_blocks {
+                Ok(SeenBlock::Slashable)
+            } else if block_already_known {
+                Ok(SeenBlock::Duplicate)
+            } else {
+                Ok(SeenBlock::UniqueNonSlashable)
+            }
+        } else {
+            Ok(SeenBlock::UniqueNonSlashable)
+        }
     }
 
     /// Returns `Ok(())` if the given `block` is sane.
@@ -112,15 +189,15 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
         }
 
         self.finalized_slot = finalized_slot;
-        self.items.retain(|slot, _set| *slot > finalized_slot);
+        self.items.retain(|key, _| key.slot > finalized_slot);
     }
 
     /// Returns `true` if the given `validator_index` has been stored in `self` at `epoch`.
     ///
     /// This is useful for doppelganger detection.
     pub fn index_seen_at_epoch(&self, validator_index: u64, epoch: Epoch) -> bool {
-        self.items.iter().any(|(slot, producers)| {
-            slot.epoch(E::slots_per_epoch()) == epoch && producers.contains(&validator_index)
+        self.items.iter().any(|(key, _)| {
+            key.slot.epoch(E::slots_per_epoch()) == epoch && key.proposer == validator_index
         })
     }
 }
@@ -148,9 +225,12 @@ mod tests {
 
         // Slot 0, proposer 0
         let block_a = get_block(0, 0);
+        let block_root = block_a.canonical_root();
 
         assert_eq!(
-            cache.observe_proposer(block_a.to_ref()),
+            cache
+                .observe_proposal(block_root, block_a.to_ref())
+                .map(SeenBlock::proposer_previously_observed),
             Ok(false),
             "can observe proposer, indicates proposer unobserved"
         );
@@ -164,7 +244,10 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .get(&Slot::new(0))
+                .get(&ProposalKey {
+                    slot: Slot::new(0),
+                    proposer: 0
+                })
                 .expect("slot zero should be present")
                 .len(),
             1,
@@ -182,7 +265,10 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .get(&Slot::new(0))
+                .get(&ProposalKey {
+                    slot: Slot::new(0),
+                    proposer: 0
+                })
                 .expect("slot zero should be present")
                 .len(),
             1,
@@ -207,9 +293,12 @@ mod tests {
 
         // First slot of finalized epoch, proposer 0
         let block_b = get_block(E::slots_per_epoch(), 0);
+        let block_root_b = block_b.canonical_root();
 
         assert_eq!(
-            cache.observe_proposer(block_b.to_ref()),
+            cache
+                .observe_proposal(block_root_b, block_b.to_ref())
+                .map(SeenBlock::proposer_previously_observed),
             Err(Error::FinalizedBlock {
                 slot: E::slots_per_epoch().into(),
                 finalized_slot: E::slots_per_epoch().into(),
@@ -229,7 +318,9 @@ mod tests {
         let block_b = get_block(three_epochs, 0);
 
         assert_eq!(
-            cache.observe_proposer(block_b.to_ref()),
+            cache
+                .observe_proposal(block_root_b, block_b.to_ref())
+                .map(SeenBlock::proposer_previously_observed),
             Ok(false),
             "can insert non-finalized block"
         );
@@ -238,7 +329,10 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .get(&Slot::new(three_epochs))
+                .get(&ProposalKey {
+                    slot: Slot::new(three_epochs),
+                    proposer: 0
+                })
                 .expect("the three epochs slot should be present")
                 .len(),
             1,
@@ -262,7 +356,10 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .get(&Slot::new(three_epochs))
+                .get(&ProposalKey {
+                    slot: Slot::new(three_epochs),
+                    proposer: 0
+                })
                 .expect("the three epochs slot should be present")
                 .len(),
             1,
@@ -276,24 +373,33 @@ mod tests {
 
         // Slot 0, proposer 0
         let block_a = get_block(0, 0);
+        let block_root_a = block_a.canonical_root();
 
         assert_eq!(
-            cache.proposer_has_been_observed(block_a.to_ref()),
+            cache
+                .proposer_has_been_observed(block_a.to_ref(), block_a.canonical_root())
+                .map(|x| x.proposer_previously_observed()),
             Ok(false),
             "no observation in empty cache"
         );
         assert_eq!(
-            cache.observe_proposer(block_a.to_ref()),
+            cache
+                .observe_proposal(block_root_a, block_a.to_ref())
+                .map(SeenBlock::proposer_previously_observed),
             Ok(false),
             "can observe proposer, indicates proposer unobserved"
         );
         assert_eq!(
-            cache.proposer_has_been_observed(block_a.to_ref()),
+            cache
+                .proposer_has_been_observed(block_a.to_ref(), block_a.canonical_root())
+                .map(|x| x.proposer_previously_observed()),
             Ok(true),
             "observed block is indicated as true"
         );
         assert_eq!(
-            cache.observe_proposer(block_a.to_ref()),
+            cache
+                .observe_proposal(block_root_a, block_a.to_ref())
+                .map(SeenBlock::proposer_previously_observed),
             Ok(true),
             "observing again indicates true"
         );
@@ -303,7 +409,10 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .get(&Slot::new(0))
+                .get(&ProposalKey {
+                    slot: Slot::new(0),
+                    proposer: 0
+                })
                 .expect("slot zero should be present")
                 .len(),
             1,
@@ -312,24 +421,33 @@ mod tests {
 
         // Slot 1, proposer 0
         let block_b = get_block(1, 0);
+        let block_root_b = block_b.canonical_root();
 
         assert_eq!(
-            cache.proposer_has_been_observed(block_b.to_ref()),
+            cache
+                .proposer_has_been_observed(block_b.to_ref(), block_b.canonical_root())
+                .map(|x| x.proposer_previously_observed()),
             Ok(false),
             "no observation for new slot"
         );
         assert_eq!(
-            cache.observe_proposer(block_b.to_ref()),
+            cache
+                .observe_proposal(block_root_b, block_b.to_ref())
+                .map(SeenBlock::proposer_previously_observed),
             Ok(false),
             "can observe proposer for new slot, indicates proposer unobserved"
         );
         assert_eq!(
-            cache.proposer_has_been_observed(block_b.to_ref()),
+            cache
+                .proposer_has_been_observed(block_b.to_ref(), block_b.canonical_root())
+                .map(|x| x.proposer_previously_observed()),
             Ok(true),
             "observed block in slot 1 is indicated as true"
         );
         assert_eq!(
-            cache.observe_proposer(block_b.to_ref()),
+            cache
+                .observe_proposal(block_root_b, block_b.to_ref())
+                .map(SeenBlock::proposer_previously_observed),
             Ok(true),
             "observing slot 1 again indicates true"
         );
@@ -339,7 +457,10 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .get(&Slot::new(0))
+                .get(&ProposalKey {
+                    slot: Slot::new(0),
+                    proposer: 0
+                })
                 .expect("slot zero should be present")
                 .len(),
             1,
@@ -348,7 +469,10 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .get(&Slot::new(1))
+                .get(&ProposalKey {
+                    slot: Slot::new(1),
+                    proposer: 0
+                })
                 .expect("slot zero should be present")
                 .len(),
             1,
@@ -357,45 +481,54 @@ mod tests {
 
         // Slot 0, proposer 1
         let block_c = get_block(0, 1);
+        let block_root_c = block_c.canonical_root();
 
         assert_eq!(
-            cache.proposer_has_been_observed(block_c.to_ref()),
+            cache
+                .proposer_has_been_observed(block_c.to_ref(), block_c.canonical_root())
+                .map(|x| x.proposer_previously_observed()),
             Ok(false),
             "no observation for new proposer"
         );
         assert_eq!(
-            cache.observe_proposer(block_c.to_ref()),
+            cache
+                .observe_proposal(block_root_c, block_c.to_ref())
+                .map(SeenBlock::proposer_previously_observed),
             Ok(false),
             "can observe new proposer, indicates proposer unobserved"
         );
         assert_eq!(
-            cache.proposer_has_been_observed(block_c.to_ref()),
+            cache
+                .proposer_has_been_observed(block_c.to_ref(), block_c.canonical_root())
+                .map(|x| x.proposer_previously_observed()),
             Ok(true),
             "observed new proposer block is indicated as true"
         );
         assert_eq!(
-            cache.observe_proposer(block_c.to_ref()),
+            cache
+                .observe_proposal(block_root_c, block_c.to_ref())
+                .map(SeenBlock::proposer_previously_observed),
             Ok(true),
             "observing new proposer again indicates true"
         );
 
         assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
-        assert_eq!(cache.items.len(), 2, "two slots should be present");
+        assert_eq!(cache.items.len(), 3, "three slots should be present");
         assert_eq!(
             cache
                 .items
-                .get(&Slot::new(0))
-                .expect("slot zero should be present")
-                .len(),
+                .iter()
+                .filter(|(k, _)| k.slot == cache.finalized_slot)
+                .count(),
             2,
             "two proposers should be present in slot 0"
         );
         assert_eq!(
             cache
                 .items
-                .get(&Slot::new(1))
-                .expect("slot zero should be present")
-                .len(),
+                .iter()
+                .filter(|(k, _)| k.slot == Slot::new(1))
+                .count(),
             1,
             "only one proposer should be present in slot 1"
         );

beacon_node/beacon_chain/src/observed_operations.rs

@@ -1,10 +1,12 @@
 use derivative::Derivative;
-use smallvec::SmallVec;
-use state_processing::{SigVerifiedOp, VerifyOperation};
+use smallvec::{smallvec, SmallVec};
+use ssz::{Decode, Encode};
+use state_processing::{SigVerifiedOp, VerifyOperation, VerifyOperationAt};
 use std::collections::HashSet;
 use std::marker::PhantomData;
 use types::{
-    AttesterSlashing, BeaconState, ChainSpec, EthSpec, ProposerSlashing, SignedVoluntaryExit,
+    AttesterSlashing, BeaconState, ChainSpec, Epoch, EthSpec, ForkName, ProposerSlashing,
+    SignedBlsToExecutionChange, SignedVoluntaryExit, Slot,
 };
 
 /// Number of validator indices to store on the stack in `observed_validators`.
@@ -24,17 +26,20 @@ pub struct ObservedOperations<T: ObservableOperation<E>, E: EthSpec> {
     /// previously seen attester slashings, i.e. those validators in the intersection of
     /// `attestation_1.attester_indices` and `attestation_2.attester_indices`.
     observed_validator_indices: HashSet<u64>,
+    /// The name of the current fork. The default will be overwritten on first use.
+    #[derivative(Default(value = "ForkName::Base"))]
+    current_fork: ForkName,
     _phantom: PhantomData<(T, E)>,
 }
 
 /// Was the observed operation new and valid for further processing, or a useless duplicate?
 #[derive(Debug, PartialEq, Eq, Clone)]
-pub enum ObservationOutcome<T> {
-    New(SigVerifiedOp<T>),
+pub enum ObservationOutcome<T: Encode + Decode, E: EthSpec> {
+    New(SigVerifiedOp<T, E>),
     AlreadyKnown,
 }
 
-/// Trait for exits and slashings which can be observed using `ObservedOperations`.
+/// Trait for operations which can be observed using `ObservedOperations`.
 pub trait ObservableOperation<E: EthSpec>: VerifyOperation<E> + Sized {
     /// The set of validator indices involved in this operation.
     ///
@@ -44,13 +49,13 @@ pub trait ObservableOperation<E: EthSpec>: VerifyOperation<E> + Sized {
 
 impl<E: EthSpec> ObservableOperation<E> for SignedVoluntaryExit {
     fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]> {
-        std::iter::once(self.message.validator_index).collect()
+        smallvec![self.message.validator_index]
     }
 }
 
 impl<E: EthSpec> ObservableOperation<E> for ProposerSlashing {
     fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]> {
-        std::iter::once(self.signed_header_1.message.proposer_index).collect()
+        smallvec![self.signed_header_1.message.proposer_index]
     }
 }
 
@@ -75,13 +80,25 @@ impl<E: EthSpec> ObservableOperation<E> for AttesterSlashing<E> {
     }
 }
 
+impl<E: EthSpec> ObservableOperation<E> for SignedBlsToExecutionChange {
+    fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]> {
+        smallvec![self.message.validator_index]
+    }
+}
+
 impl<T: ObservableOperation<E>, E: EthSpec> ObservedOperations<T, E> {
-    pub fn verify_and_observe(
+    pub fn verify_and_observe_parametric<F>(
         &mut self,
         op: T,
+        validate: F,
         head_state: &BeaconState<E>,
         spec: &ChainSpec,
-    ) -> Result<ObservationOutcome<T>, T::Error> {
+    ) -> Result<ObservationOutcome<T, E>, T::Error>
+    where
+        F: Fn(T) -> Result<SigVerifiedOp<T, E>, T::Error>,
+    {
+        self.reset_at_fork_boundary(head_state.slot(), spec);
+
         let observed_validator_indices = &mut self.observed_validator_indices;
         let new_validator_indices = op.observed_validators();
 
@@ -99,7 +116,7 @@ impl<T: ObservableOperation<E>, E: EthSpec> ObservedOperations<T, E> {
         }
 
         // Validate the op using operation-specific logic (`verify_attester_slashing`, etc).
-        let verified_op = op.validate(head_state, spec)?;
+        let verified_op = validate(op)?;
 
         // Add the relevant indices to the set of known indices to prevent processing of duplicates
         // in the future.
@@ -107,4 +124,51 @@ impl<T: ObservableOperation<E>, E: EthSpec> ObservedOperations<T, E> {
 
         Ok(ObservationOutcome::New(verified_op))
     }
+
+    pub fn verify_and_observe(
+        &mut self,
+        op: T,
+        head_state: &BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<ObservationOutcome<T, E>, T::Error> {
+        let validate = |op: T| op.validate(head_state, spec);
+        self.verify_and_observe_parametric(op, validate, head_state, spec)
+    }
+
+    /// Reset the cache when crossing a fork boundary.
+    ///
+    /// This prevents an attacker from crafting a self-slashing which is only valid before the fork
+    /// (e.g. using the Altair fork domain at a Bellatrix epoch), in order to prevent propagation of
+    /// all other slashings due to the duplicate check.
+    ///
+    /// It doesn't matter if this cache gets reset too often, as we reset it on restart anyway and a
+    /// false negative just results in propagation of messages which should have been ignored.
+    ///
+    /// In future we could check slashing relevance against the op pool itself, but that would
+    /// require indexing the attester slashings in the op pool by validator index.
+    fn reset_at_fork_boundary(&mut self, head_slot: Slot, spec: &ChainSpec) {
+        let head_fork = spec.fork_name_at_slot::<E>(head_slot);
+        if head_fork != self.current_fork {
+            self.observed_validator_indices.clear();
+            self.current_fork = head_fork;
+        }
+    }
+
+    /// Reset the cache. MUST ONLY BE USED IN TESTS.
+    pub fn __reset_for_testing_only(&mut self) {
+        self.observed_validator_indices.clear();
+    }
+}
+
+impl<T: ObservableOperation<E> + VerifyOperationAt<E>, E: EthSpec> ObservedOperations<T, E> {
+    pub fn verify_and_observe_at(
+        &mut self,
+        op: T,
+        verify_at_epoch: Epoch,
+        head_state: &BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<ObservationOutcome<T, E>, T::Error> {
+        let validate = |op: T| op.validate_at(head_state, verify_at_epoch, spec);
+        self.verify_and_observe_parametric(op, validate, head_state, spec)
+    }
 }

beacon_node/beacon_chain/src/observed_slashable.rs

@@ -0,0 +1,486 @@
+//! Provides the `ObservedSlashable` struct which tracks slashable messages seen in
+//! gossip or via RPC. Useful in supporting `broadcast_validation` in the Beacon API.
+
+use crate::observed_block_producers::Error;
+use std::collections::hash_map::Entry;
+use std::collections::{HashMap, HashSet};
+use std::marker::PhantomData;
+use types::{EthSpec, Hash256, Slot, Unsigned};
+
+#[derive(Eq, Hash, PartialEq, Debug, Default)]
+pub struct ProposalKey {
+    pub slot: Slot,
+    pub proposer: u64,
+}
+
+/// Maintains a cache of observed `(block.slot, block.proposer)`.
+///
+/// The cache supports pruning based upon the finalized epoch. It does not automatically prune, you
+/// must call `Self::prune` manually.
+///
+/// The maximum size of the cache is determined by `slots_since_finality *
+/// VALIDATOR_REGISTRY_LIMIT`. This is quite a large size, so it's important that upstream
+/// functions only use this cache for blocks with a valid signature. Only allowing valid signed
+/// blocks reduces the theoretical maximum size of this cache to `slots_since_finality *
+/// active_validator_count`, however in reality that is more like `slots_since_finality *
+/// known_distinct_shufflings` which is much smaller.
+pub struct ObservedSlashable<E: EthSpec> {
+    finalized_slot: Slot,
+    items: HashMap<ProposalKey, HashSet<Hash256>>,
+    _phantom: PhantomData<E>,
+}
+
+impl<E: EthSpec> Default for ObservedSlashable<E> {
+    /// Instantiates `Self` with `finalized_slot == 0`.
+    fn default() -> Self {
+        Self {
+            finalized_slot: Slot::new(0),
+            items: HashMap::new(),
+            _phantom: PhantomData,
+        }
+    }
+}
+
+impl<E: EthSpec> ObservedSlashable<E> {
+    /// Observe that the `header` was produced by `header.proposer_index` at `header.slot`. This will
+    /// update `self` so future calls to it indicate that this block is known.
+    ///
+    /// The supplied `block` **MUST** be signature verified (see struct-level documentation).
+    ///
+    /// ## Errors
+    ///
+    /// - `header.proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`.
+    /// - `header.slot` is equal to or less than the latest pruned `finalized_slot`.
+    pub fn observe_slashable(
+        &mut self,
+        slot: Slot,
+        proposer_index: u64,
+        block_root: Hash256,
+    ) -> Result<(), Error> {
+        self.sanitize_header(slot, proposer_index)?;
+
+        let key = ProposalKey {
+            slot,
+            proposer: proposer_index,
+        };
+
+        let entry = self.items.entry(key);
+
+        match entry {
+            Entry::Occupied(mut occupied_entry) => {
+                let block_roots = occupied_entry.get_mut();
+                block_roots.insert(block_root);
+            }
+            Entry::Vacant(vacant_entry) => {
+                let block_roots = HashSet::from([block_root]);
+                vacant_entry.insert(block_roots);
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Returns `Ok(true)` if the `block_root` is slashable, `Ok(false)` if not. Does not
+    /// update the cache, so calling this function multiple times will continue to return
+    /// `Ok(false)`, until `Self::observe_proposer` is called.
+    ///
+    /// ## Errors
+    ///
+    /// - `proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`.
+    /// - `slot` is equal to or less than the latest pruned `finalized_slot`.
+    pub fn is_slashable(
+        &self,
+        slot: Slot,
+        proposer_index: u64,
+        block_root: Hash256,
+    ) -> Result<bool, Error> {
+        self.sanitize_header(slot, proposer_index)?;
+
+        let key = ProposalKey {
+            slot,
+            proposer: proposer_index,
+        };
+
+        if let Some(block_roots) = self.items.get(&key) {
+            let no_prev_known_blocks =
+                block_roots.difference(&HashSet::from([block_root])).count() == 0;
+
+            Ok(!no_prev_known_blocks)
+        } else {
+            Ok(false)
+        }
+    }
+
+    /// Returns `Ok(())` if the given `header` is sane.
+    fn sanitize_header(&self, slot: Slot, proposer_index: u64) -> Result<(), Error> {
+        if proposer_index >= E::ValidatorRegistryLimit::to_u64() {
+            return Err(Error::ValidatorIndexTooHigh(proposer_index));
+        }
+
+        let finalized_slot = self.finalized_slot;
+        if finalized_slot > 0 && slot <= finalized_slot {
+            return Err(Error::FinalizedBlock {
+                slot,
+                finalized_slot,
+            });
+        }
+
+        Ok(())
+    }
+
+    /// Removes all observations of blocks equal to or earlier than `finalized_slot`.
+    ///
+    /// Stores `finalized_slot` in `self`, so that `self` will reject any block that has a slot
+    /// equal to or less than `finalized_slot`.
+    ///
+    /// No-op if `finalized_slot == 0`.
+    pub fn prune(&mut self, finalized_slot: Slot) {
+        if finalized_slot == 0 {
+            return;
+        }
+
+        self.finalized_slot = finalized_slot;
+        self.items.retain(|key, _| key.slot > finalized_slot);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use types::{BeaconBlock, Graffiti, MainnetEthSpec};
+
+    type E = MainnetEthSpec;
+
+    fn get_block(slot: u64, proposer: u64) -> BeaconBlock<E> {
+        let mut block = BeaconBlock::empty(&E::default_spec());
+        *block.slot_mut() = slot.into();
+        *block.proposer_index_mut() = proposer;
+        block
+    }
+
+    #[test]
+    fn pruning() {
+        let mut cache = ObservedSlashable::<E>::default();
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 0, "no slots should be present");
+
+        // Slot 0, proposer 0
+        let block_a = get_block(0, 0);
+        let block_root = block_a.canonical_root();
+
+        assert_eq!(
+            cache.observe_slashable(block_a.slot(), block_a.proposer_index(), block_root),
+            Ok(()),
+            "can observe proposer"
+        );
+
+        /*
+         * Preconditions.
+         */
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&ProposalKey {
+                    slot: Slot::new(0),
+                    proposer: 0
+                })
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        /*
+         * Check that a prune at the genesis slot does nothing.
+         */
+        cache.prune(Slot::new(0));
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&ProposalKey {
+                    slot: Slot::new(0),
+                    proposer: 0
+                })
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one block root should be present"
+        );
+
+        /*
+         * Check that a prune empties the cache
+         */
+        cache.prune(E::slots_per_epoch().into());
+        assert_eq!(
+            cache.finalized_slot,
+            Slot::from(E::slots_per_epoch()),
+            "finalized slot is updated"
+        );
+        assert_eq!(cache.items.len(), 0, "no items left");
+
+        /*
+         * Check that we can't insert a finalized block
+         */
+        // First slot of finalized epoch, proposer 0
+        let block_b = get_block(E::slots_per_epoch(), 0);
+        let block_root_b = block_b.canonical_root();
+
+        assert_eq!(
+            cache.observe_slashable(block_b.slot(), block_b.proposer_index(), block_root_b),
+            Err(Error::FinalizedBlock {
+                slot: E::slots_per_epoch().into(),
+                finalized_slot: E::slots_per_epoch().into(),
+            }),
+            "cant insert finalized block"
+        );
+
+        assert_eq!(cache.items.len(), 0, "block was not added");
+
+        /*
+         * Check that we _can_ insert a non-finalized block
+         */
+        let three_epochs = E::slots_per_epoch() * 3;
+
+        // First slot of finalized epoch, proposer 0
+        let block_b = get_block(three_epochs, 0);
+
+        assert_eq!(
+            cache.observe_slashable(block_b.slot(), block_b.proposer_index(), block_root_b),
+            Ok(()),
+            "can insert non-finalized block"
+        );
+
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&ProposalKey {
+                    slot: Slot::new(three_epochs),
+                    proposer: 0
+                })
+                .expect("the three epochs slot should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        /*
+         * Check that a prune doesnt wipe later blocks
+         */
+        let two_epochs = E::slots_per_epoch() * 2;
+        cache.prune(two_epochs.into());
+
+        assert_eq!(
+            cache.finalized_slot,
+            Slot::from(two_epochs),
+            "finalized slot is updated"
+        );
+
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&ProposalKey {
+                    slot: Slot::new(three_epochs),
+                    proposer: 0
+                })
+                .expect("the three epochs slot should be present")
+                .len(),
+            1,
+            "only one block root should be present"
+        );
+    }
+
+    #[test]
+    fn simple_observations() {
+        let mut cache = ObservedSlashable::<E>::default();
+
+        // Slot 0, proposer 0
+        let block_a = get_block(0, 0);
+        let block_root_a = block_a.canonical_root();
+
+        assert_eq!(
+            cache.is_slashable(
+                block_a.slot(),
+                block_a.proposer_index(),
+                block_a.canonical_root()
+            ),
+            Ok(false),
+            "no observation in empty cache"
+        );
+        assert_eq!(
+            cache.observe_slashable(block_a.slot(), block_a.proposer_index(), block_root_a),
+            Ok(()),
+            "can observe proposer"
+        );
+        assert_eq!(
+            cache.is_slashable(
+                block_a.slot(),
+                block_a.proposer_index(),
+                block_a.canonical_root()
+            ),
+            Ok(false),
+            "observed but unslashed block"
+        );
+        assert_eq!(
+            cache.observe_slashable(block_a.slot(), block_a.proposer_index(), block_root_a),
+            Ok(()),
+            "observing again"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&ProposalKey {
+                    slot: Slot::new(0),
+                    proposer: 0
+                })
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one block root should be present"
+        );
+
+        // Slot 1, proposer 0
+        let block_b = get_block(1, 0);
+        let block_root_b = block_b.canonical_root();
+
+        assert_eq!(
+            cache.is_slashable(
+                block_b.slot(),
+                block_b.proposer_index(),
+                block_b.canonical_root()
+            ),
+            Ok(false),
+            "not slashable for new slot"
+        );
+        assert_eq!(
+            cache.observe_slashable(block_b.slot(), block_b.proposer_index(), block_root_b),
+            Ok(()),
+            "can observe proposer for new slot"
+        );
+        assert_eq!(
+            cache.is_slashable(
+                block_b.slot(),
+                block_b.proposer_index(),
+                block_b.canonical_root()
+            ),
+            Ok(false),
+            "observed but not slashable block in slot 1"
+        );
+        assert_eq!(
+            cache.observe_slashable(block_b.slot(), block_b.proposer_index(), block_root_b),
+            Ok(()),
+            "observing slot 1 again"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&ProposalKey {
+                    slot: Slot::new(0),
+                    proposer: 0
+                })
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one block root should be present in slot 0"
+        );
+        assert_eq!(
+            cache
+                .items
+                .get(&ProposalKey {
+                    slot: Slot::new(1),
+                    proposer: 0
+                })
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one block root should be present in slot 1"
+        );
+
+        // Slot 0, proposer 1
+        let block_c = get_block(0, 1);
+        let block_root_c = block_c.canonical_root();
+
+        assert_eq!(
+            cache.is_slashable(
+                block_c.slot(),
+                block_c.proposer_index(),
+                block_c.canonical_root()
+            ),
+            Ok(false),
+            "not slashable due to new proposer"
+        );
+        assert_eq!(
+            cache.observe_slashable(block_c.slot(), block_c.proposer_index(), block_root_c),
+            Ok(()),
+            "can observe new proposer, indicates proposer unobserved"
+        );
+        assert_eq!(
+            cache.is_slashable(
+                block_c.slot(),
+                block_c.proposer_index(),
+                block_c.canonical_root()
+            ),
+            Ok(false),
+            "not slashable due to new proposer"
+        );
+        assert_eq!(
+            cache.observe_slashable(block_c.slot(), block_c.proposer_index(), block_root_c),
+            Ok(()),
+            "observing new proposer again"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 3, "three slots should be present");
+        assert_eq!(
+            cache
+                .items
+                .iter()
+                .filter(|(k, _)| k.slot == cache.finalized_slot)
+                .count(),
+            2,
+            "two proposers should be present in slot 0"
+        );
+        assert_eq!(
+            cache
+                .items
+                .iter()
+                .filter(|(k, _)| k.slot == Slot::new(1))
+                .count(),
+            1,
+            "only one proposer should be present in slot 1"
+        );
+
+        // Slot 0, proposer 1 (again)
+        let mut block_d = get_block(0, 1);
+        *block_d.body_mut().graffiti_mut() = Graffiti::from(*b"this is slashable               ");
+        let block_root_d = block_d.canonical_root();
+
+        assert_eq!(
+            cache.is_slashable(
+                block_d.slot(),
+                block_d.proposer_index(),
+                block_d.canonical_root()
+            ),
+            Ok(true),
+            "slashable due to new proposer"
+        );
+        assert_eq!(
+            cache.observe_slashable(block_d.slot(), block_d.proposer_index(), block_root_d),
+            Ok(()),
+            "can observe new proposer, indicates proposer unobserved"
+        );
+    }
+}

beacon_node/beacon_chain/src/otb_verification_service.rs

@@ -0,0 +1,381 @@
+use crate::execution_payload::{validate_merge_block, AllowOptimisticImport};
+use crate::{
+    BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, ExecutionPayloadError,
+    INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON,
+};
+use itertools::process_results;
+use proto_array::InvalidationOperation;
+use slog::{crit, debug, error, info, warn};
+use slot_clock::SlotClock;
+use ssz::{Decode, Encode};
+use ssz_derive::{Decode, Encode};
+use state_processing::per_block_processing::is_merge_transition_complete;
+use std::sync::Arc;
+use store::{DBColumn, Error as StoreError, HotColdDB, KeyValueStore, StoreItem};
+use task_executor::{ShutdownReason, TaskExecutor};
+use tokio::time::sleep;
+use tree_hash::TreeHash;
+use types::{BeaconBlockRef, EthSpec, Hash256, Slot};
+use DBColumn::OptimisticTransitionBlock as OTBColumn;
+
+#[derive(Clone, Debug, Decode, Encode, PartialEq)]
+pub struct OptimisticTransitionBlock {
+    root: Hash256,
+    slot: Slot,
+}
+
+impl OptimisticTransitionBlock {
+    // types::BeaconBlockRef<'_, <T as BeaconChainTypes>::EthSpec>
+    pub fn from_block<E: EthSpec>(block: BeaconBlockRef<E>) -> Self {
+        Self {
+            root: block.tree_hash_root(),
+            slot: block.slot(),
+        }
+    }
+
+    pub fn root(&self) -> &Hash256 {
+        &self.root
+    }
+
+    pub fn slot(&self) -> &Slot {
+        &self.slot
+    }
+
+    pub fn persist_in_store<T, A>(&self, store: A) -> Result<(), StoreError>
+    where
+        T: BeaconChainTypes,
+        A: AsRef<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    {
+        if store
+            .as_ref()
+            .item_exists::<OptimisticTransitionBlock>(&self.root)?
+        {
+            Ok(())
+        } else {
+            store.as_ref().put_item(&self.root, self)
+        }
+    }
+
+    pub fn remove_from_store<T, A>(&self, store: A) -> Result<(), StoreError>
+    where
+        T: BeaconChainTypes,
+        A: AsRef<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    {
+        store
+            .as_ref()
+            .hot_db
+            .key_delete(OTBColumn.into(), self.root.as_bytes())
+    }
+
+    fn is_canonical<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<bool, BeaconChainError> {
+        Ok(chain
+            .forwards_iter_block_roots_until(self.slot, self.slot)?
+            .next()
+            .transpose()?
+            .map(|(root, _)| root)
+            == Some(self.root))
+    }
+}
+
+impl StoreItem for OptimisticTransitionBlock {
+    fn db_column() -> DBColumn {
+        OTBColumn
+    }
+
+    fn as_store_bytes(&self) -> Vec<u8> {
+        self.as_ssz_bytes()
+    }
+
+    fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
+        Ok(Self::from_ssz_bytes(bytes)?)
+    }
+}
+
+/// The routine is expected to run once per epoch, 1/4th through the epoch.
+pub const EPOCH_DELAY_FACTOR: u32 = 4;
+
+/// Spawns a routine which checks the validity of any optimistically imported transition blocks
+///
+/// This routine will run once per epoch, at `epoch_duration / EPOCH_DELAY_FACTOR` after
+/// the start of each epoch.
+///
+/// The service will not be started if there is no `execution_layer` on the `chain`.
+pub fn start_otb_verification_service<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    chain: Arc<BeaconChain<T>>,
+) {
+    // Avoid spawning the service if there's no EL, it'll just error anyway.
+    if chain.execution_layer.is_some() {
+        executor.spawn(
+            async move { otb_verification_service(chain).await },
+            "otb_verification_service",
+        );
+    }
+}
+
+pub fn load_optimistic_transition_blocks<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+) -> Result<Vec<OptimisticTransitionBlock>, StoreError> {
+    process_results(
+        chain.store.hot_db.iter_column::<Hash256>(OTBColumn),
+        |iter| {
+            iter.map(|(_, bytes)| OptimisticTransitionBlock::from_store_bytes(&bytes))
+                .collect()
+        },
+    )?
+}
+
+#[derive(Debug)]
+pub enum Error {
+    ForkChoice(String),
+    BeaconChain(BeaconChainError),
+    StoreError(StoreError),
+    NoBlockFound(OptimisticTransitionBlock),
+}
+
+pub async fn validate_optimistic_transition_blocks<T: BeaconChainTypes>(
+    chain: &Arc<BeaconChain<T>>,
+    otbs: Vec<OptimisticTransitionBlock>,
+) -> Result<(), Error> {
+    let finalized_slot = chain
+        .canonical_head
+        .fork_choice_read_lock()
+        .get_finalized_block()
+        .map_err(|e| Error::ForkChoice(format!("{:?}", e)))?
+        .slot;
+
+    // separate otbs into
+    //     non-canonical
+    //     finalized canonical
+    //     unfinalized canonical
+    let mut non_canonical_otbs = vec![];
+    let (finalized_canonical_otbs, unfinalized_canonical_otbs) = process_results(
+        otbs.into_iter().map(|otb| {
+            otb.is_canonical(chain)
+                .map(|is_canonical| (otb, is_canonical))
+        }),
+        |pair_iter| {
+            pair_iter
+                .filter_map(|(otb, is_canonical)| {
+                    if is_canonical {
+                        Some(otb)
+                    } else {
+                        non_canonical_otbs.push(otb);
+                        None
+                    }
+                })
+                .partition::<Vec<_>, _>(|otb| *otb.slot() <= finalized_slot)
+        },
+    )
+    .map_err(Error::BeaconChain)?;
+
+    // remove non-canonical blocks that conflict with finalized checkpoint from the database
+    for otb in non_canonical_otbs {
+        if *otb.slot() <= finalized_slot {
+            otb.remove_from_store::<T, _>(&chain.store)
+                .map_err(Error::StoreError)?;
+        }
+    }
+
+    // ensure finalized canonical otb are valid, otherwise kill client
+    for otb in finalized_canonical_otbs {
+        match chain.get_block(otb.root()).await {
+            Ok(Some(block)) => {
+                match validate_merge_block(chain, block.message(), AllowOptimisticImport::No).await
+                {
+                    Ok(()) => {
+                        // merge transition block is valid, remove it from OTB
+                        otb.remove_from_store::<T, _>(&chain.store)
+                            .map_err(Error::StoreError)?;
+                        info!(
+                            chain.log,
+                            "Validated merge transition block";
+                            "block_root" => ?otb.root(),
+                            "type" => "finalized"
+                        );
+                    }
+                    // The block was not able to be verified by the EL. Leave the OTB in the
+                    // database since the EL is likely still syncing and may verify the block
+                    // later.
+                    Err(BlockError::ExecutionPayloadError(
+                        ExecutionPayloadError::UnverifiedNonOptimisticCandidate,
+                    )) => (),
+                    Err(BlockError::ExecutionPayloadError(
+                        ExecutionPayloadError::InvalidTerminalPoWBlock { .. },
+                    )) => {
+                        // Finalized Merge Transition Block is Invalid! Kill the Client!
+                        crit!(
+                            chain.log,
+                            "Finalized merge transition block is invalid!";
+                            "msg" => "You must use the `--purge-db` flag to clear the database and restart sync. \
+                            You may be on a hostile network.",
+                            "block_hash" => ?block.canonical_root()
+                        );
+                        let mut shutdown_sender = chain.shutdown_sender();
+                        if let Err(e) = shutdown_sender.try_send(ShutdownReason::Failure(
+                            INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON,
+                        )) {
+                            crit!(
+                                chain.log,
+                                "Failed to shut down client";
+                                "error" => ?e,
+                                "shutdown_reason" => INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON
+                            );
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            Ok(None) => return Err(Error::NoBlockFound(otb)),
+            // Our database has pruned the payload and the payload was unavailable on the EL since
+            // the EL is still syncing or the payload is non-canonical.
+            Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => (),
+            Err(e) => return Err(Error::BeaconChain(e)),
+        }
+    }
+
+    // attempt to validate any non-finalized canonical otb blocks
+    for otb in unfinalized_canonical_otbs {
+        match chain.get_block(otb.root()).await {
+            Ok(Some(block)) => {
+                match validate_merge_block(chain, block.message(), AllowOptimisticImport::No).await
+                {
+                    Ok(()) => {
+                        // merge transition block is valid, remove it from OTB
+                        otb.remove_from_store::<T, _>(&chain.store)
+                            .map_err(Error::StoreError)?;
+                        info!(
+                            chain.log,
+                            "Validated merge transition block";
+                            "block_root" => ?otb.root(),
+                            "type" => "not finalized"
+                        );
+                    }
+                    // The block was not able to be verified by the EL. Leave the OTB in the
+                    // database since the EL is likely still syncing and may verify the block
+                    // later.
+                    Err(BlockError::ExecutionPayloadError(
+                        ExecutionPayloadError::UnverifiedNonOptimisticCandidate,
+                    )) => (),
+                    Err(BlockError::ExecutionPayloadError(
+                        ExecutionPayloadError::InvalidTerminalPoWBlock { .. },
+                    )) => {
+                        // Unfinalized Merge Transition Block is Invalid -> Run process_invalid_execution_payload
+                        warn!(
+                            chain.log,
+                            "Merge transition block invalid";
+                            "block_root" => ?otb.root()
+                        );
+                        chain
+                            .process_invalid_execution_payload(
+                                &InvalidationOperation::InvalidateOne {
+                                    block_root: *otb.root(),
+                                },
+                            )
+                            .await
+                            .map_err(|e| {
+                                warn!(
+                                    chain.log,
+                                    "Error checking merge transition block";
+                                    "error" => ?e,
+                                    "location" => "process_invalid_execution_payload"
+                                );
+                                Error::BeaconChain(e)
+                            })?;
+                    }
+                    _ => {}
+                }
+            }
+            Ok(None) => return Err(Error::NoBlockFound(otb)),
+            // Our database has pruned the payload and the payload was unavailable on the EL since
+            // the EL is still syncing or the payload is non-canonical.
+            Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => (),
+            Err(e) => return Err(Error::BeaconChain(e)),
+        }
+    }
+
+    Ok(())
+}
+
+/// Loop until any optimistically imported merge transition blocks have been verified and
+/// the merge has been finalized.
+async fn otb_verification_service<T: BeaconChainTypes>(chain: Arc<BeaconChain<T>>) {
+    let epoch_duration = chain.slot_clock.slot_duration() * T::EthSpec::slots_per_epoch() as u32;
+    loop {
+        match chain
+            .slot_clock
+            .duration_to_next_epoch(T::EthSpec::slots_per_epoch())
+        {
+            Some(duration) => {
+                let additional_delay = epoch_duration / EPOCH_DELAY_FACTOR;
+                sleep(duration + additional_delay).await;
+
+                debug!(
+                    chain.log,
+                    "OTB verification service firing";
+                );
+
+                if !is_merge_transition_complete(
+                    &chain.canonical_head.cached_head().snapshot.beacon_state,
+                ) {
+                    // We are pre-merge. Nothing to do yet.
+                    continue;
+                }
+
+                // load all optimistically imported transition blocks from the database
+                match load_optimistic_transition_blocks(chain.as_ref()) {
+                    Ok(otbs) => {
+                        if otbs.is_empty() {
+                            if chain
+                                .canonical_head
+                                .fork_choice_read_lock()
+                                .get_finalized_block()
+                                .map_or(false, |block| {
+                                    block.execution_status.is_execution_enabled()
+                                })
+                            {
+                                // there are no optimistic blocks in the database, we can exit
+                                // the service since the merge transition is finalized and we'll
+                                // never see another transition block
+                                break;
+                            } else {
+                                debug!(
+                                    chain.log,
+                                    "No optimistic transition blocks";
+                                    "info" => "waiting for the merge transition to finalize"
+                                )
+                            }
+                        }
+                        if let Err(e) = validate_optimistic_transition_blocks(&chain, otbs).await {
+                            warn!(
+                                chain.log,
+                                "Error while validating optimistic transition blocks";
+                                "error" => ?e
+                            );
+                        }
+                    }
+                    Err(e) => {
+                        error!(
+                            chain.log,
+                            "Error loading optimistic transition blocks";
+                            "error" => ?e
+                        );
+                    }
+                };
+            }
+            None => {
+                error!(chain.log, "Failed to read slot clock");
+                // If we can't read the slot clock, just wait another slot.
+                sleep(chain.slot_clock.slot_duration()).await;
+            }
+        };
+    }
+    debug!(
+        chain.log,
+        "No optimistic transition blocks in database";
+        "msg" => "shutting down OTB verification service"
+    );
+}

beacon_node/beacon_chain/src/parallel_state_cache.rs

@@ -0,0 +1,22 @@
+use promise_cache::{PromiseCache, Protect};
+use types::{BeaconState, Hash256};
+
+#[derive(Debug, Default)]
+pub struct ParallelStateProtector;
+
+impl Protect<Hash256> for ParallelStateProtector {
+    type SortKey = usize;
+
+    /// Evict in arbitrary (hashmap) order by using the same key for every value.
+    fn sort_key(&self, _: &Hash256) -> Self::SortKey {
+        0
+    }
+
+    /// We don't care too much about preventing evictions of particular states here. All the states
+    /// in this cache should be different from the head state.
+    fn protect_from_eviction(&self, _: &Hash256) -> bool {
+        false
+    }
+}
+
+pub type ParallelStateCache<E> = PromiseCache<Hash256, BeaconState<E>, ParallelStateProtector>;

beacon_node/beacon_chain/src/persisted_fork_choice.rs

@@ -1,27 +1,41 @@
-use crate::beacon_fork_choice_store::{
-    PersistedForkChoiceStoreV1, PersistedForkChoiceStoreV7, PersistedForkChoiceStoreV8,
-};
+use crate::beacon_fork_choice_store::{PersistedForkChoiceStoreV11, PersistedForkChoiceStoreV17};
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
 use store::{DBColumn, Error, StoreItem};
 use superstruct::superstruct;
 
 // If adding a new version you should update this type alias and fix the breakages.
-pub type PersistedForkChoice = PersistedForkChoiceV8;
+pub type PersistedForkChoice = PersistedForkChoiceV17;
 
 #[superstruct(
-    variants(V1, V7, V8),
+    variants(V11, V17),
     variant_attributes(derive(Encode, Decode)),
     no_enum
 )]
 pub struct PersistedForkChoice {
     pub fork_choice: fork_choice::PersistedForkChoice,
-    #[superstruct(only(V1))]
-    pub fork_choice_store: PersistedForkChoiceStoreV1,
-    #[superstruct(only(V7))]
-    pub fork_choice_store: PersistedForkChoiceStoreV7,
-    #[superstruct(only(V8))]
-    pub fork_choice_store: PersistedForkChoiceStoreV8,
+    #[superstruct(only(V11))]
+    pub fork_choice_store: PersistedForkChoiceStoreV11,
+    #[superstruct(only(V17))]
+    pub fork_choice_store: PersistedForkChoiceStoreV17,
+}
+
+impl Into<PersistedForkChoice> for PersistedForkChoiceV11 {
+    fn into(self) -> PersistedForkChoice {
+        PersistedForkChoice {
+            fork_choice: self.fork_choice,
+            fork_choice_store: self.fork_choice_store.into(),
+        }
+    }
+}
+
+impl Into<PersistedForkChoiceV11> for PersistedForkChoice {
+    fn into(self) -> PersistedForkChoiceV11 {
+        PersistedForkChoiceV11 {
+            fork_choice: self.fork_choice,
+            fork_choice_store: self.fork_choice_store.into(),
+        }
+    }
 }
 
 macro_rules! impl_store_item {
@@ -35,13 +49,12 @@ macro_rules! impl_store_item {
                 self.as_ssz_bytes()
             }
 
-            fn from_store_bytes(bytes: &[u8]) -> std::result::Result<Self, Error> {
+            fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error> {
                 Self::from_ssz_bytes(bytes).map_err(Into::into)
             }
         }
     };
 }
 
-impl_store_item!(PersistedForkChoiceV1);
-impl_store_item!(PersistedForkChoiceV7);
-impl_store_item!(PersistedForkChoiceV8);
+impl_store_item!(PersistedForkChoiceV11);
+impl_store_item!(PersistedForkChoiceV17);

beacon_node/beacon_chain/src/pre_finalization_cache.rs

@@ -3,11 +3,13 @@ use itertools::process_results;
 use lru::LruCache;
 use parking_lot::Mutex;
 use slog::debug;
+use std::num::NonZeroUsize;
 use std::time::Duration;
+use types::non_zero_usize::new_non_zero_usize;
 use types::Hash256;
 
-const BLOCK_ROOT_CACHE_LIMIT: usize = 512;
-const LOOKUP_LIMIT: usize = 8;
+const BLOCK_ROOT_CACHE_LIMIT: NonZeroUsize = new_non_zero_usize(512);
+const LOOKUP_LIMIT: NonZeroUsize = new_non_zero_usize(8);
 const METRICS_TIMEOUT: Duration = Duration::from_millis(100);
 
 /// Cache for rejecting attestations to blocks from before finalization.
@@ -71,14 +73,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
 
         // 2. Check on disk.
-        if self.store.get_block(&block_root)?.is_some() {
+        if self.store.get_blinded_block(&block_root, None)?.is_some() {
             cache.block_roots.put(block_root, ());
             return Ok(true);
         }
 
         // 3. Check the network with a single block lookup.
         cache.in_progress_lookups.put(block_root, ());
-        if cache.in_progress_lookups.len() == LOOKUP_LIMIT {
+        if cache.in_progress_lookups.len() == LOOKUP_LIMIT.get() {
             // NOTE: we expect this to occur sometimes if a lot of blocks that we look up fail to be
             // imported for reasons other than being pre-finalization. The cache will eventually
             // self-repair in this case by replacing old entries with new ones until all the failed

beacon_node/beacon_chain/src/proposer_prep_service.rs

@@ -0,0 +1,72 @@
+use crate::{BeaconChain, BeaconChainTypes};
+use slog::{debug, error};
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use task_executor::TaskExecutor;
+use tokio::time::sleep;
+
+/// Spawns a routine which ensures the EL is provided advance notice of any block producers.
+///
+/// This routine will run once per slot, at `chain.prepare_payload_lookahead()`
+/// before the start of each slot.
+///
+/// The service will not be started if there is no `execution_layer` on the `chain`.
+pub fn start_proposer_prep_service<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    chain: Arc<BeaconChain<T>>,
+) {
+    // Avoid spawning the service if there's no EL, it'll just error anyway.
+    if chain.execution_layer.is_some() {
+        executor.clone().spawn(
+            async move { proposer_prep_service(executor, chain).await },
+            "proposer_prep_service",
+        );
+    }
+}
+
+/// Loop indefinitely, calling `BeaconChain::prepare_beacon_proposer_async` at an interval.
+async fn proposer_prep_service<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    chain: Arc<BeaconChain<T>>,
+) {
+    let slot_duration = chain.slot_clock.slot_duration();
+
+    loop {
+        match chain.slot_clock.duration_to_next_slot() {
+            Some(duration) => {
+                let additional_delay =
+                    slot_duration.saturating_sub(chain.config.prepare_payload_lookahead);
+                sleep(duration + additional_delay).await;
+
+                debug!(
+                    chain.log,
+                    "Proposer prepare routine firing";
+                );
+
+                let inner_chain = chain.clone();
+                executor.spawn(
+                    async move {
+                        if let Ok(current_slot) = inner_chain.slot() {
+                            if let Err(e) = inner_chain.prepare_beacon_proposer(current_slot).await
+                            {
+                                error!(
+                                    inner_chain.log,
+                                    "Proposer prepare routine failed";
+                                    "error" => ?e
+                                );
+                            }
+                        } else {
+                            debug!(inner_chain.log, "No slot for proposer prepare routine");
+                        }
+                    },
+                    "proposer_prep_update",
+                );
+            }
+            None => {
+                error!(chain.log, "Failed to read slot clock");
+                // If we can't read the slot clock, just wait another slot.
+                sleep(slot_duration).await;
+            }
+        };
+    }
+}

beacon_node/beacon_chain/src/schema_change.rs

@@ -1,185 +1,91 @@
 //! Utilities for managing database schema changes.
-mod migration_schema_v6;
-mod migration_schema_v7;
-mod migration_schema_v8;
-mod types;
+mod migration_schema_v17;
+mod migration_schema_v18;
+mod migration_schema_v19;
+mod migration_schema_v20;
 
-use crate::beacon_chain::{BeaconChainTypes, FORK_CHOICE_DB_KEY, OP_POOL_DB_KEY};
-use crate::persisted_fork_choice::{PersistedForkChoiceV1, PersistedForkChoiceV7};
-use crate::validator_pubkey_cache::ValidatorPubkeyCache;
-use operation_pool::{PersistedOperationPool, PersistedOperationPoolBase};
-use slog::{warn, Logger};
-use ssz::{Decode, Encode};
-use ssz_derive::{Decode, Encode};
-use std::fs;
-use std::path::Path;
+use crate::beacon_chain::BeaconChainTypes;
+use crate::types::ChainSpec;
+use slog::Logger;
 use std::sync::Arc;
-use store::config::OnDiskStoreConfig;
 use store::hot_cold_store::{HotColdDB, HotColdDBError};
-use store::metadata::{SchemaVersion, CONFIG_KEY, CURRENT_SCHEMA_VERSION};
-use store::{DBColumn, Error as StoreError, ItemStore, StoreItem};
-
-const PUBKEY_CACHE_FILENAME: &str = "pubkey_cache.ssz";
+use store::metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION};
+use store::Error as StoreError;
 
 /// Migrate the database from one schema version to another, applying all requisite mutations.
+#[allow(clippy::only_used_in_recursion)] // spec is not used but likely to be used in future
 pub fn migrate_schema<T: BeaconChainTypes>(
     db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
-    datadir: &Path,
+    deposit_contract_deploy_block: u64,
     from: SchemaVersion,
     to: SchemaVersion,
     log: Logger,
+    spec: &ChainSpec,
 ) -> Result<(), StoreError> {
     match (from, to) {
-        // Migrating from the current schema version to iself is always OK, a no-op.
+        // Migrating from the current schema version to itself is always OK, a no-op.
         (_, _) if from == to && to == CURRENT_SCHEMA_VERSION => Ok(()),
-        // Migrate across multiple versions by recursively migrating one step at a time.
+        // Upgrade for tree-states database changes.
+        (SchemaVersion(12), SchemaVersion(20)) => {
+            migration_schema_v20::upgrade_to_v20::<T>(db, log)
+        }
+        // Downgrade for tree-states database changes.
+        (SchemaVersion(20), SchemaVersion(12)) => {
+            migration_schema_v20::downgrade_from_v20::<T>(db, log)
+        }
+        // Upgrade across multiple versions by recursively migrating one step at a time.
         (_, _) if from.as_u64() + 1 < to.as_u64() => {
             let next = SchemaVersion(from.as_u64() + 1);
-            migrate_schema::<T>(db.clone(), datadir, from, next, log.clone())?;
-            migrate_schema::<T>(db, datadir, next, to, log)
+            migrate_schema::<T>(
+                db.clone(),
+                deposit_contract_deploy_block,
+                from,
+                next,
+                log.clone(),
+                spec,
+            )?;
+            migrate_schema::<T>(db, deposit_contract_deploy_block, next, to, log, spec)
         }
-        // Migration from v0.3.0 to v0.3.x, adding the temporary states column.
-        // Nothing actually needs to be done, but once a DB uses v2 it shouldn't go back.
-        (SchemaVersion(1), SchemaVersion(2)) => {
-            db.store_schema_version(to)?;
-            Ok(())
+        // Downgrade across multiple versions by recursively migrating one step at a time.
+        (_, _) if to.as_u64() + 1 < from.as_u64() => {
+            let next = SchemaVersion(from.as_u64() - 1);
+            migrate_schema::<T>(
+                db.clone(),
+                deposit_contract_deploy_block,
+                from,
+                next,
+                log.clone(),
+                spec,
+            )?;
+            migrate_schema::<T>(db, deposit_contract_deploy_block, next, to, log, spec)
         }
-        // Migration for removing the pubkey cache.
-        (SchemaVersion(2), SchemaVersion(3)) => {
-            let pk_cache_path = datadir.join(PUBKEY_CACHE_FILENAME);
 
-            // Load from file, store to DB.
-            ValidatorPubkeyCache::<T>::load_from_file(&pk_cache_path)
-                .and_then(|cache| ValidatorPubkeyCache::convert(cache, db.clone()))
-                .map_err(|e| StoreError::SchemaMigrationError(format!("{:?}", e)))?;
-
-            db.store_schema_version(to)?;
-
-            // Delete cache file now that keys are stored in the DB.
-            fs::remove_file(&pk_cache_path).map_err(|e| {
-                StoreError::SchemaMigrationError(format!(
-                    "unable to delete {}: {:?}",
-                    pk_cache_path.display(),
-                    e
-                ))
-            })?;
-
-            Ok(())
-        }
-        // Migration for adding sync committee contributions to the persisted op pool.
-        (SchemaVersion(3), SchemaVersion(4)) => {
-            // Deserialize from what exists in the database using the `PersistedOperationPoolBase`
-            // variant and convert it to the Altair variant.
-            let pool_opt = db
-                .get_item::<PersistedOperationPoolBase<T::EthSpec>>(&OP_POOL_DB_KEY)?
-                .map(PersistedOperationPool::Base)
-                .map(PersistedOperationPool::base_to_altair);
-
-            if let Some(pool) = pool_opt {
-                // Store the converted pool under the same key.
-                db.put_item::<PersistedOperationPool<T::EthSpec>>(&OP_POOL_DB_KEY, &pool)?;
-            }
-
-            db.store_schema_version(to)?;
-
-            Ok(())
-        }
-        // Migration for weak subjectivity sync support and clean up of `OnDiskStoreConfig` (#1784).
-        (SchemaVersion(4), SchemaVersion(5)) => {
-            if let Some(OnDiskStoreConfigV4 {
-                slots_per_restore_point,
-                ..
-            }) = db.hot_db.get(&CONFIG_KEY)?
-            {
-                let new_config = OnDiskStoreConfig {
-                    slots_per_restore_point,
-                };
-                db.hot_db.put(&CONFIG_KEY, &new_config)?;
-            }
-
-            db.store_schema_version(to)?;
-
-            Ok(())
-        }
-        // Migration for adding `execution_status` field to the fork choice store.
-        (SchemaVersion(5), SchemaVersion(6)) => {
-            // Database operations to be done atomically
-            let mut ops = vec![];
-
-            // The top-level `PersistedForkChoice` struct is still V1 but will have its internal
-            // bytes for the fork choice updated to V6.
-            let fork_choice_opt = db.get_item::<PersistedForkChoiceV1>(&FORK_CHOICE_DB_KEY)?;
-            if let Some(mut persisted_fork_choice) = fork_choice_opt {
-                migration_schema_v6::update_execution_statuses::<T>(&mut persisted_fork_choice)
-                    .map_err(StoreError::SchemaMigrationError)?;
-
-                // Store the converted fork choice store under the same key.
-                ops.push(persisted_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
-            }
-
-            db.store_schema_version_atomically(to, ops)?;
-
-            Ok(())
-        }
-        // 1. Add `proposer_boost_root`.
-        // 2. Update `justified_epoch` to `justified_checkpoint` and `finalized_epoch` to
-        //  `finalized_checkpoint`.
-        // 3. This migration also includes a potential update to the justified
-        //  checkpoint in case the fork choice store's justified checkpoint and finalized checkpoint
-        //  combination does not actually exist for any blocks in fork choice. This was possible in
-        //  the consensus spec prior to v1.1.6.
         //
-        // Relevant issues:
+        // Migrations from before SchemaVersion(16) are deprecated.
         //
-        // https://github.com/sigp/lighthouse/issues/2741
-        // https://github.com/ethereum/consensus-specs/pull/2727
-        // https://github.com/ethereum/consensus-specs/pull/2730
-        (SchemaVersion(6), SchemaVersion(7)) => {
-            // Database operations to be done atomically
-            let mut ops = vec![];
-
-            let fork_choice_opt = db.get_item::<PersistedForkChoiceV1>(&FORK_CHOICE_DB_KEY)?;
-            if let Some(persisted_fork_choice_v1) = fork_choice_opt {
-                // This migrates the `PersistedForkChoiceStore`, adding the `proposer_boost_root` field.
-                let mut persisted_fork_choice_v7 = persisted_fork_choice_v1.into();
-
-                let result = migration_schema_v7::update_fork_choice::<T>(
-                    &mut persisted_fork_choice_v7,
-                    db.clone(),
-                );
-
-                // Fall back to re-initializing fork choice from an anchor state if necessary.
-                if let Err(e) = result {
-                    warn!(log, "Unable to migrate to database schema 7, re-initializing fork choice"; "error" => ?e);
-                    migration_schema_v7::update_with_reinitialized_fork_choice::<T>(
-                        &mut persisted_fork_choice_v7,
-                        db.clone(),
-                    )
-                    .map_err(StoreError::SchemaMigrationError)?;
-                }
-
-                // Store the converted fork choice store under the same key.
-                ops.push(persisted_fork_choice_v7.as_kv_store_op(FORK_CHOICE_DB_KEY));
-            }
-
-            db.store_schema_version_atomically(to, ops)?;
-
-            Ok(())
+        (SchemaVersion(16), SchemaVersion(17)) => {
+            let ops = migration_schema_v17::upgrade_to_v17::<T>(db.clone(), log)?;
+            db.store_schema_version_atomically(to, ops)
         }
-        // Migration to add an `epoch` key to the fork choice's balances cache.
-        (SchemaVersion(7), SchemaVersion(8)) => {
-            let mut ops = vec![];
-            let fork_choice_opt = db.get_item::<PersistedForkChoiceV7>(&FORK_CHOICE_DB_KEY)?;
-            if let Some(fork_choice) = fork_choice_opt {
-                let updated_fork_choice =
-                    migration_schema_v8::update_fork_choice::<T>(fork_choice, db.clone())?;
-
-                ops.push(updated_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
-            }
-
-            db.store_schema_version_atomically(to, ops)?;
-
-            Ok(())
+        (SchemaVersion(17), SchemaVersion(16)) => {
+            let ops = migration_schema_v17::downgrade_from_v17::<T>(db.clone(), log)?;
+            db.store_schema_version_atomically(to, ops)
+        }
+        (SchemaVersion(17), SchemaVersion(18)) => {
+            let ops = migration_schema_v18::upgrade_to_v18::<T>(db.clone(), log)?;
+            db.store_schema_version_atomically(to, ops)
+        }
+        (SchemaVersion(18), SchemaVersion(17)) => {
+            let ops = migration_schema_v18::downgrade_from_v18::<T>(db.clone(), log)?;
+            db.store_schema_version_atomically(to, ops)
+        }
+        (SchemaVersion(18), SchemaVersion(19)) => {
+            let ops = migration_schema_v19::upgrade_to_v19::<T>(db.clone(), log)?;
+            db.store_schema_version_atomically(to, ops)
+        }
+        (SchemaVersion(19), SchemaVersion(18)) => {
+            let ops = migration_schema_v19::downgrade_from_v19::<T>(db.clone(), log)?;
+            db.store_schema_version_atomically(to, ops)
         }
         // Anything else is an error.
         (_, _) => Err(HotColdDBError::UnsupportedSchemaVersion {
@@ -189,24 +95,3 @@ pub fn migrate_schema<T: BeaconChainTypes>(
         .into()),
     }
 }
-
-// Store config used in v4 schema and earlier.
-#[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)]
-pub struct OnDiskStoreConfigV4 {
-    pub slots_per_restore_point: u64,
-    pub _block_cache_size: usize,
-}
-
-impl StoreItem for OnDiskStoreConfigV4 {
-    fn db_column() -> DBColumn {
-        DBColumn::BeaconMeta
-    }
-
-    fn as_store_bytes(&self) -> Vec<u8> {
-        self.as_ssz_bytes()
-    }
-
-    fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
-        Ok(Self::from_ssz_bytes(bytes)?)
-    }
-}

beacon_node/beacon_chain/src/schema_change/migration_schema_v17.rs

@@ -0,0 +1,88 @@
+use crate::beacon_chain::{BeaconChainTypes, FORK_CHOICE_DB_KEY};
+use crate::persisted_fork_choice::{PersistedForkChoiceV11, PersistedForkChoiceV17};
+use proto_array::core::{SszContainerV16, SszContainerV17};
+use slog::{debug, Logger};
+use ssz::{Decode, Encode};
+use std::sync::Arc;
+use store::{Error, HotColdDB, KeyValueStoreOp, StoreItem};
+
+pub fn upgrade_fork_choice(
+    mut fork_choice: PersistedForkChoiceV11,
+) -> Result<PersistedForkChoiceV17, Error> {
+    let ssz_container_v16 = SszContainerV16::from_ssz_bytes(
+        &fork_choice.fork_choice.proto_array_bytes,
+    )
+    .map_err(|e| {
+        Error::SchemaMigrationError(format!(
+            "Failed to decode ProtoArrayForkChoice during schema migration: {:?}",
+            e
+        ))
+    })?;
+
+    let ssz_container_v17: SszContainerV17 = ssz_container_v16.try_into().map_err(|e| {
+        Error::SchemaMigrationError(format!(
+            "Missing checkpoint during schema migration: {:?}",
+            e
+        ))
+    })?;
+    fork_choice.fork_choice.proto_array_bytes = ssz_container_v17.as_ssz_bytes();
+
+    Ok(fork_choice.into())
+}
+
+pub fn downgrade_fork_choice(
+    mut fork_choice: PersistedForkChoiceV17,
+) -> Result<PersistedForkChoiceV11, Error> {
+    let ssz_container_v17 = SszContainerV17::from_ssz_bytes(
+        &fork_choice.fork_choice.proto_array_bytes,
+    )
+    .map_err(|e| {
+        Error::SchemaMigrationError(format!(
+            "Failed to decode ProtoArrayForkChoice during schema migration: {:?}",
+            e
+        ))
+    })?;
+
+    let ssz_container_v16: SszContainerV16 = ssz_container_v17.into();
+    fork_choice.fork_choice.proto_array_bytes = ssz_container_v16.as_ssz_bytes();
+
+    Ok(fork_choice.into())
+}
+
+pub fn upgrade_to_v17<T: BeaconChainTypes>(
+    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    log: Logger,
+) -> Result<Vec<KeyValueStoreOp>, Error> {
+    // Get persisted_fork_choice.
+    let v11 = db
+        .get_item::<PersistedForkChoiceV11>(&FORK_CHOICE_DB_KEY)?
+        .ok_or_else(|| Error::SchemaMigrationError("fork choice missing from database".into()))?;
+
+    let v17 = upgrade_fork_choice(v11)?;
+
+    debug!(
+        log,
+        "Removing unused best_justified_checkpoint from fork choice store."
+    );
+
+    Ok(vec![v17.as_kv_store_op(FORK_CHOICE_DB_KEY)])
+}
+
+pub fn downgrade_from_v17<T: BeaconChainTypes>(
+    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    log: Logger,
+) -> Result<Vec<KeyValueStoreOp>, Error> {
+    // Get persisted_fork_choice.
+    let v17 = db
+        .get_item::<PersistedForkChoiceV17>(&FORK_CHOICE_DB_KEY)?
+        .ok_or_else(|| Error::SchemaMigrationError("fork choice missing from database".into()))?;
+
+    let v11 = downgrade_fork_choice(v17)?;
+
+    debug!(
+        log,
+        "Adding junk best_justified_checkpoint to fork choice store."
+    );
+
+    Ok(vec![v11.as_kv_store_op(FORK_CHOICE_DB_KEY)])
+}